diff --git a/Makefile b/Makefile index c89fd2f..b4bfdbf 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ virtualenv: netaddr jinja2 lint: virtualenv - .venv/bin/flake8 --exclude hooks/charmhelpers hooks unit_tests tests + .venv/bin/flake8 --exclude hooks/charmhelpers hooks unit_tests tests --ignore E402 @charm proof unit_test: virtualenv diff --git a/config.yaml b/config.yaml index 8b2e69c..07e7676 100644 --- a/config.yaml +++ b/config.yaml @@ -13,10 +13,28 @@ options: type: string default: 'juju-br0' description: The interface connected to PLUMgrid Managment network. + os-data-network: + type: string + default: + description: | + The IP address and netmask of the OpenStack Data network (e.g., + 192.168.0.0/24) + . + This network will be used for tenant network traffic in overlay + networks. + fabric-interfaces: + default: 'MANAGEMENT' + type: string + description: | + Interfaces that will provide fabric connectivity on the gateway nodes. + Provided in form of json in a string. These interfaces have to be connected + to the os-data-network specified in the config. Default value is MANAGEMENT which + will configure the management interface as the fabric interface on each + director. network-device-mtu: type: string default: '1580' - description: The MTU size for interfaces managed by director. + description: The MTU size for interfaces managed by gateway. install_sources: default: 'ppa:plumgrid-team/stable' type: string diff --git a/hooks/pg_gw_context.py b/hooks/pg_gw_context.py index d94f3df..91149bf 100644 --- a/hooks/pg_gw_context.py +++ b/hooks/pg_gw_context.py @@ -3,14 +3,17 @@ # This file contains the class that generates context for # PLUMgrid template files. +from charmhelpers.contrib.openstack import context +from charmhelpers.contrib.openstack.utils import get_host_ip from charmhelpers.core.hookenv import ( relation_ids, related_units, relation_get, ) -from charmhelpers.contrib.openstack import context -from charmhelpers.contrib.openstack.utils import get_host_ip -from socket import gethostname as get_unit_hostname +from socket import ( + gethostname, + getfqdn +) def _pg_dir_settings(): @@ -60,7 +63,7 @@ class PGGwContext(context.NeutronContext): return {} pg_dir_ips = '' - pg_dir_settings = _pg_dir_settings() + pg_dir_settings = sorted(_pg_dir_settings()) single_ip = True for ip in pg_dir_settings: if single_ip: @@ -69,10 +72,16 @@ class PGGwContext(context.NeutronContext): else: pg_dir_ips = pg_dir_ips + ',' + str(ip) pg_ctxt['local_ip'] = pg_dir_ips - unit_hostname = get_unit_hostname() + unit_hostname = gethostname() pg_ctxt['pg_hostname'] = unit_hostname - from pg_gw_utils import get_mgmt_interface, get_gw_interfaces + pg_ctxt['pg_fqdn'] = getfqdn() + from pg_gw_utils import ( + get_mgmt_interface, + get_gw_interfaces, + get_fabric_interface + ) pg_ctxt['interface'] = get_mgmt_interface() + pg_ctxt['fabric_interface'] = get_fabric_interface() pg_ctxt['label'] = unit_hostname pg_ctxt['fabric_mode'] = 'host' pg_ctxt['ext_interfaces'] = get_gw_interfaces() diff --git a/hooks/pg_gw_hooks.py b/hooks/pg_gw_hooks.py index 8aa54d3..5b37e89 100755 --- a/hooks/pg_gw_hooks.py +++ b/hooks/pg_gw_hooks.py @@ -11,6 +11,7 @@ from charmhelpers.core.hookenv import ( Hooks, UnregisteredHookError, log, + config, ) from charmhelpers.fetch import ( @@ -29,6 +30,8 @@ from pg_gw_utils import ( remove_iovisor, ensure_mtu, add_lcm_key, + fabric_interface_changed, + load_iptables, ) hooks = Hooks() @@ -40,6 +43,7 @@ def install(): ''' Install hook is run when the charm is first deployed on a node. ''' + load_iptables() configure_sources(update=True) pkgs = determine_packages() for pkg in pkgs: @@ -73,6 +77,15 @@ def config_changed(): if add_lcm_key(): log("PLUMgrid LCM Key added") return 1 + charm_config = config() + if charm_config.changed('fabric-interfaces'): + if not fabric_interface_changed(): + log("Fabric interface already set") + return 1 + if charm_config.changed('os-data-network'): + if charm_config['fabric-interfaces'] == 'MANAGEMENT': + log('Fabric running on managment network') + return 1 stop_pg() configure_sources(update=True) pkgs = determine_packages() @@ -87,6 +100,11 @@ def config_changed(): restart_pg() +@hooks.hook('upgrade-charm') +def upgrade_charm(): + load_iptables() + + @hooks.hook('stop') def stop(): ''' diff --git a/hooks/pg_gw_utils.py b/hooks/pg_gw_utils.py index d84cfa3..186870f 100644 --- a/hooks/pg_gw_utils.py +++ b/hooks/pg_gw_utils.py @@ -2,8 +2,18 @@ # This file contains functions used by the hooks to deploy PLUMgrid Gateway. -from charmhelpers.contrib.openstack.neutron import neutron_plugin_attribute +import pg_gw_context +import subprocess +import time +import os +import json +from collections import OrderedDict +from socket import gethostname as get_unit_hostname from copy import deepcopy +from charmhelpers.contrib.openstack.neutron import neutron_plugin_attribute +from charmhelpers.contrib.storage.linux.ceph import modprobe +from charmhelpers.core.host import set_nic_mtu +from charmhelpers.contrib.openstack import templating from charmhelpers.core.hookenv import ( log, config, @@ -13,6 +23,8 @@ from charmhelpers.contrib.network.ip import ( get_iface_from_addr, get_bridges, get_bridge_nics, + is_address_in_network, + get_iface_addr ) from charmhelpers.core.host import ( write_file, @@ -20,33 +32,22 @@ from charmhelpers.core.host import ( service_stop, ) from charmhelpers.fetch import ( - apt_cache + apt_cache, + apt_install ) -from charmhelpers.contrib.storage.linux.ceph import modprobe -from charmhelpers.core.host import set_nic_mtu -from charmhelpers.contrib.openstack import templating -from collections import OrderedDict from charmhelpers.contrib.openstack.utils import ( os_release, ) -from socket import gethostname as get_unit_hostname -import pg_gw_context -import subprocess -import time -import os -import json LXC_CONF = "/etc/libvirt/lxc.conf" TEMPLATES = 'templates/' PG_LXC_DATA_PATH = '/var/lib/libvirt/filesystems/plumgrid-data' - PG_CONF = '%s/conf/pg/plumgrid.conf' % PG_LXC_DATA_PATH PG_HN_CONF = '%s/conf/etc/hostname' % PG_LXC_DATA_PATH PG_HS_CONF = '%s/conf/etc/hosts' % PG_LXC_DATA_PATH PG_IFCS_CONF = '%s/conf/pg/ifcs.conf' % PG_LXC_DATA_PATH AUTH_KEY_PATH = '%s/root/.ssh/authorized_keys' % PG_LXC_DATA_PATH IFC_LIST_GW = '/var/run/plumgrid/lxc/ifc_list_gateway' - SUDOERS_CONF = '/etc/sudoers.d/ifc_ctl_sudoers' BASE_RESOURCE_MAP = OrderedDict([ @@ -139,9 +140,7 @@ def restart_pg(): ''' Stops and Starts PLUMgrid service after flushing iptables. ''' - service_stop('plumgrid') - time.sleep(30) - _exec_cmd(cmd=['iptables', '-F']) + stop_pg() service_start('plumgrid') time.sleep(30) @@ -151,7 +150,7 @@ def stop_pg(): Stops PLUMgrid service. ''' service_stop('plumgrid') - time.sleep(2) + time.sleep(30) def load_iovisor(): @@ -166,26 +165,27 @@ def remove_iovisor(): Removes iovisor kernel module. ''' _exec_cmd(cmd=['rmmod', 'iovisor'], - error_msg='Error Loading Iovisor Kernel Module') + error_msg='Error Removing IOVisor Kernel Module') time.sleep(1) +def interface_exists(interface): + ''' + Checks if interface exists on node. + ''' + try: + subprocess.check_call(['ip', 'link', 'show', interface], + stdout=open(os.devnull, 'w'), + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + return False + return True + + def get_mgmt_interface(): ''' Returns the managment interface. ''' - def interface_exists(interface): - ''' - Checks if interface exists on node. - ''' - try: - subprocess.check_call(['ip', 'link', 'show', interface], - stdout=open(os.devnull, 'w'), - stderr=subprocess.STDOUT) - except subprocess.CalledProcessError: - return False - return True - mgmt_interface = config('mgmt-interface') if interface_exists(mgmt_interface): return mgmt_interface @@ -195,20 +195,74 @@ def get_mgmt_interface(): return get_iface_from_addr(unit_get('private-address')) +def fabric_interface_changed(): + ''' + Returns true if interface for node changed. + ''' + fabric_interface = get_fabric_interface() + try: + with open(PG_IFCS_CONF, 'r') as ifcs: + for line in ifcs: + if 'fabric_core' in line: + if line.split()[0] == fabric_interface: + return False + except IOError: + return True + return True + + +def get_fabric_interface(): + ''' + Returns the fabric interface. + ''' + fabric_interfaces = config('fabric-interfaces') + if fabric_interfaces == 'MANAGEMENT': + return get_mgmt_interface() + else: + try: + all_fabric_interfaces = json.loads(fabric_interfaces) + except ValueError: + raise ValueError('Invalid json provided for fabric interfaces') + hostname = get_unit_hostname() + if hostname in all_fabric_interfaces: + node_fabric_interface = all_fabric_interfaces[hostname] + elif 'DEFAULT' in all_fabric_interfaces: + node_fabric_interface = all_fabric_interfaces['DEFAULT'] + else: + raise ValueError('No fabric interface provided for node') + if interface_exists(node_fabric_interface): + if is_address_in_network(config('os-data-network'), + get_iface_addr(node_fabric_interface)[0]): + return node_fabric_interface + else: + raise ValueError('Fabric interface not in fabric network') + else: + log('Provided fabric interface %s does not exist' + % node_fabric_interface) + raise ValueError('Provided fabric interface does not exist') + return node_fabric_interface + + def get_gw_interfaces(): ''' Gateway node can have multiple interfaces. This function parses json provided in config to get all gateway interfaces for this node. ''' - node_interfaces = ['eth1'] + node_interfaces = [] try: all_interfaces = json.loads(config('external-interfaces')) except ValueError: - log("Invalid JSON") - return node_interfaces + raise ValueError("Invalid json provided for gateway interfaces") hostname = get_unit_hostname() if hostname in all_interfaces: node_interfaces = all_interfaces[hostname].split(',') + elif 'DEFAULT' in all_interfaces: + node_interfaces = all_interfaces['DEFAULT'].split(',') + for interface in node_interfaces: + if not interface_exists(interface): + log('Provided gateway interface %s does not exist' + % interface) + raise ValueError('Provided gateway interface does not exist') return node_interfaces @@ -217,12 +271,12 @@ def ensure_mtu(): Ensures required MTU of the underlying networking of the node. ''' interface_mtu = config('network-device-mtu') - mgmt_interface = get_mgmt_interface() - if mgmt_interface in get_bridges(): - attached_interfaces = get_bridge_nics(mgmt_interface) + fabric_interface = get_fabric_interface() + if fabric_interface in get_bridges(): + attached_interfaces = get_bridge_nics(fabric_interface) for interface in attached_interfaces: set_nic_mtu(interface, interface_mtu) - set_nic_mtu(mgmt_interface, interface_mtu) + set_nic_mtu(fabric_interface, interface_mtu) def _exec_cmd(cmd=None, error_msg='Command exited with ERRORs', fatal=False): @@ -270,3 +324,48 @@ def add_lcm_key(): fa.write('\n') fa.close() return 1 + + +def load_iptables(): + ''' + Loads iptables rules to allow all PLUMgrid communication. + ''' + network = get_cidr_from_iface(get_mgmt_interface()) + if network: + _exec_cmd(['sudo', 'iptables', '-A', 'INPUT', '-p', 'tcp', + '-j', 'ACCEPT', '-s', network, '-d', + network, '-m', 'state', '--state', 'NEW']) + _exec_cmd(['sudo', 'iptables', '-A', 'INPUT', '-p', 'udp', '-j', + 'ACCEPT', '-s', network, '-d', network, + '-m', 'state', '--state', 'NEW']) + apt_install('iptables-persistent') + + +def get_cidr_from_iface(interface): + ''' + Determines Network CIDR from interface. + ''' + if not interface: + return None + apt_install('ohai') + try: + os_info = subprocess.check_output(['ohai', '-l', 'fatal']) + except OSError: + log('Unable to get operating system information') + return None + try: + os_info_json = json.loads(os_info) + except ValueError: + log('Unable to determine network') + return None + device = os_info_json['network']['interfaces'].get(interface) + if device is not None: + if device.get('routes'): + routes = device['routes'] + for net in routes: + if 'scope' in net: + return net.get('destination') + else: + return None + else: + return None diff --git a/hooks/upgrade-charm b/hooks/upgrade-charm new file mode 120000 index 0000000..3aec9ba --- /dev/null +++ b/hooks/upgrade-charm @@ -0,0 +1 @@ +pg_gw_hooks.py \ No newline at end of file diff --git a/templates/kilo/hosts b/templates/kilo/hosts index 99e3be5..e19e49d 100644 --- a/templates/kilo/hosts +++ b/templates/kilo/hosts @@ -1,5 +1,5 @@ 127.0.0.1 localhost -127.0.1.1 {{ pg_hostname }} +127.0.1.1 {{ pg_fqdn }} {{ pg_hostname }} # The following lines are desirable for IPv6 capable hosts ::1 ip6-localhost ip6-loopback diff --git a/templates/kilo/ifcs.conf b/templates/kilo/ifcs.conf index 40166fe..657fbd5 100644 --- a/templates/kilo/ifcs.conf +++ b/templates/kilo/ifcs.conf @@ -1,4 +1,4 @@ -{{ interface }} = fabric_core host +{{ fabric_interface }} = fabric_core host {% if ext_interfaces -%} {% for ip in ext_interfaces -%} {{ ip }} = access_phys diff --git a/unit_tests/test_pg_gw_context.py b/unit_tests/test_pg_gw_context.py index 6604f1f..7fa2c16 100644 --- a/unit_tests/test_pg_gw_context.py +++ b/unit_tests/test_pg_gw_context.py @@ -5,7 +5,8 @@ import pg_gw_utils as utils import charmhelpers TO_PATCH = [ - 'get_unit_hostname', + 'gethostname', + 'getfqdn' ] @@ -38,9 +39,10 @@ class PGGwContextTest(CharmTestCase): @patch.object(charmhelpers.contrib.openstack.context, 'neutron_plugin_attribute') @patch.object(utils, 'get_mgmt_interface') + @patch.object(utils, 'get_fabric_interface') @patch.object(utils, 'get_gw_interfaces') - def test_neutroncc_context_api_rel(self, _gw_int, _mgmt_int, - _npa, _pg_dir_settings, + def test_neutroncc_context_api_rel(self, _gw_int, _fabric_int, + _mgmt_int, _npa, _pg_dir_settings, _save_flag_file, _config_flag, _unit_get, _unit_priv_ip, _config, _is_clus, _https, _ens_pkgs): @@ -54,11 +56,13 @@ class PGGwContextTest(CharmTestCase): _npa.side_effect = mock_npa _unit_get.return_value = '192.168.100.201' _unit_priv_ip.return_value = '192.168.100.201' - self.get_unit_hostname.return_value = 'node0' + self.gethostname.return_value = 'node0' + self.getfqdn.return_value = 'node0' _is_clus.return_value = False _config_flag.return_value = False _pg_dir_settings.return_value = {'pg_dir_ip': '192.168.100.201'} _mgmt_int.return_value = 'juju-br0' + _fabric_int.return_value = 'juju-br0' _gw_int.return_value = ['eth1'] napi_ctxt = context.PGGwContext() expect = { @@ -71,7 +75,9 @@ class PGGwContextTest(CharmTestCase): 'neutron_security_groups': None, 'neutron_url': 'https://192.168.100.201:9696', 'pg_hostname': 'node0', + 'pg_fqdn': 'node0', 'interface': 'juju-br0', + 'fabric_interface': 'juju-br0', 'label': 'node0', 'fabric_mode': 'host', 'neutron_alchemy_flags': False, diff --git a/unit_tests/test_pg_gw_hooks.py b/unit_tests/test_pg_gw_hooks.py index bebdb9d..a1b5779 100644 --- a/unit_tests/test_pg_gw_hooks.py +++ b/unit_tests/test_pg_gw_hooks.py @@ -1,5 +1,6 @@ from mock import MagicMock, patch, call from test_utils import CharmTestCase + with patch('charmhelpers.core.hookenv.config') as config: config.return_value = 'neutron' import pg_gw_utils as utils @@ -29,6 +30,7 @@ TO_PATCH = [ 'ensure_mtu', 'add_lcm_key', 'determine_packages', + 'load_iptables' ] NEUTRON_CONF_DIR = "/etc/neutron" @@ -69,21 +71,8 @@ class PGGwHooksTests(CharmTestCase): self.restart_pg.assert_called_with() def test_config_changed_hook(self): - _pkgs = ['plumgrid-lxc', 'iovisor-dkms'] - self.add_lcm_key.return_value = 0 - self.determine_packages.return_value = [_pkgs] + self.add_lcm_key.return_value = 1 self._call_hook('config-changed') - self.stop_pg.assert_called_with() - self.configure_sources.assert_called_with(update=True) - self.apt_install.assert_has_calls([ - call(_pkgs, fatal=True, - options=['--force-yes']), - ]) - self.load_iovisor.assert_called_with() - self.ensure_mtu.assert_called_with() - self.ensure_files.assert_called_with() - self.CONFIGS.write_all.assert_called_with() - self.restart_pg.assert_called_with() def test_stop(self): _pkgs = ['plumgrid-lxc', 'iovisor-dkms']