From 19412720272a2cd148eaf4fffe53a99095219501 Mon Sep 17 00:00:00 2001 From: Zhang Hua Date: Wed, 19 Nov 2014 15:02:40 +0800 Subject: [PATCH 01/75] modify MTU of tunnel nic more than 1500 --- config.yaml | 6 ++++++ hooks/quantum_hooks.py | 5 ++++- hooks/quantum_utils.py | 23 +++++++++++++++++++++-- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/config.yaml b/config.yaml index aea9d3c6..7a316ec2 100644 --- a/config.yaml +++ b/config.yaml @@ -109,3 +109,9 @@ options: . This network will be used for tenant network traffic in overlay networks. + tunnel-nic-mtu: + type: int + default: 1500 + description: | + To improve network performance of VM, sometimes we should keep VM MTU as 1500 + and use charm to modify MTU of tunnel nic more than 1500 (e.g. 1546 for GRE) diff --git a/hooks/quantum_hooks.py b/hooks/quantum_hooks.py index eb7bfb50..53408e22 100755 --- a/hooks/quantum_hooks.py +++ b/hooks/quantum_hooks.py @@ -45,7 +45,8 @@ from quantum_utils import ( valid_plugin, configure_ovs, reassign_agent_resources, - stop_services + stop_services, + configure_mtu ) hooks = Hooks() @@ -66,6 +67,7 @@ def install(): fatal=True) apt_install(filter_installed_packages(get_packages()), fatal=True) + configure_mtu() else: log('Please provide a valid plugin config', level=ERROR) sys.exit(1) @@ -89,6 +91,7 @@ def config_changed(): if valid_plugin(): CONFIGS.write_all() configure_ovs() + configure_mtu() else: log('Please provide a valid plugin config', level=ERROR) sys.exit(1) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index e07a7e00..7c257d0a 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -2,14 +2,20 @@ from charmhelpers.core.host import ( service_running, service_stop, service_restart, - lsb_release + lsb_release, + list_nics, + set_nic_mtu +) +from charmhelpers.contrib.network.ip import ( + get_ipv4_addr ) from charmhelpers.core.hookenv import ( - log, + log, INFO, config, relations_of_type, unit_private_ip, is_relation_made, + unit_get ) from charmhelpers.fetch import ( apt_upgrade, @@ -569,3 +575,16 @@ def configure_ovs(): ext_port_ctx = ExternalPortContext()() if ext_port_ctx is not None and ext_port_ctx['ext_port']: add_bridge_port(EXT_BRIDGE, ext_port_ctx['ext_port']) + +def configure_mtu(): + tunnel_nic_mtu = config('tunnel-nic-mtu') + if tunnel_nic_mtu > 1500: + tunnel_ip = unit_get('private-address') + tunnel_nic = 'eth0' + for nic in list_nics(['eth', 'bond']): + if tunnel_ip in get_ipv4_addr(nic, fatal=False): + tunnel_nic = nic + break + set_nic_mtu(tunnel_nic, tunnel_nic_mtu) + log('set mtu={} for tunnel nic={}' + .format(tunnel_nic_mtu, tunnel_nic), level=INFO) From b2d9d9b2037b130dda8392364d22dab5c7eff342 Mon Sep 17 00:00:00 2001 From: Zhang Hua Date: Wed, 19 Nov 2014 16:12:48 +0800 Subject: [PATCH 02/75] fix error TypeError: execv() arg 2 must contain only strings --- hooks/quantum_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 7c257d0a..8b102438 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -585,6 +585,6 @@ def configure_mtu(): if tunnel_ip in get_ipv4_addr(nic, fatal=False): tunnel_nic = nic break - set_nic_mtu(tunnel_nic, tunnel_nic_mtu) + set_nic_mtu(tunnel_nic, str(tunnel_nic_mtu)) log('set mtu={} for tunnel nic={}' .format(tunnel_nic_mtu, tunnel_nic), level=INFO) From 0b151032d87a12558f19a56d71d9ebd7b532291b Mon Sep 17 00:00:00 2001 From: Zhang Hua Date: Wed, 19 Nov 2014 16:46:46 +0800 Subject: [PATCH 03/75] allow change mtu back to 1500 --- hooks/quantum_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 8b102438..ba6e10dd 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -578,7 +578,7 @@ def configure_ovs(): def configure_mtu(): tunnel_nic_mtu = config('tunnel-nic-mtu') - if tunnel_nic_mtu > 1500: + if tunnel_nic_mtu >= 1500: tunnel_ip = unit_get('private-address') tunnel_nic = 'eth0' for nic in list_nics(['eth', 'bond']): From 87b99ce676d0733783cff6d84d4bc4d9d5b90e77 Mon Sep 17 00:00:00 2001 From: Zhang Hua Date: Thu, 20 Nov 2014 18:57:45 +0800 Subject: [PATCH 04/75] fix lint and unit test error --- hooks/quantum_utils.py | 1 + unit_tests/test_quantum_hooks.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index ba6e10dd..8f5a8f43 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -576,6 +576,7 @@ def configure_ovs(): if ext_port_ctx is not None and ext_port_ctx['ext_port']: add_bridge_port(EXT_BRIDGE, ext_port_ctx['ext_port']) + def configure_mtu(): tunnel_nic_mtu = config('tunnel-nic-mtu') if tunnel_nic_mtu >= 1500: diff --git a/unit_tests/test_quantum_hooks.py b/unit_tests/test_quantum_hooks.py index c5f0520f..f26620a9 100644 --- a/unit_tests/test_quantum_hooks.py +++ b/unit_tests/test_quantum_hooks.py @@ -40,7 +40,8 @@ TO_PATCH = [ 'lsb_release', 'stop_services', 'b64decode', - 'is_relation_made' + 'is_relation_made', + 'configure_mtu' ] @@ -80,6 +81,7 @@ class TestQuantumHooks(CharmTestCase): self.assertTrue(self.get_early_packages.called) self.assertTrue(self.get_packages.called) self.assertTrue(self.execd_preinstall.called) + self.assertTrue(self.configure_mtu.called) def test_install_hook_precise_nocloudarchive(self): self.test_config.set('openstack-origin', 'distro') @@ -112,6 +114,7 @@ class TestQuantumHooks(CharmTestCase): self.assertTrue(_pgsql_db_joined.called) self.assertTrue(_amqp_joined.called) self.assertTrue(_amqp_nova_joined.called) + self.assertTrue(self.configure_mtu.called) def test_config_changed_upgrade(self): self.openstack_upgrade_available.return_value = True @@ -119,6 +122,7 @@ class TestQuantumHooks(CharmTestCase): self._call_hook('config-changed') self.assertTrue(self.do_openstack_upgrade.called) self.assertTrue(self.configure_ovs.called) + self.assertTrue(self.configure_mtu.called) def test_config_changed_n1kv(self): self.openstack_upgrade_available.return_value = False From 5125be6e7c22ea4a3ba3742adf9383e9a38fd12f Mon Sep 17 00:00:00 2001 From: Zhang Hua Date: Wed, 3 Dec 2014 21:16:58 +0800 Subject: [PATCH 05/75] sync charm-helpers --- charm-helpers-hooks.yaml | 2 +- .../charmhelpers/contrib/hahelpers/cluster.py | 23 +- hooks/charmhelpers/contrib/network/ip.py | 134 +++-- .../contrib/openstack/amulet/deployment.py | 3 +- .../contrib/openstack/amulet/utils.py | 4 +- .../charmhelpers/contrib/openstack/context.py | 545 ++++++++++-------- hooks/charmhelpers/contrib/openstack/ip.py | 68 ++- .../charmhelpers/contrib/openstack/neutron.py | 24 +- .../contrib/openstack/templating.py | 10 +- hooks/charmhelpers/contrib/openstack/utils.py | 47 +- .../contrib/storage/linux/ceph.py | 187 +++--- .../contrib/storage/linux/loopback.py | 8 +- .../charmhelpers/contrib/storage/linux/lvm.py | 1 + .../contrib/storage/linux/utils.py | 5 +- hooks/charmhelpers/core/fstab.py | 18 +- hooks/charmhelpers/core/hookenv.py | 36 +- hooks/charmhelpers/core/host.py | 71 ++- hooks/charmhelpers/core/services/__init__.py | 4 +- hooks/charmhelpers/core/services/helpers.py | 12 +- hooks/charmhelpers/core/templating.py | 3 +- hooks/charmhelpers/fetch/__init__.py | 30 +- hooks/charmhelpers/fetch/archiveurl.py | 69 ++- hooks/charmhelpers/fetch/bzrurl.py | 6 +- 23 files changed, 795 insertions(+), 515 deletions(-) diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index 1a98c81c..05e37a4e 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -1,4 +1,4 @@ -branch: lp:charm-helpers +branch: lp:~zhhuabj/charms/trusty/charm-helpers/lp74646 destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index 6d972007..52ce4b7c 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -13,9 +13,10 @@ clustering-related helpers. import subprocess import os - from socket import gethostname as get_unit_hostname +import six + from charmhelpers.core.hookenv import ( log, relation_ids, @@ -77,7 +78,7 @@ def is_crm_leader(resource): "show", resource ] try: - status = subprocess.check_output(cmd) + status = subprocess.check_output(cmd).decode('UTF-8') except subprocess.CalledProcessError: return False else: @@ -150,34 +151,42 @@ def https(): return False -def determine_api_port(public_port): +def determine_api_port(public_port, singlenode_mode=False): ''' Determine correct API server listening port based on existence of HTTPS reverse proxy and/or haproxy. public_port: int: standard public port for given service + singlenode_mode: boolean: Shuffle ports when only a single unit is present + returns: int: the correct listening port for the API service ''' i = 0 - if len(peer_units()) > 0 or is_clustered(): + if singlenode_mode: + i += 1 + elif len(peer_units()) > 0 or is_clustered(): i += 1 if https(): i += 1 return public_port - (i * 10) -def determine_apache_port(public_port): +def determine_apache_port(public_port, singlenode_mode=False): ''' Description: Determine correct apache listening port based on public IP + state of the cluster. public_port: int: standard public port for given service + singlenode_mode: boolean: Shuffle ports when only a single unit is present + returns: int: the correct listening port for the HAProxy service ''' i = 0 - if len(peer_units()) > 0 or is_clustered(): + if singlenode_mode: + i += 1 + elif len(peer_units()) > 0 or is_clustered(): i += 1 return public_port - (i * 10) @@ -197,7 +206,7 @@ def get_hacluster_config(): for setting in settings: conf[setting] = config_get(setting) missing = [] - [missing.append(s) for s, v in conf.iteritems() if v is None] + [missing.append(s) for s, v in six.iteritems(conf) if v is None] if missing: log('Insufficient config data to configure hacluster.', level=ERROR) raise HAIncompleteConfig diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index e62e5655..b6e69c19 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -1,16 +1,20 @@ import glob import re import subprocess -import sys from functools import partial from charmhelpers.core.hookenv import unit_get from charmhelpers.fetch import apt_install from charmhelpers.core.hookenv import ( - WARNING, - ERROR, - log + config, + log, + INFO +) +from charmhelpers.core.host import ( + list_nics, + get_nic_mtu, + set_nic_mtu ) try: @@ -34,31 +38,28 @@ def _validate_cidr(network): network) +def no_ip_found_error_out(network): + errmsg = ("No IP address found in network: %s" % network) + raise ValueError(errmsg) + + def get_address_in_network(network, fallback=None, fatal=False): - """ - Get an IPv4 or IPv6 address within the network from the host. + """Get an IPv4 or IPv6 address within the network from the host. :param network (str): CIDR presentation format. For example, '192.168.1.0/24'. :param fallback (str): If no address is found, return fallback. :param fatal (boolean): If no address is found, fallback is not set and fatal is True then exit(1). - """ - - def not_found_error_out(): - log("No IP address found in network: %s" % network, - level=ERROR) - sys.exit(1) - if network is None: if fallback is not None: return fallback + + if fatal: + no_ip_found_error_out(network) else: - if fatal: - not_found_error_out() - else: - return None + return None _validate_cidr(network) network = netaddr.IPNetwork(network) @@ -70,6 +71,7 @@ def get_address_in_network(network, fallback=None, fatal=False): cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) if cidr in network: return str(cidr.ip) + if network.version == 6 and netifaces.AF_INET6 in addresses: for addr in addresses[netifaces.AF_INET6]: if not addr['addr'].startswith('fe80'): @@ -82,20 +84,20 @@ def get_address_in_network(network, fallback=None, fatal=False): return fallback if fatal: - not_found_error_out() + no_ip_found_error_out(network) return None def is_ipv6(address): - '''Determine whether provided address is IPv6 or not''' + """Determine whether provided address is IPv6 or not.""" try: address = netaddr.IPAddress(address) except netaddr.AddrFormatError: # probably a hostname - so not an address at all! return False - else: - return address.version == 6 + + return address.version == 6 def is_address_in_network(network, address): @@ -113,11 +115,13 @@ def is_address_in_network(network, address): except (netaddr.core.AddrFormatError, ValueError): raise ValueError("Network (%s) is not in CIDR presentation format" % network) + try: address = netaddr.IPAddress(address) except (netaddr.core.AddrFormatError, ValueError): raise ValueError("Address (%s) is not in correct presentation format" % address) + if address in network: return True else: @@ -147,6 +151,7 @@ def _get_for_address(address, key): return iface else: return addresses[netifaces.AF_INET][0][key] + if address.version == 6 and netifaces.AF_INET6 in addresses: for addr in addresses[netifaces.AF_INET6]: if not addr['addr'].startswith('fe80'): @@ -160,41 +165,42 @@ def _get_for_address(address, key): return str(cidr).split('/')[1] else: return addr[key] + return None get_iface_for_address = partial(_get_for_address, key='iface') + get_netmask_for_address = partial(_get_for_address, key='netmask') def format_ipv6_addr(address): - """ - IPv6 needs to be wrapped with [] in url link to parse correctly. + """If address is IPv6, wrap it in '[]' otherwise return None. + + This is required by most configuration files when specifying IPv6 + addresses. """ if is_ipv6(address): - address = "[%s]" % address - else: - log("Not a valid ipv6 address: %s" % address, level=WARNING) - address = None + return "[%s]" % address - return address + return None def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, fatal=True, exc_list=None): - """ - Return the assigned IP address for a given interface, if any, or []. - """ + """Return the assigned IP address for a given interface, if any.""" # Extract nic if passed /dev/ethX if '/' in iface: iface = iface.split('/')[-1] + if not exc_list: exc_list = [] + try: inet_num = getattr(netifaces, inet_type) except AttributeError: - raise Exception('Unknown inet type ' + str(inet_type)) + raise Exception("Unknown inet type '%s'" % str(inet_type)) interfaces = netifaces.interfaces() if inc_aliases: @@ -202,15 +208,18 @@ def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, for _iface in interfaces: if iface == _iface or _iface.split(':')[0] == iface: ifaces.append(_iface) + if fatal and not ifaces: raise Exception("Invalid interface '%s'" % iface) + ifaces.sort() else: if iface not in interfaces: if fatal: - raise Exception("%s not found " % (iface)) + raise Exception("Interface '%s' not found " % (iface)) else: return [] + else: ifaces = [iface] @@ -221,10 +230,13 @@ def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, for entry in net_info[inet_num]: if 'addr' in entry and entry['addr'] not in exc_list: addresses.append(entry['addr']) + if fatal and not addresses: raise Exception("Interface '%s' doesn't have any %s addresses." % (iface, inet_type)) - return addresses + + return sorted(addresses) + get_ipv4_addr = partial(get_iface_addr, inet_type='AF_INET') @@ -241,6 +253,7 @@ def get_iface_from_addr(addr): raw = re.match(ll_key, _addr) if raw: _addr = raw.group(1) + if _addr == addr: log("Address '%s' is configured on iface '%s'" % (addr, iface)) @@ -251,8 +264,9 @@ def get_iface_from_addr(addr): def sniff_iface(f): - """If no iface provided, inject net iface inferred from unit private - address. + """Ensure decorated function is called with a value for iface. + + If no iface provided, inject net iface inferred from unit private address. """ def iface_sniffer(*args, **kwargs): if not kwargs.get('iface', None): @@ -295,7 +309,7 @@ def get_ipv6_addr(iface=None, inc_aliases=False, fatal=True, exc_list=None, if global_addrs: # Make sure any found global addresses are not temporary cmd = ['ip', 'addr', 'show', iface] - out = subprocess.check_output(cmd) + out = subprocess.check_output(cmd).decode('UTF-8') if dynamic_only: key = re.compile("inet6 (.+)/[0-9]+ scope global dynamic.*") else: @@ -317,33 +331,51 @@ def get_ipv6_addr(iface=None, inc_aliases=False, fatal=True, exc_list=None, return addrs if fatal: - raise Exception("Interface '%s' doesn't have a scope global " + raise Exception("Interface '%s' does not have a scope global " "non-temporary ipv6 address." % iface) return [] def get_bridges(vnic_dir='/sys/devices/virtual/net'): - """ - Return a list of bridges on the system or [] - """ - b_rgex = vnic_dir + '/*/bridge' - return [x.replace(vnic_dir, '').split('/')[1] for x in glob.glob(b_rgex)] + """Return a list of bridges on the system.""" + b_regex = "%s/*/bridge" % vnic_dir + return [x.replace(vnic_dir, '').split('/')[1] for x in glob.glob(b_regex)] def get_bridge_nics(bridge, vnic_dir='/sys/devices/virtual/net'): - """ - Return a list of nics comprising a given bridge on the system or [] - """ - brif_rgex = "%s/%s/brif/*" % (vnic_dir, bridge) - return [x.split('/')[-1] for x in glob.glob(brif_rgex)] + """Return a list of nics comprising a given bridge on the system.""" + brif_regex = "%s/%s/brif/*" % (vnic_dir, bridge) + return [x.split('/')[-1] for x in glob.glob(brif_regex)] def is_bridge_member(nic): - """ - Check if a given nic is a member of a bridge - """ + """Check if a given nic is a member of a bridge.""" for bridge in get_bridges(): if nic in get_bridge_nics(bridge): return True + return False + + +def configure_phy_nic_mtu(mng_ip=None): + """Configure mtu for physical nic.""" + phy_nic_mtu = config('phy-nic-mtu') + if phy_nic_mtu >= 1500: + phy_nic = None + if mng_ip is None: + mng_ip = unit_get('private-address') + for nic in list_nics(['eth', 'bond', 'br']): + if mng_ip in get_ipv4_addr(nic, fatal=False): + phy_nic = nic + # need to find the associated phy nic for bridge + if nic.startswith('br'): + for brnic in get_bridge_nics(nic): + if brnic.startswith('eth') or brnic.startswith('bond'): + phy_nic = brnic + break + break + if phy_nic is not None and phy_nic_mtu != get_nic_mtu(phy_nic): + set_nic_mtu(phy_nic, str(phy_nic_mtu)) + log('set mtu={} for phy_nic={}' + .format(phy_nic_mtu, phy_nic), level=INFO) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 3c7f422a..f3fee074 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -1,3 +1,4 @@ +import six from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment ) @@ -69,7 +70,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): def _configure_services(self, configs): """Configure all of the services.""" - for service, config in configs.iteritems(): + for service, config in six.iteritems(configs): self.d.configure(service, config) def _get_openstack_release(self): diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index 0f312b99..3e0cc61c 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -7,6 +7,8 @@ import glanceclient.v1.client as glance_client import keystoneclient.v2_0 as keystone_client import novaclient.v1_1.client as nova_client +import six + from charmhelpers.contrib.amulet.utils import ( AmuletUtils ) @@ -60,7 +62,7 @@ class OpenStackAmuletUtils(AmuletUtils): expected service catalog endpoints. """ self.log.debug('actual: {}'.format(repr(actual))) - for k, v in expected.iteritems(): + for k, v in six.iteritems(expected): if k in actual: ret = self._validate_dict_data(expected[k][0], actual[k][0]) if ret: diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 538dc913..eebe8c03 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -1,20 +1,18 @@ import json import os import time - from base64 import b64decode +from subprocess import check_call -from subprocess import ( - check_call -) +import six from charmhelpers.fetch import ( apt_install, filter_installed_packages, ) - from charmhelpers.core.hookenv import ( config, + is_relation_made, local_unit, log, relation_get, @@ -23,43 +21,40 @@ from charmhelpers.core.hookenv import ( relation_set, unit_get, unit_private_ip, + DEBUG, + INFO, + WARNING, ERROR, - INFO ) - from charmhelpers.core.host import ( mkdir, - write_file + write_file, ) - from charmhelpers.contrib.hahelpers.cluster import ( determine_apache_port, determine_api_port, https, - is_clustered + is_clustered, ) - from charmhelpers.contrib.hahelpers.apache import ( get_cert, get_ca_cert, install_ca_cert, ) - from charmhelpers.contrib.openstack.neutron import ( neutron_plugin_attribute, ) - from charmhelpers.contrib.network.ip import ( get_address_in_network, get_ipv6_addr, get_netmask_for_address, format_ipv6_addr, - is_address_in_network + is_address_in_network, ) - from charmhelpers.contrib.openstack.utils import get_host_ip CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' +ADDRESS_TYPES = ['admin', 'internal', 'public'] class OSContextError(Exception): @@ -67,7 +62,7 @@ class OSContextError(Exception): def ensure_packages(packages): - '''Install but do not upgrade required plugin packages''' + """Install but do not upgrade required plugin packages.""" required = filter_installed_packages(packages) if required: apt_install(required, fatal=True) @@ -75,20 +70,27 @@ def ensure_packages(packages): def context_complete(ctxt): _missing = [] - for k, v in ctxt.iteritems(): + for k, v in six.iteritems(ctxt): if v is None or v == '': _missing.append(k) + if _missing: - log('Missing required data: %s' % ' '.join(_missing), level='INFO') + log('Missing required data: %s' % ' '.join(_missing), level=INFO) return False + return True def config_flags_parser(config_flags): + """Parses config flags string into dict. + + The provided config_flags string may be a list of comma-separated values + which themselves may be comma-separated list of values. + """ if config_flags.find('==') >= 0: - log("config_flags is not in expected format (key=value)", - level=ERROR) + log("config_flags is not in expected format (key=value)", level=ERROR) raise OSContextError + # strip the following from each value. post_strippers = ' ,' # we strip any leading/trailing '=' or ' ' from the string then @@ -96,7 +98,7 @@ def config_flags_parser(config_flags): split = config_flags.strip(' =').split('=') limit = len(split) flags = {} - for i in xrange(0, limit - 1): + for i in range(0, limit - 1): current = split[i] next = split[i + 1] vindex = next.rfind(',') @@ -111,17 +113,18 @@ def config_flags_parser(config_flags): # if this not the first entry, expect an embedded key. index = current.rfind(',') if index < 0: - log("invalid config value(s) at index %s" % (i), - level=ERROR) + log("Invalid config value(s) at index %s" % (i), level=ERROR) raise OSContextError key = current[index + 1:] # Add to collection. flags[key.strip(post_strippers)] = value.rstrip(post_strippers) + return flags class OSContextGenerator(object): + """Base class for all context generators.""" interfaces = [] def __call__(self): @@ -133,11 +136,11 @@ class SharedDBContext(OSContextGenerator): def __init__(self, database=None, user=None, relation_prefix=None, ssl_dir=None): - ''' - Allows inspecting relation for settings prefixed with relation_prefix. - This is useful for parsing access for multiple databases returned via - the shared-db interface (eg, nova_password, quantum_password) - ''' + """Allows inspecting relation for settings prefixed with + relation_prefix. This is useful for parsing access for multiple + databases returned via the shared-db interface (eg, nova_password, + quantum_password) + """ self.relation_prefix = relation_prefix self.database = database self.user = user @@ -147,9 +150,8 @@ class SharedDBContext(OSContextGenerator): self.database = self.database or config('database') self.user = self.user or config('database-user') if None in [self.database, self.user]: - log('Could not generate shared_db context. ' - 'Missing required charm config options. ' - '(database name and user)') + log("Could not generate shared_db context. Missing required charm " + "config options. (database name and user)", level=ERROR) raise OSContextError ctxt = {} @@ -202,23 +204,24 @@ class PostgresqlDBContext(OSContextGenerator): def __call__(self): self.database = self.database or config('database') if self.database is None: - log('Could not generate postgresql_db context. ' - 'Missing required charm config options. ' - '(database name)') + log('Could not generate postgresql_db context. Missing required ' + 'charm config options. (database name)', level=ERROR) raise OSContextError - ctxt = {} + ctxt = {} for rid in relation_ids(self.interfaces[0]): for unit in related_units(rid): - ctxt = { - 'database_host': relation_get('host', rid=rid, unit=unit), - 'database': self.database, - 'database_user': relation_get('user', rid=rid, unit=unit), - 'database_password': relation_get('password', rid=rid, unit=unit), - 'database_type': 'postgresql', - } + rel_host = relation_get('host', rid=rid, unit=unit) + rel_user = relation_get('user', rid=rid, unit=unit) + rel_passwd = relation_get('password', rid=rid, unit=unit) + ctxt = {'database_host': rel_host, + 'database': self.database, + 'database_user': rel_user, + 'database_password': rel_passwd, + 'database_type': 'postgresql'} if context_complete(ctxt): return ctxt + return {} @@ -227,23 +230,29 @@ def db_ssl(rdata, ctxt, ssl_dir): ca_path = os.path.join(ssl_dir, 'db-client.ca') with open(ca_path, 'w') as fh: fh.write(b64decode(rdata['ssl_ca'])) + ctxt['database_ssl_ca'] = ca_path elif 'ssl_ca' in rdata: - log("Charm not setup for ssl support but ssl ca found") + log("Charm not setup for ssl support but ssl ca found", level=INFO) return ctxt + if 'ssl_cert' in rdata: cert_path = os.path.join( ssl_dir, 'db-client.cert') if not os.path.exists(cert_path): - log("Waiting 1m for ssl client cert validity") + log("Waiting 1m for ssl client cert validity", level=INFO) time.sleep(60) + with open(cert_path, 'w') as fh: fh.write(b64decode(rdata['ssl_cert'])) + ctxt['database_ssl_cert'] = cert_path key_path = os.path.join(ssl_dir, 'db-client.key') with open(key_path, 'w') as fh: fh.write(b64decode(rdata['ssl_key'])) + ctxt['database_ssl_key'] = key_path + return ctxt @@ -251,9 +260,8 @@ class IdentityServiceContext(OSContextGenerator): interfaces = ['identity-service'] def __call__(self): - log('Generating template context for identity-service') + log('Generating template context for identity-service', level=DEBUG) ctxt = {} - for rid in relation_ids('identity-service'): for unit in related_units(rid): rdata = relation_get(rid=rid, unit=unit) @@ -261,26 +269,24 @@ class IdentityServiceContext(OSContextGenerator): serv_host = format_ipv6_addr(serv_host) or serv_host auth_host = rdata.get('auth_host') auth_host = format_ipv6_addr(auth_host) or auth_host - - ctxt = { - 'service_port': rdata.get('service_port'), - 'service_host': serv_host, - 'auth_host': auth_host, - 'auth_port': rdata.get('auth_port'), - 'admin_tenant_name': rdata.get('service_tenant'), - 'admin_user': rdata.get('service_username'), - 'admin_password': rdata.get('service_password'), - 'service_protocol': - rdata.get('service_protocol') or 'http', - 'auth_protocol': - rdata.get('auth_protocol') or 'http', - } + svc_protocol = rdata.get('service_protocol') or 'http' + auth_protocol = rdata.get('auth_protocol') or 'http' + ctxt = {'service_port': rdata.get('service_port'), + 'service_host': serv_host, + 'auth_host': auth_host, + 'auth_port': rdata.get('auth_port'), + 'admin_tenant_name': rdata.get('service_tenant'), + 'admin_user': rdata.get('service_username'), + 'admin_password': rdata.get('service_password'), + 'service_protocol': svc_protocol, + 'auth_protocol': auth_protocol} if context_complete(ctxt): # NOTE(jamespage) this is required for >= icehouse # so a missing value just indicates keystone needs # upgrading ctxt['admin_tenant_id'] = rdata.get('service_tenant_id') return ctxt + return {} @@ -293,21 +299,23 @@ class AMQPContext(OSContextGenerator): self.interfaces = [rel_name] def __call__(self): - log('Generating template context for amqp') + log('Generating template context for amqp', level=DEBUG) conf = config() - user_setting = 'rabbit-user' - vhost_setting = 'rabbit-vhost' if self.relation_prefix: - user_setting = self.relation_prefix + '-rabbit-user' - vhost_setting = self.relation_prefix + '-rabbit-vhost' + user_setting = '%s-rabbit-user' % (self.relation_prefix) + vhost_setting = '%s-rabbit-vhost' % (self.relation_prefix) + else: + user_setting = 'rabbit-user' + vhost_setting = 'rabbit-vhost' try: username = conf[user_setting] vhost = conf[vhost_setting] except KeyError as e: - log('Could not generate shared_db context. ' - 'Missing required charm config options: %s.' % e) + log('Could not generate shared_db context. Missing required charm ' + 'config options: %s.' % e, level=ERROR) raise OSContextError + ctxt = {} for rid in relation_ids(self.rel_name): ha_vip_only = False @@ -321,6 +329,7 @@ class AMQPContext(OSContextGenerator): host = relation_get('private-address', rid=rid, unit=unit) host = format_ipv6_addr(host) or host ctxt['rabbitmq_host'] = host + ctxt.update({ 'rabbitmq_user': username, 'rabbitmq_password': relation_get('password', rid=rid, @@ -331,6 +340,7 @@ class AMQPContext(OSContextGenerator): ssl_port = relation_get('ssl_port', rid=rid, unit=unit) if ssl_port: ctxt['rabbit_ssl_port'] = ssl_port + ssl_ca = relation_get('ssl_ca', rid=rid, unit=unit) if ssl_ca: ctxt['rabbit_ssl_ca'] = ssl_ca @@ -344,41 +354,45 @@ class AMQPContext(OSContextGenerator): if context_complete(ctxt): if 'rabbit_ssl_ca' in ctxt: if not self.ssl_dir: - log(("Charm not setup for ssl support " - "but ssl ca found")) + log("Charm not setup for ssl support but ssl ca " + "found", level=INFO) break + ca_path = os.path.join( self.ssl_dir, 'rabbit-client-ca.pem') with open(ca_path, 'w') as fh: fh.write(b64decode(ctxt['rabbit_ssl_ca'])) ctxt['rabbit_ssl_ca'] = ca_path + # Sufficient information found = break out! break + # Used for active/active rabbitmq >= grizzly - if ('clustered' not in ctxt or ha_vip_only) \ - and len(related_units(rid)) > 1: + if (('clustered' not in ctxt or ha_vip_only) and + len(related_units(rid)) > 1): rabbitmq_hosts = [] for unit in related_units(rid): host = relation_get('private-address', rid=rid, unit=unit) host = format_ipv6_addr(host) or host rabbitmq_hosts.append(host) - ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts) + + ctxt['rabbitmq_hosts'] = ','.join(sorted(rabbitmq_hosts)) + if not context_complete(ctxt): return {} - else: - return ctxt + + return ctxt class CephContext(OSContextGenerator): + """Generates context for /etc/ceph/ceph.conf templates.""" interfaces = ['ceph'] def __call__(self): - '''This generates context for /etc/ceph/ceph.conf templates''' if not relation_ids('ceph'): return {} - log('Generating template context for ceph') - + log('Generating template context for ceph', level=DEBUG) mon_hosts = [] auth = None key = None @@ -387,18 +401,18 @@ class CephContext(OSContextGenerator): for unit in related_units(rid): auth = relation_get('auth', rid=rid, unit=unit) key = relation_get('key', rid=rid, unit=unit) - ceph_addr = \ - relation_get('ceph-public-address', rid=rid, unit=unit) or \ - relation_get('private-address', rid=rid, unit=unit) + ceph_pub_addr = relation_get('ceph-public-address', rid=rid, + unit=unit) + unit_priv_addr = relation_get('private-address', rid=rid, + unit=unit) + ceph_addr = ceph_pub_addr or unit_priv_addr ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr mon_hosts.append(ceph_addr) - ctxt = { - 'mon_hosts': ' '.join(mon_hosts), - 'auth': auth, - 'key': key, - 'use_syslog': use_syslog - } + ctxt = {'mon_hosts': ' '.join(sorted(mon_hosts)), + 'auth': auth, + 'key': key, + 'use_syslog': use_syslog} if not os.path.isdir('/etc/ceph'): os.mkdir('/etc/ceph') @@ -407,79 +421,68 @@ class CephContext(OSContextGenerator): return {} ensure_packages(['ceph-common']) - return ctxt -ADDRESS_TYPES = ['admin', 'internal', 'public'] - - class HAProxyContext(OSContextGenerator): + """Provides half a context for the haproxy template, which describes + all peers to be included in the cluster. Each charm needs to include + its own context generator that describes the port mapping. + """ interfaces = ['cluster'] - def __call__(self): - ''' - Builds half a context for the haproxy template, which describes - all peers to be included in the cluster. Each charm needs to include - its own context generator that describes the port mapping. - ''' - if not relation_ids('cluster'): - return {} + def __init__(self, singlenode_mode=False): + self.singlenode_mode = singlenode_mode - l_unit = local_unit().replace('/', '-') + def __call__(self): + if not relation_ids('cluster') and not self.singlenode_mode: + return {} if config('prefer-ipv6'): addr = get_ipv6_addr(exc_list=[config('vip')])[0] else: addr = get_host_ip(unit_get('private-address')) + l_unit = local_unit().replace('/', '-') cluster_hosts = {} # NOTE(jamespage): build out map of configured network endpoints # and associated backends for addr_type in ADDRESS_TYPES: - laddr = get_address_in_network( - config('os-{}-network'.format(addr_type))) + cfg_opt = 'os-{}-network'.format(addr_type) + laddr = get_address_in_network(config(cfg_opt)) if laddr: - cluster_hosts[laddr] = {} - cluster_hosts[laddr]['network'] = "{}/{}".format( - laddr, - get_netmask_for_address(laddr) - ) - cluster_hosts[laddr]['backends'] = {} - cluster_hosts[laddr]['backends'][l_unit] = laddr + netmask = get_netmask_for_address(laddr) + cluster_hosts[laddr] = {'network': "{}/{}".format(laddr, + netmask), + 'backends': {l_unit: laddr}} for rid in relation_ids('cluster'): for unit in related_units(rid): - _unit = unit.replace('/', '-') _laddr = relation_get('{}-address'.format(addr_type), rid=rid, unit=unit) if _laddr: + _unit = unit.replace('/', '-') cluster_hosts[laddr]['backends'][_unit] = _laddr # NOTE(jamespage) no split configurations found, just use # private addresses if not cluster_hosts: - cluster_hosts[addr] = {} - cluster_hosts[addr]['network'] = "{}/{}".format( - addr, - get_netmask_for_address(addr) - ) - cluster_hosts[addr]['backends'] = {} - cluster_hosts[addr]['backends'][l_unit] = addr + netmask = get_netmask_for_address(addr) + cluster_hosts[addr] = {'network': "{}/{}".format(addr, netmask), + 'backends': {l_unit: addr}} for rid in relation_ids('cluster'): for unit in related_units(rid): - _unit = unit.replace('/', '-') _laddr = relation_get('private-address', rid=rid, unit=unit) if _laddr: + _unit = unit.replace('/', '-') cluster_hosts[addr]['backends'][_unit] = _laddr - ctxt = { - 'frontends': cluster_hosts, - } + ctxt = {'frontends': cluster_hosts} if config('haproxy-server-timeout'): ctxt['haproxy_server_timeout'] = config('haproxy-server-timeout') + if config('haproxy-client-timeout'): ctxt['haproxy_client_timeout'] = config('haproxy-client-timeout') @@ -493,13 +496,18 @@ class HAProxyContext(OSContextGenerator): ctxt['stat_port'] = ':8888' for frontend in cluster_hosts: - if len(cluster_hosts[frontend]['backends']) > 1: + if (len(cluster_hosts[frontend]['backends']) > 1 or + self.singlenode_mode): # Enable haproxy when we have enough peers. - log('Ensuring haproxy enabled in /etc/default/haproxy.') + log('Ensuring haproxy enabled in /etc/default/haproxy.', + level=DEBUG) with open('/etc/default/haproxy', 'w') as out: out.write('ENABLED=1\n') + return ctxt - log('HAProxy context is incomplete, this unit has no peers.') + + log('HAProxy context is incomplete, this unit has no peers.', + level=INFO) return {} @@ -507,29 +515,28 @@ class ImageServiceContext(OSContextGenerator): interfaces = ['image-service'] def __call__(self): - ''' - Obtains the glance API server from the image-service relation. Useful - in nova and cinder (currently). - ''' - log('Generating template context for image-service.') + """Obtains the glance API server from the image-service relation. + Useful in nova and cinder (currently). + """ + log('Generating template context for image-service.', level=DEBUG) rids = relation_ids('image-service') if not rids: return {} + for rid in rids: for unit in related_units(rid): api_server = relation_get('glance-api-server', rid=rid, unit=unit) if api_server: return {'glance_api_servers': api_server} - log('ImageService context is incomplete. ' - 'Missing required relation data.') + + log("ImageService context is incomplete. Missing required relation " + "data.", level=INFO) return {} class ApacheSSLContext(OSContextGenerator): - - """ - Generates a context for an apache vhost configuration that configures + """Generates a context for an apache vhost configuration that configures HTTPS reverse proxying for one or many endpoints. Generated context looks something like:: @@ -563,6 +570,7 @@ class ApacheSSLContext(OSContextGenerator): else: cert_filename = 'cert' key_filename = 'key' + write_file(path=os.path.join(ssl_dir, cert_filename), content=b64decode(cert)) write_file(path=os.path.join(ssl_dir, key_filename), @@ -574,7 +582,8 @@ class ApacheSSLContext(OSContextGenerator): install_ca_cert(b64decode(ca_cert)) def canonical_names(self): - '''Figure out which canonical names clients will access this service''' + """Figure out which canonical names clients will access this service. + """ cns = [] for r_id in relation_ids('identity-service'): for unit in related_units(r_id): @@ -582,55 +591,80 @@ class ApacheSSLContext(OSContextGenerator): for k in rdata: if k.startswith('ssl_key_'): cns.append(k.lstrip('ssl_key_')) - return list(set(cns)) + + return sorted(list(set(cns))) + + def get_network_addresses(self): + """For each network configured, return corresponding address and vip + (if available). + + Returns a list of tuples of the form: + + [(address_in_net_a, vip_in_net_a), + (address_in_net_b, vip_in_net_b), + ...] + + or, if no vip(s) available: + + [(address_in_net_a, address_in_net_a), + (address_in_net_b, address_in_net_b), + ...] + """ + addresses = [] + if config('vip'): + vips = config('vip').split() + else: + vips = [] + + for net_type in ['os-internal-network', 'os-admin-network', + 'os-public-network']: + addr = get_address_in_network(config(net_type), + unit_get('private-address')) + if len(vips) > 1 and is_clustered(): + if not config(net_type): + log("Multiple networks configured but net_type " + "is None (%s)." % net_type, level=WARNING) + continue + + for vip in vips: + if is_address_in_network(config(net_type), vip): + addresses.append((addr, vip)) + break + + elif is_clustered() and config('vip'): + addresses.append((addr, config('vip'))) + else: + addresses.append((addr, addr)) + + return sorted(addresses) def __call__(self): - if isinstance(self.external_ports, basestring): + if isinstance(self.external_ports, six.string_types): self.external_ports = [self.external_ports] - if (not self.external_ports or not https()): + + if not self.external_ports or not https(): return {} self.configure_ca() self.enable_modules() - ctxt = { - 'namespace': self.service_namespace, - 'endpoints': [], - 'ext_ports': [] - } + ctxt = {'namespace': self.service_namespace, + 'endpoints': [], + 'ext_ports': []} for cn in self.canonical_names(): self.configure_cert(cn) - addresses = [] - vips = [] - if config('vip'): - vips = config('vip').split() - - for network_type in ['os-internal-network', - 'os-admin-network', - 'os-public-network']: - address = get_address_in_network(config(network_type), - unit_get('private-address')) - if len(vips) > 0 and is_clustered(): - for vip in vips: - if is_address_in_network(config(network_type), - vip): - addresses.append((address, vip)) - break - elif is_clustered(): - addresses.append((address, config('vip'))) - else: - addresses.append((address, address)) - - for address, endpoint in set(addresses): + addresses = self.get_network_addresses() + for address, endpoint in sorted(set(addresses)): for api_port in self.external_ports: ext_port = determine_apache_port(api_port) int_port = determine_api_port(api_port) portmap = (address, endpoint, int(ext_port), int(int_port)) ctxt['endpoints'].append(portmap) ctxt['ext_ports'].append(int(ext_port)) - ctxt['ext_ports'] = list(set(ctxt['ext_ports'])) + + ctxt['ext_ports'] = sorted(list(set(ctxt['ext_ports']))) return ctxt @@ -647,21 +681,23 @@ class NeutronContext(OSContextGenerator): @property def packages(self): - return neutron_plugin_attribute( - self.plugin, 'packages', self.network_manager) + return neutron_plugin_attribute(self.plugin, 'packages', + self.network_manager) @property def neutron_security_groups(self): return None def _ensure_packages(self): - [ensure_packages(pkgs) for pkgs in self.packages] + for pkgs in self.packages: + ensure_packages(pkgs) def _save_flag_file(self): if self.network_manager == 'quantum': _file = '/etc/nova/quantum_plugin.conf' else: _file = '/etc/nova/neutron_plugin.conf' + with open(_file, 'wb') as out: out.write(self.plugin + '\n') @@ -670,13 +706,11 @@ class NeutronContext(OSContextGenerator): self.network_manager) config = neutron_plugin_attribute(self.plugin, 'config', self.network_manager) - ovs_ctxt = { - 'core_plugin': driver, - 'neutron_plugin': 'ovs', - 'neutron_security_groups': self.neutron_security_groups, - 'local_ip': unit_private_ip(), - 'config': config - } + ovs_ctxt = {'core_plugin': driver, + 'neutron_plugin': 'ovs', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': config} return ovs_ctxt @@ -685,13 +719,11 @@ class NeutronContext(OSContextGenerator): self.network_manager) config = neutron_plugin_attribute(self.plugin, 'config', self.network_manager) - nvp_ctxt = { - 'core_plugin': driver, - 'neutron_plugin': 'nvp', - 'neutron_security_groups': self.neutron_security_groups, - 'local_ip': unit_private_ip(), - 'config': config - } + nvp_ctxt = {'core_plugin': driver, + 'neutron_plugin': 'nvp', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': config} return nvp_ctxt @@ -700,35 +732,50 @@ class NeutronContext(OSContextGenerator): self.network_manager) n1kv_config = neutron_plugin_attribute(self.plugin, 'config', self.network_manager) - n1kv_ctxt = { - 'core_plugin': driver, - 'neutron_plugin': 'n1kv', - 'neutron_security_groups': self.neutron_security_groups, - 'local_ip': unit_private_ip(), - 'config': n1kv_config, - 'vsm_ip': config('n1kv-vsm-ip'), - 'vsm_username': config('n1kv-vsm-username'), - 'vsm_password': config('n1kv-vsm-password'), - 'restrict_policy_profiles': config( - 'n1kv_restrict_policy_profiles'), - } + n1kv_user_config_flags = config('n1kv-config-flags') + restrict_policy_profiles = config('n1kv-restrict-policy-profiles') + n1kv_ctxt = {'core_plugin': driver, + 'neutron_plugin': 'n1kv', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': n1kv_config, + 'vsm_ip': config('n1kv-vsm-ip'), + 'vsm_username': config('n1kv-vsm-username'), + 'vsm_password': config('n1kv-vsm-password'), + 'restrict_policy_profiles': restrict_policy_profiles} + + if n1kv_user_config_flags: + flags = config_flags_parser(n1kv_user_config_flags) + n1kv_ctxt['user_config_flags'] = flags return n1kv_ctxt + def calico_ctxt(self): + driver = neutron_plugin_attribute(self.plugin, 'driver', + self.network_manager) + config = neutron_plugin_attribute(self.plugin, 'config', + self.network_manager) + calico_ctxt = {'core_plugin': driver, + 'neutron_plugin': 'Calico', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': config} + + return calico_ctxt + def neutron_ctxt(self): if https(): proto = 'https' else: proto = 'http' + if is_clustered(): host = config('vip') else: host = unit_get('private-address') - url = '%s://%s:%s' % (proto, host, '9696') - ctxt = { - 'network_manager': self.network_manager, - 'neutron_url': url, - } + + ctxt = {'network_manager': self.network_manager, + 'neutron_url': '%s://%s:%s' % (proto, host, '9696')} return ctxt def __call__(self): @@ -748,6 +795,8 @@ class NeutronContext(OSContextGenerator): ctxt.update(self.nvp_ctxt()) elif self.plugin == 'n1kv': ctxt.update(self.n1kv_ctxt()) + elif self.plugin == 'Calico': + ctxt.update(self.calico_ctxt()) alchemy_flags = config('neutron-alchemy-flags') if alchemy_flags: @@ -759,23 +808,40 @@ class NeutronContext(OSContextGenerator): class OSConfigFlagContext(OSContextGenerator): + """Provides support for user-defined config flags. - """ - Responsible for adding user-defined config-flags in charm config to a - template context. + Users can define a comma-seperated list of key=value pairs + in the charm configuration and apply them at any point in + any file by using a template flag. + + Sometimes users might want config flags inserted within a + specific section so this class allows users to specify the + template flag name, allowing for multiple template flags + (sections) within the same context. NOTE: the value of config-flags may be a comma-separated list of key=value pairs and some Openstack config files support comma-separated lists as values. """ + def __init__(self, charm_flag='config-flags', + template_flag='user_config_flags'): + """ + :param charm_flag: config flags in charm configuration. + :param template_flag: insert point for user-defined flags in template + file. + """ + super(OSConfigFlagContext, self).__init__() + self._charm_flag = charm_flag + self._template_flag = template_flag + def __call__(self): - config_flags = config('config-flags') + config_flags = config(self._charm_flag) if not config_flags: return {} - flags = config_flags_parser(config_flags) - return {'user_config_flags': flags} + return {self._template_flag: + config_flags_parser(config_flags)} class SubordinateConfigContext(OSContextGenerator): @@ -819,7 +885,6 @@ class SubordinateConfigContext(OSContextGenerator): }, } } - """ def __init__(self, service, config_file, interface): @@ -849,26 +914,28 @@ class SubordinateConfigContext(OSContextGenerator): if self.service not in sub_config: log('Found subordinate_config on %s but it contained' - 'nothing for %s service' % (rid, self.service)) + 'nothing for %s service' % (rid, self.service), + level=INFO) continue sub_config = sub_config[self.service] if self.config_file not in sub_config: log('Found subordinate_config on %s but it contained' - 'nothing for %s' % (rid, self.config_file)) + 'nothing for %s' % (rid, self.config_file), + level=INFO) continue sub_config = sub_config[self.config_file] - for k, v in sub_config.iteritems(): + for k, v in six.iteritems(sub_config): if k == 'sections': - for section, config_dict in v.iteritems(): - log("adding section '%s'" % (section)) + for section, config_dict in six.iteritems(v): + log("adding section '%s'" % (section), + level=DEBUG) ctxt[k][section] = config_dict else: ctxt[k] = v - log("%d section(s) found" % (len(ctxt['sections'])), level=INFO) - + log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG) return ctxt @@ -880,15 +947,14 @@ class LogLevelContext(OSContextGenerator): False if config('debug') is None else config('debug') ctxt['verbose'] = \ False if config('verbose') is None else config('verbose') + return ctxt class SyslogContext(OSContextGenerator): def __call__(self): - ctxt = { - 'use_syslog': config('use-syslog') - } + ctxt = {'use_syslog': config('use-syslog')} return ctxt @@ -896,13 +962,9 @@ class BindHostContext(OSContextGenerator): def __call__(self): if config('prefer-ipv6'): - return { - 'bind_host': '::' - } + return {'bind_host': '::'} else: - return { - 'bind_host': '0.0.0.0' - } + return {'bind_host': '0.0.0.0'} class WorkerConfigContext(OSContextGenerator): @@ -914,11 +976,42 @@ class WorkerConfigContext(OSContextGenerator): except ImportError: apt_install('python-psutil', fatal=True) from psutil import NUM_CPUS + return NUM_CPUS def __call__(self): - multiplier = config('worker-multiplier') or 1 - ctxt = { - "workers": self.num_cpus * multiplier - } + multiplier = config('worker-multiplier') or 0 + ctxt = {"workers": self.num_cpus * multiplier} + return ctxt + + +class ZeroMQContext(OSContextGenerator): + interfaces = ['zeromq-configuration'] + + def __call__(self): + ctxt = {} + if is_relation_made('zeromq-configuration', 'host'): + for rid in relation_ids('zeromq-configuration'): + for unit in related_units(rid): + ctxt['zmq_nonce'] = relation_get('nonce', unit, rid) + ctxt['zmq_host'] = relation_get('host', unit, rid) + + return ctxt + + +class NotificationDriverContext(OSContextGenerator): + + def __init__(self, zmq_relation='zeromq-configuration', + amqp_relation='amqp'): + """ + :param zmq_relation: Name of Zeromq relation to check + """ + self.zmq_relation = zmq_relation + self.amqp_relation = amqp_relation + + def __call__(self): + ctxt = {'notifications': 'False'} + if is_relation_made(self.amqp_relation): + ctxt['notifications'] = "True" + return ctxt diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py index bc84fc45..f062c807 100644 --- a/hooks/charmhelpers/contrib/openstack/ip.py +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -2,21 +2,19 @@ from charmhelpers.core.hookenv import ( config, unit_get, ) - from charmhelpers.contrib.network.ip import ( get_address_in_network, is_address_in_network, is_ipv6, get_ipv6_addr, ) - from charmhelpers.contrib.hahelpers.cluster import is_clustered PUBLIC = 'public' INTERNAL = 'int' ADMIN = 'admin' -_address_map = { +ADDRESS_MAP = { PUBLIC: { 'config': 'os-public-network', 'fallback': 'public-address' @@ -33,16 +31,14 @@ _address_map = { def canonical_url(configs, endpoint_type=PUBLIC): - ''' - Returns the correct HTTP URL to this host given the state of HTTPS + """Returns the correct HTTP URL to this host given the state of HTTPS configuration, hacluster and charm configuration. - :configs OSTemplateRenderer: A config tempating object to inspect for - a complete https context. - :endpoint_type str: The endpoint type to resolve. - - :returns str: Base URL for services on the current service unit. - ''' + :param configs: OSTemplateRenderer config templating object to inspect + for a complete https context. + :param endpoint_type: str endpoint type to resolve. + :param returns: str base URL for services on the current service unit. + """ scheme = 'http' if 'https' in configs.complete_contexts(): scheme = 'https' @@ -53,27 +49,45 @@ def canonical_url(configs, endpoint_type=PUBLIC): def resolve_address(endpoint_type=PUBLIC): + """Return unit address depending on net config. + + If unit is clustered with vip(s) and has net splits defined, return vip on + correct network. If clustered with no nets defined, return primary vip. + + If not clustered, return unit address ensuring address is on configured net + split if one is configured. + + :param endpoint_type: Network endpoing type + """ resolved_address = None - if is_clustered(): - if config(_address_map[endpoint_type]['config']) is None: - # Assume vip is simple and pass back directly - resolved_address = config('vip') + vips = config('vip') + if vips: + vips = vips.split() + + net_type = ADDRESS_MAP[endpoint_type]['config'] + net_addr = config(net_type) + net_fallback = ADDRESS_MAP[endpoint_type]['fallback'] + clustered = is_clustered() + if clustered: + if not net_addr: + # If no net-splits defined, we expect a single vip + resolved_address = vips[0] else: - for vip in config('vip').split(): - if is_address_in_network( - config(_address_map[endpoint_type]['config']), - vip): + for vip in vips: + if is_address_in_network(net_addr, vip): resolved_address = vip + break else: if config('prefer-ipv6'): - fallback_addr = get_ipv6_addr(exc_list=[config('vip')])[0] + fallback_addr = get_ipv6_addr(exc_list=vips)[0] else: - fallback_addr = unit_get(_address_map[endpoint_type]['fallback']) - resolved_address = get_address_in_network( - config(_address_map[endpoint_type]['config']), fallback_addr) + fallback_addr = unit_get(net_fallback) + + resolved_address = get_address_in_network(net_addr, fallback_addr) if resolved_address is None: - raise ValueError('Unable to resolve a suitable IP address' - ' based on charm state and configuration') - else: - return resolved_address + raise ValueError("Unable to resolve a suitable IP address based on " + "charm state and configuration. (net_type=%s, " + "clustered=%s)" % (net_type, clustered)) + + return resolved_address diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 84d97bca..1446f637 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -14,7 +14,7 @@ from charmhelpers.contrib.openstack.utils import os_release def headers_package(): """Ensures correct linux-headers for running kernel are installed, for building DKMS package""" - kver = check_output(['uname', '-r']).strip() + kver = check_output(['uname', '-r']).decode('UTF-8').strip() return 'linux-headers-%s' % kver QUANTUM_CONF_DIR = '/etc/quantum' @@ -22,7 +22,7 @@ QUANTUM_CONF_DIR = '/etc/quantum' def kernel_version(): """ Retrieve the current major kernel version as a tuple e.g. (3, 13) """ - kver = check_output(['uname', '-r']).strip() + kver = check_output(['uname', '-r']).decode('UTF-8').strip() kver = kver.split('.') return (int(kver[0]), int(kver[1])) @@ -138,10 +138,25 @@ def neutron_plugins(): relation_prefix='neutron', ssl_dir=NEUTRON_CONF_DIR)], 'services': [], - 'packages': [['neutron-plugin-cisco']], + 'packages': [[headers_package()] + determine_dkms_package(), + ['neutron-plugin-cisco']], 'server_packages': ['neutron-server', 'neutron-plugin-cisco'], 'server_services': ['neutron-server'] + }, + 'Calico': { + 'config': '/etc/neutron/plugins/ml2/ml2_conf.ini', + 'driver': 'neutron.plugins.ml2.plugin.Ml2Plugin', + 'contexts': [ + context.SharedDBContext(user=config('neutron-database-user'), + database=config('neutron-database'), + relation_prefix='neutron', + ssl_dir=NEUTRON_CONF_DIR)], + 'services': ['calico-compute', 'bird', 'neutron-dhcp-agent'], + 'packages': [[headers_package()] + determine_dkms_package(), + ['calico-compute', 'bird', 'neutron-dhcp-agent']], + 'server_packages': ['neutron-server', 'calico-control'], + 'server_services': ['neutron-server'] } } if release >= 'icehouse': @@ -162,7 +177,8 @@ def neutron_plugin_attribute(plugin, attr, net_manager=None): elif manager == 'neutron': plugins = neutron_plugins() else: - log('Error: Network manager does not support plugins.') + log("Network manager '%s' does not support plugins." % (manager), + level=ERROR) raise Exception try: diff --git a/hooks/charmhelpers/contrib/openstack/templating.py b/hooks/charmhelpers/contrib/openstack/templating.py index f5442712..33df0675 100644 --- a/hooks/charmhelpers/contrib/openstack/templating.py +++ b/hooks/charmhelpers/contrib/openstack/templating.py @@ -1,13 +1,13 @@ import os -from charmhelpers.fetch import apt_install +import six +from charmhelpers.fetch import apt_install from charmhelpers.core.hookenv import ( log, ERROR, INFO ) - from charmhelpers.contrib.openstack.utils import OPENSTACK_CODENAMES try: @@ -43,7 +43,7 @@ def get_loader(templates_dir, os_release): order by OpenStack release. """ tmpl_dirs = [(rel, os.path.join(templates_dir, rel)) - for rel in OPENSTACK_CODENAMES.itervalues()] + for rel in six.itervalues(OPENSTACK_CODENAMES)] if not os.path.isdir(templates_dir): log('Templates directory not found @ %s.' % templates_dir, @@ -258,7 +258,7 @@ class OSConfigRenderer(object): """ Write out all registered config files. """ - [self.write(k) for k in self.templates.iterkeys()] + [self.write(k) for k in six.iterkeys(self.templates)] def set_release(self, openstack_release): """ @@ -275,5 +275,5 @@ class OSConfigRenderer(object): ''' interfaces = [] [interfaces.extend(i.complete_contexts()) - for i in self.templates.itervalues()] + for i in six.itervalues(self.templates)] return interfaces diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index b0d1b03a..f8de1aa3 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -2,6 +2,7 @@ # Common python helper functions used for OpenStack charms. from collections import OrderedDict +from functools import wraps import subprocess import json @@ -9,11 +10,12 @@ import os import socket import sys +import six + from charmhelpers.core.hookenv import ( config, log as juju_log, charm_dir, - ERROR, INFO, relation_ids, relation_set @@ -112,7 +114,7 @@ def get_os_codename_install_source(src): # Best guess match based on deb string provided if src.startswith('deb') or src.startswith('ppa'): - for k, v in OPENSTACK_CODENAMES.iteritems(): + for k, v in six.iteritems(OPENSTACK_CODENAMES): if v in src: return v @@ -133,7 +135,7 @@ def get_os_codename_version(vers): def get_os_version_codename(codename): '''Determine OpenStack version number from codename.''' - for k, v in OPENSTACK_CODENAMES.iteritems(): + for k, v in six.iteritems(OPENSTACK_CODENAMES): if v == codename: return k e = 'Could not derive OpenStack version for '\ @@ -193,7 +195,7 @@ def get_os_version_package(pkg, fatal=True): else: vers_map = OPENSTACK_CODENAMES - for version, cname in vers_map.iteritems(): + for version, cname in six.iteritems(vers_map): if cname == codename: return version # e = "Could not determine OpenStack version for package: %s" % pkg @@ -317,7 +319,7 @@ def save_script_rc(script_path="scripts/scriptrc", **env_vars): rc_script.write( "#!/bin/bash\n") [rc_script.write('export %s=%s\n' % (u, p)) - for u, p in env_vars.iteritems() if u != "script_path"] + for u, p in six.iteritems(env_vars) if u != "script_path"] def openstack_upgrade_available(package): @@ -350,8 +352,8 @@ def ensure_block_device(block_device): ''' _none = ['None', 'none', None] if (block_device in _none): - error_out('prepare_storage(): Missing required input: ' - 'block_device=%s.' % block_device, level=ERROR) + error_out('prepare_storage(): Missing required input: block_device=%s.' + % block_device) if block_device.startswith('/dev/'): bdev = block_device @@ -367,8 +369,7 @@ def ensure_block_device(block_device): bdev = '/dev/%s' % block_device if not is_block_device(bdev): - error_out('Failed to locate valid block device at %s' % bdev, - level=ERROR) + error_out('Failed to locate valid block device at %s' % bdev) return bdev @@ -417,7 +418,7 @@ def ns_query(address): if isinstance(address, dns.name.Name): rtype = 'PTR' - elif isinstance(address, basestring): + elif isinstance(address, six.string_types): rtype = 'A' else: return None @@ -468,6 +469,14 @@ def get_hostname(address, fqdn=True): return result.split('.')[0] +def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'): + mm_map = {} + if os.path.isfile(mm_file): + with open(mm_file, 'r') as f: + mm_map = json.load(f) + return mm_map + + def sync_db_with_multi_ipv6_addresses(database, database_user, relation_prefix=None): hosts = get_ipv6_addr(dynamic_only=False) @@ -477,10 +486,24 @@ def sync_db_with_multi_ipv6_addresses(database, database_user, 'hostname': json.dumps(hosts)} if relation_prefix: - keys = kwargs.keys() - for key in keys: + for key in list(kwargs.keys()): kwargs["%s_%s" % (relation_prefix, key)] = kwargs[key] del kwargs[key] for rid in relation_ids('shared-db'): relation_set(relation_id=rid, **kwargs) + + +def os_requires_version(ostack_release, pkg): + """ + Decorator for hook to specify minimum supported release + """ + def wrap(f): + @wraps(f) + def wrapped_f(*args): + if os_release(pkg) < ostack_release: + raise Exception("This hook is not supported on releases" + " before %s" % ostack_release) + f(*args) + return wrapped_f + return wrap diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 768438a4..d47dc228 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -16,19 +16,18 @@ import time from subprocess import ( check_call, check_output, - CalledProcessError + CalledProcessError, ) - from charmhelpers.core.hookenv import ( relation_get, relation_ids, related_units, log, + DEBUG, INFO, WARNING, - ERROR + ERROR, ) - from charmhelpers.core.host import ( mount, mounts, @@ -37,7 +36,6 @@ from charmhelpers.core.host import ( service_running, umount, ) - from charmhelpers.fetch import ( apt_install, ) @@ -56,99 +54,85 @@ CEPH_CONF = """[global] def install(): - ''' Basic Ceph client installation ''' + """Basic Ceph client installation.""" ceph_dir = "/etc/ceph" if not os.path.exists(ceph_dir): os.mkdir(ceph_dir) + apt_install('ceph-common', fatal=True) def rbd_exists(service, pool, rbd_img): - ''' Check to see if a RADOS block device exists ''' + """Check to see if a RADOS block device exists.""" try: - out = check_output(['rbd', 'list', '--id', service, - '--pool', pool]) + out = check_output(['rbd', 'list', '--id', + service, '--pool', pool]).decode('UTF-8') except CalledProcessError: return False - else: - return rbd_img in out + + return rbd_img in out def create_rbd_image(service, pool, image, sizemb): - ''' Create a new RADOS block device ''' - cmd = [ - 'rbd', - 'create', - image, - '--size', - str(sizemb), - '--id', - service, - '--pool', - pool - ] + """Create a new RADOS block device.""" + cmd = ['rbd', 'create', image, '--size', str(sizemb), '--id', service, + '--pool', pool] check_call(cmd) def pool_exists(service, name): - ''' Check to see if a RADOS pool already exists ''' + """Check to see if a RADOS pool already exists.""" try: - out = check_output(['rados', '--id', service, 'lspools']) + out = check_output(['rados', '--id', service, + 'lspools']).decode('UTF-8') except CalledProcessError: return False - else: - return name in out + + return name in out def get_osds(service): - ''' - Return a list of all Ceph Object Storage Daemons - currently in the cluster - ''' + """Return a list of all Ceph Object Storage Daemons currently in the + cluster. + """ version = ceph_version() if version and version >= '0.56': return json.loads(check_output(['ceph', '--id', service, - 'osd', 'ls', '--format=json'])) - else: - return None + 'osd', 'ls', + '--format=json']).decode('UTF-8')) + + return None -def create_pool(service, name, replicas=2): - ''' Create a new RADOS pool ''' +def create_pool(service, name, replicas=3): + """Create a new RADOS pool.""" if pool_exists(service, name): log("Ceph pool {} already exists, skipping creation".format(name), level=WARNING) return + # Calculate the number of placement groups based # on upstream recommended best practices. osds = get_osds(service) if osds: - pgnum = (len(osds) * 100 / replicas) + pgnum = (len(osds) * 100 // replicas) else: # NOTE(james-page): Default to 200 for older ceph versions # which don't support OSD query from cli pgnum = 200 - cmd = [ - 'ceph', '--id', service, - 'osd', 'pool', 'create', - name, str(pgnum) - ] + + cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pgnum)] check_call(cmd) - cmd = [ - 'ceph', '--id', service, - 'osd', 'pool', 'set', name, - 'size', str(replicas) - ] + + cmd = ['ceph', '--id', service, 'osd', 'pool', 'set', name, 'size', + str(replicas)] check_call(cmd) def delete_pool(service, name): - ''' Delete a RADOS pool from ceph ''' - cmd = [ - 'ceph', '--id', service, - 'osd', 'pool', 'delete', - name, '--yes-i-really-really-mean-it' - ] + """Delete a RADOS pool from ceph.""" + cmd = ['ceph', '--id', service, 'osd', 'pool', 'delete', name, + '--yes-i-really-really-mean-it'] check_call(cmd) @@ -161,44 +145,43 @@ def _keyring_path(service): def create_keyring(service, key): - ''' Create a new Ceph keyring containing key''' + """Create a new Ceph keyring containing key.""" keyring = _keyring_path(service) if os.path.exists(keyring): - log('ceph: Keyring exists at %s.' % keyring, level=WARNING) + log('Ceph keyring exists at %s.' % keyring, level=WARNING) return - cmd = [ - 'ceph-authtool', - keyring, - '--create-keyring', - '--name=client.{}'.format(service), - '--add-key={}'.format(key) - ] + + cmd = ['ceph-authtool', keyring, '--create-keyring', + '--name=client.{}'.format(service), '--add-key={}'.format(key)] check_call(cmd) - log('ceph: Created new ring at %s.' % keyring, level=INFO) + log('Created new ceph keyring at %s.' % keyring, level=DEBUG) def create_key_file(service, key): - ''' Create a file containing key ''' + """Create a file containing key.""" keyfile = _keyfile_path(service) if os.path.exists(keyfile): - log('ceph: Keyfile exists at %s.' % keyfile, level=WARNING) + log('Keyfile exists at %s.' % keyfile, level=WARNING) return + with open(keyfile, 'w') as fd: fd.write(key) - log('ceph: Created new keyfile at %s.' % keyfile, level=INFO) + + log('Created new keyfile at %s.' % keyfile, level=INFO) def get_ceph_nodes(): - ''' Query named relation 'ceph' to detemine current nodes ''' + """Query named relation 'ceph' to determine current nodes.""" hosts = [] for r_id in relation_ids('ceph'): for unit in related_units(r_id): hosts.append(relation_get('private-address', unit=unit, rid=r_id)) + return hosts def configure(service, key, auth, use_syslog): - ''' Perform basic configuration of Ceph ''' + """Perform basic configuration of Ceph.""" create_keyring(service, key) create_key_file(service, key) hosts = get_ceph_nodes() @@ -211,17 +194,17 @@ def configure(service, key, auth, use_syslog): def image_mapped(name): - ''' Determine whether a RADOS block device is mapped locally ''' + """Determine whether a RADOS block device is mapped locally.""" try: - out = check_output(['rbd', 'showmapped']) + out = check_output(['rbd', 'showmapped']).decode('UTF-8') except CalledProcessError: return False - else: - return name in out + + return name in out def map_block_storage(service, pool, image): - ''' Map a RADOS block device for local use ''' + """Map a RADOS block device for local use.""" cmd = [ 'rbd', 'map', @@ -235,31 +218,32 @@ def map_block_storage(service, pool, image): def filesystem_mounted(fs): - ''' Determine whether a filesytems is already mounted ''' + """Determine whether a filesytems is already mounted.""" return fs in [f for f, m in mounts()] def make_filesystem(blk_device, fstype='ext4', timeout=10): - ''' Make a new filesystem on the specified block device ''' + """Make a new filesystem on the specified block device.""" count = 0 e_noent = os.errno.ENOENT while not os.path.exists(blk_device): if count >= timeout: - log('ceph: gave up waiting on block device %s' % blk_device, + log('Gave up waiting on block device %s' % blk_device, level=ERROR) raise IOError(e_noent, os.strerror(e_noent), blk_device) - log('ceph: waiting for block device %s to appear' % blk_device, - level=INFO) + + log('Waiting for block device %s to appear' % blk_device, + level=DEBUG) count += 1 time.sleep(1) else: - log('ceph: Formatting block device %s as filesystem %s.' % + log('Formatting block device %s as filesystem %s.' % (blk_device, fstype), level=INFO) check_call(['mkfs', '-t', fstype, blk_device]) def place_data_on_block_device(blk_device, data_src_dst): - ''' Migrate data in data_src_dst to blk_device and then remount ''' + """Migrate data in data_src_dst to blk_device and then remount.""" # mount block device into /mnt mount(blk_device, '/mnt') # copy data to /mnt @@ -279,8 +263,8 @@ def place_data_on_block_device(blk_device, data_src_dst): # TODO: re-use def modprobe(module): - ''' Load a kernel module and configure for auto-load on reboot ''' - log('ceph: Loading kernel module', level=INFO) + """Load a kernel module and configure for auto-load on reboot.""" + log('Loading kernel module', level=INFO) cmd = ['modprobe', module] check_call(cmd) with open('/etc/modules', 'r+') as modules: @@ -289,7 +273,7 @@ def modprobe(module): def copy_files(src, dst, symlinks=False, ignore=None): - ''' Copy files from src to dst ''' + """Copy files from src to dst.""" for item in os.listdir(src): s = os.path.join(src, item) d = os.path.join(dst, item) @@ -300,9 +284,9 @@ def copy_files(src, dst, symlinks=False, ignore=None): def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point, - blk_device, fstype, system_services=[]): - """ - NOTE: This function must only be called from a single service unit for + blk_device, fstype, system_services=[], + replicas=3): + """NOTE: This function must only be called from a single service unit for the same rbd_img otherwise data loss will occur. Ensures given pool and RBD image exists, is mapped to a block device, @@ -316,15 +300,16 @@ def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point, """ # Ensure pool, RBD image, RBD mappings are in place. if not pool_exists(service, pool): - log('ceph: Creating new pool {}.'.format(pool)) - create_pool(service, pool) + log('Creating new pool {}.'.format(pool), level=INFO) + create_pool(service, pool, replicas=replicas) if not rbd_exists(service, pool, rbd_img): - log('ceph: Creating RBD image ({}).'.format(rbd_img)) + log('Creating RBD image ({}).'.format(rbd_img), level=INFO) create_rbd_image(service, pool, rbd_img, sizemb) if not image_mapped(rbd_img): - log('ceph: Mapping RBD Image {} as a Block Device.'.format(rbd_img)) + log('Mapping RBD Image {} as a Block Device.'.format(rbd_img), + level=INFO) map_block_storage(service, pool, rbd_img) # make file system @@ -339,45 +324,47 @@ def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point, for svc in system_services: if service_running(svc): - log('ceph: Stopping services {} prior to migrating data.' - .format(svc)) + log('Stopping services {} prior to migrating data.' + .format(svc), level=DEBUG) service_stop(svc) place_data_on_block_device(blk_device, mount_point) for svc in system_services: - log('ceph: Starting service {} after migrating data.' - .format(svc)) + log('Starting service {} after migrating data.' + .format(svc), level=DEBUG) service_start(svc) def ensure_ceph_keyring(service, user=None, group=None): - ''' - Ensures a ceph keyring is created for a named service - and optionally ensures user and group ownership. + """Ensures a ceph keyring is created for a named service and optionally + ensures user and group ownership. Returns False if no ceph key is available in relation state. - ''' + """ key = None for rid in relation_ids('ceph'): for unit in related_units(rid): key = relation_get('key', rid=rid, unit=unit) if key: break + if not key: return False + create_keyring(service=service, key=key) keyring = _keyring_path(service) if user and group: check_call(['chown', '%s.%s' % (user, group), keyring]) + return True def ceph_version(): - ''' Retrieve the local version of ceph ''' + """Retrieve the local version of ceph.""" if os.path.exists('/usr/bin/ceph'): cmd = ['ceph', '-v'] - output = check_output(cmd) + output = check_output(cmd).decode('US-ASCII') output = output.split() if len(output) > 3: return output[2] diff --git a/hooks/charmhelpers/contrib/storage/linux/loopback.py b/hooks/charmhelpers/contrib/storage/linux/loopback.py index 38957ef0..a22c3d7b 100644 --- a/hooks/charmhelpers/contrib/storage/linux/loopback.py +++ b/hooks/charmhelpers/contrib/storage/linux/loopback.py @@ -1,12 +1,12 @@ - import os import re - from subprocess import ( check_call, check_output, ) +import six + ################################################## # loopback device helpers. @@ -37,7 +37,7 @@ def create_loopback(file_path): ''' file_path = os.path.abspath(file_path) check_call(['losetup', '--find', file_path]) - for d, f in loopback_devices().iteritems(): + for d, f in six.iteritems(loopback_devices()): if f == file_path: return d @@ -51,7 +51,7 @@ def ensure_loopback_device(path, size): :returns: str: Full path to the ensured loopback device (eg, /dev/loop0) ''' - for d, f in loopback_devices().iteritems(): + for d, f in six.iteritems(loopback_devices()): if f == path: return d diff --git a/hooks/charmhelpers/contrib/storage/linux/lvm.py b/hooks/charmhelpers/contrib/storage/linux/lvm.py index 8ac7fecc..0aa65f4f 100644 --- a/hooks/charmhelpers/contrib/storage/linux/lvm.py +++ b/hooks/charmhelpers/contrib/storage/linux/lvm.py @@ -61,6 +61,7 @@ def list_lvm_volume_group(block_device): vg = None pvd = check_output(['pvdisplay', block_device]).splitlines() for l in pvd: + l = l.decode('UTF-8') if l.strip().startswith('VG Name'): vg = ' '.join(l.strip().split()[2:]) return vg diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index 1b958712..c6a15e14 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -30,7 +30,8 @@ def zap_disk(block_device): # sometimes sgdisk exits non-zero; this is OK, dd will clean up call(['sgdisk', '--zap-all', '--mbrtogpt', '--clear', block_device]) - dev_end = check_output(['blockdev', '--getsz', block_device]) + dev_end = check_output(['blockdev', '--getsz', + block_device]).decode('UTF-8') gpt_end = int(dev_end.split()[0]) - 100 check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device), 'bs=1M', 'count=1']) @@ -47,7 +48,7 @@ def is_device_mounted(device): it doesn't. ''' is_partition = bool(re.search(r".*[0-9]+\b", device)) - out = check_output(['mount']) + out = check_output(['mount']).decode('UTF-8') if is_partition: return bool(re.search(device + r"\b", out)) return bool(re.search(device + r"[0-9]+\b", out)) diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py index cfaf0a65..0adf0db3 100644 --- a/hooks/charmhelpers/core/fstab.py +++ b/hooks/charmhelpers/core/fstab.py @@ -3,10 +3,11 @@ __author__ = 'Jorge Niedbalski R. ' +import io import os -class Fstab(file): +class Fstab(io.FileIO): """This class extends file in order to implement a file reader/writer for file `/etc/fstab` """ @@ -24,8 +25,8 @@ class Fstab(file): options = "defaults" self.options = options - self.d = d - self.p = p + self.d = int(d) + self.p = int(p) def __eq__(self, o): return str(self) == str(o) @@ -45,7 +46,7 @@ class Fstab(file): self._path = path else: self._path = self.DEFAULT_PATH - file.__init__(self, self._path, 'r+') + super(Fstab, self).__init__(self._path, 'rb+') def _hydrate_entry(self, line): # NOTE: use split with no arguments to split on any @@ -58,8 +59,9 @@ class Fstab(file): def entries(self): self.seek(0) for line in self.readlines(): + line = line.decode('us-ascii') try: - if not line.startswith("#"): + if line.strip() and not line.startswith("#"): yield self._hydrate_entry(line) except ValueError: pass @@ -75,14 +77,14 @@ class Fstab(file): if self.get_entry_by_attr('device', entry.device): return False - self.write(str(entry) + '\n') + self.write((str(entry) + '\n').encode('us-ascii')) self.truncate() return entry def remove_entry(self, entry): self.seek(0) - lines = self.readlines() + lines = [l.decode('us-ascii') for l in self.readlines()] found = False for index, line in enumerate(lines): @@ -97,7 +99,7 @@ class Fstab(file): lines.remove(line) self.seek(0) - self.write(''.join(lines)) + self.write(''.join(lines).encode('us-ascii')) self.truncate() return True diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index af8fe2db..99e5d208 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -9,9 +9,14 @@ import json import yaml import subprocess import sys -import UserDict from subprocess import CalledProcessError +import six +if not six.PY3: + from UserDict import UserDict +else: + from collections import UserDict + CRITICAL = "CRITICAL" ERROR = "ERROR" WARNING = "WARNING" @@ -67,12 +72,12 @@ def log(message, level=None): subprocess.call(command) -class Serializable(UserDict.IterableUserDict): +class Serializable(UserDict): """Wrapper, an object that can be serialized to yaml or json""" def __init__(self, obj): # wrap the object - UserDict.IterableUserDict.__init__(self) + UserDict.__init__(self) self.data = obj def __getattr__(self, attr): @@ -214,6 +219,12 @@ class Config(dict): except KeyError: return (self._prev_dict or {})[key] + def keys(self): + prev_keys = [] + if self._prev_dict is not None: + prev_keys = self._prev_dict.keys() + return list(set(prev_keys + list(dict.keys(self)))) + def load_previous(self, path=None): """Load previous copy of config from disk. @@ -263,7 +274,7 @@ class Config(dict): """ if self._prev_dict: - for k, v in self._prev_dict.iteritems(): + for k, v in six.iteritems(self._prev_dict): if k not in self: self[k] = v with open(self.path, 'w') as f: @@ -278,7 +289,8 @@ def config(scope=None): config_cmd_line.append(scope) config_cmd_line.append('--format=json') try: - config_data = json.loads(subprocess.check_output(config_cmd_line)) + config_data = json.loads( + subprocess.check_output(config_cmd_line).decode('UTF-8')) if scope is not None: return config_data return Config(config_data) @@ -297,10 +309,10 @@ def relation_get(attribute=None, unit=None, rid=None): if unit: _args.append(unit) try: - return json.loads(subprocess.check_output(_args)) + return json.loads(subprocess.check_output(_args).decode('UTF-8')) except ValueError: return None - except CalledProcessError, e: + except CalledProcessError as e: if e.returncode == 2: return None raise @@ -312,7 +324,7 @@ def relation_set(relation_id=None, relation_settings=None, **kwargs): relation_cmd_line = ['relation-set'] if relation_id is not None: relation_cmd_line.extend(('-r', relation_id)) - for k, v in (relation_settings.items() + kwargs.items()): + for k, v in (list(relation_settings.items()) + list(kwargs.items())): if v is None: relation_cmd_line.append('{}='.format(k)) else: @@ -329,7 +341,8 @@ def relation_ids(reltype=None): relid_cmd_line = ['relation-ids', '--format=json'] if reltype is not None: relid_cmd_line.append(reltype) - return json.loads(subprocess.check_output(relid_cmd_line)) or [] + return json.loads( + subprocess.check_output(relid_cmd_line).decode('UTF-8')) or [] return [] @@ -340,7 +353,8 @@ def related_units(relid=None): units_cmd_line = ['relation-list', '--format=json'] if relid is not None: units_cmd_line.extend(('-r', relid)) - return json.loads(subprocess.check_output(units_cmd_line)) or [] + return json.loads( + subprocess.check_output(units_cmd_line).decode('UTF-8')) or [] @cached @@ -449,7 +463,7 @@ def unit_get(attribute): """Get the unit ID for the remote unit""" _args = ['unit-get', '--format=json', attribute] try: - return json.loads(subprocess.check_output(_args)) + return json.loads(subprocess.check_output(_args).decode('UTF-8')) except ValueError: return None diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 8a91a5d6..78dcd440 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -13,13 +13,13 @@ import random import string import subprocess import hashlib -import shutil from contextlib import contextmanager - from collections import OrderedDict -from hookenv import log -from fstab import Fstab +import six + +from .hookenv import log +from .fstab import Fstab def service_start(service_name): @@ -55,7 +55,9 @@ def service(action, service_name): def service_running(service): """Determine whether a system service is running""" try: - output = subprocess.check_output(['service', service, 'status'], stderr=subprocess.STDOUT) + output = subprocess.check_output( + ['service', service, 'status'], + stderr=subprocess.STDOUT).decode('UTF-8') except subprocess.CalledProcessError: return False else: @@ -68,7 +70,9 @@ def service_running(service): def service_available(service_name): """Determine whether a system service is available""" try: - subprocess.check_output(['service', service_name, 'status'], stderr=subprocess.STDOUT) + subprocess.check_output( + ['service', service_name, 'status'], + stderr=subprocess.STDOUT).decode('UTF-8') except subprocess.CalledProcessError as e: return 'unrecognized service' not in e.output else: @@ -116,7 +120,7 @@ def rsync(from_path, to_path, flags='-r', options=None): cmd.append(from_path) cmd.append(to_path) log(" ".join(cmd)) - return subprocess.check_output(cmd).strip() + return subprocess.check_output(cmd).decode('UTF-8').strip() def symlink(source, destination): @@ -131,7 +135,7 @@ def symlink(source, destination): subprocess.check_call(cmd) -def mkdir(path, owner='root', group='root', perms=0555, force=False): +def mkdir(path, owner='root', group='root', perms=0o555, force=False): """Create a directory""" log("Making dir {} {}:{} {:o}".format(path, owner, group, perms)) @@ -147,7 +151,7 @@ def mkdir(path, owner='root', group='root', perms=0555, force=False): os.chown(realpath, uid, gid) -def write_file(path, content, owner='root', group='root', perms=0444): +def write_file(path, content, owner='root', group='root', perms=0o444): """Create or overwrite a file with the contents of a string""" log("Writing file {} {}:{} {:o}".format(path, owner, group, perms)) uid = pwd.getpwnam(owner).pw_uid @@ -178,7 +182,7 @@ def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"): cmd_args.extend([device, mountpoint]) try: subprocess.check_output(cmd_args) - except subprocess.CalledProcessError, e: + except subprocess.CalledProcessError as e: log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output)) return False @@ -192,7 +196,7 @@ def umount(mountpoint, persist=False): cmd_args = ['umount', mountpoint] try: subprocess.check_output(cmd_args) - except subprocess.CalledProcessError, e: + except subprocess.CalledProcessError as e: log('Error unmounting {}\n{}'.format(mountpoint, e.output)) return False @@ -219,8 +223,8 @@ def file_hash(path, hash_type='md5'): """ if os.path.exists(path): h = getattr(hashlib, hash_type)() - with open(path, 'r') as source: - h.update(source.read()) # IGNORE:E1101 - it does have update + with open(path, 'rb') as source: + h.update(source.read()) return h.hexdigest() else: return None @@ -298,7 +302,7 @@ def pwgen(length=None): if length is None: length = random.choice(range(35, 45)) alphanumeric_chars = [ - l for l in (string.letters + string.digits) + l for l in (string.ascii_letters + string.digits) if l not in 'l0QD1vAEIOUaeiou'] random_chars = [ random.choice(alphanumeric_chars) for _ in range(length)] @@ -307,14 +311,14 @@ def pwgen(length=None): def list_nics(nic_type): '''Return a list of nics of given type(s)''' - if isinstance(nic_type, basestring): + if isinstance(nic_type, six.string_types): int_types = [nic_type] else: int_types = nic_type interfaces = [] for int_type in int_types: cmd = ['ip', 'addr', 'show', 'label', int_type + '*'] - ip_output = subprocess.check_output(cmd).split('\n') + ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') ip_output = (line for line in ip_output if line) for line in ip_output: if line.split()[1].startswith(int_type): @@ -328,15 +332,44 @@ def list_nics(nic_type): return interfaces -def set_nic_mtu(nic, mtu): +def set_nic_mtu(nic, mtu, persistence=False): '''Set MTU on a network interface''' cmd = ['ip', 'link', 'set', nic, 'mtu', mtu] subprocess.check_call(cmd) + # persistence mtu configuration + if not persistence: + return + if os.path.exists("/etc/network/interfaces.d/%s.cfg"): + nic_cfg_file = "/etc/network/interfaces.d/%s.cfg" % nic + else: + nic_cfg_file = "/etc/network/interfaces" + f = open(nic_cfg_file,"r") + lines = f.readlines() + found = False + length = len(lines) + for i in range(len(lines)): + lines[i] = lines[i].replace('\n', '') + if lines[i].startswith("iface %s" % nic): + found = True + lines.insert(i+1, " up ip link set $IFACE mtu %s" % mtu) + lines.insert(i+2, " down ip link set $IFACE mtu 1500") + if length>i+2 and lines[i+3].startswith(" up ip link set $IFACE mtu"): + del lines[i+3] + if length>i+2 and lines[i+3].startswith(" down ip link set $IFACE mtu"): + del lines[i+3] + break + if not found: + lines.insert(length+1, "") + lines.insert(length+2, "auto %s" % nic) + lines.insert(length+3, "iface %s inet dhcp" % nic) + lines.insert(length+4, " up ip link set $IFACE mtu %s" % mtu) + lines.insert(length+5, " down ip link set $IFACE mtu 1500") + write_file(path=nic_cfg_file, content="\n".join(lines), perms=0o666) def get_nic_mtu(nic): cmd = ['ip', 'addr', 'show', nic] - ip_output = subprocess.check_output(cmd).split('\n') + ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') mtu = "" for line in ip_output: words = line.split() @@ -347,7 +380,7 @@ def get_nic_mtu(nic): def get_nic_hwaddr(nic): cmd = ['ip', '-o', '-0', 'addr', 'show', nic] - ip_output = subprocess.check_output(cmd) + ip_output = subprocess.check_output(cmd).decode('UTF-8') hwaddr = "" words = ip_output.split() if 'link/ether' in words: diff --git a/hooks/charmhelpers/core/services/__init__.py b/hooks/charmhelpers/core/services/__init__.py index e8039a84..69dde79a 100644 --- a/hooks/charmhelpers/core/services/__init__.py +++ b/hooks/charmhelpers/core/services/__init__.py @@ -1,2 +1,2 @@ -from .base import * -from .helpers import * +from .base import * # NOQA +from .helpers import * # NOQA diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py index 7067b94b..163a7932 100644 --- a/hooks/charmhelpers/core/services/helpers.py +++ b/hooks/charmhelpers/core/services/helpers.py @@ -196,7 +196,7 @@ class StoredContext(dict): if not os.path.isabs(file_name): file_name = os.path.join(hookenv.charm_dir(), file_name) with open(file_name, 'w') as file_stream: - os.fchmod(file_stream.fileno(), 0600) + os.fchmod(file_stream.fileno(), 0o600) yaml.dump(config_data, file_stream) def read_context(self, file_name): @@ -211,15 +211,19 @@ class StoredContext(dict): class TemplateCallback(ManagerCallback): """ - Callback class that will render a Jinja2 template, for use as a ready action. + Callback class that will render a Jinja2 template, for use as a ready + action. + + :param str source: The template source file, relative to + `$CHARM_DIR/templates` - :param str source: The template source file, relative to `$CHARM_DIR/templates` :param str target: The target to write the rendered template to :param str owner: The owner of the rendered file :param str group: The group of the rendered file :param int perms: The permissions of the rendered file """ - def __init__(self, source, target, owner='root', group='root', perms=0444): + def __init__(self, source, target, + owner='root', group='root', perms=0o444): self.source = source self.target = target self.owner = owner diff --git a/hooks/charmhelpers/core/templating.py b/hooks/charmhelpers/core/templating.py index 2c638853..83133fa4 100644 --- a/hooks/charmhelpers/core/templating.py +++ b/hooks/charmhelpers/core/templating.py @@ -4,7 +4,8 @@ from charmhelpers.core import host from charmhelpers.core import hookenv -def render(source, target, context, owner='root', group='root', perms=0444, templates_dir=None): +def render(source, target, context, owner='root', group='root', + perms=0o444, templates_dir=None): """ Render a template. diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 32a673d6..0a126fc3 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -5,10 +5,6 @@ from yaml import safe_load from charmhelpers.core.host import ( lsb_release ) -from urlparse import ( - urlparse, - urlunparse, -) import subprocess from charmhelpers.core.hookenv import ( config, @@ -16,6 +12,12 @@ from charmhelpers.core.hookenv import ( ) import os +import six +if six.PY3: + from urllib.parse import urlparse, urlunparse +else: + from urlparse import urlparse, urlunparse + CLOUD_ARCHIVE = """# Ubuntu Cloud Archive deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main @@ -72,6 +74,7 @@ CLOUD_ARCHIVE_POCKETS = { FETCH_HANDLERS = ( 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler', 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler', + 'charmhelpers.fetch.giturl.GitUrlFetchHandler', ) APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. @@ -148,7 +151,7 @@ def apt_install(packages, options=None, fatal=False): cmd = ['apt-get', '--assume-yes'] cmd.extend(options) cmd.append('install') - if isinstance(packages, basestring): + if isinstance(packages, six.string_types): cmd.append(packages) else: cmd.extend(packages) @@ -181,7 +184,7 @@ def apt_update(fatal=False): def apt_purge(packages, fatal=False): """Purge one or more packages""" cmd = ['apt-get', '--assume-yes', 'purge'] - if isinstance(packages, basestring): + if isinstance(packages, six.string_types): cmd.append(packages) else: cmd.extend(packages) @@ -192,7 +195,7 @@ def apt_purge(packages, fatal=False): def apt_hold(packages, fatal=False): """Hold one or more packages""" cmd = ['apt-mark', 'hold'] - if isinstance(packages, basestring): + if isinstance(packages, six.string_types): cmd.append(packages) else: cmd.extend(packages) @@ -218,6 +221,7 @@ def add_source(source, key=None): pocket for the release. 'cloud:' may be used to activate official cloud archive pockets, such as 'cloud:icehouse' + 'distro' may be used as a noop @param key: A key to be added to the system's APT keyring and used to verify the signatures on packages. Ideally, this should be an @@ -251,12 +255,14 @@ def add_source(source, key=None): release = lsb_release()['DISTRIB_CODENAME'] with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt: apt.write(PROPOSED_POCKET.format(release)) + elif source == 'distro': + pass else: - raise SourceConfigError("Unknown source: {!r}".format(source)) + log("Unknown source: {!r}".format(source)) if key: if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key: - with NamedTemporaryFile() as key_file: + with NamedTemporaryFile('w+') as key_file: key_file.write(key) key_file.flush() key_file.seek(0) @@ -293,14 +299,14 @@ def configure_sources(update=False, sources = safe_load((config(sources_var) or '').strip()) or [] keys = safe_load((config(keys_var) or '').strip()) or None - if isinstance(sources, basestring): + if isinstance(sources, six.string_types): sources = [sources] if keys is None: for source in sources: add_source(source, None) else: - if isinstance(keys, basestring): + if isinstance(keys, six.string_types): keys = [keys] if len(sources) != len(keys): @@ -397,7 +403,7 @@ def _run_apt_command(cmd, fatal=False): while result is None or result == APT_NO_LOCK: try: result = subprocess.check_call(cmd, env=env) - except subprocess.CalledProcessError, e: + except subprocess.CalledProcessError as e: retry_count = retry_count + 1 if retry_count > APT_NO_LOCK_RETRY_COUNT: raise diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py index 8c045650..8a4624b2 100644 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ b/hooks/charmhelpers/fetch/archiveurl.py @@ -1,8 +1,23 @@ import os -import urllib2 -from urllib import urlretrieve -import urlparse import hashlib +import re + +import six +if six.PY3: + from urllib.request import ( + build_opener, install_opener, urlopen, urlretrieve, + HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, + ) + from urllib.parse import urlparse, urlunparse, parse_qs + from urllib.error import URLError +else: + from urllib import urlretrieve + from urllib2 import ( + build_opener, install_opener, urlopen, + HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, + URLError + ) + from urlparse import urlparse, urlunparse, parse_qs from charmhelpers.fetch import ( BaseFetchHandler, @@ -15,6 +30,24 @@ from charmhelpers.payload.archive import ( from charmhelpers.core.host import mkdir, check_hash +def splituser(host): + '''urllib.splituser(), but six's support of this seems broken''' + _userprog = re.compile('^(.*)@(.*)$') + match = _userprog.match(host) + if match: + return match.group(1, 2) + return None, host + + +def splitpasswd(user): + '''urllib.splitpasswd(), but six's support of this is missing''' + _passwdprog = re.compile('^([^:]*):(.*)$', re.S) + match = _passwdprog.match(user) + if match: + return match.group(1, 2) + return user, None + + class ArchiveUrlFetchHandler(BaseFetchHandler): """ Handler to download archive files from arbitrary URLs. @@ -42,20 +75,20 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): """ # propogate all exceptions # URLError, OSError, etc - proto, netloc, path, params, query, fragment = urlparse.urlparse(source) + proto, netloc, path, params, query, fragment = urlparse(source) if proto in ('http', 'https'): - auth, barehost = urllib2.splituser(netloc) + auth, barehost = splituser(netloc) if auth is not None: - source = urlparse.urlunparse((proto, barehost, path, params, query, fragment)) - username, password = urllib2.splitpasswd(auth) - passman = urllib2.HTTPPasswordMgrWithDefaultRealm() + source = urlunparse((proto, barehost, path, params, query, fragment)) + username, password = splitpasswd(auth) + passman = HTTPPasswordMgrWithDefaultRealm() # Realm is set to None in add_password to force the username and password # to be used whatever the realm passman.add_password(None, source, username, password) - authhandler = urllib2.HTTPBasicAuthHandler(passman) - opener = urllib2.build_opener(authhandler) - urllib2.install_opener(opener) - response = urllib2.urlopen(source) + authhandler = HTTPBasicAuthHandler(passman) + opener = build_opener(authhandler) + install_opener(opener) + response = urlopen(source) try: with open(dest, 'w') as dest_file: dest_file.write(response.read()) @@ -91,17 +124,21 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): url_parts = self.parse_url(source) dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched') if not os.path.exists(dest_dir): - mkdir(dest_dir, perms=0755) + mkdir(dest_dir, perms=0o755) dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path)) try: self.download(source, dld_file) - except urllib2.URLError as e: + except URLError as e: raise UnhandledSource(e.reason) except OSError as e: raise UnhandledSource(e.strerror) - options = urlparse.parse_qs(url_parts.fragment) + options = parse_qs(url_parts.fragment) for key, value in options.items(): - if key in hashlib.algorithms: + if not six.PY3: + algorithms = hashlib.algorithms + else: + algorithms = hashlib.algorithms_available + if key in algorithms: check_hash(dld_file, value, key) if checksum: check_hash(dld_file, checksum, hash_type) diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py index 0e580e47..8ef48f30 100644 --- a/hooks/charmhelpers/fetch/bzrurl.py +++ b/hooks/charmhelpers/fetch/bzrurl.py @@ -5,6 +5,10 @@ from charmhelpers.fetch import ( ) from charmhelpers.core.host import mkdir +import six +if six.PY3: + raise ImportError('bzrlib does not support Python3') + try: from bzrlib.branch import Branch except ImportError: @@ -42,7 +46,7 @@ class BzrUrlFetchHandler(BaseFetchHandler): dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", branch_name) if not os.path.exists(dest_dir): - mkdir(dest_dir, perms=0755) + mkdir(dest_dir, perms=0o755) try: self.branch(source, dest_dir) except OSError as e: From 3870da9c3f04fa5397b0d1d5a22afdbba3becc56 Mon Sep 17 00:00:00 2001 From: Zhang Hua Date: Wed, 3 Dec 2014 21:23:26 +0800 Subject: [PATCH 06/75] fix charm-helpers-hooks --- charm-helpers-hooks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index 05e37a4e..1a98c81c 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -1,4 +1,4 @@ -branch: lp:~zhhuabj/charms/trusty/charm-helpers/lp74646 +branch: lp:charm-helpers destination: hooks/charmhelpers include: - core From 1f72ac62728c354f6a03e1ea185566c0aaf7c52c Mon Sep 17 00:00:00 2001 From: Zhang Hua Date: Wed, 3 Dec 2014 21:34:25 +0800 Subject: [PATCH 07/75] change to use the method of charm-helpers --- config.yaml | 2 +- hooks/quantum_hooks.py | 10 ++++++---- hooks/quantum_utils.py | 22 +--------------------- unit_tests/test_quantum_hooks.py | 8 ++++---- 4 files changed, 12 insertions(+), 30 deletions(-) diff --git a/config.yaml b/config.yaml index 7a316ec2..c0018f3e 100644 --- a/config.yaml +++ b/config.yaml @@ -109,7 +109,7 @@ options: . This network will be used for tenant network traffic in overlay networks. - tunnel-nic-mtu: + phy-nic-mtu: type: int default: 1500 description: | diff --git a/hooks/quantum_hooks.py b/hooks/quantum_hooks.py index 53408e22..2b8a14ac 100755 --- a/hooks/quantum_hooks.py +++ b/hooks/quantum_hooks.py @@ -22,6 +22,9 @@ from charmhelpers.core.host import ( restart_on_change, lsb_release, ) +from charmhelpers.contrib.network.ip import ( + configure_phy_nic_mtu +) from charmhelpers.contrib.hahelpers.cluster import( eligible_leader ) @@ -45,8 +48,7 @@ from quantum_utils import ( valid_plugin, configure_ovs, reassign_agent_resources, - stop_services, - configure_mtu + stop_services ) hooks = Hooks() @@ -67,7 +69,7 @@ def install(): fatal=True) apt_install(filter_installed_packages(get_packages()), fatal=True) - configure_mtu() + configure_phy_nic_mtu() else: log('Please provide a valid plugin config', level=ERROR) sys.exit(1) @@ -91,7 +93,7 @@ def config_changed(): if valid_plugin(): CONFIGS.write_all() configure_ovs() - configure_mtu() + configure_phy_nic_mtu() else: log('Please provide a valid plugin config', level=ERROR) sys.exit(1) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 8f5a8f43..14472256 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -3,19 +3,13 @@ from charmhelpers.core.host import ( service_stop, service_restart, lsb_release, - list_nics, - set_nic_mtu -) -from charmhelpers.contrib.network.ip import ( - get_ipv4_addr ) from charmhelpers.core.hookenv import ( - log, INFO, + log, config, relations_of_type, unit_private_ip, is_relation_made, - unit_get ) from charmhelpers.fetch import ( apt_upgrade, @@ -575,17 +569,3 @@ def configure_ovs(): ext_port_ctx = ExternalPortContext()() if ext_port_ctx is not None and ext_port_ctx['ext_port']: add_bridge_port(EXT_BRIDGE, ext_port_ctx['ext_port']) - - -def configure_mtu(): - tunnel_nic_mtu = config('tunnel-nic-mtu') - if tunnel_nic_mtu >= 1500: - tunnel_ip = unit_get('private-address') - tunnel_nic = 'eth0' - for nic in list_nics(['eth', 'bond']): - if tunnel_ip in get_ipv4_addr(nic, fatal=False): - tunnel_nic = nic - break - set_nic_mtu(tunnel_nic, str(tunnel_nic_mtu)) - log('set mtu={} for tunnel nic={}' - .format(tunnel_nic_mtu, tunnel_nic), level=INFO) diff --git a/unit_tests/test_quantum_hooks.py b/unit_tests/test_quantum_hooks.py index f26620a9..e7452940 100644 --- a/unit_tests/test_quantum_hooks.py +++ b/unit_tests/test_quantum_hooks.py @@ -41,7 +41,7 @@ TO_PATCH = [ 'stop_services', 'b64decode', 'is_relation_made', - 'configure_mtu' + 'configure_phy_nic_mtu' ] @@ -81,7 +81,7 @@ class TestQuantumHooks(CharmTestCase): self.assertTrue(self.get_early_packages.called) self.assertTrue(self.get_packages.called) self.assertTrue(self.execd_preinstall.called) - self.assertTrue(self.configure_mtu.called) + self.assertTrue(self.configure_phy_nic_mtu.called) def test_install_hook_precise_nocloudarchive(self): self.test_config.set('openstack-origin', 'distro') @@ -114,7 +114,7 @@ class TestQuantumHooks(CharmTestCase): self.assertTrue(_pgsql_db_joined.called) self.assertTrue(_amqp_joined.called) self.assertTrue(_amqp_nova_joined.called) - self.assertTrue(self.configure_mtu.called) + self.assertTrue(self.configure_phy_nic_mtu.called) def test_config_changed_upgrade(self): self.openstack_upgrade_available.return_value = True @@ -122,7 +122,7 @@ class TestQuantumHooks(CharmTestCase): self._call_hook('config-changed') self.assertTrue(self.do_openstack_upgrade.called) self.assertTrue(self.configure_ovs.called) - self.assertTrue(self.configure_mtu.called) + self.assertTrue(self.configure_phy_nic_mtu.called) def test_config_changed_n1kv(self): self.openstack_upgrade_available.return_value = False From 9fa87d89b483571b211779c97f2fe73af7fc86e6 Mon Sep 17 00:00:00 2001 From: Zhang Hua Date: Thu, 4 Dec 2014 12:56:33 +0800 Subject: [PATCH 08/75] enable network-device-mtu --- hooks/quantum_contexts.py | 4 +++- templates/icehouse/neutron.conf | 1 + unit_tests/test_quantum_contexts.py | 6 ++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index bf7312eb..3297aee6 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -111,8 +111,8 @@ def _neutron_api_settings(): ''' neutron_settings = { 'l2_population': False, + 'network_device_mtu': 1500, 'overlay_network_type': 'gre', - } for rid in relation_ids('neutron-plugin-api'): for unit in related_units(rid): @@ -122,6 +122,7 @@ def _neutron_api_settings(): neutron_settings = { 'l2_population': rdata['l2-population'], 'overlay_network_type': rdata['overlay-network-type'], + 'network_device_mtu': rdata['network-device-mtu'], } return neutron_settings return neutron_settings @@ -223,6 +224,7 @@ class QuantumGatewayContext(OSContextGenerator): 'verbose': config('verbose'), 'instance_mtu': config('instance-mtu'), 'l2_population': neutron_api_settings['l2_population'], + 'network_device_mtu': neutron_api_settings['network_device_mtu'], 'overlay_network_type': neutron_api_settings['overlay_network_type'], } diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index 6db71eb4..78814822 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -11,5 +11,6 @@ core_plugin = {{ core_plugin }} control_exchange = neutron notification_driver = neutron.openstack.common.notifier.list_notifier list_notifier_drivers = neutron.openstack.common.notifier.rabbit_notifier +network_device_mtu = {{ network_device_mtu }} [agent] root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf \ No newline at end of file diff --git a/unit_tests/test_quantum_contexts.py b/unit_tests/test_quantum_contexts.py index 451aff08..80ad60dd 100644 --- a/unit_tests/test_quantum_contexts.py +++ b/unit_tests/test_quantum_contexts.py @@ -236,6 +236,7 @@ class TestQuantumGatewayContext(CharmTestCase): 'debug': False, 'verbose': True, 'l2_population': False, + 'network_device_mtu': 1500, 'overlay_network_type': 'gre', }) @@ -362,24 +363,29 @@ class TestMisc(CharmTestCase): self.relation_ids.return_value = ['foo'] self.related_units.return_value = ['bar'] self.test_relation.set({'l2-population': True, + 'network-device-mtu': 1500, 'overlay-network-type': 'gre', }) self.relation_get.side_effect = self.test_relation.get self.assertEquals(quantum_contexts._neutron_api_settings(), {'l2_population': True, + 'network_device_mtu': 1500, 'overlay_network_type': 'gre'}) def test_neutron_api_settings2(self): self.relation_ids.return_value = ['foo'] self.related_units.return_value = ['bar'] self.test_relation.set({'l2-population': True, + 'network-device-mtu': 1500, 'overlay-network-type': 'gre', }) self.relation_get.side_effect = self.test_relation.get self.assertEquals(quantum_contexts._neutron_api_settings(), {'l2_population': True, + 'network_device_mtu': 1500, 'overlay_network_type': 'gre'}) def test_neutron_api_settings_no_apiplugin(self): self.relation_ids.return_value = [] self.assertEquals(quantum_contexts._neutron_api_settings(), {'l2_population': False, + 'network_device_mtu': 1500, 'overlay_network_type': 'gre', }) From 938fcc32be494e59a983935d227f9d63d7b407ea Mon Sep 17 00:00:00 2001 From: Zhang Hua Date: Thu, 4 Dec 2014 13:01:42 +0800 Subject: [PATCH 09/75] fix unit test --- unit_tests/test_quantum_contexts.py | 65 +++++++++++++---------------- 1 file changed, 30 insertions(+), 35 deletions(-) diff --git a/unit_tests/test_quantum_contexts.py b/unit_tests/test_quantum_contexts.py index 80ad60dd..71af62f6 100644 --- a/unit_tests/test_quantum_contexts.py +++ b/unit_tests/test_quantum_contexts.py @@ -46,11 +46,39 @@ def patch_open(): yield mock_open, mock_file -class _TestQuantumContext(CharmTestCase): +class TestNetworkServiceContext(CharmTestCase): def setUp(self): - super(_TestQuantumContext, self).setUp(quantum_contexts, TO_PATCH) + super(TestNetworkServiceContext, self).setUp(quantum_contexts, + TO_PATCH) self.config.side_effect = self.test_config.get + self.context = quantum_contexts.NetworkServiceContext() + self.test_relation.set( + {'keystone_host': '10.5.0.1', + 'service_port': '5000', + 'auth_port': '20000', + 'service_tenant': 'tenant', + 'service_username': 'username', + 'service_password': 'password', + 'quantum_host': '10.5.0.2', + 'quantum_port': '9696', + 'quantum_url': 'http://10.5.0.2:9696/v2', + 'region': 'aregion'} + ) + self.data_result = { + 'keystone_host': '10.5.0.1', + 'service_port': '5000', + 'auth_port': '20000', + 'service_tenant': 'tenant', + 'service_username': 'username', + 'service_password': 'password', + 'quantum_host': '10.5.0.2', + 'quantum_port': '9696', + 'quantum_url': 'http://10.5.0.2:9696/v2', + 'region': 'aregion', + 'service_protocol': 'http', + 'auth_protocol': 'http', + } def test_not_related(self): self.relation_ids.return_value = [] @@ -84,39 +112,6 @@ class _TestQuantumContext(CharmTestCase): self.assertEquals(self.context(), self.data_result) -class TestNetworkServiceContext(_TestQuantumContext): - - def setUp(self): - super(TestNetworkServiceContext, self).setUp() - self.context = quantum_contexts.NetworkServiceContext() - self.test_relation.set( - {'keystone_host': '10.5.0.1', - 'service_port': '5000', - 'auth_port': '20000', - 'service_tenant': 'tenant', - 'service_username': 'username', - 'service_password': 'password', - 'quantum_host': '10.5.0.2', - 'quantum_port': '9696', - 'quantum_url': 'http://10.5.0.2:9696/v2', - 'region': 'aregion'} - ) - self.data_result = { - 'keystone_host': '10.5.0.1', - 'service_port': '5000', - 'auth_port': '20000', - 'service_tenant': 'tenant', - 'service_username': 'username', - 'service_password': 'password', - 'quantum_host': '10.5.0.2', - 'quantum_port': '9696', - 'quantum_url': 'http://10.5.0.2:9696/v2', - 'region': 'aregion', - 'service_protocol': 'http', - 'auth_protocol': 'http', - } - - class TestExternalPortContext(CharmTestCase): def setUp(self): From 45c5a3de6d35a4be3b46bb81338ad6bfc0c7aeb1 Mon Sep 17 00:00:00 2001 From: Zhang Hua Date: Thu, 4 Dec 2014 16:21:30 +0800 Subject: [PATCH 10/75] enable persistence --- hooks/charmhelpers/contrib/network/ip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index b6e69c19..0d785057 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -376,6 +376,6 @@ def configure_phy_nic_mtu(mng_ip=None): break break if phy_nic is not None and phy_nic_mtu != get_nic_mtu(phy_nic): - set_nic_mtu(phy_nic, str(phy_nic_mtu)) + set_nic_mtu(phy_nic, str(phy_nic_mtu), persistence=True) log('set mtu={} for phy_nic={}' .format(phy_nic_mtu, phy_nic), level=INFO) From 4d3cec7df55349e6f1073ef23678067c809b242b Mon Sep 17 00:00:00 2001 From: Zhang Hua Date: Thu, 4 Dec 2014 17:28:37 +0800 Subject: [PATCH 11/75] sync charm-helpers --- hooks/charmhelpers/core/host.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 78dcd440..b9f74467 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -339,7 +339,7 @@ def set_nic_mtu(nic, mtu, persistence=False): # persistence mtu configuration if not persistence: return - if os.path.exists("/etc/network/interfaces.d/%s.cfg"): + if os.path.exists("/etc/network/interfaces.d/%s.cfg" % nic): nic_cfg_file = "/etc/network/interfaces.d/%s.cfg" % nic else: nic_cfg_file = "/etc/network/interfaces" @@ -364,7 +364,7 @@ def set_nic_mtu(nic, mtu, persistence=False): lines.insert(length+3, "iface %s inet dhcp" % nic) lines.insert(length+4, " up ip link set $IFACE mtu %s" % mtu) lines.insert(length+5, " down ip link set $IFACE mtu 1500") - write_file(path=nic_cfg_file, content="\n".join(lines), perms=0o666) + write_file(path=nic_cfg_file, content="\n".join(lines), perms=0o644) def get_nic_mtu(nic): From 30ea6896bbc19a128b31329701ef06402f9c686b Mon Sep 17 00:00:00 2001 From: Zhang Hua Date: Fri, 5 Dec 2014 15:17:53 +0800 Subject: [PATCH 12/75] sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 112 +++++++++++++++++- hooks/charmhelpers/core/hookenv.py | 2 + hooks/charmhelpers/core/host.py | 22 +++- 3 files changed, 134 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index f8de1aa3..44179679 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -11,6 +11,7 @@ import socket import sys import six +import yaml from charmhelpers.core.hookenv import ( config, @@ -32,7 +33,8 @@ from charmhelpers.contrib.network.ip import ( ) from charmhelpers.core.host import lsb_release, mounts, umount -from charmhelpers.fetch import apt_install, apt_cache +from charmhelpers.fetch import apt_install, apt_cache, install_remote +from charmhelpers.contrib.python.packages import pip_install from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device @@ -507,3 +509,111 @@ def os_requires_version(ostack_release, pkg): f(*args) return wrapped_f return wrap + + +def git_install_requested(): + """Returns true if openstack-origin-git is specified.""" + return config('openstack-origin-git') != "None" + + +requirements_dir = None + + +def git_clone_and_install(file_name, core_project): + """Clone/install all OpenStack repos specified in yaml config file.""" + global requirements_dir + + if file_name == "None": + return + + yaml_file = os.path.join(charm_dir(), file_name) + + # clone/install the requirements project first + installed = _git_clone_and_install_subset(yaml_file, + whitelist=['requirements']) + if 'requirements' not in installed: + error_out('requirements git repository must be specified') + + # clone/install all other projects except requirements and the core project + blacklist = ['requirements', core_project] + _git_clone_and_install_subset(yaml_file, blacklist=blacklist, + update_requirements=True) + + # clone/install the core project + whitelist = [core_project] + installed = _git_clone_and_install_subset(yaml_file, whitelist=whitelist, + update_requirements=True) + if core_project not in installed: + error_out('{} git repository must be specified'.format(core_project)) + + +def _git_clone_and_install_subset(yaml_file, whitelist=[], blacklist=[], + update_requirements=False): + """Clone/install subset of OpenStack repos specified in yaml config file.""" + global requirements_dir + installed = [] + + with open(yaml_file, 'r') as fd: + projects = yaml.load(fd) + for proj, val in projects.items(): + # The project subset is chosen based on the following 3 rules: + # 1) If project is in blacklist, we don't clone/install it, period. + # 2) If whitelist is empty, we clone/install everything else. + # 3) If whitelist is not empty, we clone/install everything in the + # whitelist. + if proj in blacklist: + continue + if whitelist and proj not in whitelist: + continue + repo = val['repository'] + branch = val['branch'] + repo_dir = _git_clone_and_install_single(repo, branch, + update_requirements) + if proj == 'requirements': + requirements_dir = repo_dir + installed.append(proj) + return installed + + +def _git_clone_and_install_single(repo, branch, update_requirements=False): + """Clone and install a single git repository.""" + dest_parent_dir = "/mnt/openstack-git/" + dest_dir = os.path.join(dest_parent_dir, os.path.basename(repo)) + + if not os.path.exists(dest_parent_dir): + juju_log('Host dir not mounted at {}. ' + 'Creating directory there instead.'.format(dest_parent_dir)) + os.mkdir(dest_parent_dir) + + if not os.path.exists(dest_dir): + juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) + repo_dir = install_remote(repo, dest=dest_parent_dir, branch=branch) + else: + repo_dir = dest_dir + + if update_requirements: + if not requirements_dir: + error_out('requirements repo must be cloned before ' + 'updating from global requirements.') + _git_update_requirements(repo_dir, requirements_dir) + + juju_log('Installing git repo from dir: {}'.format(repo_dir)) + pip_install(repo_dir) + + return repo_dir + + +def _git_update_requirements(package_dir, reqs_dir): + """Update from global requirements. + + Update an OpenStack git directory's requirements.txt and + test-requirements.txt from global-requirements.txt.""" + orig_dir = os.getcwd() + os.chdir(reqs_dir) + cmd = "python update.py {}".format(package_dir) + try: + subprocess.check_call(cmd.split(' ')) + except subprocess.CalledProcessError: + package = os.path.basename(package_dir) + error_out("Error updating {} from global-requirements.txt".format(package)) + os.chdir(orig_dir) diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index 99e5d208..07d1f690 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -68,6 +68,8 @@ def log(message, level=None): command = ['juju-log'] if level: command += ['-l', level] + if not isinstance(message, six.string_types): + message = repr(message) command += [message] subprocess.call(command) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index b9f74467..d9b4dd41 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -101,6 +101,26 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False): return user_info +def add_group(group_name, system_group=False): + """Add a group to the system""" + try: + group_info = grp.getgrnam(group_name) + log('group {0} already exists!'.format(group_name)) + except KeyError: + log('creating group {0}'.format(group_name)) + cmd = ['addgroup'] + if system_group: + cmd.append('--system') + else: + cmd.extend([ + '--group', + ]) + cmd.append(group_name) + subprocess.check_call(cmd) + group_info = grp.getgrnam(group_name) + return group_info + + def add_user_to_group(username, group): """Add a user to a group""" cmd = [ @@ -397,8 +417,8 @@ def cmp_pkgrevno(package, revno, pkgcache=None): ''' import apt_pkg - from charmhelpers.fetch import apt_cache if not pkgcache: + from charmhelpers.fetch import apt_cache pkgcache = apt_cache() pkg = pkgcache[package] return apt_pkg.version_compare(pkg.current_ver.ver_str, revno) From db4d1389701fe9322159ffd86ef360571aedf076 Mon Sep 17 00:00:00 2001 From: Zhang Hua Date: Wed, 10 Dec 2014 15:48:26 +0800 Subject: [PATCH 13/75] sync charm-helpers to inclue contrib.python to fix unit test error --- charm-helpers-hooks.yaml | 1 + hooks/charmhelpers/contrib/network/ufw.py | 182 ++++++++++++++++++ hooks/charmhelpers/contrib/python/__init__.py | 0 hooks/charmhelpers/contrib/python/debug.py | 40 ++++ hooks/charmhelpers/contrib/python/packages.py | 77 ++++++++ hooks/charmhelpers/contrib/python/rpdb.py | 42 ++++ hooks/charmhelpers/contrib/python/version.py | 18 ++ hooks/charmhelpers/fetch/giturl.py | 51 +++++ 8 files changed, 411 insertions(+) create mode 100644 hooks/charmhelpers/contrib/network/ufw.py create mode 100644 hooks/charmhelpers/contrib/python/__init__.py create mode 100644 hooks/charmhelpers/contrib/python/debug.py create mode 100644 hooks/charmhelpers/contrib/python/packages.py create mode 100644 hooks/charmhelpers/contrib/python/rpdb.py create mode 100644 hooks/charmhelpers/contrib/python/version.py create mode 100644 hooks/charmhelpers/fetch/giturl.py diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index 1a98c81c..fbd0f037 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -7,4 +7,5 @@ include: - contrib.hahelpers - contrib.network - contrib.storage.linux + - contrib.python - payload.execd diff --git a/hooks/charmhelpers/contrib/network/ufw.py b/hooks/charmhelpers/contrib/network/ufw.py new file mode 100644 index 00000000..2199623a --- /dev/null +++ b/hooks/charmhelpers/contrib/network/ufw.py @@ -0,0 +1,182 @@ +""" +This module contains helpers to add and remove ufw rules. + +Examples: + +- open SSH port for subnet 10.0.3.0/24: + + >>> from charmhelpers.contrib.network import ufw + >>> ufw.enable() + >>> ufw.grant_access(src='10.0.3.0/24', dst='any', port='22', proto='tcp') + +- open service by name as defined in /etc/services: + + >>> from charmhelpers.contrib.network import ufw + >>> ufw.enable() + >>> ufw.service('ssh', 'open') + +- close service by port number: + + >>> from charmhelpers.contrib.network import ufw + >>> ufw.enable() + >>> ufw.service('4949', 'close') # munin +""" + +__author__ = "Felipe Reyes " + +import re +import subprocess +from charmhelpers.core import hookenv + + +def is_enabled(): + """ + Check if `ufw` is enabled + + :returns: True if ufw is enabled + """ + output = subprocess.check_output(['ufw', 'status'], env={'LANG': 'en_US'}) + + m = re.findall(r'^Status: active\n', output, re.M) + + return len(m) >= 1 + + +def enable(): + """ + Enable ufw + + :returns: True if ufw is successfully enabled + """ + if is_enabled(): + return True + + output = subprocess.check_output(['ufw', 'enable'], env={'LANG': 'en_US'}) + + m = re.findall('^Firewall is active and enabled on system startup\n', + output, re.M) + hookenv.log(output, level='DEBUG') + + if len(m) == 0: + hookenv.log("ufw couldn't be enabled", level='WARN') + return False + else: + hookenv.log("ufw enabled", level='INFO') + return True + + +def disable(): + """ + Disable ufw + + :returns: True if ufw is successfully disabled + """ + if not is_enabled(): + return True + + output = subprocess.check_output(['ufw', 'disable'], env={'LANG': 'en_US'}) + + m = re.findall(r'^Firewall stopped and disabled on system startup\n', + output, re.M) + hookenv.log(output, level='DEBUG') + + if len(m) == 0: + hookenv.log("ufw couldn't be disabled", level='WARN') + return False + else: + hookenv.log("ufw disabled", level='INFO') + return True + + +def modify_access(src, dst='any', port=None, proto=None, action='allow'): + """ + Grant access to an address or subnet + + :param src: address (e.g. 192.168.1.234) or subnet + (e.g. 192.168.1.0/24). + :param dst: destiny of the connection, if the machine has multiple IPs and + connections to only one of those have to accepted this is the + field has to be set. + :param port: destiny port + :param proto: protocol (tcp or udp) + :param action: `allow` or `delete` + """ + if not is_enabled(): + hookenv.log('ufw is disabled, skipping modify_access()', level='WARN') + return + + if action == 'delete': + cmd = ['ufw', 'delete', 'allow'] + else: + cmd = ['ufw', action] + + if src is not None: + cmd += ['from', src] + + if dst is not None: + cmd += ['to', dst] + + if port is not None: + cmd += ['port', port] + + if proto is not None: + cmd += ['proto', proto] + + hookenv.log('ufw {}: {}'.format(action, ' '.join(cmd)), level='DEBUG') + p = subprocess.Popen(cmd, stdout=subprocess.PIPE) + (stdout, stderr) = p.communicate() + + hookenv.log(stdout, level='INFO') + + if p.returncode != 0: + hookenv.log(stderr, level='ERROR') + hookenv.log('Error running: {}, exit code: {}'.format(' '.join(cmd), + p.returncode), + level='ERROR') + + +def grant_access(src, dst='any', port=None, proto=None): + """ + Grant access to an address or subnet + + :param src: address (e.g. 192.168.1.234) or subnet + (e.g. 192.168.1.0/24). + :param dst: destiny of the connection, if the machine has multiple IPs and + connections to only one of those have to accepted this is the + field has to be set. + :param port: destiny port + :param proto: protocol (tcp or udp) + """ + return modify_access(src, dst=dst, port=port, proto=proto, action='allow') + + +def revoke_access(src, dst='any', port=None, proto=None): + """ + Revoke access to an address or subnet + + :param src: address (e.g. 192.168.1.234) or subnet + (e.g. 192.168.1.0/24). + :param dst: destiny of the connection, if the machine has multiple IPs and + connections to only one of those have to accepted this is the + field has to be set. + :param port: destiny port + :param proto: protocol (tcp or udp) + """ + return modify_access(src, dst=dst, port=port, proto=proto, action='delete') + + +def service(name, action): + """ + Open/close access to a service + + :param name: could be a service name defined in `/etc/services` or a port + number. + :param action: `open` or `close` + """ + if action == 'open': + subprocess.check_output(['ufw', 'allow', name]) + elif action == 'close': + subprocess.check_output(['ufw', 'delete', 'allow', name]) + else: + raise Exception(("'{}' not supported, use 'allow' " + "or 'delete'").format(action)) diff --git a/hooks/charmhelpers/contrib/python/__init__.py b/hooks/charmhelpers/contrib/python/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hooks/charmhelpers/contrib/python/debug.py b/hooks/charmhelpers/contrib/python/debug.py new file mode 100644 index 00000000..b821d416 --- /dev/null +++ b/hooks/charmhelpers/contrib/python/debug.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# coding: utf-8 + +from __future__ import print_function + +__author__ = "Jorge Niedbalski " + +import atexit +import sys + +from charmhelpers.contrib.python.rpdb import Rpdb +from charmhelpers.core.hookenv import ( + open_port, + close_port, + ERROR, + log +) + +DEFAULT_ADDR = "0.0.0.0" +DEFAULT_PORT = 4444 + + +def _error(message): + log(message, level=ERROR) + + +def set_trace(addr=DEFAULT_ADDR, port=DEFAULT_PORT): + """ + Set a trace point using the remote debugger + """ + atexit.register(close_port, port) + try: + log("Starting a remote python debugger session on %s:%s" % (addr, + port)) + open_port(port) + debugger = Rpdb(addr=addr, port=port) + debugger.set_trace(sys._getframe().f_back) + except: + _error("Cannot start a remote debug session on %s:%s" % (addr, + port)) diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py new file mode 100644 index 00000000..78162b1b --- /dev/null +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# coding: utf-8 + +__author__ = "Jorge Niedbalski " + +from charmhelpers.fetch import apt_install, apt_update +from charmhelpers.core.hookenv import log + +try: + from pip import main as pip_execute +except ImportError: + apt_update() + apt_install('python-pip') + from pip import main as pip_execute + + +def parse_options(given, available): + """Given a set of options, check if available""" + for key, value in sorted(given.items()): + if key in available: + yield "--{0}={1}".format(key, value) + + +def pip_install_requirements(requirements, **options): + """Install a requirements file """ + command = ["install"] + + available_options = ('proxy', 'src', 'log', ) + for option in parse_options(options, available_options): + command.append(option) + + command.append("-r {0}".format(requirements)) + log("Installing from file: {} with options: {}".format(requirements, + command)) + pip_execute(command) + + +def pip_install(package, fatal=False, **options): + """Install a python package""" + command = ["install"] + + available_options = ('proxy', 'src', 'log', "index-url", ) + for option in parse_options(options, available_options): + command.append(option) + + if isinstance(package, list): + command.extend(package) + else: + command.append(package) + + log("Installing {} package with options: {}".format(package, + command)) + pip_execute(command) + + +def pip_uninstall(package, **options): + """Uninstall a python package""" + command = ["uninstall", "-q", "-y"] + + available_options = ('proxy', 'log', ) + for option in parse_options(options, available_options): + command.append(option) + + if isinstance(package, list): + command.extend(package) + else: + command.append(package) + + log("Uninstalling {} package with options: {}".format(package, + command)) + pip_execute(command) + + +def pip_list(): + """Returns the list of current python installed packages + """ + return pip_execute(["list"]) diff --git a/hooks/charmhelpers/contrib/python/rpdb.py b/hooks/charmhelpers/contrib/python/rpdb.py new file mode 100644 index 00000000..e9892995 --- /dev/null +++ b/hooks/charmhelpers/contrib/python/rpdb.py @@ -0,0 +1,42 @@ +"""Remote Python Debugger (pdb wrapper).""" + +__author__ = "Bertrand Janin " +__version__ = "0.1.3" + +import pdb +import socket +import sys + + +class Rpdb(pdb.Pdb): + + def __init__(self, addr="127.0.0.1", port=4444): + """Initialize the socket and initialize pdb.""" + + # Backup stdin and stdout before replacing them by the socket handle + self.old_stdout = sys.stdout + self.old_stdin = sys.stdin + + # Open a 'reusable' socket to let the webapp reload on the same port + self.skt = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) + self.skt.bind((addr, port)) + self.skt.listen(1) + (clientsocket, address) = self.skt.accept() + handle = clientsocket.makefile('rw') + pdb.Pdb.__init__(self, completekey='tab', stdin=handle, stdout=handle) + sys.stdout = sys.stdin = handle + + def shutdown(self): + """Revert stdin and stdout, close the socket.""" + sys.stdout = self.old_stdout + sys.stdin = self.old_stdin + self.skt.close() + self.set_continue() + + def do_continue(self, arg): + """Stop all operation on ``continue``.""" + self.shutdown() + return 1 + + do_EOF = do_quit = do_exit = do_c = do_cont = do_continue diff --git a/hooks/charmhelpers/contrib/python/version.py b/hooks/charmhelpers/contrib/python/version.py new file mode 100644 index 00000000..e74dc9bc --- /dev/null +++ b/hooks/charmhelpers/contrib/python/version.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +# coding: utf-8 + +__author__ = "Jorge Niedbalski " + +import sys + + +def current_version(): + """Current system python version""" + return sys.version_info + + +def current_version_string(): + """Current system python version as string major.minor.micro""" + return "{0}.{1}.{2}".format(sys.version_info.major, + sys.version_info.minor, + sys.version_info.micro) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py new file mode 100644 index 00000000..f3aa2821 --- /dev/null +++ b/hooks/charmhelpers/fetch/giturl.py @@ -0,0 +1,51 @@ +import os +from charmhelpers.fetch import ( + BaseFetchHandler, + UnhandledSource +) +from charmhelpers.core.host import mkdir + +import six +if six.PY3: + raise ImportError('GitPython does not support Python 3') + +try: + from git import Repo +except ImportError: + from charmhelpers.fetch import apt_install + apt_install("python-git") + from git import Repo + + +class GitUrlFetchHandler(BaseFetchHandler): + """Handler for git branches via generic and github URLs""" + def can_handle(self, source): + url_parts = self.parse_url(source) + # TODO (mattyw) no support for ssh git@ yet + if url_parts.scheme not in ('http', 'https', 'git'): + return False + else: + return True + + def clone(self, source, dest, branch): + if not self.can_handle(source): + raise UnhandledSource("Cannot handle {}".format(source)) + + repo = Repo.clone_from(source, dest) + repo.git.checkout(branch) + + def install(self, source, branch="master", dest=None): + url_parts = self.parse_url(source) + branch_name = url_parts.path.strip("/").split("/")[-1] + if dest: + dest_dir = os.path.join(dest, branch_name) + else: + dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", + branch_name) + if not os.path.exists(dest_dir): + mkdir(dest_dir, perms=0o755) + try: + self.clone(source, dest_dir, branch) + except OSError as e: + raise UnhandledSource(e.strerror) + return dest_dir From ff6971fed49e484f934f2b260c5ec02810b3fc4f Mon Sep 17 00:00:00 2001 From: Zhang Hua Date: Thu, 11 Dec 2014 21:25:23 +0800 Subject: [PATCH 14/75] fix KeyError: network-device-mtu --- hooks/quantum_contexts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index 3297aee6..a5310981 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -122,7 +122,8 @@ def _neutron_api_settings(): neutron_settings = { 'l2_population': rdata['l2-population'], 'overlay_network_type': rdata['overlay-network-type'], - 'network_device_mtu': rdata['network-device-mtu'], + 'network_device_mtu': rdata['network-device-mtu'] + if 'network-device-mtu' in rdata else 1500, } return neutron_settings return neutron_settings From d0e81fbc7de7a2463b37c353bf2d1cc037d04499 Mon Sep 17 00:00:00 2001 From: Zhang Hua Date: Thu, 11 Dec 2014 21:51:37 +0800 Subject: [PATCH 15/75] fix hanging indent --- hooks/quantum_contexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index a5310981..f097f134 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -123,7 +123,7 @@ def _neutron_api_settings(): 'l2_population': rdata['l2-population'], 'overlay_network_type': rdata['overlay-network-type'], 'network_device_mtu': rdata['network-device-mtu'] - if 'network-device-mtu' in rdata else 1500, + if 'network-device-mtu' in rdata else 1500, } return neutron_settings return neutron_settings From d2fa08858ebeba0878311d48793e0ed9a045b7b7 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 2 Feb 2015 13:32:22 +0000 Subject: [PATCH 16/75] Add dvr support --- hooks/quantum_contexts.py | 17 ++++++++++++----- hooks/quantum_utils.py | 3 +++ templates/juno/l3_agent.ini | 25 +++++++++++++++++++++++++ templates/juno/ml2_conf.ini | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 templates/juno/l3_agent.ini create mode 100644 templates/juno/ml2_conf.ini diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index 508bdac1..78f4f519 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -111,6 +111,7 @@ def _neutron_api_settings(): ''' neutron_settings = { 'l2_population': False, + 'enable_dvr': False, 'overlay_network_type': 'gre', } @@ -119,10 +120,11 @@ def _neutron_api_settings(): rdata = relation_get(rid=rid, unit=unit) if 'l2-population' not in rdata: continue - neutron_settings = { - 'l2_population': rdata['l2-population'], - 'overlay_network_type': rdata['overlay-network-type'], - } + neutron_settings['l2_population'] = rdata['l2-population'] + if 'overlay-network-type' in rdata: + neutron_settings['overlay_network_type'] = rdata['overlay-network-type'] + if 'enable-dvr' in rdata: + neutron_settings['enable_dvr'] = rdata['enable-dvr'] return neutron_settings return neutron_settings @@ -158,6 +160,7 @@ class NetworkServiceContext(OSContextGenerator): class L3AgentContext(OSContextGenerator): def __call__(self): + neutron_api_settings = _neutron_api_settings() ctxt = {} if config('run-internal-router') == 'leader': ctxt['handle_internal_only_router'] = eligible_leader(None) @@ -170,9 +173,12 @@ class L3AgentContext(OSContextGenerator): if config('external-network-id'): ctxt['ext_net_id'] = config('external-network-id') - if config('plugin'): ctxt['plugin'] = config('plugin') + if neutron_api_settings['enable_dvr'] == 'True': + ctxt['agent_mode'] = 'dvr_snat' + else: + ctxt['agent_mode'] = 'legacy' return ctxt @@ -243,6 +249,7 @@ class QuantumGatewayContext(OSContextGenerator): 'verbose': config('verbose'), 'instance_mtu': config('instance-mtu'), 'l2_population': neutron_api_settings['l2_population'], + 'enable_dvr': neutron_api_settings['enable_dvr'], 'overlay_network_type': neutron_api_settings['overlay_network_type'], } diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 857f8d9a..eac92d50 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -413,6 +413,9 @@ def register_configs(): drop_config = NEUTRON_ML2_PLUGIN_CONF if release >= 'icehouse': drop_config = NEUTRON_OVS_PLUGIN_CONF + # NOTE(gnuoy) neutron-vpn-agent supercedes l3-agent for icehouse + CONFIG_FILES[name][plugin][NEUTRON_L3_AGENT_CONF]['services'] = \ + ['neutron-vpn-agent'] if drop_config in CONFIG_FILES[name][plugin]: CONFIG_FILES[name][plugin].pop(drop_config) diff --git a/templates/juno/l3_agent.ini b/templates/juno/l3_agent.ini new file mode 100644 index 00000000..6fd6cd4b --- /dev/null +++ b/templates/juno/l3_agent.ini @@ -0,0 +1,25 @@ +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +############################################################################### +[DEFAULT] +interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver +auth_url = {{ service_protocol }}://{{ keystone_host }}:{{ service_port }}/v2.0 +auth_region = {{ region }} +admin_tenant_name = {{ service_tenant }} +admin_user = {{ service_username }} +admin_password = {{ service_password }} +root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf +handle_internal_only_routers = {{ handle_internal_only_router }} +{% if plugin == 'n1kv' %} +l3_agent_manager = neutron.agent.l3_agent.L3NATAgentWithStateReport +external_network_bridge = br-int +ovs_use_veth = False +use_namespaces = True +{% else %} +ovs_use_veth = True +{% endif %} +{% if ext_net_id -%} +gateway_external_network_id = {{ ext_net_id }} +{% endif -%} +agent_mode = {{ agent_mode }} diff --git a/templates/juno/ml2_conf.ini b/templates/juno/ml2_conf.ini new file mode 100644 index 00000000..5354b4c2 --- /dev/null +++ b/templates/juno/ml2_conf.ini @@ -0,0 +1,33 @@ +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +############################################################################### +[ml2] +type_drivers = gre,vxlan,vlan,flat +tenant_network_types = gre,vxlan,vlan,flat +mechanism_drivers = openvswitch,l2population + +[ml2_type_gre] +tunnel_id_ranges = 1:1000 + +[ml2_type_vxlan] +vni_ranges = 1001:2000 + +[ml2_type_vlan] +network_vlan_ranges = physnet1:1000:2000 + +[ml2_type_flat] +flat_networks = physnet1 + +[ovs] +enable_tunneling = True +local_ip = {{ local_ip }} +bridge_mappings = physnet1:br-data + +[agent] +tunnel_types = {{ overlay_network_type }} +l2_population = {{ l2_population }} +enable_distributed_routing = {{ enable_dvr }} + +[securitygroup] +firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver From 06f8796f935fd7f44dd1672f5a3626c2c4d718ba Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 4 Feb 2015 16:21:36 +0000 Subject: [PATCH 17/75] Restart neutron-openvswitch-agent on changes to relevant files --- hooks/quantum_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index eac92d50..b77e6a7e 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -307,7 +307,8 @@ NEUTRON_OVS_CONFIG_FILES = { 'neutron-metering-agent', 'neutron-lbaas-agent', 'neutron-plugin-vpn-agent', - 'neutron-vpn-agent'] + 'neutron-vpn-agent', + 'neutron-openvswitch-agent'] }, NEUTRON_L3_AGENT_CONF: { 'hook_contexts': [NetworkServiceContext(), @@ -339,7 +340,8 @@ NEUTRON_OVS_CONFIG_FILES = { }, NEUTRON_ML2_PLUGIN_CONF: { 'hook_contexts': [QuantumGatewayContext()], - 'services': ['neutron-plugin-openvswitch-agent'] + 'services': ['neutron-plugin-openvswitch-agent', + 'neutron-openvswitch-agent'] }, EXT_PORT_CONF: { 'hook_contexts': [ExternalPortContext()], From 37d85b0296ba18f51d0b045110a286d13146d756 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 9 Feb 2015 16:14:27 +0000 Subject: [PATCH 18/75] set sync file --- charm-helpers-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charm-helpers-tests.yaml b/charm-helpers-tests.yaml index 48b12f6f..62c87f0e 100644 --- a/charm-helpers-tests.yaml +++ b/charm-helpers-tests.yaml @@ -1,4 +1,4 @@ -branch: lp:charm-helpers +branch: lp:~cts-engineering/charm-helpers/neutron-mtu destination: tests/charmhelpers include: - contrib.amulet From 8319879b7b1467cc152a1408ad87de85778a70ec Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 9 Feb 2015 16:40:08 +0000 Subject: [PATCH 19/75] synced charm-helpers --- config.yaml | 1 + hooks/charmhelpers/contrib/network/ufw.py | 67 +++++++++++++++++++---- hooks/charmhelpers/core/host.py | 8 +-- hooks/charmhelpers/core/templating.py | 6 +- 4 files changed, 63 insertions(+), 19 deletions(-) diff --git a/config.yaml b/config.yaml index ded59252..984b1cf1 100644 --- a/config.yaml +++ b/config.yaml @@ -157,5 +157,6 @@ options: phy-nic-mtu: type: int default: 1500 + description: | To improve network performance of VM, sometimes we should keep VM MTU as 1500 and use charm to modify MTU of tunnel nic more than 1500 (e.g. 1546 for GRE) diff --git a/hooks/charmhelpers/contrib/network/ufw.py b/hooks/charmhelpers/contrib/network/ufw.py index 72b42953..1e79a0ca 100644 --- a/hooks/charmhelpers/contrib/network/ufw.py +++ b/hooks/charmhelpers/contrib/network/ufw.py @@ -46,6 +46,10 @@ import subprocess from charmhelpers.core import hookenv +class UFWError(Exception): + pass + + def is_enabled(): """ Check if `ufw` is enabled @@ -62,6 +66,53 @@ def is_enabled(): return len(m) >= 1 +def is_ipv6_ok(): + """ + Check if IPv6 support is present and ip6tables functional + + :returns: True if IPv6 is working, False otherwise + """ + + # do we have IPv6 in the machine? + if os.path.isdir('/proc/sys/net/ipv6'): + # is ip6tables kernel module loaded? + lsmod = subprocess.check_output(['lsmod'], universal_newlines=True) + matches = re.findall('^ip6_tables[ ]+', lsmod, re.M) + if len(matches) == 0: + # ip6tables support isn't complete, let's try to load it + try: + subprocess.check_output(['modprobe', 'ip6_tables'], + universal_newlines=True) + # great, we could load the module + return True + except subprocess.CalledProcessError as ex: + hookenv.log("Couldn't load ip6_tables module: %s" % ex.output, + level="WARN") + # we are in a world where ip6tables isn't working + # so we inform that the machine doesn't have IPv6 + return False + else: + # the module is present :) + return True + + else: + # the system doesn't have IPv6 + return False + + +def disable_ipv6(): + """ + Disable ufw IPv6 support in /etc/default/ufw + """ + exit_code = subprocess.call(['sed', '-i', 's/IPV6=.*/IPV6=no/g', + '/etc/default/ufw']) + if exit_code == 0: + hookenv.log('IPv6 support in ufw disabled', level='INFO') + else: + hookenv.log("Couldn't disable IPv6 support in ufw", level="ERROR") + raise UFWError("Couldn't disable IPv6 support in ufw") + + def enable(): """ Enable ufw @@ -71,16 +122,8 @@ def enable(): if is_enabled(): return True - if not os.path.isdir('/proc/sys/net/ipv6'): - # disable IPv6 support in ufw - hookenv.log("This machine doesn't have IPv6 enabled", level="INFO") - exit_code = subprocess.call(['sed', '-i', 's/IPV6=yes/IPV6=no/g', - '/etc/default/ufw']) - if exit_code == 0: - hookenv.log('IPv6 support in ufw disabled', level='INFO') - else: - hookenv.log("Couldn't disable IPv6 support in ufw", level="ERROR") - raise Exception("Couldn't disable IPv6 support in ufw") + if not is_ipv6_ok(): + disable_ipv6() output = subprocess.check_output(['ufw', 'enable'], universal_newlines=True, @@ -217,5 +260,5 @@ def service(name, action): subprocess.check_output(['ufw', 'delete', 'allow', str(name)], universal_newlines=True) else: - raise Exception(("'{}' not supported, use 'allow' " - "or 'delete'").format(action)) + raise UFWError(("'{}' not supported, use 'allow' " + "or 'delete'").format(action)) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 2ec26b0a..3c37db5e 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -191,11 +191,11 @@ def mkdir(path, owner='root', group='root', perms=0o555, force=False): def write_file(path, content, owner='root', group='root', perms=0o444): - """Create or overwrite a file with the contents of a string""" + """Create or overwrite a file with the contents of a byte string.""" log("Writing file {} {}:{} {:o}".format(path, owner, group, perms)) uid = pwd.getpwnam(owner).pw_uid gid = grp.getgrnam(group).gr_gid - with open(path, 'w') as target: + with open(path, 'wb') as target: os.fchown(target.fileno(), uid, gid) os.fchmod(target.fileno(), perms) target.write(content) @@ -305,11 +305,11 @@ def restart_on_change(restart_map, stopstart=False): ceph_client_changed function. """ def wrap(f): - def wrapped_f(*args): + def wrapped_f(*args, **kwargs): checksums = {} for path in restart_map: checksums[path] = file_hash(path) - f(*args) + f(*args, **kwargs) restarts = [] for path in restart_map: if checksums[path] != file_hash(path): diff --git a/hooks/charmhelpers/core/templating.py b/hooks/charmhelpers/core/templating.py index 97669092..45319998 100644 --- a/hooks/charmhelpers/core/templating.py +++ b/hooks/charmhelpers/core/templating.py @@ -21,7 +21,7 @@ from charmhelpers.core import hookenv def render(source, target, context, owner='root', group='root', - perms=0o444, templates_dir=None): + perms=0o444, templates_dir=None, encoding='UTF-8'): """ Render a template. @@ -64,5 +64,5 @@ def render(source, target, context, owner='root', group='root', level=hookenv.ERROR) raise e content = template.render(context) - host.mkdir(os.path.dirname(target), owner, group) - host.write_file(target, content, owner, group, perms) + host.mkdir(os.path.dirname(target), owner, group, perms=0o755) + host.write_file(target, content.encode(encoding), owner, group, perms) From 4387974a3a45084c67ec3ce3a869357c58687882 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 9 Feb 2015 18:19:40 +0000 Subject: [PATCH 20/75] synced ch --- hooks/charmhelpers/contrib/network/ip.py | 27 +++++--- hooks/charmhelpers/core/host.py | 79 +++++++++++++++++------- 2 files changed, 74 insertions(+), 32 deletions(-) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index a84beb3d..3431e29f 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -374,24 +374,33 @@ def is_bridge_member(nic): return False -def configure_phy_nic_mtu(mng_ip=None): +def configure_phy_nic_mtu(mgmt_ip=None): """Configure mtu for physical nic.""" phy_nic_mtu = config('phy-nic-mtu') if phy_nic_mtu >= 1500: phy_nic = None - if mng_ip is None: - mng_ip = unit_get('private-address') + if mgmt_ip is None: + mgmt_ip = unit_get('private-address') + for nic in list_nics(['eth', 'bond', 'br']): - if mng_ip in get_ipv4_addr(nic, fatal=False): + if config('prefer-ipv6'): + addrs = get_ipv6_addr(iface=nic, fatal=False) + else: + addrs = get_ipv4_addr(iface=nic, fatal=False) + + if mgmt_ip in addrs: phy_nic = nic - # need to find the associated phy nic for bridge + # If bridge iface, find the associated phy nic if nic.startswith('br'): for brnic in get_bridge_nics(nic): if brnic.startswith('eth') or brnic.startswith('bond'): phy_nic = brnic break + + if phy_nic and phy_nic_mtu != get_nic_mtu(phy_nic): + set_nic_mtu(phy_nic, str(phy_nic_mtu), persistence=True) + log('Setting mtu=%s for phy_nic=%s' % (phy_nic_mtu, + phy_nic), + level=INFO) + break - if phy_nic is not None and phy_nic_mtu != get_nic_mtu(phy_nic): - set_nic_mtu(phy_nic, str(phy_nic_mtu), persistence=True) - log('set mtu={} for phy_nic={}' - .format(phy_nic_mtu, phy_nic), level=INFO) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 3c37db5e..e2d6c655 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -372,39 +372,72 @@ def list_nics(nic_type): def set_nic_mtu(nic, mtu, persistence=False): - '''Set MTU on a network interface''' + """Set MTU on a network interface.""" cmd = ['ip', 'link', 'set', nic, 'mtu', mtu] subprocess.check_call(cmd) + # persistence mtu configuration if not persistence: return - if os.path.exists("/etc/network/interfaces.d/%s.cfg" % nic): - nic_cfg_file = "/etc/network/interfaces.d/%s.cfg" % nic - else: - nic_cfg_file = "/etc/network/interfaces" - f = open(nic_cfg_file, "r") - lines = f.readlines() + nic_cfg_path = "/etc/network/interfaces" + nicd_cfg_path = "%s.d/%s.cfg" % (nic_cfg_path, nic) + if os.path.exists(nicd_cfg_path): + nic_cfg_path = nicd_cfg_path + + with open(nic_cfg_path, "r") as fd: + lines = fd.readlines() + found = False length = len(lines) - for i in range(len(lines)): - lines[i] = lines[i].replace('\n', '') - if lines[i].startswith("iface %s" % nic): + for i, line in enumerate(lines): + lines[i] = line.strip() + if line.strip().startswith("iface %s" % nic): found = True - lines.insert(i + 1, " up ip link set $IFACE mtu %s" % mtu) - lines.insert(i + 2, " down ip link set $IFACE mtu 1500") - if length > i + 2 and lines[i + 3].startswith(" up ip link set $IFACE mtu"): - del lines[i + 3] - if length > i + 2 and lines[i + 3].startswith(" down ip link set $IFACE mtu"): - del lines[i + 3] + index = i break - if not found: - lines.insert(length + 1, "") - lines.insert(length + 2, "auto %s" % nic) - lines.insert(length + 3, "iface %s inet dhcp" % nic) - lines.insert(length + 4, " up ip link set $IFACE mtu %s" % mtu) - lines.insert(length + 5, " down ip link set $IFACE mtu 1500") - write_file(path=nic_cfg_file, content="\n".join(lines), perms=0o644) + + inserts_cfg_found = """ + up ip link set $IFACE mtu %s + down ip link set $IFACE mtu 1500 + """ % (mtu) + + inserts_cfg_not_found = """ + auto %s + iface %s inet dhcp + up ip link set $IFACE mtu %s + down ip link set $IFACE mtu 1500 + """ % (nic, nic, mtu) + + deletes = """ + up ip link set $IFACE mtu + down ip link set $IFACE mtu + """ + + if found: + for line in inserts_cfg_found.split('\n'): + if not line.strip(): + continue + + index += 1 + lines.insert(index, " %s" % line.strip()) + + for line in deletes.split('\n'): + if not line.strip(): + continue + + if length > index and lines[index + 1].startswith(line): + del lines[index + 1] + else: + index = length + for line in inserts_cfg_not_found.split('\n'): + if not line.strip(): + continue + + index += 1 + lines.insert(index, " %s" % line.strip()) + + write_file(path=nic_cfg_path, content="\n".join(lines), perms=0o644) def get_nic_mtu(nic): From 717248440c4636277c58ae18b9438a122bf9e2a8 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 9 Feb 2015 20:20:00 +0000 Subject: [PATCH 21/75] synced ch and fixed tests --- config.yaml | 5 +- .../charmhelpers/contrib/openstack/context.py | 39 +++++++ hooks/quantum_contexts.py | 53 ++------- hooks/quantum_hooks.py | 5 - templates/ext-port.conf | 1 + unit_tests/test_quantum_contexts.py | 104 ++++++++++++------ unit_tests/test_quantum_hooks.py | 4 - 7 files changed, 122 insertions(+), 89 deletions(-) diff --git a/config.yaml b/config.yaml index 984b1cf1..ac543d3c 100644 --- a/config.yaml +++ b/config.yaml @@ -158,5 +158,6 @@ options: type: int default: 1500 description: | - To improve network performance of VM, sometimes we should keep VM MTU as 1500 - and use charm to modify MTU of tunnel nic more than 1500 (e.g. 1546 for GRE) + To improve network performance of VM, sometimes we should keep VM MTU as + 1500 and use charm to modify MTU of tunnel nic more than 1500 (e.g. 1546 + for GRE). diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index c7c4cd4a..612762c6 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -16,6 +16,7 @@ import json import os +import re import time from base64 import b64decode from subprocess import check_call @@ -47,6 +48,8 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.sysctl import create as sysctl_create from charmhelpers.core.host import ( + list_nics, + get_nic_hwaddr, mkdir, write_file, ) @@ -66,10 +69,12 @@ from charmhelpers.contrib.openstack.neutron import ( ) from charmhelpers.contrib.network.ip import ( get_address_in_network, + get_ipv4_addr, get_ipv6_addr, get_netmask_for_address, format_ipv6_addr, is_address_in_network, + is_bridge_member, ) from charmhelpers.contrib.openstack.utils import get_host_ip @@ -833,6 +838,40 @@ class NeutronContext(OSContextGenerator): return ctxt +class NeutronPortContext(OSContextGenerator): + def resolve_port(self, config_key): + if not config(config_key): + return None + + hwaddr_to_nic = {} + hwaddr_to_ip = {} + for nic in list_nics(['eth', 'bond']): + hwaddr = get_nic_hwaddr(nic) + hwaddr_to_nic[hwaddr] = nic + addresses = get_ipv4_addr(nic, fatal=False) + \ + get_ipv6_addr(iface=nic, fatal=False) + hwaddr_to_ip[hwaddr] = addresses + + mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I) + for entry in config(config_key).split(): + entry = entry.strip() + if re.match(mac_regex, entry): + if entry in hwaddr_to_nic and len(hwaddr_to_ip[entry]) == 0: + # If the nic is part of a bridge then don't use it + if is_bridge_member(hwaddr_to_nic[entry]): + continue + # Entry is a MAC address for a valid interface that doesn't + # have an IP address assigned yet. + return hwaddr_to_nic[entry] + else: + # If the passed entry is not a MAC address, assume it's a valid + # interface, and that the user put it there on purpose (we can + # trust it to be the real external network). + return entry + + return None + + class OSConfigFlagContext(OSContextGenerator): """Provides support for user-defined config flags. diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index c3bc7401..f05c1fc0 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -2,10 +2,6 @@ import os import uuid import socket -from charmhelpers.core.host import ( - list_nics, - get_nic_hwaddr -) from charmhelpers.core.hookenv import ( config, relation_ids, @@ -20,6 +16,7 @@ from charmhelpers.fetch import ( from charmhelpers.contrib.openstack.context import ( OSContextGenerator, context_complete, + NeutronPortContext, ) from charmhelpers.contrib.openstack.utils import ( get_os_codename_install_source @@ -27,12 +24,8 @@ from charmhelpers.contrib.openstack.utils import ( from charmhelpers.contrib.hahelpers.cluster import( eligible_leader ) -import re from charmhelpers.contrib.network.ip import ( get_address_in_network, - get_ipv4_addr, - get_ipv6_addr, - is_bridge_member, ) DB_USER = "quantum" @@ -111,8 +104,8 @@ def _neutron_api_settings(): ''' neutron_settings = { 'l2_population': False, - 'network_device_mtu': 1500, 'overlay_network_type': 'gre', + 'network_device_mtu': 1500, } for rid in relation_ids('neutron-plugin-api'): for unit in related_units(rid): @@ -122,8 +115,7 @@ def _neutron_api_settings(): neutron_settings = { 'l2_population': rdata['l2-population'], 'overlay_network_type': rdata['overlay-network-type'], - 'network_device_mtu': rdata['network-device-mtu'] - if 'network-device-mtu' in rdata else 1500, + 'network_device_mtu': rdata.get('network-device-mtu', 1500) } return neutron_settings return neutron_settings @@ -178,44 +170,13 @@ class L3AgentContext(OSContextGenerator): return ctxt -class NeutronPortContext(OSContextGenerator): - - def _resolve_port(self, config_key): - if not config(config_key): - return None - hwaddr_to_nic = {} - hwaddr_to_ip = {} - for nic in list_nics(['eth', 'bond']): - hwaddr = get_nic_hwaddr(nic) - hwaddr_to_nic[hwaddr] = nic - addresses = get_ipv4_addr(nic, fatal=False) + \ - get_ipv6_addr(iface=nic, fatal=False) - hwaddr_to_ip[hwaddr] = addresses - mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I) - for entry in config(config_key).split(): - entry = entry.strip() - if re.match(mac_regex, entry): - if entry in hwaddr_to_nic and len(hwaddr_to_ip[entry]) == 0: - # If the nic is part of a bridge then don't use it - if is_bridge_member(hwaddr_to_nic[entry]): - continue - # Entry is a MAC address for a valid interface that doesn't - # have an IP address assigned yet. - return hwaddr_to_nic[entry] - else: - # If the passed entry is not a MAC address, assume it's a valid - # interface, and that the user put it there on purpose (we can - # trust it to be the real external network). - return entry - return None - - class ExternalPortContext(NeutronPortContext): def __call__(self): - port = self._resolve_port('ext-port') + port = self.resolve_port('ext-port') if port: - return {"ext_port": port} + return {"ext_port": port, + "mtu": config('phy-nic-mtu')} else: return None @@ -223,7 +184,7 @@ class ExternalPortContext(NeutronPortContext): class DataPortContext(NeutronPortContext): def __call__(self): - port = self._resolve_port('data-port') + port = self.resolve_port('data-port') if port: return {"data_port": port} else: diff --git a/hooks/quantum_hooks.py b/hooks/quantum_hooks.py index d8f2bee3..e8b74861 100755 --- a/hooks/quantum_hooks.py +++ b/hooks/quantum_hooks.py @@ -22,9 +22,6 @@ from charmhelpers.core.host import ( restart_on_change, lsb_release, ) -from charmhelpers.contrib.network.ip import ( - configure_phy_nic_mtu -) from charmhelpers.contrib.hahelpers.cluster import( get_hacluster_config, eligible_leader @@ -81,7 +78,6 @@ def install(): fatal=True) apt_install(filter_installed_packages(get_packages()), fatal=True) - configure_phy_nic_mtu() else: log('Please provide a valid plugin config', level=ERROR) sys.exit(1) @@ -114,7 +110,6 @@ def config_changed(): if valid_plugin(): CONFIGS.write_all() configure_ovs() - configure_phy_nic_mtu() else: log('Please provide a valid plugin config', level=ERROR) sys.exit(1) diff --git a/templates/ext-port.conf b/templates/ext-port.conf index 6080c30e..6a492f3c 100644 --- a/templates/ext-port.conf +++ b/templates/ext-port.conf @@ -6,4 +6,5 @@ task script ip link set {{ ext_port }} up + ip link set {{ ext_port }} mtu {{ mtu }} end script \ No newline at end of file diff --git a/unit_tests/test_quantum_contexts.py b/unit_tests/test_quantum_contexts.py index ca3a59f0..443319a2 100644 --- a/unit_tests/test_quantum_contexts.py +++ b/unit_tests/test_quantum_contexts.py @@ -16,11 +16,7 @@ TO_PATCH = [ 'config', 'context_complete', 'eligible_leader', - 'get_ipv4_addr', - 'get_ipv6_addr', - 'get_nic_hwaddr', 'get_os_codename_install_source', - 'list_nics', 'relation_get', 'relation_ids', 'related_units', @@ -131,14 +127,12 @@ class TestNeutronPortContext(CharmTestCase): } self.absent_macs = "aa:a5:ae:ae:ab:a4 " - def test_no_ext_port(self): - self.config.return_value = None - self.assertIsNone(quantum_contexts.ExternalPortContext()()) + def fake_config(self, cfgdict): - def test_ext_port_eth(self): - self.config.return_value = 'eth1010' - self.assertEquals(quantum_contexts.ExternalPortContext()(), - {'ext_port': 'eth1010'}) + def _fake_config(key): + return cfgdict.get(key) + + return _fake_config def _fake_get_hwaddr(self, arg): return self.machine_macs[arg] @@ -146,31 +140,77 @@ class TestNeutronPortContext(CharmTestCase): def _fake_get_ipv4(self, arg, fatal=False): return self.machine_nics[arg] - def test_ext_port_mac(self): - config_macs = self.absent_macs + " " + self.machine_macs['eth2'] - self.get_ipv4_addr.side_effect = self._fake_get_ipv4 - self.get_ipv6_addr.return_value = [] - self.config.return_value = config_macs - self.list_nics.return_value = self.machine_macs.keys() - self.get_nic_hwaddr.side_effect = self._fake_get_hwaddr - self.assertEquals(quantum_contexts.ExternalPortContext()(), - {'ext_port': 'eth2'}) - self.config.return_value = self.absent_macs + @patch('charmhelpers.contrib.openstack.context.config') + def test_no_ext_port(self, mock_config): + self.config.side_effect = config = self.fake_config({}) + mock_config.side_effect = config self.assertIsNone(quantum_contexts.ExternalPortContext()()) - def test_ext_port_mac_one_used_nic(self): - config_macs = self.machine_macs['eth1'] + " " + \ - self.machine_macs['eth2'] - self.get_ipv4_addr.side_effect = self._fake_get_ipv4 - self.get_ipv6_addr.return_value = [] - self.config.return_value = config_macs - self.list_nics.return_value = self.machine_macs.keys() - self.get_nic_hwaddr.side_effect = self._fake_get_hwaddr + @patch('charmhelpers.contrib.openstack.context.config') + def test_ext_port_eth(self, mock_config): + config = self.fake_config({'ext-port': 'eth1010'}) + self.config.side_effect = config + mock_config.side_effect = config self.assertEquals(quantum_contexts.ExternalPortContext()(), - {'ext_port': 'eth2'}) + {'ext_port': 'eth1010', 'mtu': None}) - def test_data_port_eth(self): - self.config.return_value = 'eth1010' + @patch('charmhelpers.contrib.openstack.context.get_nic_hwaddr') + @patch('charmhelpers.contrib.openstack.context.list_nics') + @patch('charmhelpers.contrib.openstack.context.get_ipv6_addr') + @patch('charmhelpers.contrib.openstack.context.get_ipv4_addr') + @patch('charmhelpers.contrib.openstack.context.config') + def test_ext_port_mac(self, mock_config, mock_get_ipv4_addr, + mock_get_ipv6_addr, mock_list_nics, + mock_get_nic_hwaddr): + config_macs = self.absent_macs + " " + self.machine_macs['eth2'] + config = self.fake_config({'ext-port': config_macs}) + self.config.side_effect = config + mock_config.side_effect = config + + mock_get_ipv4_addr.side_effect = self._fake_get_ipv4 + mock_get_ipv6_addr.return_value = [] + mock_list_nics.return_value = self.machine_macs.keys() + mock_get_nic_hwaddr.side_effect = self._fake_get_hwaddr + + self.assertEquals(quantum_contexts.ExternalPortContext()(), + {'ext_port': 'eth2', 'mtu': None}) + + config = self.fake_config({'ext-port': self.absent_macs}) + self.config.side_effect = config + mock_config.side_effect = config + + self.assertIsNone(quantum_contexts.ExternalPortContext()()) + self.assertTrue(mock_config.called) + + @patch('charmhelpers.contrib.openstack.context.get_nic_hwaddr') + @patch('charmhelpers.contrib.openstack.context.list_nics') + @patch('charmhelpers.contrib.openstack.context.get_ipv6_addr') + @patch('charmhelpers.contrib.openstack.context.get_ipv4_addr') + @patch('charmhelpers.contrib.openstack.context.config') + def test_ext_port_mac_one_used_nic(self, mock_config, + mock_get_ipv4_addr, + mock_get_ipv6_addr, mock_list_nics, + mock_get_nic_hwaddr): + + config_macs = "%s %s" % (self.machine_macs['eth1'], + self.machine_macs['eth2']) + + mock_get_ipv4_addr.side_effect = self._fake_get_ipv4 + mock_get_ipv6_addr.return_value = [] + mock_list_nics.return_value = self.machine_macs.keys() + mock_get_nic_hwaddr.side_effect = self._fake_get_hwaddr + + config = self.fake_config({'ext-port': config_macs, + 'phy-nic-mtu': 1234}) + self.config.side_effect = config + mock_config.side_effect = config + self.assertEquals(quantum_contexts.ExternalPortContext()(), + {'ext_port': 'eth2', 'mtu': 1234}) + + @patch('charmhelpers.contrib.openstack.context.config') + def test_data_port_eth(self, mock_config): + config = self.fake_config({'data-port': 'eth1010'}) + mock_config.side_effect = config self.assertEquals(quantum_contexts.DataPortContext()(), {'data_port': 'eth1010'}) diff --git a/unit_tests/test_quantum_hooks.py b/unit_tests/test_quantum_hooks.py index c2a02c79..fd555736 100644 --- a/unit_tests/test_quantum_hooks.py +++ b/unit_tests/test_quantum_hooks.py @@ -48,7 +48,6 @@ TO_PATCH = [ 'remove_legacy_ha_files', 'cleanup_ovs_netns', 'stop_neutron_ha_monitor_daemon', - 'configure_phy_nic_mtu' ] @@ -88,7 +87,6 @@ class TestQuantumHooks(CharmTestCase): self.assertTrue(self.get_early_packages.called) self.assertTrue(self.get_packages.called) self.assertTrue(self.execd_preinstall.called) - self.assertTrue(self.configure_phy_nic_mtu.called) def test_install_hook_precise_nocloudarchive(self): self.test_config.set('openstack-origin', 'distro') @@ -123,7 +121,6 @@ class TestQuantumHooks(CharmTestCase): self.assertTrue(_amqp_joined.called) self.assertTrue(_amqp_nova_joined.called) self.create_sysctl.assert_called() - self.assertTrue(self.configure_phy_nic_mtu.called) def test_config_changed_upgrade(self): self.openstack_upgrade_available.return_value = True @@ -131,7 +128,6 @@ class TestQuantumHooks(CharmTestCase): self._call_hook('config-changed') self.assertTrue(self.do_openstack_upgrade.called) self.assertTrue(self.configure_ovs.called) - self.assertTrue(self.configure_phy_nic_mtu.called) def test_config_changed_n1kv(self): self.openstack_upgrade_available.return_value = False From c92def8a3c3a5453febf06a3415a81c0a9058c32 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 9 Feb 2015 20:27:53 +0000 Subject: [PATCH 22/75] sync ch --- hooks/charmhelpers/contrib/network/ip.py | 41 +-------------- hooks/charmhelpers/core/host.py | 67 +----------------------- 2 files changed, 3 insertions(+), 105 deletions(-) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 3431e29f..98b17544 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -23,14 +23,7 @@ from functools import partial from charmhelpers.core.hookenv import unit_get from charmhelpers.fetch import apt_install from charmhelpers.core.hookenv import ( - config, - log, - INFO -) -from charmhelpers.core.host import ( - list_nics, - get_nic_mtu, - set_nic_mtu + log ) try: @@ -372,35 +365,3 @@ def is_bridge_member(nic): return True return False - - -def configure_phy_nic_mtu(mgmt_ip=None): - """Configure mtu for physical nic.""" - phy_nic_mtu = config('phy-nic-mtu') - if phy_nic_mtu >= 1500: - phy_nic = None - if mgmt_ip is None: - mgmt_ip = unit_get('private-address') - - for nic in list_nics(['eth', 'bond', 'br']): - if config('prefer-ipv6'): - addrs = get_ipv6_addr(iface=nic, fatal=False) - else: - addrs = get_ipv4_addr(iface=nic, fatal=False) - - if mgmt_ip in addrs: - phy_nic = nic - # If bridge iface, find the associated phy nic - if nic.startswith('br'): - for brnic in get_bridge_nics(nic): - if brnic.startswith('eth') or brnic.startswith('bond'): - phy_nic = brnic - break - - if phy_nic and phy_nic_mtu != get_nic_mtu(phy_nic): - set_nic_mtu(phy_nic, str(phy_nic_mtu), persistence=True) - log('Setting mtu=%s for phy_nic=%s' % (phy_nic_mtu, - phy_nic), - level=INFO) - - break diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index e2d6c655..b771c611 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -371,74 +371,11 @@ def list_nics(nic_type): return interfaces -def set_nic_mtu(nic, mtu, persistence=False): - """Set MTU on a network interface.""" +def set_nic_mtu(nic, mtu): + '''Set MTU on a network interface''' cmd = ['ip', 'link', 'set', nic, 'mtu', mtu] subprocess.check_call(cmd) - # persistence mtu configuration - if not persistence: - return - - nic_cfg_path = "/etc/network/interfaces" - nicd_cfg_path = "%s.d/%s.cfg" % (nic_cfg_path, nic) - if os.path.exists(nicd_cfg_path): - nic_cfg_path = nicd_cfg_path - - with open(nic_cfg_path, "r") as fd: - lines = fd.readlines() - - found = False - length = len(lines) - for i, line in enumerate(lines): - lines[i] = line.strip() - if line.strip().startswith("iface %s" % nic): - found = True - index = i - break - - inserts_cfg_found = """ - up ip link set $IFACE mtu %s - down ip link set $IFACE mtu 1500 - """ % (mtu) - - inserts_cfg_not_found = """ - auto %s - iface %s inet dhcp - up ip link set $IFACE mtu %s - down ip link set $IFACE mtu 1500 - """ % (nic, nic, mtu) - - deletes = """ - up ip link set $IFACE mtu - down ip link set $IFACE mtu - """ - - if found: - for line in inserts_cfg_found.split('\n'): - if not line.strip(): - continue - - index += 1 - lines.insert(index, " %s" % line.strip()) - - for line in deletes.split('\n'): - if not line.strip(): - continue - - if length > index and lines[index + 1].startswith(line): - del lines[index + 1] - else: - index = length - for line in inserts_cfg_not_found.split('\n'): - if not line.strip(): - continue - - index += 1 - lines.insert(index, " %s" % line.strip()) - - write_file(path=nic_cfg_path, content="\n".join(lines), perms=0o644) - def get_nic_mtu(nic): cmd = ['ip', 'addr', 'show', nic] From 33674e07a62435805cfd089891a5098e10bb560d Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 10 Feb 2015 09:53:54 +0000 Subject: [PATCH 23/75] default to no network-device-mtu --- hooks/quantum_contexts.py | 5 ++++- templates/ext-port.conf | 12 +++++++++--- unit_tests/test_quantum_contexts.py | 2 -- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index f05c1fc0..f10ab921 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -115,8 +115,11 @@ def _neutron_api_settings(): neutron_settings = { 'l2_population': rdata['l2-population'], 'overlay_network_type': rdata['overlay-network-type'], - 'network_device_mtu': rdata.get('network-device-mtu', 1500) } + mtu = rdata.get('network-device-mtu') + if mtu: + neutron_settings['network_device_mtu'] = mtu + return neutron_settings return neutron_settings diff --git a/templates/ext-port.conf b/templates/ext-port.conf index 6a492f3c..41bafc49 100644 --- a/templates/ext-port.conf +++ b/templates/ext-port.conf @@ -5,6 +5,12 @@ start on runlevel [2345] task script - ip link set {{ ext_port }} up - ip link set {{ ext_port }} mtu {{ mtu }} -end script \ No newline at end of file + EXT_PORT="{{ ext_port }}" + MTU="{{ mtu }}" + if [ -n "$EXT_PORT" ]; then + ip link set $EXT_PORT up + if [ -n "$MTU" ]; then + ip link set $EXT_PORT mtu $MTU + fi + fi +end script diff --git a/unit_tests/test_quantum_contexts.py b/unit_tests/test_quantum_contexts.py index 443319a2..be8c23bb 100644 --- a/unit_tests/test_quantum_contexts.py +++ b/unit_tests/test_quantum_contexts.py @@ -415,12 +415,10 @@ class TestMisc(CharmTestCase): self.relation_ids.return_value = ['foo'] self.related_units.return_value = ['bar'] self.test_relation.set({'l2-population': True, - 'network-device-mtu': 1500, 'overlay-network-type': 'gre', }) self.relation_get.side_effect = self.test_relation.get self.assertEquals(quantum_contexts._neutron_api_settings(), {'l2_population': True, - 'network_device_mtu': 1500, 'overlay_network_type': 'gre'}) def test_neutron_api_settings_no_apiplugin(self): From a39353d623c4283185f61aeef692b166d84e808a Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 10 Feb 2015 10:50:32 +0000 Subject: [PATCH 24/75] default to no network-device-mtu --- hooks/quantum_contexts.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index f10ab921..eb52dca4 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -116,9 +116,9 @@ def _neutron_api_settings(): 'l2_population': rdata['l2-population'], 'overlay_network_type': rdata['overlay-network-type'], } - mtu = rdata.get('network-device-mtu') - if mtu: - neutron_settings['network_device_mtu'] = mtu + net_dev_mtu = rdata.get('network-device-mtu') + if net_dev_mtu: + neutron_settings['network_device_mtu'] = net_dev_mtu return neutron_settings return neutron_settings @@ -209,10 +209,14 @@ class QuantumGatewayContext(OSContextGenerator): 'verbose': config('verbose'), 'instance_mtu': config('instance-mtu'), 'l2_population': neutron_api_settings['l2_population'], - 'network_device_mtu': neutron_api_settings['network_device_mtu'], 'overlay_network_type': neutron_api_settings['overlay_network_type'], } + + net_dev_mtu = neutron_api_settings.get('network_device_mtu') + if net_dev_mtu: + ctxt['network_device_mtu'] = net_dev_mtu + return ctxt From f9f95ae0ac8e737210df6d21ed80699c1f4861c2 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 10 Feb 2015 12:00:24 +0000 Subject: [PATCH 25/75] default to no network-device-mtu --- config.yaml | 4 ++-- hooks/quantum_contexts.py | 11 +++++++---- templates/ext-port.conf | 2 +- unit_tests/test_quantum_contexts.py | 10 +++++----- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/config.yaml b/config.yaml index ac543d3c..580987cd 100644 --- a/config.yaml +++ b/config.yaml @@ -156,8 +156,8 @@ options: HA Cluster nodes. phy-nic-mtu: type: int - default: 1500 + default: 0 description: | To improve network performance of VM, sometimes we should keep VM MTU as 1500 and use charm to modify MTU of tunnel nic more than 1500 (e.g. 1546 - for GRE). + for GRE). A value of zero means no mtu will be set/modified. diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index eb52dca4..6e8769a1 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -176,12 +176,15 @@ class L3AgentContext(OSContextGenerator): class ExternalPortContext(NeutronPortContext): def __call__(self): + ctxt = {} port = self.resolve_port('ext-port') if port: - return {"ext_port": port, - "mtu": config('phy-nic-mtu')} - else: - return None + ctxt = {"ext_port": port} + mtu = config('phy-nic-mtu') + if mtu: + ctxt['ext_port_mtu'] = mtu + + return ctxt class DataPortContext(NeutronPortContext): diff --git a/templates/ext-port.conf b/templates/ext-port.conf index 41bafc49..1d850240 100644 --- a/templates/ext-port.conf +++ b/templates/ext-port.conf @@ -6,7 +6,7 @@ task script EXT_PORT="{{ ext_port }}" - MTU="{{ mtu }}" + MTU="{{ ext_port_mtu }}" if [ -n "$EXT_PORT" ]; then ip link set $EXT_PORT up if [ -n "$MTU" ]; then diff --git a/unit_tests/test_quantum_contexts.py b/unit_tests/test_quantum_contexts.py index be8c23bb..f3f8b00e 100644 --- a/unit_tests/test_quantum_contexts.py +++ b/unit_tests/test_quantum_contexts.py @@ -144,7 +144,7 @@ class TestNeutronPortContext(CharmTestCase): def test_no_ext_port(self, mock_config): self.config.side_effect = config = self.fake_config({}) mock_config.side_effect = config - self.assertIsNone(quantum_contexts.ExternalPortContext()()) + self.assertEquals(quantum_contexts.ExternalPortContext()(), {}) @patch('charmhelpers.contrib.openstack.context.config') def test_ext_port_eth(self, mock_config): @@ -152,7 +152,7 @@ class TestNeutronPortContext(CharmTestCase): self.config.side_effect = config mock_config.side_effect = config self.assertEquals(quantum_contexts.ExternalPortContext()(), - {'ext_port': 'eth1010', 'mtu': None}) + {'ext_port': 'eth1010'}) @patch('charmhelpers.contrib.openstack.context.get_nic_hwaddr') @patch('charmhelpers.contrib.openstack.context.list_nics') @@ -173,13 +173,13 @@ class TestNeutronPortContext(CharmTestCase): mock_get_nic_hwaddr.side_effect = self._fake_get_hwaddr self.assertEquals(quantum_contexts.ExternalPortContext()(), - {'ext_port': 'eth2', 'mtu': None}) + {'ext_port': 'eth2'}) config = self.fake_config({'ext-port': self.absent_macs}) self.config.side_effect = config mock_config.side_effect = config - self.assertIsNone(quantum_contexts.ExternalPortContext()()) + self.assertEquals(quantum_contexts.ExternalPortContext()(), {}) self.assertTrue(mock_config.called) @patch('charmhelpers.contrib.openstack.context.get_nic_hwaddr') @@ -205,7 +205,7 @@ class TestNeutronPortContext(CharmTestCase): self.config.side_effect = config mock_config.side_effect = config self.assertEquals(quantum_contexts.ExternalPortContext()(), - {'ext_port': 'eth2', 'mtu': 1234}) + {'ext_port': 'eth2', 'ext_port_mtu': 1234}) @patch('charmhelpers.contrib.openstack.context.config') def test_data_port_eth(self, mock_config): From d801d52e3138ce13be2e6909baa66140949d5da6 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 10 Feb 2015 12:08:44 +0000 Subject: [PATCH 26/75] default to no network-device-mtu --- hooks/quantum_contexts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index 6e8769a1..31e31324 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -105,7 +105,6 @@ def _neutron_api_settings(): neutron_settings = { 'l2_population': False, 'overlay_network_type': 'gre', - 'network_device_mtu': 1500, } for rid in relation_ids('neutron-plugin-api'): for unit in related_units(rid): From 3d8c48d0f4b99b9696eba4fbdbb097c91f2d6e19 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 10 Feb 2015 12:33:46 +0000 Subject: [PATCH 27/75] restart ext-port on change --- hooks/quantum_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 857f8d9a..cffd45ec 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -289,7 +289,7 @@ QUANTUM_OVS_CONFIG_FILES = { }, EXT_PORT_CONF: { 'hook_contexts': [ExternalPortContext()], - 'services': [] + 'services': ['ext-port'] } } QUANTUM_OVS_CONFIG_FILES.update(QUANTUM_SHARED_CONFIG_FILES) From 83ae5cbbe915b63109b55e62827783cc86d75232 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 10 Feb 2015 12:41:46 +0000 Subject: [PATCH 28/75] restart ext-port on change --- hooks/quantum_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index cffd45ec..7fb4477a 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -343,7 +343,7 @@ NEUTRON_OVS_CONFIG_FILES = { }, EXT_PORT_CONF: { 'hook_contexts': [ExternalPortContext()], - 'services': [] + 'services': ['ext-port'] } } NEUTRON_OVS_CONFIG_FILES.update(NEUTRON_SHARED_CONFIG_FILES) From bcb49ed3aa9f2b5f00ac0f71a63ce32413e7adb7 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 10 Feb 2015 14:37:52 +0000 Subject: [PATCH 29/75] allow net_dev_mtu local override --- config.yaml | 6 ++++++ hooks/quantum_contexts.py | 12 +++++++++--- unit_tests/test_quantum_contexts.py | 4 +++- unit_tests/test_quantum_utils.py | 1 + 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/config.yaml b/config.yaml index 580987cd..7418fd54 100644 --- a/config.yaml +++ b/config.yaml @@ -161,3 +161,9 @@ options: To improve network performance of VM, sometimes we should keep VM MTU as 1500 and use charm to modify MTU of tunnel nic more than 1500 (e.g. 1546 for GRE). A value of zero means no mtu will be set/modified. + network-device-mtu: + type: int + default: 0 + description: | + The MTU size for the interfaces managed by neutron. If set to 0, no value + will be applied. diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index 31e31324..060688d8 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -120,6 +120,12 @@ def _neutron_api_settings(): neutron_settings['network_device_mtu'] = net_dev_mtu return neutron_settings + + # Override if locally provided + cfg_net_dev_mtu = config('network-device-mtu') + if cfg_net_dev_mtu: + neutron_settings['network_device_mtu'] = cfg_net_dev_mtu + return neutron_settings @@ -215,9 +221,9 @@ class QuantumGatewayContext(OSContextGenerator): neutron_api_settings['overlay_network_type'], } - net_dev_mtu = neutron_api_settings.get('network_device_mtu') - if net_dev_mtu: - ctxt['network_device_mtu'] = net_dev_mtu + if 'network_device_mtu' in neutron_api_settings: + ctxt['network_device_mtu'] = \ + neutron_api_settings['network_device_mtu'] return ctxt diff --git a/unit_tests/test_quantum_contexts.py b/unit_tests/test_quantum_contexts.py index f3f8b00e..f1ee85d0 100644 --- a/unit_tests/test_quantum_contexts.py +++ b/unit_tests/test_quantum_contexts.py @@ -263,11 +263,13 @@ class TestQuantumGatewayContext(CharmTestCase): self.test_config.set('debug', False) self.test_config.set('verbose', True) self.test_config.set('instance-mtu', 1420) + self.test_config.set('network-device-mtu', 1500) self.get_os_codename_install_source.return_value = 'folsom' _host_ip.return_value = '10.5.0.1' _secret.return_value = 'testsecret' self.assertEquals(quantum_contexts.QuantumGatewayContext()(), { 'shared_secret': 'testsecret', + 'network_device_mtu': 1500, 'local_ip': '10.5.0.1', 'instance_mtu': 1420, 'core_plugin': "quantum.plugins.openvswitch.ovs_quantum_plugin." @@ -276,7 +278,6 @@ class TestQuantumGatewayContext(CharmTestCase): 'debug': False, 'verbose': True, 'l2_population': False, - 'network_device_mtu': 1500, 'overlay_network_type': 'gre', }) @@ -422,6 +423,7 @@ class TestMisc(CharmTestCase): 'overlay_network_type': 'gre'}) def test_neutron_api_settings_no_apiplugin(self): + self.config.return_value = 1500 self.relation_ids.return_value = [] self.assertEquals(quantum_contexts._neutron_api_settings(), {'l2_population': False, diff --git a/unit_tests/test_quantum_utils.py b/unit_tests/test_quantum_utils.py index 8c319761..775783b8 100644 --- a/unit_tests/test_quantum_utils.py +++ b/unit_tests/test_quantum_utils.py @@ -267,6 +267,7 @@ class TestQuantumUtils(CharmTestCase): quantum_utils.NEUTRON_METERING_AGENT_CONF: ['neutron-metering-agent', 'neutron-plugin-metering-agent'], quantum_utils.NOVA_CONF: ['nova-api-metadata'], + quantum_utils.EXT_PORT_CONF: ['ext-port'], } self.assertDictEqual(quantum_utils.restart_map(), ex_map) From cda5a85f39d9c111da16e0a13ee0a4cb452853b0 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 10 Feb 2015 19:28:49 +0000 Subject: [PATCH 30/75] support multiple phy nics for mtu --- hooks/quantum_contexts.py | 14 ++++++++++++++ hooks/quantum_utils.py | 10 ++++++++++ unit_tests/test_quantum_utils.py | 1 + 3 files changed, 25 insertions(+) diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index 060688d8..86bed27c 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -192,6 +192,20 @@ class ExternalPortContext(NeutronPortContext): return ctxt +class PhyNICMTUContext(NeutronPortContext): + + def __call__(self): + ctxt = {} + port = config('phy-nics') + if port: + ctxt = {"devs": port.replace(' ', '\n')} + mtu = config('phy-nic-mtu') + if mtu: + ctxt['mtu'] = mtu + + return ctxt + + class DataPortContext(NeutronPortContext): def __call__(self): diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 7fb4477a..2cdd7419 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -53,6 +53,7 @@ from quantum_contexts import ( NetworkServiceContext, L3AgentContext, ExternalPortContext, + PhyNICMTUContext, DataPortContext, remap_plugin ) @@ -207,6 +208,7 @@ def get_common_package(): return 'neutron-common' EXT_PORT_CONF = '/etc/init/ext-port.conf' +PHY_NIC_MTU_CONF = 'os-charm-phy-nic-mtu.conf' TEMPLATES = 'templates' QUANTUM_CONF = "/etc/quantum/quantum.conf" @@ -290,6 +292,10 @@ QUANTUM_OVS_CONFIG_FILES = { EXT_PORT_CONF: { 'hook_contexts': [ExternalPortContext()], 'services': ['ext-port'] + }, + PHY_NIC_MTU_CONF: { + 'hook_contexts': [PhyNICMTUContext()], + 'services': ['os-charm-phy-nic-mtu'] } } QUANTUM_OVS_CONFIG_FILES.update(QUANTUM_SHARED_CONFIG_FILES) @@ -344,6 +350,10 @@ NEUTRON_OVS_CONFIG_FILES = { EXT_PORT_CONF: { 'hook_contexts': [ExternalPortContext()], 'services': ['ext-port'] + }, + PHY_NIC_MTU_CONF: { + 'hook_contexts': [PhyNICMTUContext()], + 'services': ['os-charm-phy-nic-mtu'] } } NEUTRON_OVS_CONFIG_FILES.update(NEUTRON_SHARED_CONFIG_FILES) diff --git a/unit_tests/test_quantum_utils.py b/unit_tests/test_quantum_utils.py index 775783b8..cf57c45e 100644 --- a/unit_tests/test_quantum_utils.py +++ b/unit_tests/test_quantum_utils.py @@ -268,6 +268,7 @@ class TestQuantumUtils(CharmTestCase): ['neutron-metering-agent', 'neutron-plugin-metering-agent'], quantum_utils.NOVA_CONF: ['nova-api-metadata'], quantum_utils.EXT_PORT_CONF: ['ext-port'], + quantum_utils.PHY_NIC_MTU_CONF: ['os-charm-phy-nic-mtu'], } self.assertDictEqual(quantum_utils.restart_map(), ex_map) From c45a96d687bb0f22505ede768177c008ebf92163 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 10 Feb 2015 21:05:32 +0000 Subject: [PATCH 31/75] more --- config.yaml | 5 +++++ templates/os-charm-phy-nic-mtu.conf | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 templates/os-charm-phy-nic-mtu.conf diff --git a/config.yaml b/config.yaml index 7418fd54..1feb9b17 100644 --- a/config.yaml +++ b/config.yaml @@ -154,6 +154,11 @@ options: description: | Default multicast port number that will be used to communicate between HA Cluster nodes. + phy-nics: + type: string + default: + description: | + A space-separated list of NICs that we want phy-nic-mtu applied to. phy-nic-mtu: type: int default: 0 diff --git a/templates/os-charm-phy-nic-mtu.conf b/templates/os-charm-phy-nic-mtu.conf new file mode 100644 index 00000000..09c52aa7 --- /dev/null +++ b/templates/os-charm-phy-nic-mtu.conf @@ -0,0 +1,19 @@ +description "Enabling Quantum external networking port" + +start on runlevel [2345] + +task + +script + devs="{{ devs }}" + mtu="{{ mtu }}" + tmpfile=`mktemp` + echo $devs > $tmpfile + if [ -n "$mtu" ]; then + while read -r dev; do + [ -n "$dev" ] || continue + ip link set $dev mtu $mtu + done < echo $devs; + rm $tmpfile + fi +end script From 59a26e2a6f017c92e8c29a26f9af09793eb5c6c1 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 10 Feb 2015 21:24:51 +0000 Subject: [PATCH 32/75] more --- templates/os-charm-phy-nic-mtu.conf | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/templates/os-charm-phy-nic-mtu.conf b/templates/os-charm-phy-nic-mtu.conf index 09c52aa7..9bf66a78 100644 --- a/templates/os-charm-phy-nic-mtu.conf +++ b/templates/os-charm-phy-nic-mtu.conf @@ -5,15 +5,18 @@ start on runlevel [2345] task script - devs="{{ devs }}" - mtu="{{ mtu }}" + devs="eth1" + mtu="1550" tmpfile=`mktemp` echo $devs > $tmpfile if [ -n "$mtu" ]; then while read -r dev; do [ -n "$dev" ] || continue - ip link set $dev mtu $mtu - done < echo $devs; + rc=0 + ip link set $dev mtu $mtu || rc=$? + [ $rc = 0 ] || break + done < $tmpfile rm $tmpfile + [ $rc = 0 ] || exit $rc fi -end script +end script \ No newline at end of file From cf0cf01060007e11ac1fc7d9141cacb64dbd50c2 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 10 Feb 2015 21:32:48 +0000 Subject: [PATCH 33/75] more --- hooks/quantum_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 2cdd7419..62daefc4 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -208,7 +208,7 @@ def get_common_package(): return 'neutron-common' EXT_PORT_CONF = '/etc/init/ext-port.conf' -PHY_NIC_MTU_CONF = 'os-charm-phy-nic-mtu.conf' +PHY_NIC_MTU_CONF = '/etc/init/os-charm-phy-nic-mtu.conf' TEMPLATES = 'templates' QUANTUM_CONF = "/etc/quantum/quantum.conf" From a100c37ff04778cf7aafe01aed25b664f80e5810 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Wed, 11 Feb 2015 13:45:11 +0000 Subject: [PATCH 34/75] more --- hooks/quantum_contexts.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index 86bed27c..abb4b843 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -105,7 +105,14 @@ def _neutron_api_settings(): neutron_settings = { 'l2_population': False, 'overlay_network_type': 'gre', + } + + # Override if locally provided + cfg_net_dev_mtu = config('network-device-mtu') + if cfg_net_dev_mtu: + neutron_settings['network_device_mtu'] = cfg_net_dev_mtu + for rid in relation_ids('neutron-plugin-api'): for unit in related_units(rid): rdata = relation_get(rid=rid, unit=unit) @@ -115,17 +122,12 @@ def _neutron_api_settings(): 'l2_population': rdata['l2-population'], 'overlay_network_type': rdata['overlay-network-type'], } + # Don't override locally provided value if there is one. net_dev_mtu = rdata.get('network-device-mtu') - if net_dev_mtu: + if net_dev_mtu and 'network_device_mtu' not in neutron_settings: neutron_settings['network_device_mtu'] = net_dev_mtu return neutron_settings - - # Override if locally provided - cfg_net_dev_mtu = config('network-device-mtu') - if cfg_net_dev_mtu: - neutron_settings['network_device_mtu'] = cfg_net_dev_mtu - return neutron_settings @@ -235,9 +237,9 @@ class QuantumGatewayContext(OSContextGenerator): neutron_api_settings['overlay_network_type'], } - if 'network_device_mtu' in neutron_api_settings: - ctxt['network_device_mtu'] = \ - neutron_api_settings['network_device_mtu'] + net_dev_mtu = neutron_api_settings.get('network_device_mtu') + if net_dev_mtu: + ctxt['network_device_mtu'] = net_dev_mtu return ctxt From 33f67c083fd839c242bde42632f0169be335d02e Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Wed, 11 Feb 2015 15:36:39 +0000 Subject: [PATCH 35/75] more --- templates/os-charm-phy-nic-mtu.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/os-charm-phy-nic-mtu.conf b/templates/os-charm-phy-nic-mtu.conf index 9bf66a78..6684aceb 100644 --- a/templates/os-charm-phy-nic-mtu.conf +++ b/templates/os-charm-phy-nic-mtu.conf @@ -5,8 +5,8 @@ start on runlevel [2345] task script - devs="eth1" - mtu="1550" + devs="{{ devs }}" + mtu="{{ mtu }}" tmpfile=`mktemp` echo $devs > $tmpfile if [ -n "$mtu" ]; then From 3c15e1e168343fdbb4e8867bdfc3cbab9c27c65d Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Wed, 11 Feb 2015 15:47:28 +0000 Subject: [PATCH 36/75] more --- hooks/quantum_contexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index abb4b843..e387dc91 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -200,7 +200,7 @@ class PhyNICMTUContext(NeutronPortContext): ctxt = {} port = config('phy-nics') if port: - ctxt = {"devs": port.replace(' ', '\n')} + ctxt = {"devs": port.replace(' ', '\\n')} mtu = config('phy-nic-mtu') if mtu: ctxt['mtu'] = mtu From 16b941c1b334cf0335908da2b07e36ef17338684 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Wed, 11 Feb 2015 15:52:33 +0000 Subject: [PATCH 37/75] more --- templates/os-charm-phy-nic-mtu.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/os-charm-phy-nic-mtu.conf b/templates/os-charm-phy-nic-mtu.conf index 6684aceb..06d1967b 100644 --- a/templates/os-charm-phy-nic-mtu.conf +++ b/templates/os-charm-phy-nic-mtu.conf @@ -13,8 +13,8 @@ script while read -r dev; do [ -n "$dev" ] || continue rc=0 + # Try all devices before exiting with error ip link set $dev mtu $mtu || rc=$? - [ $rc = 0 ] || break done < $tmpfile rm $tmpfile [ $rc = 0 ] || exit $rc From babccfb5ffd2ee5f5118908de1720d18bb2c82d6 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Wed, 11 Feb 2015 16:28:12 +0000 Subject: [PATCH 38/75] more --- config.yaml | 6 ++++++ hooks/quantum_contexts.py | 4 ++++ templates/icehouse/ml2_conf.ini | 2 +- unit_tests/test_quantum_contexts.py | 1 + 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/config.yaml b/config.yaml index 1feb9b17..96c507ba 100644 --- a/config.yaml +++ b/config.yaml @@ -114,6 +114,12 @@ options: juju-myservice-0 If you're running multiple environments with the same services in them this allows you to differentiate between them. + bridge-mappings: + type: string + default: 'physnet1:br-data' + description: | + Space-separated list of ML2 data bridge mappings with format + :. # Network configuration options # by default all access is over 'private-address' os-data-network: diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index e387dc91..1620f6d6 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -237,6 +237,10 @@ class QuantumGatewayContext(OSContextGenerator): neutron_api_settings['overlay_network_type'], } + mappings = config('bridge-mappings') + if mappings: + ctxt['bridge_mappings'] = mappings + net_dev_mtu = neutron_api_settings.get('network_device_mtu') if net_dev_mtu: ctxt['network_device_mtu'] = net_dev_mtu diff --git a/templates/icehouse/ml2_conf.ini b/templates/icehouse/ml2_conf.ini index b7245222..2d9101e7 100644 --- a/templates/icehouse/ml2_conf.ini +++ b/templates/icehouse/ml2_conf.ini @@ -22,7 +22,7 @@ flat_networks = physnet1 [ovs] enable_tunneling = True local_ip = {{ local_ip }} -bridge_mappings = physnet1:br-data +bridge_mappings = {{ bridge_mappings }} [agent] tunnel_types = {{ overlay_network_type }} diff --git a/unit_tests/test_quantum_contexts.py b/unit_tests/test_quantum_contexts.py index f1ee85d0..3d5d12c1 100644 --- a/unit_tests/test_quantum_contexts.py +++ b/unit_tests/test_quantum_contexts.py @@ -279,6 +279,7 @@ class TestQuantumGatewayContext(CharmTestCase): 'verbose': True, 'l2_population': False, 'overlay_network_type': 'gre', + 'bridge_mappings': 'physnet1:br-data' }) From 99788cdb2700029ca247fe9306f01e4a47d61e06 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Wed, 11 Feb 2015 18:47:51 +0000 Subject: [PATCH 39/75] more --- config.yaml | 4 ++-- hooks/quantum_utils.py | 32 ++++++++++++++++++++++++++------ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/config.yaml b/config.yaml index 96c507ba..ade1f5fa 100644 --- a/config.yaml +++ b/config.yaml @@ -21,8 +21,8 @@ options: type: string default: description: | - The data port will be added to br-data and will allow usage of flat or VLAN - network types with Neutron. + The data port will be added to bridges provided in bridge-mappings and + will allow usage of flat or VLAN network types with Neutron. openstack-origin: type: string default: distro diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 62daefc4..954c81b7 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -475,7 +475,6 @@ def restart_map(): INT_BRIDGE = "br-int" EXT_BRIDGE = "br-ex" -DATA_BRIDGE = 'br-data' DHCP_AGENT = "DHCP Agent" L3_AGENT = "L3 Agent" @@ -598,6 +597,23 @@ def do_openstack_upgrade(): return configs +def get_bridges_from_mapping(): + """If a bridge mapping is provided, extract the bridge names. + + Returns list of bridges from mapping. + """ + bridges = [] + mappings = config('bridge-mappings') + if mappings: + mappings = mappings.split(' ') + for m in mappings: + p = m.partition(':') + if p[1] == ':': + bridges.append(p[2]) + + return bridges + + def configure_ovs(): if config('plugin') == OVS: if not service_running('openvswitch-switch'): @@ -608,11 +624,15 @@ def configure_ovs(): if ext_port_ctx and ext_port_ctx['ext_port']: add_bridge_port(EXT_BRIDGE, ext_port_ctx['ext_port']) - add_bridge(DATA_BRIDGE) - data_port_ctx = DataPortContext()() - if data_port_ctx and data_port_ctx['data_port']: - add_bridge_port(DATA_BRIDGE, data_port_ctx['data_port'], - promisc=True) + for br in get_bridges_from_mapping(): + add_bridge(br) + data_port_ctx = DataPortContext()() + if data_port_ctx and data_port_ctx['data_port']: + add_bridge_port(br, data_port_ctx['data_port'], + promisc=True) + + # Ensure this runs so that any new bridges have correct mtu + service_restart('os-charm-phy-nic-mtu') def copy_file(src, dst, perms=None, force=False): From 62177edae4d5f8f6398610ade81bf4e1bf47910d Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Thu, 12 Feb 2015 11:17:55 +0000 Subject: [PATCH 40/75] more --- hooks/quantum_contexts.py | 1 + templates/icehouse/ml2_conf.ini | 3 +++ 2 files changed, 4 insertions(+) diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index 1620f6d6..5d161fcd 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -244,6 +244,7 @@ class QuantumGatewayContext(OSContextGenerator): net_dev_mtu = neutron_api_settings.get('network_device_mtu') if net_dev_mtu: ctxt['network_device_mtu'] = net_dev_mtu + ctxt['veth_mtu'] = net_dev_mtu return ctxt diff --git a/templates/icehouse/ml2_conf.ini b/templates/icehouse/ml2_conf.ini index 2d9101e7..e7318b39 100644 --- a/templates/icehouse/ml2_conf.ini +++ b/templates/icehouse/ml2_conf.ini @@ -23,6 +23,9 @@ flat_networks = physnet1 enable_tunneling = True local_ip = {{ local_ip }} bridge_mappings = {{ bridge_mappings }} +{% if veth_mtu -%} +veth_mtu = {{ veth_mtu }} +{% endif %} [agent] tunnel_types = {{ overlay_network_type }} From 84ffe0a3f03a07982fbcc7cfe70079cb975ab001 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Thu, 12 Feb 2015 11:18:46 +0000 Subject: [PATCH 41/75] more --- unit_tests/test_quantum_contexts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/unit_tests/test_quantum_contexts.py b/unit_tests/test_quantum_contexts.py index 3d5d12c1..e5d21401 100644 --- a/unit_tests/test_quantum_contexts.py +++ b/unit_tests/test_quantum_contexts.py @@ -279,7 +279,8 @@ class TestQuantumGatewayContext(CharmTestCase): 'verbose': True, 'l2_population': False, 'overlay_network_type': 'gre', - 'bridge_mappings': 'physnet1:br-data' + 'bridge_mappings': 'physnet1:br-data', + 'veth_mtu': 1500, }) From 781c0344e72823b81eea03675196a4020dc39d38 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Thu, 12 Feb 2015 12:18:30 +0000 Subject: [PATCH 42/75] more --- templates/havana/ovs_neutron_plugin.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/templates/havana/ovs_neutron_plugin.ini b/templates/havana/ovs_neutron_plugin.ini index 96d0f824..e58afd65 100644 --- a/templates/havana/ovs_neutron_plugin.ini +++ b/templates/havana/ovs_neutron_plugin.ini @@ -7,3 +7,8 @@ local_ip = {{ local_ip }} tenant_network_type = gre enable_tunneling = True tunnel_id_ranges = 1:1000 + +[agent] +{% if veth_mtu -%} +veth_mtu = {{ veth_mtu }} +{% endif %} From 7840e1f141bfccaf720e95a7b955ed52cd6207d3 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Thu, 12 Feb 2015 13:50:21 +0000 Subject: [PATCH 43/75] move veth_mtu to correct section --- templates/icehouse/ml2_conf.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/icehouse/ml2_conf.ini b/templates/icehouse/ml2_conf.ini index e7318b39..4275b303 100644 --- a/templates/icehouse/ml2_conf.ini +++ b/templates/icehouse/ml2_conf.ini @@ -23,13 +23,13 @@ flat_networks = physnet1 enable_tunneling = True local_ip = {{ local_ip }} bridge_mappings = {{ bridge_mappings }} -{% if veth_mtu -%} -veth_mtu = {{ veth_mtu }} -{% endif %} [agent] tunnel_types = {{ overlay_network_type }} l2_population = {{ l2_population }} +{% if veth_mtu -%} +veth_mtu = {{ veth_mtu }} +{% endif %} [securitygroup] firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver From f24dc04e46099e95532cd8ba1916a123856083be Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 19 Feb 2015 16:16:07 +0000 Subject: [PATCH 44/75] Fix unit tests --- hooks/quantum_contexts.py | 3 ++- hooks/quantum_utils.py | 3 +-- unit_tests/test_quantum_contexts.py | 19 +++++++++++++------ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index 78f4f519..5cb55fff 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -122,7 +122,8 @@ def _neutron_api_settings(): continue neutron_settings['l2_population'] = rdata['l2-population'] if 'overlay-network-type' in rdata: - neutron_settings['overlay_network_type'] = rdata['overlay-network-type'] + neutron_settings['overlay_network_type'] = \ + rdata['overlay-network-type'] if 'enable-dvr' in rdata: neutron_settings['enable_dvr'] = rdata['enable-dvr'] return neutron_settings diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index b77e6a7e..4c9fe9d7 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -307,8 +307,7 @@ NEUTRON_OVS_CONFIG_FILES = { 'neutron-metering-agent', 'neutron-lbaas-agent', 'neutron-plugin-vpn-agent', - 'neutron-vpn-agent', - 'neutron-openvswitch-agent'] + 'neutron-vpn-agent'] }, NEUTRON_L3_AGENT_CONF: { 'hook_contexts': [NetworkServiceContext(), diff --git a/unit_tests/test_quantum_contexts.py b/unit_tests/test_quantum_contexts.py index 88a961ff..d1c6a1f1 100644 --- a/unit_tests/test_quantum_contexts.py +++ b/unit_tests/test_quantum_contexts.py @@ -192,7 +192,8 @@ class TestL3AgentContext(CharmTestCase): self.test_config.set('external-network-id', '') self.eligible_leader.return_value = False self.assertEquals(quantum_contexts.L3AgentContext()(), - {'handle_internal_only_router': False, + {'agent_mode': 'legacy', + 'handle_internal_only_router': False, 'plugin': 'ovs'}) def test_hior_leader(self): @@ -200,7 +201,8 @@ class TestL3AgentContext(CharmTestCase): self.test_config.set('external-network-id', 'netid') self.eligible_leader.return_value = True self.assertEquals(quantum_contexts.L3AgentContext()(), - {'handle_internal_only_router': True, + {'agent_mode': 'legacy', + 'handle_internal_only_router': True, 'ext_net_id': 'netid', 'plugin': 'ovs'}) @@ -209,7 +211,8 @@ class TestL3AgentContext(CharmTestCase): self.test_config.set('external-network-id', 'netid') self.eligible_leader.return_value = True self.assertEquals(quantum_contexts.L3AgentContext()(), - {'handle_internal_only_router': True, + {'agent_mode': 'legacy', + 'handle_internal_only_router': True, 'ext_net_id': 'netid', 'plugin': 'ovs'}) @@ -233,6 +236,7 @@ class TestQuantumGatewayContext(CharmTestCase): _secret.return_value = 'testsecret' self.assertEquals(quantum_contexts.QuantumGatewayContext()(), { 'shared_secret': 'testsecret', + 'enable_dvr': False, 'local_ip': '10.5.0.1', 'instance_mtu': 1420, 'core_plugin': "quantum.plugins.openvswitch.ovs_quantum_plugin." @@ -370,7 +374,8 @@ class TestMisc(CharmTestCase): 'overlay-network-type': 'gre', }) self.relation_get.side_effect = self.test_relation.get self.assertEquals(quantum_contexts._neutron_api_settings(), - {'l2_population': True, + {'enable_dvr': False, + 'l2_population': True, 'overlay_network_type': 'gre'}) def test_neutron_api_settings2(self): @@ -380,11 +385,13 @@ class TestMisc(CharmTestCase): 'overlay-network-type': 'gre', }) self.relation_get.side_effect = self.test_relation.get self.assertEquals(quantum_contexts._neutron_api_settings(), - {'l2_population': True, + {'enable_dvr': False, + 'l2_population': True, 'overlay_network_type': 'gre'}) def test_neutron_api_settings_no_apiplugin(self): self.relation_ids.return_value = [] self.assertEquals(quantum_contexts._neutron_api_settings(), - {'l2_population': False, + {'enable_dvr': False, + 'l2_population': False, 'overlay_network_type': 'gre', }) From 6d74a7126dad72d78a8d51756d3c2a910bfe2f38 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 20 Feb 2015 11:50:46 +0000 Subject: [PATCH 45/75] Add vrrp ha support --- hooks/quantum_contexts.py | 4 ++++ hooks/quantum_hooks.py | 11 +++++++++-- hooks/quantum_utils.py | 2 ++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index 5cb55fff..51971353 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -112,6 +112,7 @@ def _neutron_api_settings(): neutron_settings = { 'l2_population': False, 'enable_dvr': False, + 'enable_l3ha': False, 'overlay_network_type': 'gre', } @@ -126,6 +127,8 @@ def _neutron_api_settings(): rdata['overlay-network-type'] if 'enable-dvr' in rdata: neutron_settings['enable_dvr'] = rdata['enable-dvr'] + if 'enable-l3ha' in rdata: + neutron_settings['enable_l3ha'] = rdata['enable-l3ha'] return neutron_settings return neutron_settings @@ -251,6 +254,7 @@ class QuantumGatewayContext(OSContextGenerator): 'instance_mtu': config('instance-mtu'), 'l2_population': neutron_api_settings['l2_population'], 'enable_dvr': neutron_api_settings['enable_dvr'], + 'enable_l3ha': neutron_api_settings['enable_l3ha'], 'overlay_network_type': neutron_api_settings['overlay_network_type'], } diff --git a/hooks/quantum_hooks.py b/hooks/quantum_hooks.py index e8b74861..722998ec 100755 --- a/hooks/quantum_hooks.py +++ b/hooks/quantum_hooks.py @@ -193,13 +193,20 @@ def amqp_departed(): 'pgsql-db-relation-changed', 'amqp-relation-changed', 'cluster-relation-changed', - 'cluster-relation-joined', - 'neutron-plugin-api-relation-changed') + 'cluster-relation-joined') @restart_on_change(restart_map()) def db_amqp_changed(): CONFIGS.write_all() +@hooks.hook('neutron-plugin-api-relation-changed') +@restart_on_change(restart_map()) +def neutron_plugin_api_changed(): + apt_install(filter_installed_packages(get_packages()), + fatal=True) + CONFIGS.write_all() + + @hooks.hook('quantum-network-service-relation-changed') @restart_on_change(restart_map()) def nm_changed(): diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 4c9fe9d7..9a768617 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -197,6 +197,8 @@ def get_packages(): packages.append('openswan') if source >= 'kilo': packages.append('python-neutron-fwaas') + if QuantumGatewayContext()()['enable_l3ha']: + packages.append('keepalived') return packages From a5af21f87747010f72f0356ab8af4041739b2589 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 20 Feb 2015 13:11:12 +0000 Subject: [PATCH 46/75] Fix bug blocking install --- hooks/quantum_contexts.py | 16 ++++++++-------- hooks/quantum_utils.py | 5 +++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index 51971353..226fc743 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -104,7 +104,7 @@ def core_plugin(): return CORE_PLUGIN[networking_name()][plugin] -def _neutron_api_settings(): +def neutron_api_settings(): ''' Inspects current neutron-plugin-api relation for neutron settings. Return defaults if it is not present @@ -164,7 +164,7 @@ class NetworkServiceContext(OSContextGenerator): class L3AgentContext(OSContextGenerator): def __call__(self): - neutron_api_settings = _neutron_api_settings() + api_settings = neutron_api_settings() ctxt = {} if config('run-internal-router') == 'leader': ctxt['handle_internal_only_router'] = eligible_leader(None) @@ -179,7 +179,7 @@ class L3AgentContext(OSContextGenerator): ctxt['ext_net_id'] = config('external-network-id') if config('plugin'): ctxt['plugin'] = config('plugin') - if neutron_api_settings['enable_dvr'] == 'True': + if api_settings['enable_dvr'] == 'True': ctxt['agent_mode'] = 'dvr_snat' else: ctxt['agent_mode'] = 'legacy' @@ -241,7 +241,7 @@ class DataPortContext(NeutronPortContext): class QuantumGatewayContext(OSContextGenerator): def __call__(self): - neutron_api_settings = _neutron_api_settings() + api_settings = neutron_api_settings() ctxt = { 'shared_secret': get_shared_secret(), 'local_ip': @@ -252,11 +252,11 @@ class QuantumGatewayContext(OSContextGenerator): 'debug': config('debug'), 'verbose': config('verbose'), 'instance_mtu': config('instance-mtu'), - 'l2_population': neutron_api_settings['l2_population'], - 'enable_dvr': neutron_api_settings['enable_dvr'], - 'enable_l3ha': neutron_api_settings['enable_l3ha'], + 'l2_population': api_settings['l2_population'], + 'enable_dvr': api_settings['enable_dvr'], + 'enable_l3ha': api_settings['enable_l3ha'], 'overlay_network_type': - neutron_api_settings['overlay_network_type'], + api_settings['overlay_network_type'], } return ctxt diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 9a768617..308e9e5d 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -54,7 +54,8 @@ from quantum_contexts import ( L3AgentContext, ExternalPortContext, DataPortContext, - remap_plugin + remap_plugin, + neutron_api_settings, ) from copy import deepcopy @@ -197,7 +198,7 @@ def get_packages(): packages.append('openswan') if source >= 'kilo': packages.append('python-neutron-fwaas') - if QuantumGatewayContext()()['enable_l3ha']: + if neutron_api_settings()['enable_l3ha']: packages.append('keepalived') return packages From 8bcff7847dde1f122bd90eef2472ba0caae82127 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 24 Feb 2015 11:42:07 +0000 Subject: [PATCH 47/75] synced charm-helpers --- hooks/charmhelpers/contrib/python/debug.py | 4 ++-- hooks/charmhelpers/contrib/python/rpdb.py | 6 +++--- hooks/charmhelpers/contrib/python/version.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/hooks/charmhelpers/contrib/python/debug.py b/hooks/charmhelpers/contrib/python/debug.py index 960770f2..871cd6f3 100644 --- a/hooks/charmhelpers/contrib/python/debug.py +++ b/hooks/charmhelpers/contrib/python/debug.py @@ -19,8 +19,6 @@ from __future__ import print_function -__author__ = "Jorge Niedbalski " - import atexit import sys @@ -32,6 +30,8 @@ from charmhelpers.core.hookenv import ( log ) +__author__ = "Jorge Niedbalski " + DEFAULT_ADDR = "0.0.0.0" DEFAULT_PORT = 4444 diff --git a/hooks/charmhelpers/contrib/python/rpdb.py b/hooks/charmhelpers/contrib/python/rpdb.py index 80bf66aa..d503f88f 100644 --- a/hooks/charmhelpers/contrib/python/rpdb.py +++ b/hooks/charmhelpers/contrib/python/rpdb.py @@ -16,13 +16,13 @@ """Remote Python Debugger (pdb wrapper).""" -__author__ = "Bertrand Janin " -__version__ = "0.1.3" - import pdb import socket import sys +__author__ = "Bertrand Janin " +__version__ = "0.1.3" + class Rpdb(pdb.Pdb): diff --git a/hooks/charmhelpers/contrib/python/version.py b/hooks/charmhelpers/contrib/python/version.py index 6dc11296..c39fcbf7 100644 --- a/hooks/charmhelpers/contrib/python/version.py +++ b/hooks/charmhelpers/contrib/python/version.py @@ -17,10 +17,10 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . -__author__ = "Jorge Niedbalski " - import sys +__author__ = "Jorge Niedbalski " + def current_version(): """Current system python version""" From 1f228745aab74e4a62e474c29d7573d7ea08cf97 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 26 Feb 2015 09:09:27 +0000 Subject: [PATCH 48/75] Unit tests and lint fixes --- .../charmhelpers/contrib/charmsupport/nrpe.py | 48 ++++++- .../charmhelpers/contrib/hahelpers/cluster.py | 6 +- hooks/charmhelpers/contrib/network/ufw.py | 28 ++-- .../contrib/openstack/amulet/deployment.py | 7 +- .../charmhelpers/contrib/openstack/context.py | 37 ++++-- .../contrib/openstack/files/__init__.py | 18 +++ hooks/charmhelpers/contrib/openstack/ip.py | 37 ++++++ hooks/charmhelpers/contrib/openstack/utils.py | 1 + hooks/charmhelpers/contrib/python/packages.py | 4 +- hooks/charmhelpers/core/fstab.py | 8 +- hooks/charmhelpers/core/strutils.py | 42 ++++++ hooks/charmhelpers/core/sysctl.py | 4 +- hooks/charmhelpers/core/unitdata.py | 2 +- hooks/charmhelpers/fetch/archiveurl.py | 20 +-- hooks/charmhelpers/fetch/giturl.py | 2 +- hooks/quantum_contexts.py | 12 +- hooks/quantum_utils.py | 10 +- tests/charmhelpers/contrib/amulet/utils.py | 124 +++++++++++++++++- .../contrib/openstack/amulet/deployment.py | 7 +- unit_tests/test_quantum_contexts.py | 55 ++++++-- unit_tests/test_quantum_hooks.py | 6 + unit_tests/test_quantum_utils.py | 15 ++- 22 files changed, 420 insertions(+), 73 deletions(-) create mode 100644 hooks/charmhelpers/contrib/openstack/files/__init__.py create mode 100644 hooks/charmhelpers/core/strutils.py diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py index 0fd0a9d8..9d961cfb 100644 --- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py +++ b/hooks/charmhelpers/contrib/charmsupport/nrpe.py @@ -24,6 +24,8 @@ import subprocess import pwd import grp import os +import glob +import shutil import re import shlex import yaml @@ -161,7 +163,7 @@ define service {{ log('Check command not found: {}'.format(parts[0])) return '' - def write(self, nagios_context, hostname, nagios_servicegroups=None): + def write(self, nagios_context, hostname, nagios_servicegroups): nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format( self.command) with open(nrpe_check_file, 'w') as nrpe_check_config: @@ -177,14 +179,11 @@ define service {{ nagios_servicegroups) def write_service_config(self, nagios_context, hostname, - nagios_servicegroups=None): + nagios_servicegroups): for f in os.listdir(NRPE.nagios_exportdir): if re.search('.*{}.cfg'.format(self.command), f): os.remove(os.path.join(NRPE.nagios_exportdir, f)) - if not nagios_servicegroups: - nagios_servicegroups = nagios_context - templ_vars = { 'nagios_hostname': hostname, 'nagios_servicegroup': nagios_servicegroups, @@ -211,10 +210,10 @@ class NRPE(object): super(NRPE, self).__init__() self.config = config() self.nagios_context = self.config['nagios_context'] - if 'nagios_servicegroups' in self.config: + if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']: self.nagios_servicegroups = self.config['nagios_servicegroups'] else: - self.nagios_servicegroups = 'juju' + self.nagios_servicegroups = self.nagios_context self.unit_name = local_unit().replace('/', '-') if hostname: self.hostname = hostname @@ -322,3 +321,38 @@ def add_init_service_checks(nrpe, services, unit_name): check_cmd='check_status_file.py -f ' '/var/lib/nagios/service-check-%s.txt' % svc, ) + + +def copy_nrpe_checks(): + """ + Copy the nrpe checks into place + + """ + NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins' + nrpe_files_dir = os.path.join(os.getenv('CHARM_DIR'), 'hooks', + 'charmhelpers', 'contrib', 'openstack', + 'files') + + if not os.path.exists(NAGIOS_PLUGINS): + os.makedirs(NAGIOS_PLUGINS) + for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")): + if os.path.isfile(fname): + shutil.copy2(fname, + os.path.join(NAGIOS_PLUGINS, os.path.basename(fname))) + + +def add_haproxy_checks(nrpe, unit_name): + """ + Add checks for each service in list + + :param NRPE nrpe: NRPE object to add check to + :param str unit_name: Unit name to use in check description + """ + nrpe.add_check( + shortname='haproxy_servers', + description='Check HAProxy {%s}' % unit_name, + check_cmd='check_haproxy.sh') + nrpe.add_check( + shortname='haproxy_queue', + description='Check HAProxy queue depth {%s}' % unit_name, + check_cmd='check_haproxy_queue_depth.sh') diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index 9a2588b6..9333efc3 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -48,6 +48,9 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.decorators import ( retry_on_exception, ) +from charmhelpers.core.strutils import ( + bool_from_string, +) class HAIncompleteConfig(Exception): @@ -164,7 +167,8 @@ def https(): . returns: boolean ''' - if config_get('use-https') == "yes": + use_https = config_get('use-https') + if use_https and bool_from_string(use_https): return True if config_get('ssl_cert') and config_get('ssl_key'): return True diff --git a/hooks/charmhelpers/contrib/network/ufw.py b/hooks/charmhelpers/contrib/network/ufw.py index 1e79a0ca..560e6a03 100644 --- a/hooks/charmhelpers/contrib/network/ufw.py +++ b/hooks/charmhelpers/contrib/network/ufw.py @@ -37,19 +37,22 @@ Examples: >>> ufw.enable() >>> ufw.service('4949', 'close') # munin """ - -__author__ = "Felipe Reyes " - import re import os import subprocess from charmhelpers.core import hookenv +__author__ = "Felipe Reyes " + class UFWError(Exception): pass +class UFWIPv6Error(UFWError): + pass + + def is_enabled(): """ Check if `ufw` is enabled @@ -66,10 +69,13 @@ def is_enabled(): return len(m) >= 1 -def is_ipv6_ok(): +def is_ipv6_ok(soft_fail=False): """ Check if IPv6 support is present and ip6tables functional + :param soft_fail: If set to True and IPv6 support is broken, then reports + that the host doesn't have IPv6 support, otherwise a + UFWIPv6Error exception is raised. :returns: True if IPv6 is working, False otherwise """ @@ -89,8 +95,11 @@ def is_ipv6_ok(): hookenv.log("Couldn't load ip6_tables module: %s" % ex.output, level="WARN") # we are in a world where ip6tables isn't working - # so we inform that the machine doesn't have IPv6 - return False + if soft_fail: + # so we inform that the machine doesn't have IPv6 + return False + else: + raise UFWIPv6Error("IPv6 firewall support broken") else: # the module is present :) return True @@ -113,16 +122,19 @@ def disable_ipv6(): raise UFWError("Couldn't disable IPv6 support in ufw") -def enable(): +def enable(soft_fail=False): """ Enable ufw + :param soft_fail: If set to True silently disables IPv6 support in ufw, + otherwise a UFWIPv6Error exception is raised when IP6 + support is broken. :returns: True if ufw is successfully enabled """ if is_enabled(): return True - if not is_ipv6_ok(): + if not is_ipv6_ok(soft_fail): disable_ipv6() output = subprocess.check_output(['ufw', 'enable'], diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index c50d3ec6..0cfeaa4c 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -71,16 +71,19 @@ class OpenStackAmuletDeployment(AmuletDeployment): services.append(this_service) use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', 'ceph-osd', 'ceph-radosgw'] + # Openstack subordinate charms do not expose an origin option as that + # is controlled by the principle + ignore = ['neutron-openvswitch'] if self.openstack: for svc in services: - if svc['name'] not in use_source: + if svc['name'] not in use_source + ignore: config = {'openstack-origin': self.openstack} self.d.configure(svc['name'], config) if self.source: for svc in services: - if svc['name'] in use_source: + if svc['name'] in use_source and svc['name'] not in ignore: config = {'source': self.source} self.d.configure(svc['name'], config) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index c7c4cd4a..d268ea8f 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -279,9 +279,25 @@ def db_ssl(rdata, ctxt, ssl_dir): class IdentityServiceContext(OSContextGenerator): interfaces = ['identity-service'] + def __init__(self, service=None, service_user=None): + self.service = service + self.service_user = service_user + def __call__(self): log('Generating template context for identity-service', level=DEBUG) ctxt = {} + + if self.service and self.service_user: + # This is required for pki token signing if we don't want /tmp to + # be used. + cachedir = '/var/cache/%s' % (self.service) + if not os.path.isdir(cachedir): + log("Creating service cache dir %s" % (cachedir), level=DEBUG) + mkdir(path=cachedir, owner=self.service_user, + group=self.service_user, perms=0o700) + + ctxt['signing_dir'] = cachedir + for rid in relation_ids('identity-service'): for unit in related_units(rid): rdata = relation_get(rid=rid, unit=unit) @@ -291,15 +307,16 @@ class IdentityServiceContext(OSContextGenerator): auth_host = format_ipv6_addr(auth_host) or auth_host svc_protocol = rdata.get('service_protocol') or 'http' auth_protocol = rdata.get('auth_protocol') or 'http' - ctxt = {'service_port': rdata.get('service_port'), - 'service_host': serv_host, - 'auth_host': auth_host, - 'auth_port': rdata.get('auth_port'), - 'admin_tenant_name': rdata.get('service_tenant'), - 'admin_user': rdata.get('service_username'), - 'admin_password': rdata.get('service_password'), - 'service_protocol': svc_protocol, - 'auth_protocol': auth_protocol} + ctxt.update({'service_port': rdata.get('service_port'), + 'service_host': serv_host, + 'auth_host': auth_host, + 'auth_port': rdata.get('auth_port'), + 'admin_tenant_name': rdata.get('service_tenant'), + 'admin_user': rdata.get('service_username'), + 'admin_password': rdata.get('service_password'), + 'service_protocol': svc_protocol, + 'auth_protocol': auth_protocol}) + if context_complete(ctxt): # NOTE(jamespage) this is required for >= icehouse # so a missing value just indicates keystone needs @@ -1021,6 +1038,8 @@ class ZeroMQContext(OSContextGenerator): for unit in related_units(rid): ctxt['zmq_nonce'] = relation_get('nonce', unit, rid) ctxt['zmq_host'] = relation_get('host', unit, rid) + ctxt['zmq_redis_address'] = relation_get( + 'zmq_redis_address', unit, rid) return ctxt diff --git a/hooks/charmhelpers/contrib/openstack/files/__init__.py b/hooks/charmhelpers/contrib/openstack/files/__init__.py new file mode 100644 index 00000000..75876796 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/files/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +# dummy __init__.py to fool syncer into thinking this is a syncable python +# module diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py index 9eabed73..29bbddcb 100644 --- a/hooks/charmhelpers/contrib/openstack/ip.py +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -26,6 +26,8 @@ from charmhelpers.contrib.network.ip import ( ) from charmhelpers.contrib.hahelpers.cluster import is_clustered +from functools import partial + PUBLIC = 'public' INTERNAL = 'int' ADMIN = 'admin' @@ -107,3 +109,38 @@ def resolve_address(endpoint_type=PUBLIC): "clustered=%s)" % (net_type, clustered)) return resolved_address + + +def endpoint_url(configs, url_template, port, endpoint_type=PUBLIC, + override=None): + """Returns the correct endpoint URL to advertise to Keystone. + + This method provides the correct endpoint URL which should be advertised to + the keystone charm for endpoint creation. This method allows for the url to + be overridden to force a keystone endpoint to have specific URL for any of + the defined scopes (admin, internal, public). + + :param configs: OSTemplateRenderer config templating object to inspect + for a complete https context. + :param url_template: str format string for creating the url template. Only + two values will be passed - the scheme+hostname + returned by the canonical_url and the port. + :param endpoint_type: str endpoint type to resolve. + :param override: str the name of the config option which overrides the + endpoint URL defined by the charm itself. None will + disable any overrides (default). + """ + if override: + # Return any user-defined overrides for the keystone endpoint URL. + user_value = config(override) + if user_value: + return user_value.strip() + + return url_template % (canonical_url(configs, endpoint_type), port) + + +public_endpoint = partial(endpoint_url, endpoint_type=PUBLIC) + +internal_endpoint = partial(endpoint_url, endpoint_type=INTERNAL) + +admin_endpoint = partial(endpoint_url, endpoint_type=ADMIN) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 26259a03..af2b3596 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -103,6 +103,7 @@ SWIFT_CODENAMES = OrderedDict([ ('2.1.0', 'juno'), ('2.2.0', 'juno'), ('2.2.1', 'kilo'), + ('2.2.2', 'kilo'), ]) DEFAULT_LOOPBACK_SIZE = '5G' diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index d848a120..8659516b 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -17,8 +17,6 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . -__author__ = "Jorge Niedbalski " - from charmhelpers.fetch import apt_install, apt_update from charmhelpers.core.hookenv import log @@ -29,6 +27,8 @@ except ImportError: apt_install('python-pip') from pip import main as pip_execute +__author__ = "Jorge Niedbalski " + def parse_options(given, available): """Given a set of options, check if available""" diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py index be7de248..3056fbac 100644 --- a/hooks/charmhelpers/core/fstab.py +++ b/hooks/charmhelpers/core/fstab.py @@ -17,11 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . -__author__ = 'Jorge Niedbalski R. ' - import io import os +__author__ = 'Jorge Niedbalski R. ' + class Fstab(io.FileIO): """This class extends file in order to implement a file reader/writer @@ -77,7 +77,7 @@ class Fstab(io.FileIO): for line in self.readlines(): line = line.decode('us-ascii') try: - if line.strip() and not line.startswith("#"): + if line.strip() and not line.strip().startswith("#"): yield self._hydrate_entry(line) except ValueError: pass @@ -104,7 +104,7 @@ class Fstab(io.FileIO): found = False for index, line in enumerate(lines): - if not line.startswith("#"): + if line.strip() and not line.strip().startswith("#"): if self._hydrate_entry(line) == entry: found = True break diff --git a/hooks/charmhelpers/core/strutils.py b/hooks/charmhelpers/core/strutils.py new file mode 100644 index 00000000..efc4402e --- /dev/null +++ b/hooks/charmhelpers/core/strutils.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +import six + + +def bool_from_string(value): + """Interpret string value as boolean. + + Returns True if value translates to True otherwise False. + """ + if isinstance(value, six.string_types): + value = six.text_type(value) + else: + msg = "Unable to interpret non-string value '%s' as boolean" % (value) + raise ValueError(msg) + + value = value.strip().lower() + + if value in ['y', 'yes', 'true', 't']: + return True + elif value in ['n', 'no', 'false', 'f']: + return False + + msg = "Unable to interpret string value '%s' as boolean" % (value) + raise ValueError(msg) diff --git a/hooks/charmhelpers/core/sysctl.py b/hooks/charmhelpers/core/sysctl.py index 8e1b9eeb..21cc8ab2 100644 --- a/hooks/charmhelpers/core/sysctl.py +++ b/hooks/charmhelpers/core/sysctl.py @@ -17,8 +17,6 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . -__author__ = 'Jorge Niedbalski R. ' - import yaml from subprocess import check_call @@ -29,6 +27,8 @@ from charmhelpers.core.hookenv import ( ERROR, ) +__author__ = 'Jorge Niedbalski R. ' + def create(sysctl_dict, sysctl_file): """Creates a sysctl.conf file from a YAML associative array diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py index 01329ab7..3000134a 100644 --- a/hooks/charmhelpers/core/unitdata.py +++ b/hooks/charmhelpers/core/unitdata.py @@ -435,7 +435,7 @@ class HookData(object): os.path.join(charm_dir, 'revision')).read().strip() charm_rev = charm_rev or '0' revs = self.kv.get('charm_revisions', []) - if not charm_rev in revs: + if charm_rev not in revs: revs.append(charm_rev.strip() or '0') self.kv.set('charm_revisions', revs) diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py index d25a0ddd..8dfce505 100644 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ b/hooks/charmhelpers/fetch/archiveurl.py @@ -18,6 +18,16 @@ import os import hashlib import re +from charmhelpers.fetch import ( + BaseFetchHandler, + UnhandledSource +) +from charmhelpers.payload.archive import ( + get_archive_handler, + extract, +) +from charmhelpers.core.host import mkdir, check_hash + import six if six.PY3: from urllib.request import ( @@ -35,16 +45,6 @@ else: ) from urlparse import urlparse, urlunparse, parse_qs -from charmhelpers.fetch import ( - BaseFetchHandler, - UnhandledSource -) -from charmhelpers.payload.archive import ( - get_archive_handler, - extract, -) -from charmhelpers.core.host import mkdir, check_hash - def splituser(host): '''urllib.splituser(), but six's support of this seems broken''' diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 5376786b..93aae87b 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -32,7 +32,7 @@ except ImportError: apt_install("python-git") from git import Repo -from git.exc import GitCommandError +from git.exc import GitCommandError # noqa E402 class GitUrlFetchHandler(BaseFetchHandler): diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index 226fc743..d1123710 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -34,6 +34,7 @@ from charmhelpers.contrib.network.ip import ( get_ipv6_addr, is_bridge_member, ) +from charmhelpers.core.strutils import bool_from_string DB_USER = "quantum" QUANTUM_DB = "quantum" @@ -121,14 +122,17 @@ def neutron_api_settings(): rdata = relation_get(rid=rid, unit=unit) if 'l2-population' not in rdata: continue - neutron_settings['l2_population'] = rdata['l2-population'] + neutron_settings['l2_population'] = \ + bool_from_string(rdata['l2-population']) if 'overlay-network-type' in rdata: neutron_settings['overlay_network_type'] = \ rdata['overlay-network-type'] if 'enable-dvr' in rdata: - neutron_settings['enable_dvr'] = rdata['enable-dvr'] + neutron_settings['enable_dvr'] = \ + bool_from_string(rdata['enable-dvr']) if 'enable-l3ha' in rdata: - neutron_settings['enable_l3ha'] = rdata['enable-l3ha'] + neutron_settings['enable_l3ha'] = \ + bool_from_string(rdata['enable-l3ha']) return neutron_settings return neutron_settings @@ -179,7 +183,7 @@ class L3AgentContext(OSContextGenerator): ctxt['ext_net_id'] = config('external-network-id') if config('plugin'): ctxt['plugin'] = config('plugin') - if api_settings['enable_dvr'] == 'True': + if api_settings['enable_dvr']: ctxt['agent_mode'] = 'dvr_snat' else: ctxt['agent_mode'] = 'legacy' diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 308e9e5d..6b7d1bcc 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -316,7 +316,7 @@ NEUTRON_OVS_CONFIG_FILES = { 'hook_contexts': [NetworkServiceContext(), L3AgentContext(), QuantumGatewayContext()], - 'services': ['neutron-l3-agent'] + 'services': ['neutron-l3-agent', 'neutron-vpn-agent'] }, NEUTRON_METERING_AGENT_CONF: { 'hook_contexts': [QuantumGatewayContext()], @@ -334,7 +334,7 @@ NEUTRON_OVS_CONFIG_FILES = { }, NEUTRON_FWAAS_CONF: { 'hook_contexts': [QuantumGatewayContext()], - 'services': ['neutron-l3-agent'] + 'services': ['neutron-l3-agent', 'neutron-vpn-agent'] }, NEUTRON_OVS_PLUGIN_CONF: { 'hook_contexts': [QuantumGatewayContext()], @@ -342,8 +342,7 @@ NEUTRON_OVS_CONFIG_FILES = { }, NEUTRON_ML2_PLUGIN_CONF: { 'hook_contexts': [QuantumGatewayContext()], - 'services': ['neutron-plugin-openvswitch-agent', - 'neutron-openvswitch-agent'] + 'services': ['neutron-plugin-openvswitch-agent'] }, EXT_PORT_CONF: { 'hook_contexts': [ExternalPortContext()], @@ -417,9 +416,6 @@ def register_configs(): drop_config = NEUTRON_ML2_PLUGIN_CONF if release >= 'icehouse': drop_config = NEUTRON_OVS_PLUGIN_CONF - # NOTE(gnuoy) neutron-vpn-agent supercedes l3-agent for icehouse - CONFIG_FILES[name][plugin][NEUTRON_L3_AGENT_CONF]['services'] = \ - ['neutron-vpn-agent'] if drop_config in CONFIG_FILES[name][plugin]: CONFIG_FILES[name][plugin].pop(drop_config) diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 3464b873..65219d33 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -169,8 +169,13 @@ class AmuletUtils(object): cmd = 'pgrep -o -f {}'.format(service) else: cmd = 'pgrep -o {}'.format(service) - proc_dir = '/proc/{}'.format(sentry_unit.run(cmd)[0].strip()) - return self._get_dir_mtime(sentry_unit, proc_dir) + cmd = cmd + ' | grep -v pgrep || exit 0' + cmd_out = sentry_unit.run(cmd) + self.log.debug('CMDout: ' + str(cmd_out)) + if cmd_out[0]: + self.log.debug('Pid for %s %s' % (service, str(cmd_out[0]))) + proc_dir = '/proc/{}'.format(cmd_out[0].strip()) + return self._get_dir_mtime(sentry_unit, proc_dir) def service_restarted(self, sentry_unit, service, filename, pgrep_full=False, sleep_time=20): @@ -187,6 +192,121 @@ class AmuletUtils(object): else: return False + def service_restarted_since(self, sentry_unit, mtime, service, + pgrep_full=False, sleep_time=20, + retry_count=2): + """Check if service was been started after a given time. + + Args: + sentry_unit (sentry): The sentry unit to check for the service on + mtime (float): The epoch time to check against + service (string): service name to look for in process table + pgrep_full (boolean): Use full command line search mode with pgrep + sleep_time (int): Seconds to sleep before looking for process + retry_count (int): If service is not found, how many times to retry + + Returns: + bool: True if service found and its start time it newer than mtime, + False if service is older than mtime or if service was + not found. + """ + self.log.debug('Checking %s restarted since %s' % (service, mtime)) + time.sleep(sleep_time) + proc_start_time = self._get_proc_start_time(sentry_unit, service, + pgrep_full) + while retry_count > 0 and not proc_start_time: + self.log.debug('No pid file found for service %s, will retry %i ' + 'more times' % (service, retry_count)) + time.sleep(30) + proc_start_time = self._get_proc_start_time(sentry_unit, service, + pgrep_full) + retry_count = retry_count - 1 + + if not proc_start_time: + self.log.warn('No proc start time found, assuming service did ' + 'not start') + return False + if proc_start_time >= mtime: + self.log.debug('proc start time is newer than provided mtime' + '(%s >= %s)' % (proc_start_time, mtime)) + return True + else: + self.log.warn('proc start time (%s) is older than provided mtime ' + '(%s), service did not restart' % (proc_start_time, + mtime)) + return False + + def config_updated_since(self, sentry_unit, filename, mtime, + sleep_time=20): + """Check if file was modified after a given time. + + Args: + sentry_unit (sentry): The sentry unit to check the file mtime on + filename (string): The file to check mtime of + mtime (float): The epoch time to check against + sleep_time (int): Seconds to sleep before looking for process + + Returns: + bool: True if file was modified more recently than mtime, False if + file was modified before mtime, + """ + self.log.debug('Checking %s updated since %s' % (filename, mtime)) + time.sleep(sleep_time) + file_mtime = self._get_file_mtime(sentry_unit, filename) + if file_mtime >= mtime: + self.log.debug('File mtime is newer than provided mtime ' + '(%s >= %s)' % (file_mtime, mtime)) + return True + else: + self.log.warn('File mtime %s is older than provided mtime %s' + % (file_mtime, mtime)) + return False + + def validate_service_config_changed(self, sentry_unit, mtime, service, + filename, pgrep_full=False, + sleep_time=20, retry_count=2): + """Check service and file were updated after mtime + + Args: + sentry_unit (sentry): The sentry unit to check for the service on + mtime (float): The epoch time to check against + service (string): service name to look for in process table + filename (string): The file to check mtime of + pgrep_full (boolean): Use full command line search mode with pgrep + sleep_time (int): Seconds to sleep before looking for process + retry_count (int): If service is not found, how many times to retry + + Typical Usage: + u = OpenStackAmuletUtils(ERROR) + ... + mtime = u.get_sentry_time(self.cinder_sentry) + self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'}) + if not u.validate_service_config_changed(self.cinder_sentry, + mtime, + 'cinder-api', + '/etc/cinder/cinder.conf') + amulet.raise_status(amulet.FAIL, msg='update failed') + Returns: + bool: True if both service and file where updated/restarted after + mtime, False if service is older than mtime or if service was + not found or if filename was modified before mtime. + """ + self.log.debug('Checking %s restarted since %s' % (service, mtime)) + time.sleep(sleep_time) + service_restart = self.service_restarted_since(sentry_unit, mtime, + service, + pgrep_full=pgrep_full, + sleep_time=0, + retry_count=retry_count) + config_update = self.config_updated_since(sentry_unit, filename, mtime, + sleep_time=0) + return service_restart and config_update + + def get_sentry_time(self, sentry_unit): + """Return current epoch time on a sentry""" + cmd = "date +'%s'" + return float(sentry_unit.run(cmd)[0]) + def relation_error(self, name, data): return 'unexpected relation data in {} - {}'.format(name, data) diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index c50d3ec6..0cfeaa4c 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -71,16 +71,19 @@ class OpenStackAmuletDeployment(AmuletDeployment): services.append(this_service) use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', 'ceph-osd', 'ceph-radosgw'] + # Openstack subordinate charms do not expose an origin option as that + # is controlled by the principle + ignore = ['neutron-openvswitch'] if self.openstack: for svc in services: - if svc['name'] not in use_source: + if svc['name'] not in use_source + ignore: config = {'openstack-origin': self.openstack} self.d.configure(svc['name'], config) if self.source: for svc in services: - if svc['name'] in use_source: + if svc['name'] in use_source and svc['name'] not in ignore: config = {'source': self.source} self.d.configure(svc['name'], config) diff --git a/unit_tests/test_quantum_contexts.py b/unit_tests/test_quantum_contexts.py index d1c6a1f1..a2b62f05 100644 --- a/unit_tests/test_quantum_contexts.py +++ b/unit_tests/test_quantum_contexts.py @@ -216,6 +216,12 @@ class TestL3AgentContext(CharmTestCase): 'ext_net_id': 'netid', 'plugin': 'ovs'}) + @patch.object(quantum_contexts, 'neutron_api_settings') + def test_dvr(self, _napi_settings): + _napi_settings.return_value = {'enable_dvr': True} + self.assertEquals(quantum_contexts.L3AgentContext()()['agent_mode'], + 'dvr_snat') + class TestQuantumGatewayContext(CharmTestCase): @@ -224,9 +230,14 @@ class TestQuantumGatewayContext(CharmTestCase): TO_PATCH) self.config.side_effect = self.test_config.get + @patch.object(quantum_contexts, 'neutron_api_settings') @patch.object(quantum_contexts, 'get_shared_secret') @patch.object(quantum_contexts, 'get_host_ip') - def test_all(self, _host_ip, _secret): + def test_all(self, _host_ip, _secret, napi_settings): + napi_settings.return_value = {'l2_population': True, + 'enable_dvr': True, + 'overlay_network_type': 'gre', + 'enable_l3ha': True} self.test_config.set('plugin', 'ovs') self.test_config.set('debug', False) self.test_config.set('verbose', True) @@ -236,7 +247,8 @@ class TestQuantumGatewayContext(CharmTestCase): _secret.return_value = 'testsecret' self.assertEquals(quantum_contexts.QuantumGatewayContext()(), { 'shared_secret': 'testsecret', - 'enable_dvr': False, + 'enable_dvr': True, + 'enable_l3ha': True, 'local_ip': '10.5.0.1', 'instance_mtu': 1420, 'core_plugin': "quantum.plugins.openvswitch.ovs_quantum_plugin." @@ -244,7 +256,7 @@ class TestQuantumGatewayContext(CharmTestCase): 'plugin': 'ovs', 'debug': False, 'verbose': True, - 'l2_population': False, + 'l2_population': True, 'overlay_network_type': 'gre', }) @@ -370,28 +382,55 @@ class TestMisc(CharmTestCase): def test_neutron_api_settings(self): self.relation_ids.return_value = ['foo'] self.related_units.return_value = ['bar'] - self.test_relation.set({'l2-population': True, + self.test_relation.set({'l2-population': 'True', 'overlay-network-type': 'gre', }) self.relation_get.side_effect = self.test_relation.get - self.assertEquals(quantum_contexts._neutron_api_settings(), + self.assertEquals(quantum_contexts.neutron_api_settings(), {'enable_dvr': False, + 'enable_l3ha': False, 'l2_population': True, 'overlay_network_type': 'gre'}) def test_neutron_api_settings2(self): self.relation_ids.return_value = ['foo'] self.related_units.return_value = ['bar'] - self.test_relation.set({'l2-population': True, + self.test_relation.set({'l2-population': 'True', 'overlay-network-type': 'gre', }) self.relation_get.side_effect = self.test_relation.get - self.assertEquals(quantum_contexts._neutron_api_settings(), + self.assertEquals(quantum_contexts.neutron_api_settings(), {'enable_dvr': False, + 'enable_l3ha': False, 'l2_population': True, 'overlay_network_type': 'gre'}) def test_neutron_api_settings_no_apiplugin(self): self.relation_ids.return_value = [] - self.assertEquals(quantum_contexts._neutron_api_settings(), + self.assertEquals(quantum_contexts.neutron_api_settings(), {'enable_dvr': False, + 'enable_l3ha': False, 'l2_population': False, 'overlay_network_type': 'gre', }) + + def test_neutron_api_dvr(self): + self.relation_ids.return_value = ['foo'] + self.related_units.return_value = ['bar'] + self.test_relation.set({'l2-population': 'True', + 'enable-dvr': 'True'}) + self.relation_get.side_effect = self.test_relation.get + self.assertEquals(quantum_contexts.neutron_api_settings(), + {'enable_dvr': True, + 'enable_l3ha': False, + 'l2_population': True, + 'overlay_network_type': 'gre'}) + + def test_neutron_api_l3ha(self): + self.relation_ids.return_value = ['foo'] + self.related_units.return_value = ['bar'] + self.test_relation.set({'l2-population': 'False', + 'enable-l3ha': 'True'}) + self.relation_get.side_effect = self.test_relation.get + self.assertEquals(quantum_contexts.neutron_api_settings(), + {'enable_dvr': False, + 'enable_l3ha': True, + 'l2_population': False, + 'overlay_network_type': 'gre'}) diff --git a/unit_tests/test_quantum_hooks.py b/unit_tests/test_quantum_hooks.py index 7ce1651f..2465273a 100644 --- a/unit_tests/test_quantum_hooks.py +++ b/unit_tests/test_quantum_hooks.py @@ -246,6 +246,12 @@ class TestQuantumHooks(CharmTestCase): self.assertTrue(self.CONFIGS.write_all.called) self.install_ca_cert.assert_called_with('cert') + def test_neutron_plugin_changed(self): + self.filter_installed_packages.return_value = ['foo'] + self._call_hook('neutron-plugin-api-relation-changed') + self.apt_install.assert_called_with(['foo'], fatal=True) + self.assertTrue(self.CONFIGS.write_all.called) + def test_cluster_departed_nvp(self): self.test_config.set('plugin', 'nvp') self._call_hook('cluster-relation-departed') diff --git a/unit_tests/test_quantum_utils.py b/unit_tests/test_quantum_utils.py index 8c319761..0249aaf8 100644 --- a/unit_tests/test_quantum_utils.py +++ b/unit_tests/test_quantum_utils.py @@ -46,7 +46,8 @@ TO_PATCH = [ 'is_relation_made', 'lsb_release', 'mkdir', - 'copy2' + 'copy2', + 'neutron_api_settings', ] @@ -139,6 +140,11 @@ class TestQuantumUtils(CharmTestCase): self.get_os_codename_install_source.return_value = 'kilo' self.assertTrue('python-neutron-fwaas' in quantum_utils.get_packages()) + def test_get_packages_l3ha(self): + self.config.return_value = 'ovs' + self.get_os_codename_install_source.return_value = 'juno' + self.assertTrue('keepalived' in quantum_utils.get_packages()) + def test_configure_ovs_starts_service_if_required(self): self.config.return_value = 'ovs' self.service_running.return_value = False @@ -241,6 +247,7 @@ class TestQuantumUtils(CharmTestCase): def test_restart_map_ovs(self): self.config.return_value = 'ovs' + self.get_os_codename_install_source.return_value = 'havana' ex_map = { quantum_utils.NEUTRON_CONF: ['neutron-l3-agent', 'neutron-dhcp-agent', @@ -261,9 +268,11 @@ class TestQuantumUtils(CharmTestCase): quantum_utils.NEUTRON_VPNAAS_AGENT_CONF: [ 'neutron-plugin-vpn-agent', 'neutron-vpn-agent'], - quantum_utils.NEUTRON_L3_AGENT_CONF: ['neutron-l3-agent'], + quantum_utils.NEUTRON_L3_AGENT_CONF: ['neutron-l3-agent', + 'neutron-vpn-agent'], quantum_utils.NEUTRON_DHCP_AGENT_CONF: ['neutron-dhcp-agent'], - quantum_utils.NEUTRON_FWAAS_CONF: ['neutron-l3-agent'], + quantum_utils.NEUTRON_FWAAS_CONF: ['neutron-l3-agent', + 'neutron-vpn-agent'], quantum_utils.NEUTRON_METERING_AGENT_CONF: ['neutron-metering-agent', 'neutron-plugin-metering-agent'], quantum_utils.NOVA_CONF: ['nova-api-metadata'], From d2ecfad238315525ec6c69c08010da3b657e5e0b Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Fri, 27 Feb 2015 16:55:07 +0000 Subject: [PATCH 49/75] more --- config.yaml | 6 --- .../charmhelpers/contrib/openstack/context.py | 30 ++++++++----- .../charmhelpers/contrib/openstack/neutron.py | 33 +++++++++++++++ hooks/quantum_contexts.py | 42 ++++++++++--------- hooks/quantum_utils.py | 34 ++++++--------- unit_tests/test_quantum_contexts.py | 18 +++----- unit_tests/test_quantum_utils.py | 3 +- 7 files changed, 94 insertions(+), 72 deletions(-) diff --git a/config.yaml b/config.yaml index ade1f5fa..fe151717 100644 --- a/config.yaml +++ b/config.yaml @@ -172,9 +172,3 @@ options: To improve network performance of VM, sometimes we should keep VM MTU as 1500 and use charm to modify MTU of tunnel nic more than 1500 (e.g. 1546 for GRE). A value of zero means no mtu will be set/modified. - network-device-mtu: - type: int - default: 0 - description: | - The MTU size for the interfaces managed by neutron. If set to 0, no value - will be applied. diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 612762c6..6f109864 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -839,37 +839,45 @@ class NeutronContext(OSContextGenerator): class NeutronPortContext(OSContextGenerator): - def resolve_port(self, config_key): - if not config(config_key): + NIC_PREFIXES = ['eth', 'bond'] + + def resolve_ports(self, ports): + """Resolve NICs not yet bound to bridge(s) + + If hwaddress provided then returns resolved hwaddress otherwise NIC. + """ + if not ports: return None hwaddr_to_nic = {} hwaddr_to_ip = {} - for nic in list_nics(['eth', 'bond']): + for nic in list_nics(self.NIC_PREFIXES): hwaddr = get_nic_hwaddr(nic) hwaddr_to_nic[hwaddr] = nic - addresses = get_ipv4_addr(nic, fatal=False) + \ - get_ipv6_addr(iface=nic, fatal=False) + addresses = get_ipv4_addr(nic, fatal=False) + addresses += get_ipv6_addr(iface=nic, fatal=False) hwaddr_to_ip[hwaddr] = addresses + resolved = [] mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I) - for entry in config(config_key).split(): - entry = entry.strip() + for entry in ports: if re.match(mac_regex, entry): - if entry in hwaddr_to_nic and len(hwaddr_to_ip[entry]) == 0: + # NIC is in known NICs and does NOT hace an IP address + if entry in hwaddr_to_nic and not hwaddr_to_ip[entry]: # If the nic is part of a bridge then don't use it if is_bridge_member(hwaddr_to_nic[entry]): continue + # Entry is a MAC address for a valid interface that doesn't # have an IP address assigned yet. - return hwaddr_to_nic[entry] + resolved.append(hwaddr_to_nic[entry]) else: # If the passed entry is not a MAC address, assume it's a valid # interface, and that the user put it there on purpose (we can # trust it to be the real external network). - return entry + resolved.append(entry) - return None + return resolved class OSConfigFlagContext(OSContextGenerator): diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 902757fe..6d9fdff6 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -237,3 +237,36 @@ def network_manager(): else: # ensure accurate naming for all releases post-H return 'neutron' + + +def parse_mappings(mappings): + parsed = {} + if mappings: + mappings = mappings.split(' ') + for m in mappings: + p = m.partition(':') + if p[1] == ':': + parsed[p[0].strip()] = p[2].strip() + + return parsed + + +def parse_bridge_mappings(mappings): + """Parse bridge mappings. + + Mappings must be a space-delimited list of provider:bridge mappings. + + Returns dict of the form {provider:bridge}. + """ + return parse_mappings(mappings) + + +def parse_data_port_mappings(mappings): + """Parse data port mappings. + + Mappings must be a space-delimited list of provider:port mappings. + + Returns dict of the form {provider:port}. + """ + return parse_mappings(mappings) + diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index 5d161fcd..ed103a76 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -27,6 +27,9 @@ from charmhelpers.contrib.hahelpers.cluster import( from charmhelpers.contrib.network.ip import ( get_address_in_network, ) +from charmhelpers.contrib.openstack.neutron import ( + parse_data_port_mappings, +) DB_USER = "quantum" QUANTUM_DB = "quantum" @@ -108,11 +111,6 @@ def _neutron_api_settings(): } - # Override if locally provided - cfg_net_dev_mtu = config('network-device-mtu') - if cfg_net_dev_mtu: - neutron_settings['network_device_mtu'] = cfg_net_dev_mtu - for rid in relation_ids('neutron-plugin-api'): for unit in related_units(rid): rdata = relation_get(rid=rid, unit=unit) @@ -122,9 +120,8 @@ def _neutron_api_settings(): 'l2_population': rdata['l2-population'], 'overlay_network_type': rdata['overlay-network-type'], } - # Don't override locally provided value if there is one. net_dev_mtu = rdata.get('network-device-mtu') - if net_dev_mtu and 'network_device_mtu' not in neutron_settings: + if net_dev_mtu: neutron_settings['network_device_mtu'] = net_dev_mtu return neutron_settings @@ -184,17 +181,20 @@ class ExternalPortContext(NeutronPortContext): def __call__(self): ctxt = {} - port = self.resolve_port('ext-port') - if port: - ctxt = {"ext_port": port} - mtu = config('phy-nic-mtu') - if mtu: - ctxt['ext_port_mtu'] = mtu + ports = config('ext-port') + if ports: + ports = [p.strip() for p in ports.split()] + ports = self.resolve_ports(ports) + if ports: + ctxt = {"ext_port": ports[0]} + mtu = config('phy-nic-mtu') + if mtu: + ctxt['ext_port_mtu'] = mtu return ctxt -class PhyNICMTUContext(NeutronPortContext): +class PhyNICMTUContext(OSContextGenerator): def __call__(self): ctxt = {} @@ -211,11 +211,15 @@ class PhyNICMTUContext(NeutronPortContext): class DataPortContext(NeutronPortContext): def __call__(self): - port = self.resolve_port('data-port') - if port: - return {"data_port": port} - else: - return None + ports = config('data-port') + if ports: + portmap = parse_data_port_mappings(ports) + ports = self.resolve_ports(portmap.values()) + if ports: + return {provider: port for provider, port in + portmap.iteritems() if port in ports} + + return None class QuantumGatewayContext(OSContextGenerator): diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 954c81b7..c20b9c56 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -57,6 +57,9 @@ from quantum_contexts import ( DataPortContext, remap_plugin ) +from charmhelpers.contrib.openstack.neutron import ( + parse_bridge_mappings, +) from copy import deepcopy @@ -597,23 +600,6 @@ def do_openstack_upgrade(): return configs -def get_bridges_from_mapping(): - """If a bridge mapping is provided, extract the bridge names. - - Returns list of bridges from mapping. - """ - bridges = [] - mappings = config('bridge-mappings') - if mappings: - mappings = mappings.split(' ') - for m in mappings: - p = m.partition(':') - if p[1] == ':': - bridges.append(p[2]) - - return bridges - - def configure_ovs(): if config('plugin') == OVS: if not service_running('openvswitch-switch'): @@ -624,12 +610,16 @@ def configure_ovs(): if ext_port_ctx and ext_port_ctx['ext_port']: add_bridge_port(EXT_BRIDGE, ext_port_ctx['ext_port']) - for br in get_bridges_from_mapping(): + portmaps = DataPortContext()() + bridgemaps = parse_bridge_mappings(config('bridge-mappings')) + for provider, br in bridgemaps.iteritems(): add_bridge(br) - data_port_ctx = DataPortContext()() - if data_port_ctx and data_port_ctx['data_port']: - add_bridge_port(br, data_port_ctx['data_port'], - promisc=True) + + if not portmaps or provider not in portmaps: + continue + + add_bridge_port(br, data_port_ctx['data_port'], + promisc=True) # Ensure this runs so that any new bridges have correct mtu service_restart('os-charm-phy-nic-mtu') diff --git a/unit_tests/test_quantum_contexts.py b/unit_tests/test_quantum_contexts.py index e5d21401..7f6a8706 100644 --- a/unit_tests/test_quantum_contexts.py +++ b/unit_tests/test_quantum_contexts.py @@ -180,7 +180,6 @@ class TestNeutronPortContext(CharmTestCase): mock_config.side_effect = config self.assertEquals(quantum_contexts.ExternalPortContext()(), {}) - self.assertTrue(mock_config.called) @patch('charmhelpers.contrib.openstack.context.get_nic_hwaddr') @patch('charmhelpers.contrib.openstack.context.list_nics') @@ -207,12 +206,13 @@ class TestNeutronPortContext(CharmTestCase): self.assertEquals(quantum_contexts.ExternalPortContext()(), {'ext_port': 'eth2', 'ext_port_mtu': 1234}) - @patch('charmhelpers.contrib.openstack.context.config') - def test_data_port_eth(self, mock_config): - config = self.fake_config({'data-port': 'eth1010'}) - mock_config.side_effect = config + @patch('charmhelpers.contrib.openstack.context.NeutronPortContext.' + 'resolve_ports') + def test_data_port_eth(self, mock_resolve): + self.config.side_effect = self.fake_config({'data-port': 'phybr1:eth1010'}) + mock_resolve.side_effect = lambda ports: ports self.assertEquals(quantum_contexts.DataPortContext()(), - {'data_port': 'eth1010'}) + {'phybr1': 'eth1010'}) class TestL3AgentContext(CharmTestCase): @@ -263,13 +263,11 @@ class TestQuantumGatewayContext(CharmTestCase): self.test_config.set('debug', False) self.test_config.set('verbose', True) self.test_config.set('instance-mtu', 1420) - self.test_config.set('network-device-mtu', 1500) self.get_os_codename_install_source.return_value = 'folsom' _host_ip.return_value = '10.5.0.1' _secret.return_value = 'testsecret' self.assertEquals(quantum_contexts.QuantumGatewayContext()(), { 'shared_secret': 'testsecret', - 'network_device_mtu': 1500, 'local_ip': '10.5.0.1', 'instance_mtu': 1420, 'core_plugin': "quantum.plugins.openvswitch.ovs_quantum_plugin." @@ -280,7 +278,6 @@ class TestQuantumGatewayContext(CharmTestCase): 'l2_population': False, 'overlay_network_type': 'gre', 'bridge_mappings': 'physnet1:br-data', - 'veth_mtu': 1500, }) @@ -406,12 +403,10 @@ class TestMisc(CharmTestCase): self.relation_ids.return_value = ['foo'] self.related_units.return_value = ['bar'] self.test_relation.set({'l2-population': True, - 'network-device-mtu': 1500, 'overlay-network-type': 'gre', }) self.relation_get.side_effect = self.test_relation.get self.assertEquals(quantum_contexts._neutron_api_settings(), {'l2_population': True, - 'network_device_mtu': 1500, 'overlay_network_type': 'gre'}) def test_neutron_api_settings2(self): @@ -429,5 +424,4 @@ class TestMisc(CharmTestCase): self.relation_ids.return_value = [] self.assertEquals(quantum_contexts._neutron_api_settings(), {'l2_population': False, - 'network_device_mtu': 1500, 'overlay_network_type': 'gre', }) diff --git a/unit_tests/test_quantum_utils.py b/unit_tests/test_quantum_utils.py index cf57c45e..ecd25240 100644 --- a/unit_tests/test_quantum_utils.py +++ b/unit_tests/test_quantum_utils.py @@ -180,8 +180,7 @@ class TestQuantumUtils(CharmTestCase): call('br-ex'), call('br-data') ]) - self.add_bridge_port.assert_called_with('br-data', 'eth0', - promisc=True) + self.assertFalse(self.add_bridge_port.called) def test_do_openstack_upgrade(self): self.config.side_effect = self.test_config.get From 6a0f330305a03ab39346d1544adb212fabfca46a Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Fri, 27 Feb 2015 18:25:33 +0000 Subject: [PATCH 50/75] more --- hooks/quantum_contexts.py | 16 ++++++++++++---- hooks/quantum_utils.py | 3 +-- unit_tests/test_quantum_contexts.py | 3 ++- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index ed103a76..6cffca87 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -30,6 +30,9 @@ from charmhelpers.contrib.network.ip import ( from charmhelpers.contrib.openstack.neutron import ( parse_data_port_mappings, ) +from charmhelpers.core.host import ( + get_nic_hwaddr, +) DB_USER = "quantum" QUANTUM_DB = "quantum" @@ -214,10 +217,15 @@ class DataPortContext(NeutronPortContext): ports = config('data-port') if ports: portmap = parse_data_port_mappings(ports) - ports = self.resolve_ports(portmap.values()) - if ports: - return {provider: port for provider, port in - portmap.iteritems() if port in ports} + ports = portmap.values() + resolved = self.resolve_ports(ports) + normalized = {get_nic_hwaddr(port): port for port in resolved + if port not in ports} + normalized.update({port: port for port in resolved + if port in ports}) + if resolved: + return {provider: normalized[port] for provider, port in + portmap.iteritems() if port in normalized.keys()} return None diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index c20b9c56..d66c9e8d 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -618,8 +618,7 @@ def configure_ovs(): if not portmaps or provider not in portmaps: continue - add_bridge_port(br, data_port_ctx['data_port'], - promisc=True) + add_bridge_port(br, portmaps[provider], promisc=True) # Ensure this runs so that any new bridges have correct mtu service_restart('os-charm-phy-nic-mtu') diff --git a/unit_tests/test_quantum_contexts.py b/unit_tests/test_quantum_contexts.py index 7f6a8706..d572507d 100644 --- a/unit_tests/test_quantum_contexts.py +++ b/unit_tests/test_quantum_contexts.py @@ -209,7 +209,8 @@ class TestNeutronPortContext(CharmTestCase): @patch('charmhelpers.contrib.openstack.context.NeutronPortContext.' 'resolve_ports') def test_data_port_eth(self, mock_resolve): - self.config.side_effect = self.fake_config({'data-port': 'phybr1:eth1010'}) + self.config.side_effect = self.fake_config({'data-port': + 'phybr1:eth1010'}) mock_resolve.side_effect = lambda ports: ports self.assertEquals(quantum_contexts.DataPortContext()(), {'phybr1': 'eth1010'}) From 17461b15ade0b88db20a7d1b6f4e0d8b6e181bd5 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 3 Mar 2015 10:24:40 +0800 Subject: [PATCH 51/75] synced ch --- hooks/charmhelpers/contrib/network/ip.py | 85 ++++++++++++++++++- .../charmhelpers/contrib/openstack/context.py | 41 ++++++--- .../charmhelpers/contrib/openstack/neutron.py | 1 - hooks/charmhelpers/contrib/openstack/utils.py | 78 ++--------------- hooks/charmhelpers/core/services/helpers.py | 16 +++- 5 files changed, 132 insertions(+), 89 deletions(-) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 98b17544..fff6d5ca 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -17,13 +17,16 @@ import glob import re import subprocess +import six +import socket from functools import partial from charmhelpers.core.hookenv import unit_get from charmhelpers.fetch import apt_install from charmhelpers.core.hookenv import ( - log + log, + WARNING, ) try: @@ -365,3 +368,83 @@ def is_bridge_member(nic): return True return False + + +def is_ip(address): + """ + Returns True if address is a valid IP address. + """ + try: + # Test to see if already an IPv4 address + socket.inet_aton(address) + return True + except socket.error: + return False + + +def ns_query(address): + try: + import dns.resolver + except ImportError: + apt_install('python-dnspython') + import dns.resolver + + if isinstance(address, dns.name.Name): + rtype = 'PTR' + elif isinstance(address, six.string_types): + rtype = 'A' + else: + return None + + answers = dns.resolver.query(address, rtype) + if answers: + return str(answers[0]) + return None + + +def get_host_ip(hostname, fallback=None): + """ + Resolves the IP for a given hostname, or returns + the input if it is already an IP. + """ + if is_ip(hostname): + return hostname + + ip_addr = ns_query(hostname) + if not ip_addr: + try: + ip_addr = socket.gethostbyname(hostname) + except: + log("Failed to resolve hostname '%s'" % (hostname), + level=WARNING) + return fallback + return ip_addr + + +def get_hostname(address, fqdn=True): + """ + Resolves hostname for given IP, or returns the input + if it is already a hostname. + """ + if is_ip(address): + try: + import dns.reversename + except ImportError: + apt_install("python-dnspython") + import dns.reversename + + rev = dns.reversename.from_address(address) + result = ns_query(rev) + if not result: + return None + else: + result = address + + if fqdn: + # strip trailing . + if result.endswith('.'): + return result[:-1] + else: + return result + else: + return result.split('.')[0] diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 6f109864..21bfde88 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -196,7 +196,7 @@ class SharedDBContext(OSContextGenerator): unit=local_unit()) if set_hostname != access_hostname: relation_set(relation_settings={hostname_key: access_hostname}) - return ctxt # Defer any further hook execution for now.... + return None # Defer any further hook execution for now.... password_setting = 'password' if self.relation_prefix: @@ -284,9 +284,25 @@ def db_ssl(rdata, ctxt, ssl_dir): class IdentityServiceContext(OSContextGenerator): interfaces = ['identity-service'] + def __init__(self, service=None, service_user=None): + self.service = service + self.service_user = service_user + def __call__(self): log('Generating template context for identity-service', level=DEBUG) ctxt = {} + + if self.service and self.service_user: + # This is required for pki token signing if we don't want /tmp to + # be used. + cachedir = '/var/cache/%s' % (self.service) + if not os.path.isdir(cachedir): + log("Creating service cache dir %s" % (cachedir), level=DEBUG) + mkdir(path=cachedir, owner=self.service_user, + group=self.service_user, perms=0o700) + + ctxt['signing_dir'] = cachedir + for rid in relation_ids('identity-service'): for unit in related_units(rid): rdata = relation_get(rid=rid, unit=unit) @@ -296,15 +312,16 @@ class IdentityServiceContext(OSContextGenerator): auth_host = format_ipv6_addr(auth_host) or auth_host svc_protocol = rdata.get('service_protocol') or 'http' auth_protocol = rdata.get('auth_protocol') or 'http' - ctxt = {'service_port': rdata.get('service_port'), - 'service_host': serv_host, - 'auth_host': auth_host, - 'auth_port': rdata.get('auth_port'), - 'admin_tenant_name': rdata.get('service_tenant'), - 'admin_user': rdata.get('service_username'), - 'admin_password': rdata.get('service_password'), - 'service_protocol': svc_protocol, - 'auth_protocol': auth_protocol} + ctxt.update({'service_port': rdata.get('service_port'), + 'service_host': serv_host, + 'auth_host': auth_host, + 'auth_port': rdata.get('auth_port'), + 'admin_tenant_name': rdata.get('service_tenant'), + 'admin_user': rdata.get('service_username'), + 'admin_password': rdata.get('service_password'), + 'service_protocol': svc_protocol, + 'auth_protocol': auth_protocol}) + if context_complete(ctxt): # NOTE(jamespage) this is required for >= icehouse # so a missing value just indicates keystone needs @@ -844,7 +861,7 @@ class NeutronPortContext(OSContextGenerator): def resolve_ports(self, ports): """Resolve NICs not yet bound to bridge(s) - If hwaddress provided then returns resolved hwaddress otherwise NIC. + If hwaddress provided then returns resolved hwaddress otherwise NIC. """ if not ports: return None @@ -1068,6 +1085,8 @@ class ZeroMQContext(OSContextGenerator): for unit in related_units(rid): ctxt['zmq_nonce'] = relation_get('nonce', unit, rid) ctxt['zmq_host'] = relation_get('host', unit, rid) + ctxt['zmq_redis_address'] = relation_get( + 'zmq_redis_address', unit, rid) return ctxt diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 6d9fdff6..76a90011 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -269,4 +269,3 @@ def parse_data_port_mappings(mappings): Returns dict of the form {provider:port}. """ return parse_mappings(mappings) - diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index af2b3596..4f110c63 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -23,12 +23,13 @@ from functools import wraps import subprocess import json import os -import socket import sys import six import yaml +from charmhelpers.contrib.network import ip + from charmhelpers.core.hookenv import ( config, log as juju_log, @@ -421,77 +422,10 @@ def clean_storage(block_device): else: zap_disk(block_device) - -def is_ip(address): - """ - Returns True if address is a valid IP address. - """ - try: - # Test to see if already an IPv4 address - socket.inet_aton(address) - return True - except socket.error: - return False - - -def ns_query(address): - try: - import dns.resolver - except ImportError: - apt_install('python-dnspython') - import dns.resolver - - if isinstance(address, dns.name.Name): - rtype = 'PTR' - elif isinstance(address, six.string_types): - rtype = 'A' - else: - return None - - answers = dns.resolver.query(address, rtype) - if answers: - return str(answers[0]) - return None - - -def get_host_ip(hostname): - """ - Resolves the IP for a given hostname, or returns - the input if it is already an IP. - """ - if is_ip(hostname): - return hostname - - return ns_query(hostname) - - -def get_hostname(address, fqdn=True): - """ - Resolves hostname for given IP, or returns the input - if it is already a hostname. - """ - if is_ip(address): - try: - import dns.reversename - except ImportError: - apt_install('python-dnspython') - import dns.reversename - - rev = dns.reversename.from_address(address) - result = ns_query(rev) - if not result: - return None - else: - result = address - - if fqdn: - # strip trailing . - if result.endswith('.'): - return result[:-1] - else: - return result - else: - return result.split('.')[0] +is_ip = ip.is_ip +ns_query = ip.ns_query +get_host_ip = ip.get_host_ip +get_hostname = ip.get_hostname def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'): diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py index 5e3af9da..15b21664 100644 --- a/hooks/charmhelpers/core/services/helpers.py +++ b/hooks/charmhelpers/core/services/helpers.py @@ -45,12 +45,14 @@ class RelationContext(dict): """ name = None interface = None - required_keys = [] def __init__(self, name=None, additional_required_keys=None): + if not hasattr(self, 'required_keys'): + self.required_keys = [] + if name is not None: self.name = name - if additional_required_keys is not None: + if additional_required_keys: self.required_keys.extend(additional_required_keys) self.get_data() @@ -134,7 +136,10 @@ class MysqlRelation(RelationContext): """ name = 'db' interface = 'mysql' - required_keys = ['host', 'user', 'password', 'database'] + + def __init__(self, *args, **kwargs): + self.required_keys = ['host', 'user', 'password', 'database'] + super(HttpRelation).__init__(self, *args, **kwargs) class HttpRelation(RelationContext): @@ -146,7 +151,10 @@ class HttpRelation(RelationContext): """ name = 'website' interface = 'http' - required_keys = ['host', 'port'] + + def __init__(self, *args, **kwargs): + self.required_keys = ['host', 'port'] + super(HttpRelation).__init__(self, *args, **kwargs) def provide_data(self): return { From eebdd55c9d300379eab9b46e06e87d925217566c Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 3 Mar 2015 11:21:38 +0800 Subject: [PATCH 52/75] more --- config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.yaml b/config.yaml index fe151717..e964222f 100644 --- a/config.yaml +++ b/config.yaml @@ -167,8 +167,8 @@ options: A space-separated list of NICs that we want phy-nic-mtu applied to. phy-nic-mtu: type: int - default: 0 + default: description: | To improve network performance of VM, sometimes we should keep VM MTU as 1500 and use charm to modify MTU of tunnel nic more than 1500 (e.g. 1546 - for GRE). A value of zero means no mtu will be set/modified. + for GRE). From 59fba68718dad51bdeae9800a2d80d0a209e2c4d Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 3 Mar 2015 13:47:59 +0800 Subject: [PATCH 53/75] more --- hooks/charmhelpers/contrib/openstack/neutron.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 76a90011..818ece57 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -264,8 +264,19 @@ def parse_bridge_mappings(mappings): def parse_data_port_mappings(mappings): """Parse data port mappings. - Mappings must be a space-delimited list of provider:port mappings. + Mappings must be a space-delimited list of bridge:port mappings. - Returns dict of the form {provider:port}. + Returns dict of the form {bridge:port}. """ - return parse_mappings(mappings) + mappings = parse_mappings(mappings) + bridges = mappings.keys() + ports = mappings.values() + if len(set(bridges)) != len(bridges): + raise Exception("It is not allowed to have more than one port " + "configured on the same bridge") + + if len(set(ports)) != len(ports): + raise Exception("It is not allowed to have the same port configured " + "on more than one bridge") + + return mappings From 44d2c1e462064a5205974bae47e4252186c849f0 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 3 Mar 2015 13:55:27 +0800 Subject: [PATCH 54/75] more --- hooks/quantum_contexts.py | 2 +- hooks/quantum_utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index 6cffca87..9a14da63 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -224,7 +224,7 @@ class DataPortContext(NeutronPortContext): normalized.update({port: port for port in resolved if port in ports}) if resolved: - return {provider: normalized[port] for provider, port in + return {bridge: normalized[port] for bridge, port in portmap.iteritems() if port in normalized.keys()} return None diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index d66c9e8d..b23d951d 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -615,10 +615,10 @@ def configure_ovs(): for provider, br in bridgemaps.iteritems(): add_bridge(br) - if not portmaps or provider not in portmaps: + if not portmaps or br not in portmaps: continue - add_bridge_port(br, portmaps[provider], promisc=True) + add_bridge_port(br, portmaps[br], promisc=True) # Ensure this runs so that any new bridges have correct mtu service_restart('os-charm-phy-nic-mtu') From 9c7a35104dbf934972905f4848302dfa0743e2ac Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 3 Mar 2015 14:06:48 +0800 Subject: [PATCH 55/75] more --- .../charmhelpers/contrib/openstack/neutron.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 818ece57..a0e0d836 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -239,10 +239,10 @@ def network_manager(): return 'neutron' -def parse_mappings(mappings): +def parse_mappings(mappings, delimiter=' '): parsed = {} if mappings: - mappings = mappings.split(' ') + mappings = mappings.split(delimiter) for m in mappings: p = m.partition(':') if p[1] == ':': @@ -280,3 +280,18 @@ def parse_data_port_mappings(mappings): "on more than one bridge") return mappings + + +def parse_vlan_range_mappings(mappings): + """Parse vlan range mappings. + + Mappings must be a space-delimited list of provider:start:end mappings. + + Returns dict of the form {provider: (start, end)}. + """ + _mappings = parse_mappings(mappings) + mappings = {} + for p, r in _mappings.iteritems(): + mappings[p] = tuple(r.split(':')) + + return mappings From c809638c0cf60f69cdafb70a6fe247951abb288a Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 3 Mar 2015 14:24:07 +0800 Subject: [PATCH 56/75] more --- config.yaml | 5 +++++ hooks/quantum_contexts.py | 7 +++++++ templates/icehouse/ml2_conf.ini | 4 ++-- unit_tests/test_quantum_contexts.py | 2 ++ 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/config.yaml b/config.yaml index e964222f..930c2e78 100644 --- a/config.yaml +++ b/config.yaml @@ -120,6 +120,11 @@ options: description: | Space-separated list of ML2 data bridge mappings with format :. + vlan-ranges: + type: string + default: "physnet1:1000:2000" + description: | + Space-delimited list of network provider vlan id ranges. # Network configuration options # by default all access is over 'private-address' os-data-network: diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index 9a14da63..be6e7fee 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -29,6 +29,7 @@ from charmhelpers.contrib.network.ip import ( ) from charmhelpers.contrib.openstack.neutron import ( parse_data_port_mappings, + parse_vlan_range_mappings, ) from charmhelpers.core.host import ( get_nic_hwaddr, @@ -253,6 +254,12 @@ class QuantumGatewayContext(OSContextGenerator): if mappings: ctxt['bridge_mappings'] = mappings + vlan_ranges = config('vlan-ranges') + vlan_range_mappings = parse_vlan_range_mappings(config('vlan-ranges')) + if vlan_ranges: + ctxt['network_providers'] = ' '.join(vlan_range_mappings.keys()) + ctxt['vlan_ranges'] = vlan_ranges + net_dev_mtu = neutron_api_settings.get('network_device_mtu') if net_dev_mtu: ctxt['network_device_mtu'] = net_dev_mtu diff --git a/templates/icehouse/ml2_conf.ini b/templates/icehouse/ml2_conf.ini index 4275b303..c9c04982 100644 --- a/templates/icehouse/ml2_conf.ini +++ b/templates/icehouse/ml2_conf.ini @@ -14,10 +14,10 @@ tunnel_id_ranges = 1:1000 vni_ranges = 1001:2000 [ml2_type_vlan] -network_vlan_ranges = physnet1:1000:2000 +network_vlan_ranges = {{ vlan_ranges }} [ml2_type_flat] -flat_networks = physnet1 +flat_networks = {{ network_providers }} [ovs] enable_tunneling = True diff --git a/unit_tests/test_quantum_contexts.py b/unit_tests/test_quantum_contexts.py index d572507d..03ef3247 100644 --- a/unit_tests/test_quantum_contexts.py +++ b/unit_tests/test_quantum_contexts.py @@ -279,6 +279,8 @@ class TestQuantumGatewayContext(CharmTestCase): 'l2_population': False, 'overlay_network_type': 'gre', 'bridge_mappings': 'physnet1:br-data', + 'network_providers': 'physnet1', + 'vlan_ranges': 'physnet1:1000:2000' }) From 43bc6fd54ebe7d4993b183d695d6598139d492cb Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 3 Mar 2015 15:03:53 +0800 Subject: [PATCH 57/75] more --- config.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/config.yaml b/config.yaml index 930c2e78..63795597 100644 --- a/config.yaml +++ b/config.yaml @@ -13,7 +13,7 @@ options: type: string default: description: | - A space-separated list of external ports to use for routing of instance + Space-delimited list of external ports to use for routing of instance traffic to the external public network. Valid values are either MAC addresses (in which case only MAC addresses for interfaces without an IP address already assigned will be used), or interfaces (eth0) @@ -21,8 +21,10 @@ options: type: string default: description: | - The data port will be added to bridges provided in bridge-mappings and - will allow usage of flat or VLAN network types with Neutron. + Space-delimited list of bridge:port mappings. Ports will be added to + their corresponding bridge. The bridges will allow usage of flat or + VLAN network types with Neutron and should match this defined in + bridge-mappings. openstack-origin: type: string default: distro From fac3c0198df917af36ba106a25c0753147a1b035 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 3 Mar 2015 15:21:23 +0800 Subject: [PATCH 58/75] more --- .../charmhelpers/contrib/openstack/neutron.py | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index a0e0d836..89ca435a 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -239,10 +239,10 @@ def network_manager(): return 'neutron' -def parse_mappings(mappings, delimiter=' '): +def parse_mappings(mappings): parsed = {} if mappings: - mappings = mappings.split(delimiter) + mappings = mappings.split(' ') for m in mappings: p = m.partition(':') if p[1] == ':': @@ -261,16 +261,24 @@ def parse_bridge_mappings(mappings): return parse_mappings(mappings) -def parse_data_port_mappings(mappings): +def parse_data_port_mappings(mappings, default_bridge='br-data'): """Parse data port mappings. Mappings must be a space-delimited list of bridge:port mappings. Returns dict of the form {bridge:port}. """ - mappings = parse_mappings(mappings) - bridges = mappings.keys() - ports = mappings.values() + _mappings = parse_mappings(mappings) + if not _mappings: + if not mappings: + return {} + + # For backwards-compatibility we need to support port-only provided in + # config. + _mappings = {default_bridge: mappings.split(' ')[0]} + + bridges = _mappings.keys() + ports = _mappings.values() if len(set(bridges)) != len(bridges): raise Exception("It is not allowed to have more than one port " "configured on the same bridge") @@ -279,7 +287,7 @@ def parse_data_port_mappings(mappings): raise Exception("It is not allowed to have the same port configured " "on more than one bridge") - return mappings + return _mappings def parse_vlan_range_mappings(mappings): @@ -290,6 +298,9 @@ def parse_vlan_range_mappings(mappings): Returns dict of the form {provider: (start, end)}. """ _mappings = parse_mappings(mappings) + if not _mappings: + return {} + mappings = {} for p, r in _mappings.iteritems(): mappings[p] = tuple(r.split(':')) From a0bc348fbb668f56ba6bf1a0acb5fef9934a690c Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 3 Mar 2015 10:40:41 +0000 Subject: [PATCH 59/75] Purge keepalived if l3ha is turned off --- hooks/quantum_hooks.py | 11 ++++++++--- hooks/quantum_utils.py | 13 +++++++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/hooks/quantum_hooks.py b/hooks/quantum_hooks.py index 722998ec..ca3fcd17 100755 --- a/hooks/quantum_hooks.py +++ b/hooks/quantum_hooks.py @@ -40,6 +40,7 @@ from charmhelpers.contrib.charmsupport import nrpe import sys from quantum_utils import ( + L3HA_PACKAGES, register_configs, restart_map, services, @@ -56,7 +57,8 @@ from quantum_utils import ( install_legacy_ha_files, cleanup_ovs_netns, reassign_agent_resources, - stop_neutron_ha_monitor_daemon + stop_neutron_ha_monitor_daemon, + use_l3ha, ) hooks = Hooks() @@ -202,8 +204,11 @@ def db_amqp_changed(): @hooks.hook('neutron-plugin-api-relation-changed') @restart_on_change(restart_map()) def neutron_plugin_api_changed(): - apt_install(filter_installed_packages(get_packages()), - fatal=True) + if use_l3ha(): + apt_update() + apt_install(L3HA_PACKAGES, fatal=True) + else: + apt_purge(L3HA_PACKAGES, fatal=True) CONFIGS.write_all() diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 6b7d1bcc..1c1f74cf 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -168,7 +168,7 @@ LEGACY_FILES_MAP = { }, } LEGACY_RES_MAP = ['res_monitor'] - +L3HA_PACKAGES = ['keepalived'] def get_early_packages(): '''Return a list of package for pre-install based on configured plugin''' @@ -182,7 +182,6 @@ def get_early_packages(): return pkgs + [headers_package()] return pkgs - def get_packages(): '''Return a list of packages for install based on the configured plugin''' plugin = remap_plugin(config('plugin')) @@ -198,10 +197,13 @@ def get_packages(): packages.append('openswan') if source >= 'kilo': packages.append('python-neutron-fwaas') - if neutron_api_settings()['enable_l3ha']: - packages.append('keepalived') + packages.extend(determine_l3ha_packages()) return packages +def determine_l3ha_packages(): + if use_l3ha(): + return L3HA_PACKAGES + return [] def get_common_package(): if get_os_codename_package('quantum-common', fatal=False) is not None: @@ -209,6 +211,9 @@ def get_common_package(): else: return 'neutron-common' +def use_l3ha(): + return neutron_api_settings()['enable_l3ha'] + EXT_PORT_CONF = '/etc/init/ext-port.conf' TEMPLATES = 'templates' From 0a5e106ed472a893da58c7a0bda240db3702e3b9 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 3 Mar 2015 11:22:28 +0000 Subject: [PATCH 60/75] Fix lint and unit tests --- hooks/quantum_utils.py | 5 +++++ unit_tests/test_quantum_hooks.py | 7 ++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 1c1f74cf..b6080b8e 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -170,6 +170,7 @@ LEGACY_FILES_MAP = { LEGACY_RES_MAP = ['res_monitor'] L3HA_PACKAGES = ['keepalived'] + def get_early_packages(): '''Return a list of package for pre-install based on configured plugin''' if config('plugin') in [OVS]: @@ -182,6 +183,7 @@ def get_early_packages(): return pkgs + [headers_package()] return pkgs + def get_packages(): '''Return a list of packages for install based on the configured plugin''' plugin = remap_plugin(config('plugin')) @@ -200,17 +202,20 @@ def get_packages(): packages.extend(determine_l3ha_packages()) return packages + def determine_l3ha_packages(): if use_l3ha(): return L3HA_PACKAGES return [] + def get_common_package(): if get_os_codename_package('quantum-common', fatal=False) is not None: return 'quantum-common' else: return 'neutron-common' + def use_l3ha(): return neutron_api_settings()['enable_l3ha'] diff --git a/unit_tests/test_quantum_hooks.py b/unit_tests/test_quantum_hooks.py index 2465273a..3146b771 100644 --- a/unit_tests/test_quantum_hooks.py +++ b/unit_tests/test_quantum_hooks.py @@ -47,7 +47,8 @@ TO_PATCH = [ 'get_hacluster_config', 'remove_legacy_ha_files', 'cleanup_ovs_netns', - 'stop_neutron_ha_monitor_daemon' + 'stop_neutron_ha_monitor_daemon', + 'use_l3ha', ] @@ -247,9 +248,9 @@ class TestQuantumHooks(CharmTestCase): self.install_ca_cert.assert_called_with('cert') def test_neutron_plugin_changed(self): - self.filter_installed_packages.return_value = ['foo'] + self.use_l3ha.return_value = True self._call_hook('neutron-plugin-api-relation-changed') - self.apt_install.assert_called_with(['foo'], fatal=True) + self.apt_install.assert_called_with(['keepalived'], fatal=True) self.assertTrue(self.CONFIGS.write_all.called) def test_cluster_departed_nvp(self): From df0a7279879b56d8f7f4097f6b281ec39f1a0721 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Thu, 12 Mar 2015 10:33:36 +0000 Subject: [PATCH 61/75] phy-nic* > get value from data-port config param --- config.yaml | 12 ----------- hooks/quantum_contexts.py | 33 ++++++++++++++++------------- unit_tests/test_quantum_contexts.py | 8 ++++--- 3 files changed, 23 insertions(+), 30 deletions(-) diff --git a/config.yaml b/config.yaml index 3d92376b..6bf27e47 100644 --- a/config.yaml +++ b/config.yaml @@ -173,15 +173,3 @@ options: description: | Default multicast port number that will be used to communicate between HA Cluster nodes. - phy-nics: - type: string - default: - description: | - A space-separated list of NICs that we want phy-nic-mtu applied to. - phy-nic-mtu: - type: int - default: - description: | - To improve network performance of VM, sometimes we should keep VM MTU as - 1500 and use charm to modify MTU of tunnel nic more than 1500 (e.g. 1546 - for GRE). diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index be6e7fee..03459142 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -191,27 +191,14 @@ class ExternalPortContext(NeutronPortContext): ports = self.resolve_ports(ports) if ports: ctxt = {"ext_port": ports[0]} - mtu = config('phy-nic-mtu') + neutron_api_settings = _neutron_api_settings() + mtu = neutron_api_settings.get('network_device_mtu') if mtu: ctxt['ext_port_mtu'] = mtu return ctxt -class PhyNICMTUContext(OSContextGenerator): - - def __call__(self): - ctxt = {} - port = config('phy-nics') - if port: - ctxt = {"devs": port.replace(' ', '\\n')} - mtu = config('phy-nic-mtu') - if mtu: - ctxt['mtu'] = mtu - - return ctxt - - class DataPortContext(NeutronPortContext): def __call__(self): @@ -231,6 +218,22 @@ class DataPortContext(NeutronPortContext): return None +class PhyNICMTUContext(DataPortContext): + + def __call__(self): + ctxt = {} + mappings = super(PhyNICMTUContext, self).__call__() + if mappings and mappings.values(): + ports = mappings.values() + neutron_api_settings = _neutron_api_settings() + mtu = neutron_api_settings.get('network_device_mtu') + if mtu: + ctxt["devs"] = '\\n'.join(ports) + ctxt['mtu'] = mtu + + return ctxt + + class QuantumGatewayContext(OSContextGenerator): def __call__(self): diff --git a/unit_tests/test_quantum_contexts.py b/unit_tests/test_quantum_contexts.py index 03ef3247..d983cde3 100644 --- a/unit_tests/test_quantum_contexts.py +++ b/unit_tests/test_quantum_contexts.py @@ -181,6 +181,7 @@ class TestNeutronPortContext(CharmTestCase): self.assertEquals(quantum_contexts.ExternalPortContext()(), {}) + @patch.object(quantum_contexts, '_neutron_api_settings') @patch('charmhelpers.contrib.openstack.context.get_nic_hwaddr') @patch('charmhelpers.contrib.openstack.context.list_nics') @patch('charmhelpers.contrib.openstack.context.get_ipv6_addr') @@ -189,8 +190,10 @@ class TestNeutronPortContext(CharmTestCase): def test_ext_port_mac_one_used_nic(self, mock_config, mock_get_ipv4_addr, mock_get_ipv6_addr, mock_list_nics, - mock_get_nic_hwaddr): + mock_get_nic_hwaddr, + mock_neutron_api_settings): + mock_neutron_api_settings.return_value = {'network_device_mtu': 1234} config_macs = "%s %s" % (self.machine_macs['eth1'], self.machine_macs['eth2']) @@ -199,8 +202,7 @@ class TestNeutronPortContext(CharmTestCase): mock_list_nics.return_value = self.machine_macs.keys() mock_get_nic_hwaddr.side_effect = self._fake_get_hwaddr - config = self.fake_config({'ext-port': config_macs, - 'phy-nic-mtu': 1234}) + config = self.fake_config({'ext-port': config_macs}) self.config.side_effect = config mock_config.side_effect = config self.assertEquals(quantum_contexts.ExternalPortContext()(), From d049c7e5e15723cb6abdeceb5420b9fc2312e6d5 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 16 Mar 2015 16:37:23 +0100 Subject: [PATCH 62/75] cleanup --- charm-helpers-hooks.yaml | 2 +- charm-helpers-tests.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index 7fc452c2..84cc6c76 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -1,4 +1,4 @@ -branch: lp:~cts-engineering/charm-helpers/neutron-mtu +branch: lp:charm-helpers destination: hooks/charmhelpers include: - core diff --git a/charm-helpers-tests.yaml b/charm-helpers-tests.yaml index 62c87f0e..48b12f6f 100644 --- a/charm-helpers-tests.yaml +++ b/charm-helpers-tests.yaml @@ -1,4 +1,4 @@ -branch: lp:~cts-engineering/charm-helpers/neutron-mtu +branch: lp:charm-helpers destination: tests/charmhelpers include: - contrib.amulet From 42270d52ee4b44e57d9f5a54e43bd1047994907a Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 16 Mar 2015 18:44:02 +0000 Subject: [PATCH 63/75] added vlan-ranges config --- hooks/quantum_contexts.py | 20 ++++++++--------- hooks/quantum_utils.py | 3 ++- unit_tests/test_quantum_contexts.py | 17 ++++++++++++--- unit_tests/test_quantum_utils.py | 34 +++++++++++++++++++++-------- 4 files changed, 51 insertions(+), 23 deletions(-) diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index 03459142..7856bf06 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -118,17 +118,16 @@ def _neutron_api_settings(): for rid in relation_ids('neutron-plugin-api'): for unit in related_units(rid): rdata = relation_get(rid=rid, unit=unit) - if 'l2-population' not in rdata: - continue - neutron_settings = { - 'l2_population': rdata['l2-population'], - 'overlay_network_type': rdata['overlay-network-type'], - } + if 'l2-population' in rdata: + neutron_settings.update({ + 'l2_population': rdata['l2-population'], + 'overlay_network_type': rdata['overlay-network-type'], + }) + net_dev_mtu = rdata.get('network-device-mtu') if net_dev_mtu: neutron_settings['network_device_mtu'] = net_dev_mtu - return neutron_settings return neutron_settings @@ -258,9 +257,10 @@ class QuantumGatewayContext(OSContextGenerator): ctxt['bridge_mappings'] = mappings vlan_ranges = config('vlan-ranges') - vlan_range_mappings = parse_vlan_range_mappings(config('vlan-ranges')) - if vlan_ranges: - ctxt['network_providers'] = ' '.join(vlan_range_mappings.keys()) + vlan_range_mappings = parse_vlan_range_mappings(vlan_ranges) + if vlan_range_mappings: + providers = sorted(vlan_range_mappings.keys()) + ctxt['network_providers'] = ' '.join(providers) ctxt['vlan_ranges'] = vlan_ranges net_dev_mtu = neutron_api_settings.get('network_device_mtu') diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index b23d951d..e7a0f6c6 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -620,7 +620,8 @@ def configure_ovs(): add_bridge_port(br, portmaps[br], promisc=True) - # Ensure this runs so that any new bridges have correct mtu + # Ensure this runs so that mtu is applied to data-port interfaces if + # provided. service_restart('os-charm-phy-nic-mtu') diff --git a/unit_tests/test_quantum_contexts.py b/unit_tests/test_quantum_contexts.py index d983cde3..5244ec89 100644 --- a/unit_tests/test_quantum_contexts.py +++ b/unit_tests/test_quantum_contexts.py @@ -266,10 +266,19 @@ class TestQuantumGatewayContext(CharmTestCase): self.test_config.set('debug', False) self.test_config.set('verbose', True) self.test_config.set('instance-mtu', 1420) + self.test_config.set('vlan-ranges', + 'physnet1:1000:2000 physnet2:2001:3000') + # Provided by neutron-api relation + self.relation_ids.return_value = ['neutron-plugin-api:0'] + self.related_units.return_value = ['neutron-api/0'] + rdata = {'network-device-mtu': 9000} + self.relation_get.side_effect = lambda *args, **kwargs: rdata self.get_os_codename_install_source.return_value = 'folsom' _host_ip.return_value = '10.5.0.1' _secret.return_value = 'testsecret' - self.assertEquals(quantum_contexts.QuantumGatewayContext()(), { + ctxt = quantum_contexts.QuantumGatewayContext()() + self.relation_ids.assert_called_with('neutron-plugin-api') + self.assertEquals(ctxt, { 'shared_secret': 'testsecret', 'local_ip': '10.5.0.1', 'instance_mtu': 1420, @@ -281,8 +290,10 @@ class TestQuantumGatewayContext(CharmTestCase): 'l2_population': False, 'overlay_network_type': 'gre', 'bridge_mappings': 'physnet1:br-data', - 'network_providers': 'physnet1', - 'vlan_ranges': 'physnet1:1000:2000' + 'network_providers': 'physnet1 physnet2', + 'vlan_ranges': 'physnet1:1000:2000 physnet2:2001:3000', + 'network_device_mtu': 9000, + 'veth_mtu': 9000, }) diff --git a/unit_tests/test_quantum_utils.py b/unit_tests/test_quantum_utils.py index ecd25240..60a87d03 100644 --- a/unit_tests/test_quantum_utils.py +++ b/unit_tests/test_quantum_utils.py @@ -36,7 +36,6 @@ TO_PATCH = [ 'service_running', 'NetworkServiceContext', 'ExternalPortContext', - 'DataPortContext', 'unit_private_ip', 'relations_of_type', 'service_stop', @@ -139,7 +138,9 @@ class TestQuantumUtils(CharmTestCase): self.get_os_codename_install_source.return_value = 'kilo' self.assertTrue('python-neutron-fwaas' in quantum_utils.get_packages()) - def test_configure_ovs_starts_service_if_required(self): + @patch('quantum_contexts.config') + def test_configure_ovs_starts_service_if_required(self, mock_config): + mock_config.side_effect = self.test_config.get self.config.return_value = 'ovs' self.service_running.return_value = False quantum_utils.configure_ovs() @@ -150,14 +151,14 @@ class TestQuantumUtils(CharmTestCase): quantum_utils.configure_ovs() self.assertFalse(self.full_restart.called) - def test_configure_ovs_ovs_ext_port(self): + @patch('quantum_contexts.config') + def test_configure_ovs_ovs_ext_port(self, mock_config): + mock_config.side_effect = self.test_config.get self.config.side_effect = self.test_config.get self.test_config.set('plugin', 'ovs') self.test_config.set('ext-port', 'eth0') self.ExternalPortContext.return_value = \ DummyExternalPortContext(return_value={'ext_port': 'eth0'}) - self.DataPortContext.return_value = \ - DummyExternalPortContext(return_value=None) quantum_utils.configure_ovs() self.add_bridge.assert_has_calls([ call('br-int'), @@ -166,20 +167,35 @@ class TestQuantumUtils(CharmTestCase): ]) self.add_bridge_port.assert_called_with('br-ex', 'eth0') - def test_configure_ovs_ovs_data_port(self): + @patch('quantum_contexts.config') + def test_configure_ovs_ovs_data_port(self, mock_config): + mock_config.side_effect = self.test_config.get self.config.side_effect = self.test_config.get self.test_config.set('plugin', 'ovs') - self.test_config.set('data-port', 'eth0') self.ExternalPortContext.return_value = \ DummyExternalPortContext(return_value=None) - self.DataPortContext.return_value = \ - DummyExternalPortContext(return_value={'data_port': 'eth0'}) + # Test back-compatibility i.e. port but no bridge (so br-data is + # assumed) + self.test_config.set('data-port', 'eth0') quantum_utils.configure_ovs() self.add_bridge.assert_has_calls([ call('br-int'), call('br-ex'), call('br-data') ]) + self.assertTrue(self.add_bridge_port.called) + + # Now test with bridge:port format + self.test_config.set('data-port', 'br-foo:eth0') + self.add_bridge.reset_mock() + self.add_bridge_port.reset_mock() + quantum_utils.configure_ovs() + self.add_bridge.assert_has_calls([ + call('br-int'), + call('br-ex'), + call('br-data') + ]) + # Not called since we have a bogus bridge in data-ports self.assertFalse(self.add_bridge_port.called) def test_do_openstack_upgrade(self): From 9cb8b2fb4655f8c1686b3af9a829d10aaaf56dce Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 24 Mar 2015 12:07:21 +0000 Subject: [PATCH 64/75] Moved contexts out into charmhelpers --- .../charmhelpers/contrib/openstack/context.py | 141 ++++++++++- hooks/quantum_contexts.py | 120 +--------- hooks/quantum_utils.py | 10 +- unit_tests/test_quantum_contexts.py | 221 +----------------- unit_tests/test_quantum_utils.py | 6 +- 5 files changed, 158 insertions(+), 340 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 90ac6d69..7c52bd73 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -47,6 +47,7 @@ from charmhelpers.core.hookenv import ( ) from charmhelpers.core.sysctl import create as sysctl_create +from charmhelpers.core.strutils import bool_from_string from charmhelpers.core.host import ( list_nics, @@ -67,6 +68,7 @@ from charmhelpers.contrib.hahelpers.apache import ( ) from charmhelpers.contrib.openstack.neutron import ( neutron_plugin_attribute, + parse_data_port_mappings, ) from charmhelpers.contrib.openstack.ip import ( resolve_address, @@ -82,7 +84,6 @@ from charmhelpers.contrib.network.ip import ( is_bridge_member, ) from charmhelpers.contrib.openstack.utils import get_host_ip - CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' ADDRESS_TYPES = ['admin', 'internal', 'public'] @@ -1162,3 +1163,141 @@ class SysctlContext(OSContextGenerator): sysctl_create(sysctl_dict, '/etc/sysctl.d/50-{0}.conf'.format(charm_name())) return {'sysctl': sysctl_dict} + + +class NeutronAPIContext(OSContextGenerator): + ''' + Inspects current neutron-plugin-api relation for neutron settings. Return + defaults if it is not present. + ''' + interfaces = ['neutron-plugin-api'] + + def __call__(self): + self.neutron_defaults = { + 'l2_population': { + 'rel_key': 'l2-population', + 'default': False + }, + 'enable_dvr': { + 'rel_key': 'enable-dvr', + 'default': False + }, + 'enable_l3ha': { + 'rel_key': 'enable-l3ha', + 'default': False + }, + 'overlay_network_type': { + 'rel_key': 'overlay-network-type', + 'default': 'gre' + }, + 'network_device_mtu': { + 'rel_key': 'network-device-mtu', + 'default': None + }, + } + ctxt = self.get_neutron_options({}) + for rid in relation_ids('neutron-plugin-api'): + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + if 'l2-population' in rdata: + ctxt.update(self.get_neutron_options(rdata)) + + return ctxt + + def get_neutron_options(self, rdata): + settings = {} + for nkey in self.neutron_defaults.keys(): + defv = self.neutron_defaults[nkey]['default'] + rkey = self.neutron_defaults[nkey]['rel_key'] + if rkey in rdata.keys(): + if type(defv) is bool: + settings[nkey] = bool_from_string(rdata[rkey]) + else: + settings[nkey] = rdata[rkey] + else: + settings[nkey] = defv + return settings + + +class ExternalPortContext(NeutronPortContext): + + def __call__(self): + ctxt = {} + ports = config('ext-port') + if ports: + ports = [p.strip() for p in ports.split()] + ports = self.resolve_ports(ports) + if ports: + ctxt = {"ext_port": ports[0]} + napi_settings = NeutronAPIContext()() + mtu = napi_settings.get('network_device_mtu') + if mtu: + ctxt['ext_port_mtu'] = mtu + + return ctxt + + +class DataPortContext(NeutronPortContext): + + def __call__(self): + ports = config('data-port') + if ports: + portmap = parse_data_port_mappings(ports) + ports = portmap.values() + resolved = self.resolve_ports(ports) + normalized = {get_nic_hwaddr(port): port for port in resolved + if port not in ports} + normalized.update({port: port for port in resolved + if port in ports}) + if resolved: + return {bridge: normalized[port] for bridge, port in + portmap.iteritems() if port in normalized.keys()} + + return None + + +class PhyNICMTUContext(DataPortContext): + + def __call__(self): + ctxt = {} + mappings = super(PhyNICMTUContext, self).__call__() + if mappings and mappings.values(): + ports = mappings.values() + napi_settings = NeutronAPIContext()() + mtu = napi_settings.get('network_device_mtu') + if mtu: + ctxt["devs"] = '\\n'.join(ports) + ctxt['mtu'] = mtu + + return ctxt + + +class NetworkServiceContext(OSContextGenerator): + + def __init__(self, rel_name='quantum-network-service'): + self.rel_name = rel_name + self.interfaces = [rel_name] + + def __call__(self): + for rid in relation_ids(self.rel_name): + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + ctxt = { + 'keystone_host': rdata.get('keystone_host'), + 'service_port': rdata.get('service_port'), + 'auth_port': rdata.get('auth_port'), + 'service_tenant': rdata.get('service_tenant'), + 'service_username': rdata.get('service_username'), + 'service_password': rdata.get('service_password'), + 'quantum_host': rdata.get('quantum_host'), + 'quantum_port': rdata.get('quantum_port'), + 'quantum_url': rdata.get('quantum_url'), + 'region': rdata.get('region'), + 'service_protocol': + rdata.get('service_protocol') or 'http', + 'auth_protocol': + rdata.get('auth_protocol') or 'http', + } + if context_complete(ctxt): + return ctxt + return {} diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index 7856bf06..b262a019 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -4,9 +4,6 @@ import uuid import socket from charmhelpers.core.hookenv import ( config, - relation_ids, - related_units, - relation_get, unit_get, cached ) @@ -15,8 +12,7 @@ from charmhelpers.fetch import ( ) from charmhelpers.contrib.openstack.context import ( OSContextGenerator, - context_complete, - NeutronPortContext, + NeutronAPIContext, ) from charmhelpers.contrib.openstack.utils import ( get_os_codename_install_source @@ -28,12 +24,8 @@ from charmhelpers.contrib.network.ip import ( get_address_in_network, ) from charmhelpers.contrib.openstack.neutron import ( - parse_data_port_mappings, parse_vlan_range_mappings, ) -from charmhelpers.core.host import ( - get_nic_hwaddr, -) DB_USER = "quantum" QUANTUM_DB = "quantum" @@ -104,61 +96,6 @@ def core_plugin(): return CORE_PLUGIN[networking_name()][plugin] -def _neutron_api_settings(): - ''' - Inspects current neutron-plugin-api relation for neutron settings. Return - defaults if it is not present - ''' - neutron_settings = { - 'l2_population': False, - 'overlay_network_type': 'gre', - - } - - for rid in relation_ids('neutron-plugin-api'): - for unit in related_units(rid): - rdata = relation_get(rid=rid, unit=unit) - if 'l2-population' in rdata: - neutron_settings.update({ - 'l2_population': rdata['l2-population'], - 'overlay_network_type': rdata['overlay-network-type'], - }) - - net_dev_mtu = rdata.get('network-device-mtu') - if net_dev_mtu: - neutron_settings['network_device_mtu'] = net_dev_mtu - - return neutron_settings - - -class NetworkServiceContext(OSContextGenerator): - interfaces = ['quantum-network-service'] - - def __call__(self): - for rid in relation_ids('quantum-network-service'): - for unit in related_units(rid): - rdata = relation_get(rid=rid, unit=unit) - ctxt = { - 'keystone_host': rdata.get('keystone_host'), - 'service_port': rdata.get('service_port'), - 'auth_port': rdata.get('auth_port'), - 'service_tenant': rdata.get('service_tenant'), - 'service_username': rdata.get('service_username'), - 'service_password': rdata.get('service_password'), - 'quantum_host': rdata.get('quantum_host'), - 'quantum_port': rdata.get('quantum_port'), - 'quantum_url': rdata.get('quantum_url'), - 'region': rdata.get('region'), - 'service_protocol': - rdata.get('service_protocol') or 'http', - 'auth_protocol': - rdata.get('auth_protocol') or 'http', - } - if context_complete(ctxt): - return ctxt - return {} - - class L3AgentContext(OSContextGenerator): def __call__(self): @@ -180,63 +117,10 @@ class L3AgentContext(OSContextGenerator): return ctxt -class ExternalPortContext(NeutronPortContext): - - def __call__(self): - ctxt = {} - ports = config('ext-port') - if ports: - ports = [p.strip() for p in ports.split()] - ports = self.resolve_ports(ports) - if ports: - ctxt = {"ext_port": ports[0]} - neutron_api_settings = _neutron_api_settings() - mtu = neutron_api_settings.get('network_device_mtu') - if mtu: - ctxt['ext_port_mtu'] = mtu - - return ctxt - - -class DataPortContext(NeutronPortContext): - - def __call__(self): - ports = config('data-port') - if ports: - portmap = parse_data_port_mappings(ports) - ports = portmap.values() - resolved = self.resolve_ports(ports) - normalized = {get_nic_hwaddr(port): port for port in resolved - if port not in ports} - normalized.update({port: port for port in resolved - if port in ports}) - if resolved: - return {bridge: normalized[port] for bridge, port in - portmap.iteritems() if port in normalized.keys()} - - return None - - -class PhyNICMTUContext(DataPortContext): - - def __call__(self): - ctxt = {} - mappings = super(PhyNICMTUContext, self).__call__() - if mappings and mappings.values(): - ports = mappings.values() - neutron_api_settings = _neutron_api_settings() - mtu = neutron_api_settings.get('network_device_mtu') - if mtu: - ctxt["devs"] = '\\n'.join(ports) - ctxt['mtu'] = mtu - - return ctxt - - class QuantumGatewayContext(OSContextGenerator): def __call__(self): - neutron_api_settings = _neutron_api_settings() + neutron_api_settings = NeutronAPIContext()() ctxt = { 'shared_secret': get_shared_secret(), 'local_ip': diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index e7a0f6c6..eaec1a87 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -41,7 +41,11 @@ from charmhelpers.contrib.openstack.neutron import ( import charmhelpers.contrib.openstack.context as context from charmhelpers.contrib.openstack.context import ( - SyslogContext + SyslogContext, + NetworkServiceContext, + ExternalPortContext, + PhyNICMTUContext, + DataPortContext, ) import charmhelpers.contrib.openstack.templating as templating from charmhelpers.contrib.openstack.neutron import headers_package @@ -50,11 +54,7 @@ from quantum_contexts import ( NEUTRON, QUANTUM, networking_name, QuantumGatewayContext, - NetworkServiceContext, L3AgentContext, - ExternalPortContext, - PhyNICMTUContext, - DataPortContext, remap_plugin ) from charmhelpers.contrib.openstack.neutron import ( diff --git a/unit_tests/test_quantum_contexts.py b/unit_tests/test_quantum_contexts.py index 5244ec89..e183c119 100644 --- a/unit_tests/test_quantum_contexts.py +++ b/unit_tests/test_quantum_contexts.py @@ -14,12 +14,8 @@ from test_utils import ( TO_PATCH = [ 'apt_install', 'config', - 'context_complete', 'eligible_leader', 'get_os_codename_install_source', - 'relation_get', - 'relation_ids', - 'related_units', 'unit_get', ] @@ -42,182 +38,6 @@ def patch_open(): yield mock_open, mock_file -class TestNetworkServiceContext(CharmTestCase): - - def setUp(self): - super(TestNetworkServiceContext, self).setUp(quantum_contexts, - TO_PATCH) - self.config.side_effect = self.test_config.get - self.context = quantum_contexts.NetworkServiceContext() - self.test_relation.set( - {'keystone_host': '10.5.0.1', - 'service_port': '5000', - 'auth_port': '20000', - 'service_tenant': 'tenant', - 'service_username': 'username', - 'service_password': 'password', - 'quantum_host': '10.5.0.2', - 'quantum_port': '9696', - 'quantum_url': 'http://10.5.0.2:9696/v2', - 'region': 'aregion'} - ) - self.data_result = { - 'keystone_host': '10.5.0.1', - 'service_port': '5000', - 'auth_port': '20000', - 'service_tenant': 'tenant', - 'service_username': 'username', - 'service_password': 'password', - 'quantum_host': '10.5.0.2', - 'quantum_port': '9696', - 'quantum_url': 'http://10.5.0.2:9696/v2', - 'region': 'aregion', - 'service_protocol': 'http', - 'auth_protocol': 'http', - } - - def test_not_related(self): - self.relation_ids.return_value = [] - self.assertEquals(self.context(), {}) - - def test_no_units(self): - self.relation_ids.return_value = [] - self.relation_ids.return_value = ['foo'] - self.related_units.return_value = [] - self.assertEquals(self.context(), {}) - - def test_no_data(self): - self.relation_ids.return_value = ['foo'] - self.related_units.return_value = ['bar'] - self.relation_get.side_effect = self.test_relation.get - self.context_complete.return_value = False - self.assertEquals(self.context(), {}) - - def test_data_multi_unit(self): - self.relation_ids.return_value = ['foo'] - self.related_units.return_value = ['bar', 'baz'] - self.context_complete.return_value = True - self.relation_get.side_effect = self.test_relation.get - self.assertEquals(self.context(), self.data_result) - - def test_data_single_unit(self): - self.relation_ids.return_value = ['foo'] - self.related_units.return_value = ['bar'] - self.context_complete.return_value = True - self.relation_get.side_effect = self.test_relation.get - self.assertEquals(self.context(), self.data_result) - - -class TestNeutronPortContext(CharmTestCase): - - def setUp(self): - super(TestNeutronPortContext, self).setUp(quantum_contexts, - TO_PATCH) - self.machine_macs = { - 'eth0': 'fe:c5:ce:8e:2b:00', - 'eth1': 'fe:c5:ce:8e:2b:01', - 'eth2': 'fe:c5:ce:8e:2b:02', - 'eth3': 'fe:c5:ce:8e:2b:03', - } - self.machine_nics = { - 'eth0': ['192.168.0.1'], - 'eth1': ['192.168.0.2'], - 'eth2': [], - 'eth3': [], - } - self.absent_macs = "aa:a5:ae:ae:ab:a4 " - - def fake_config(self, cfgdict): - - def _fake_config(key): - return cfgdict.get(key) - - return _fake_config - - def _fake_get_hwaddr(self, arg): - return self.machine_macs[arg] - - def _fake_get_ipv4(self, arg, fatal=False): - return self.machine_nics[arg] - - @patch('charmhelpers.contrib.openstack.context.config') - def test_no_ext_port(self, mock_config): - self.config.side_effect = config = self.fake_config({}) - mock_config.side_effect = config - self.assertEquals(quantum_contexts.ExternalPortContext()(), {}) - - @patch('charmhelpers.contrib.openstack.context.config') - def test_ext_port_eth(self, mock_config): - config = self.fake_config({'ext-port': 'eth1010'}) - self.config.side_effect = config - mock_config.side_effect = config - self.assertEquals(quantum_contexts.ExternalPortContext()(), - {'ext_port': 'eth1010'}) - - @patch('charmhelpers.contrib.openstack.context.get_nic_hwaddr') - @patch('charmhelpers.contrib.openstack.context.list_nics') - @patch('charmhelpers.contrib.openstack.context.get_ipv6_addr') - @patch('charmhelpers.contrib.openstack.context.get_ipv4_addr') - @patch('charmhelpers.contrib.openstack.context.config') - def test_ext_port_mac(self, mock_config, mock_get_ipv4_addr, - mock_get_ipv6_addr, mock_list_nics, - mock_get_nic_hwaddr): - config_macs = self.absent_macs + " " + self.machine_macs['eth2'] - config = self.fake_config({'ext-port': config_macs}) - self.config.side_effect = config - mock_config.side_effect = config - - mock_get_ipv4_addr.side_effect = self._fake_get_ipv4 - mock_get_ipv6_addr.return_value = [] - mock_list_nics.return_value = self.machine_macs.keys() - mock_get_nic_hwaddr.side_effect = self._fake_get_hwaddr - - self.assertEquals(quantum_contexts.ExternalPortContext()(), - {'ext_port': 'eth2'}) - - config = self.fake_config({'ext-port': self.absent_macs}) - self.config.side_effect = config - mock_config.side_effect = config - - self.assertEquals(quantum_contexts.ExternalPortContext()(), {}) - - @patch.object(quantum_contexts, '_neutron_api_settings') - @patch('charmhelpers.contrib.openstack.context.get_nic_hwaddr') - @patch('charmhelpers.contrib.openstack.context.list_nics') - @patch('charmhelpers.contrib.openstack.context.get_ipv6_addr') - @patch('charmhelpers.contrib.openstack.context.get_ipv4_addr') - @patch('charmhelpers.contrib.openstack.context.config') - def test_ext_port_mac_one_used_nic(self, mock_config, - mock_get_ipv4_addr, - mock_get_ipv6_addr, mock_list_nics, - mock_get_nic_hwaddr, - mock_neutron_api_settings): - - mock_neutron_api_settings.return_value = {'network_device_mtu': 1234} - config_macs = "%s %s" % (self.machine_macs['eth1'], - self.machine_macs['eth2']) - - mock_get_ipv4_addr.side_effect = self._fake_get_ipv4 - mock_get_ipv6_addr.return_value = [] - mock_list_nics.return_value = self.machine_macs.keys() - mock_get_nic_hwaddr.side_effect = self._fake_get_hwaddr - - config = self.fake_config({'ext-port': config_macs}) - self.config.side_effect = config - mock_config.side_effect = config - self.assertEquals(quantum_contexts.ExternalPortContext()(), - {'ext_port': 'eth2', 'ext_port_mtu': 1234}) - - @patch('charmhelpers.contrib.openstack.context.NeutronPortContext.' - 'resolve_ports') - def test_data_port_eth(self, mock_resolve): - self.config.side_effect = self.fake_config({'data-port': - 'phybr1:eth1010'}) - mock_resolve.side_effect = lambda ports: ports - self.assertEquals(quantum_contexts.DataPortContext()(), - {'phybr1': 'eth1010'}) - - class TestL3AgentContext(CharmTestCase): def setUp(self): @@ -259,9 +79,12 @@ class TestQuantumGatewayContext(CharmTestCase): TO_PATCH) self.config.side_effect = self.test_config.get + @patch('charmhelpers.contrib.openstack.context.relation_get') + @patch('charmhelpers.contrib.openstack.context.related_units') + @patch('charmhelpers.contrib.openstack.context.relation_ids') @patch.object(quantum_contexts, 'get_shared_secret') @patch.object(quantum_contexts, 'get_host_ip') - def test_all(self, _host_ip, _secret): + def test_all(self, _host_ip, _secret, _rids, _runits, _rget): self.test_config.set('plugin', 'ovs') self.test_config.set('debug', False) self.test_config.set('verbose', True) @@ -269,15 +92,14 @@ class TestQuantumGatewayContext(CharmTestCase): self.test_config.set('vlan-ranges', 'physnet1:1000:2000 physnet2:2001:3000') # Provided by neutron-api relation - self.relation_ids.return_value = ['neutron-plugin-api:0'] - self.related_units.return_value = ['neutron-api/0'] - rdata = {'network-device-mtu': 9000} - self.relation_get.side_effect = lambda *args, **kwargs: rdata + _rids.return_value = ['neutron-plugin-api:0'] + _runits.return_value = ['neutron-api/0'] + rdata = {'network-device-mtu': 9000, 'l2-population': 'False'} + _rget.side_effect = lambda *args, **kwargs: rdata self.get_os_codename_install_source.return_value = 'folsom' _host_ip.return_value = '10.5.0.1' _secret.return_value = 'testsecret' ctxt = quantum_contexts.QuantumGatewayContext()() - self.relation_ids.assert_called_with('neutron-plugin-api') self.assertEquals(ctxt, { 'shared_secret': 'testsecret', 'local_ip': '10.5.0.1', @@ -414,30 +236,3 @@ class TestMisc(CharmTestCase): self.config.return_value = 'ovs' self.assertEquals(quantum_contexts.core_plugin(), quantum_contexts.NEUTRON_ML2_PLUGIN) - - def test_neutron_api_settings(self): - self.relation_ids.return_value = ['foo'] - self.related_units.return_value = ['bar'] - self.test_relation.set({'l2-population': True, - 'overlay-network-type': 'gre', }) - self.relation_get.side_effect = self.test_relation.get - self.assertEquals(quantum_contexts._neutron_api_settings(), - {'l2_population': True, - 'overlay_network_type': 'gre'}) - - def test_neutron_api_settings2(self): - self.relation_ids.return_value = ['foo'] - self.related_units.return_value = ['bar'] - self.test_relation.set({'l2-population': True, - 'overlay-network-type': 'gre', }) - self.relation_get.side_effect = self.test_relation.get - self.assertEquals(quantum_contexts._neutron_api_settings(), - {'l2_population': True, - 'overlay_network_type': 'gre'}) - - def test_neutron_api_settings_no_apiplugin(self): - self.config.return_value = 1500 - self.relation_ids.return_value = [] - self.assertEquals(quantum_contexts._neutron_api_settings(), - {'l2_population': False, - 'overlay_network_type': 'gre', }) diff --git a/unit_tests/test_quantum_utils.py b/unit_tests/test_quantum_utils.py index 60a87d03..1ceaa423 100644 --- a/unit_tests/test_quantum_utils.py +++ b/unit_tests/test_quantum_utils.py @@ -138,7 +138,7 @@ class TestQuantumUtils(CharmTestCase): self.get_os_codename_install_source.return_value = 'kilo' self.assertTrue('python-neutron-fwaas' in quantum_utils.get_packages()) - @patch('quantum_contexts.config') + @patch('charmhelpers.contrib.openstack.context.config') def test_configure_ovs_starts_service_if_required(self, mock_config): mock_config.side_effect = self.test_config.get self.config.return_value = 'ovs' @@ -151,7 +151,7 @@ class TestQuantumUtils(CharmTestCase): quantum_utils.configure_ovs() self.assertFalse(self.full_restart.called) - @patch('quantum_contexts.config') + @patch('charmhelpers.contrib.openstack.context.config') def test_configure_ovs_ovs_ext_port(self, mock_config): mock_config.side_effect = self.test_config.get self.config.side_effect = self.test_config.get @@ -167,7 +167,7 @@ class TestQuantumUtils(CharmTestCase): ]) self.add_bridge_port.assert_called_with('br-ex', 'eth0') - @patch('quantum_contexts.config') + @patch('charmhelpers.contrib.openstack.context.config') def test_configure_ovs_ovs_data_port(self, mock_config): mock_config.side_effect = self.test_config.get self.config.side_effect = self.test_config.get From 484416045d15aa26b15e78c77fe9095810f78308 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 24 Mar 2015 13:29:12 +0000 Subject: [PATCH 65/75] Sync charmhelpers --- charm-helpers-hooks.yaml | 3 +- .../charmhelpers/contrib/openstack/context.py | 28 +++++++++++-------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index 84cc6c76..e1644103 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -1,4 +1,5 @@ -branch: lp:charm-helpers +branch: lp:~gnuoy/charm-helpers/neutron-shuffle +#branch: lp:charm-helpers destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 7c52bd73..45e65790 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -1176,23 +1176,27 @@ class NeutronAPIContext(OSContextGenerator): self.neutron_defaults = { 'l2_population': { 'rel_key': 'l2-population', - 'default': False - }, - 'enable_dvr': { - 'rel_key': 'enable-dvr', - 'default': False - }, - 'enable_l3ha': { - 'rel_key': 'enable-l3ha', - 'default': False + 'default': False, }, 'overlay_network_type': { 'rel_key': 'overlay-network-type', - 'default': 'gre' + 'default': 'gre', + }, + 'neutron_security_groups': { + 'rel_key': 'neutron-security-groups', + 'default': False, }, 'network_device_mtu': { 'rel_key': 'network-device-mtu', - 'default': None + 'default': None, + }, + 'enable_dvr': { + 'rel_key': 'enable-dvr', + 'default': False, + }, + 'enable_l3ha': { + 'rel_key': 'enable-l3ha', + 'default': False, }, } ctxt = self.get_neutron_options({}) @@ -1251,7 +1255,7 @@ class DataPortContext(NeutronPortContext): if port in ports}) if resolved: return {bridge: normalized[port] for bridge, port in - portmap.iteritems() if port in normalized.keys()} + six.iteritems(portmap) if port in normalized.keys()} return None From 7868fdf9e96c2b556fc369a796f03cfcd99b30e7 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 24 Mar 2015 14:39:26 +0000 Subject: [PATCH 66/75] Only render network_device_mtu if the option is set --- templates/icehouse/neutron.conf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index 78814822..aa7ca65a 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -11,6 +11,8 @@ core_plugin = {{ core_plugin }} control_exchange = neutron notification_driver = neutron.openstack.common.notifier.list_notifier list_notifier_drivers = neutron.openstack.common.notifier.rabbit_notifier +{% if network_device_mtu -%} network_device_mtu = {{ network_device_mtu }} +{% endif -%} [agent] -root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf \ No newline at end of file +root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf From 33419fec1d149c41d22f80d96cef0452a16ddfb6 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 24 Mar 2015 17:20:26 +0000 Subject: [PATCH 67/75] Fixup nova conf for kilo --- templates/kilo/nova.conf | 9 +++++---- templates/parts/section-rabbitmq | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 templates/parts/section-rabbitmq diff --git a/templates/kilo/nova.conf b/templates/kilo/nova.conf index 47489f8b..14a6e68c 100644 --- a/templates/kilo/nova.conf +++ b/templates/kilo/nova.conf @@ -6,16 +6,12 @@ [DEFAULT] logdir=/var/log/nova state_path=/var/lib/nova -lock_path=/var/lock/nova root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf verbose= {{ verbose }} use_syslog = {{ use_syslog }} api_paste_config=/etc/nova/api-paste.ini enabled_apis=metadata multi_host=True -{% include "parts/database" %} -# Access to message bus -{% include "parts/rabbitmq" %} # Access to neutron API services network_api_class=nova.network.neutronv2.api.API [neutron] @@ -27,3 +23,8 @@ admin_password={{ service_password }} admin_auth_url={{ service_protocol }}://{{ keystone_host }}:{{ service_port }}/v2.0 service_metadata_proxy=True metadata_proxy_shared_secret={{ shared_secret }} + +{% include "parts/section-rabbitmq" %} + +[oslo_concurrency] +lock_path=/var/lock/nova diff --git a/templates/parts/section-rabbitmq b/templates/parts/section-rabbitmq new file mode 100644 index 00000000..df21178b --- /dev/null +++ b/templates/parts/section-rabbitmq @@ -0,0 +1,22 @@ +{% if rabbitmq_host or rabbitmq_hosts -%} +[oslo_messaging_rabbit] +rabbit_userid = {{ rabbitmq_user }} +rabbit_virtual_host = {{ rabbitmq_virtual_host }} +rabbit_password = {{ rabbitmq_password }} +{% if rabbitmq_hosts -%} +rabbit_hosts = {{ rabbitmq_hosts }} +{% if rabbitmq_ha_queues -%} +rabbit_ha_queues = True +rabbit_durable_queues = False +{% endif -%} +{% else -%} +rabbit_host = {{ rabbitmq_host }} +{% endif -%} +{% if rabbit_ssl_port -%} +rabbit_use_ssl = True +rabbit_port = {{ rabbit_ssl_port }} +{% if rabbit_ssl_ca -%} +kombu_ssl_ca_certs = {{ rabbit_ssl_ca }} +{% endif -%} +{% endif -%} +{% endif -%} \ No newline at end of file From d1efa609c160225be7a4c5daa0149530e89d3de2 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 24 Mar 2015 17:25:18 +0000 Subject: [PATCH 68/75] Add neutron config for kilo --- templates/kilo/fwaas_driver.ini | 1 + templates/kilo/lbaas_agent.ini | 5 +++++ templates/kilo/neutron.conf | 23 +++++++++++++++++++++++ templates/kilo/vpn_agent.ini | 1 + 4 files changed, 30 insertions(+) create mode 100644 templates/kilo/neutron.conf diff --git a/templates/kilo/fwaas_driver.ini b/templates/kilo/fwaas_driver.ini index 8ce9e542..b31a5008 100644 --- a/templates/kilo/fwaas_driver.ini +++ b/templates/kilo/fwaas_driver.ini @@ -1,3 +1,4 @@ +# kilo ############################################################################### # [ WARNING ] # Configuration file maintained by Juju. Local changes may be overwritten. diff --git a/templates/kilo/lbaas_agent.ini b/templates/kilo/lbaas_agent.ini index b37b7e1d..1c541b85 100644 --- a/templates/kilo/lbaas_agent.ini +++ b/templates/kilo/lbaas_agent.ini @@ -1,3 +1,8 @@ +# kilo +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +############################################################################### [DEFAULT] periodic_interval = 10 interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver diff --git a/templates/kilo/neutron.conf b/templates/kilo/neutron.conf new file mode 100644 index 00000000..b6ae7361 --- /dev/null +++ b/templates/kilo/neutron.conf @@ -0,0 +1,23 @@ +# kilo +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +############################################################################### +[DEFAULT] +verbose = {{ verbose }} +debug = {{ debug }} +core_plugin = {{ core_plugin }} +control_exchange = neutron +notification_driver = neutron.openstack.common.notifier.list_notifier +list_notifier_drivers = neutron.openstack.common.notifier.rabbit_notifier +{% if network_device_mtu -%} +network_device_mtu = {{ network_device_mtu }} +{% endif -%} + +[agent] +root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf + +{% include "parts/section-rabbitmq" %} + +[oslo_concurrency] +lock_path = /var/lock/neutron diff --git a/templates/kilo/vpn_agent.ini b/templates/kilo/vpn_agent.ini index 90874fd0..95c41926 100644 --- a/templates/kilo/vpn_agent.ini +++ b/templates/kilo/vpn_agent.ini @@ -1,3 +1,4 @@ +# kilo ############################################################################### # [ WARNING ] # Configuration file maintained by Juju. Local changes may be overwritten. From 877b10431a4aa763c1b409c76c12bd70add31dd8 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 25 Mar 2015 07:55:25 +0000 Subject: [PATCH 69/75] Fixes for contexts moving into charmhelpers --- charm-helpers-hooks.yaml | 3 +- .../charmhelpers/contrib/openstack/context.py | 145 +++++++++++++++++- hooks/quantum_contexts.py | 125 +-------------- hooks/quantum_utils.py | 14 +- unit_tests/test_quantum_utils.py | 8 +- 5 files changed, 161 insertions(+), 134 deletions(-) diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index 84cc6c76..0b5106d1 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -1,4 +1,5 @@ -branch: lp:charm-helpers +#branch: lp:charm-helpers +branch: lp:~gnuoy/charm-helpers/neutron-shuffle destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 90ac6d69..45e65790 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -47,6 +47,7 @@ from charmhelpers.core.hookenv import ( ) from charmhelpers.core.sysctl import create as sysctl_create +from charmhelpers.core.strutils import bool_from_string from charmhelpers.core.host import ( list_nics, @@ -67,6 +68,7 @@ from charmhelpers.contrib.hahelpers.apache import ( ) from charmhelpers.contrib.openstack.neutron import ( neutron_plugin_attribute, + parse_data_port_mappings, ) from charmhelpers.contrib.openstack.ip import ( resolve_address, @@ -82,7 +84,6 @@ from charmhelpers.contrib.network.ip import ( is_bridge_member, ) from charmhelpers.contrib.openstack.utils import get_host_ip - CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' ADDRESS_TYPES = ['admin', 'internal', 'public'] @@ -1162,3 +1163,145 @@ class SysctlContext(OSContextGenerator): sysctl_create(sysctl_dict, '/etc/sysctl.d/50-{0}.conf'.format(charm_name())) return {'sysctl': sysctl_dict} + + +class NeutronAPIContext(OSContextGenerator): + ''' + Inspects current neutron-plugin-api relation for neutron settings. Return + defaults if it is not present. + ''' + interfaces = ['neutron-plugin-api'] + + def __call__(self): + self.neutron_defaults = { + 'l2_population': { + 'rel_key': 'l2-population', + 'default': False, + }, + 'overlay_network_type': { + 'rel_key': 'overlay-network-type', + 'default': 'gre', + }, + 'neutron_security_groups': { + 'rel_key': 'neutron-security-groups', + 'default': False, + }, + 'network_device_mtu': { + 'rel_key': 'network-device-mtu', + 'default': None, + }, + 'enable_dvr': { + 'rel_key': 'enable-dvr', + 'default': False, + }, + 'enable_l3ha': { + 'rel_key': 'enable-l3ha', + 'default': False, + }, + } + ctxt = self.get_neutron_options({}) + for rid in relation_ids('neutron-plugin-api'): + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + if 'l2-population' in rdata: + ctxt.update(self.get_neutron_options(rdata)) + + return ctxt + + def get_neutron_options(self, rdata): + settings = {} + for nkey in self.neutron_defaults.keys(): + defv = self.neutron_defaults[nkey]['default'] + rkey = self.neutron_defaults[nkey]['rel_key'] + if rkey in rdata.keys(): + if type(defv) is bool: + settings[nkey] = bool_from_string(rdata[rkey]) + else: + settings[nkey] = rdata[rkey] + else: + settings[nkey] = defv + return settings + + +class ExternalPortContext(NeutronPortContext): + + def __call__(self): + ctxt = {} + ports = config('ext-port') + if ports: + ports = [p.strip() for p in ports.split()] + ports = self.resolve_ports(ports) + if ports: + ctxt = {"ext_port": ports[0]} + napi_settings = NeutronAPIContext()() + mtu = napi_settings.get('network_device_mtu') + if mtu: + ctxt['ext_port_mtu'] = mtu + + return ctxt + + +class DataPortContext(NeutronPortContext): + + def __call__(self): + ports = config('data-port') + if ports: + portmap = parse_data_port_mappings(ports) + ports = portmap.values() + resolved = self.resolve_ports(ports) + normalized = {get_nic_hwaddr(port): port for port in resolved + if port not in ports} + normalized.update({port: port for port in resolved + if port in ports}) + if resolved: + return {bridge: normalized[port] for bridge, port in + six.iteritems(portmap) if port in normalized.keys()} + + return None + + +class PhyNICMTUContext(DataPortContext): + + def __call__(self): + ctxt = {} + mappings = super(PhyNICMTUContext, self).__call__() + if mappings and mappings.values(): + ports = mappings.values() + napi_settings = NeutronAPIContext()() + mtu = napi_settings.get('network_device_mtu') + if mtu: + ctxt["devs"] = '\\n'.join(ports) + ctxt['mtu'] = mtu + + return ctxt + + +class NetworkServiceContext(OSContextGenerator): + + def __init__(self, rel_name='quantum-network-service'): + self.rel_name = rel_name + self.interfaces = [rel_name] + + def __call__(self): + for rid in relation_ids(self.rel_name): + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + ctxt = { + 'keystone_host': rdata.get('keystone_host'), + 'service_port': rdata.get('service_port'), + 'auth_port': rdata.get('auth_port'), + 'service_tenant': rdata.get('service_tenant'), + 'service_username': rdata.get('service_username'), + 'service_password': rdata.get('service_password'), + 'quantum_host': rdata.get('quantum_host'), + 'quantum_port': rdata.get('quantum_port'), + 'quantum_url': rdata.get('quantum_url'), + 'region': rdata.get('region'), + 'service_protocol': + rdata.get('service_protocol') or 'http', + 'auth_protocol': + rdata.get('auth_protocol') or 'http', + } + if context_complete(ctxt): + return ctxt + return {} diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index 853f178b..b6b1ef19 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -17,6 +17,7 @@ from charmhelpers.contrib.openstack.context import ( OSContextGenerator, context_complete, NeutronPortContext, + NeutronAPIContext, ) from charmhelpers.contrib.openstack.utils import ( get_os_codename_install_source @@ -106,75 +107,10 @@ def core_plugin(): return CORE_PLUGIN[networking_name()][plugin] -def neutron_api_settings(): - ''' - Inspects current neutron-plugin-api relation for neutron settings. Return - defaults if it is not present - ''' - NEUTRON_DEFAULTS = { - 'l2_population': {'rel_key': 'l2-population', 'default': False}, - 'enable_dvr': {'rel_key': 'enable-dvr', 'default': False}, - 'enable_l3ha': {'rel_key': 'enable-l3ha', 'default': False}, - 'overlay_network_type': {'rel_key': 'overlay-network-type', 'default': 'gre'}, - 'network_device_mtu': {'rel_key': 'network-device-mtu', 'default': None}, - } - - def get_neutron_options(rdata): - settings = {} - for nkey in NEUTRON_DEFAULTS.keys(): - defv = NEUTRON_DEFAULTS[nkey]['default'] - rkey = NEUTRON_DEFAULTS[nkey]['rel_key'] - if rkey in rdata.keys(): - if type(defv) is bool: - settings[nkey] = bool_from_string(rdata[rkey]) - else: - settings[nkey] = rdata[rkey] - elif defv is not None: - settings[nkey] = defv - return settings - - neutron_settings = get_neutron_options({}) - for rid in relation_ids('neutron-plugin-api'): - for unit in related_units(rid): - rdata = relation_get(rid=rid, unit=unit) - if 'l2-population' in relation_get(rid=rid, unit=unit): - neutron_settings.update(get_neutron_options(rdata)) - - return neutron_settings - - -class NetworkServiceContext(OSContextGenerator): - interfaces = ['quantum-network-service'] - - def __call__(self): - for rid in relation_ids('quantum-network-service'): - for unit in related_units(rid): - rdata = relation_get(rid=rid, unit=unit) - ctxt = { - 'keystone_host': rdata.get('keystone_host'), - 'service_port': rdata.get('service_port'), - 'auth_port': rdata.get('auth_port'), - 'service_tenant': rdata.get('service_tenant'), - 'service_username': rdata.get('service_username'), - 'service_password': rdata.get('service_password'), - 'quantum_host': rdata.get('quantum_host'), - 'quantum_port': rdata.get('quantum_port'), - 'quantum_url': rdata.get('quantum_url'), - 'region': rdata.get('region'), - 'service_protocol': - rdata.get('service_protocol') or 'http', - 'auth_protocol': - rdata.get('auth_protocol') or 'http', - } - if context_complete(ctxt): - return ctxt - return {} - - class L3AgentContext(OSContextGenerator): def __call__(self): - api_settings = neutron_api_settings() + api_settings = NeutronAPIContext()() ctxt = {} if config('run-internal-router') == 'leader': ctxt['handle_internal_only_router'] = eligible_leader(None) @@ -196,63 +132,10 @@ class L3AgentContext(OSContextGenerator): return ctxt -class ExternalPortContext(NeutronPortContext): - - def __call__(self): - ctxt = {} - ports = config('ext-port') - if ports: - ports = [p.strip() for p in ports.split()] - ports = self.resolve_ports(ports) - if ports: - ctxt = {"ext_port": ports[0]} - napi_settings = neutron_api_settings() - mtu = napi_settings.get('network_device_mtu') - if mtu: - ctxt['ext_port_mtu'] = mtu - - return ctxt - - -class DataPortContext(NeutronPortContext): - - def __call__(self): - ports = config('data-port') - if ports: - portmap = parse_data_port_mappings(ports) - ports = portmap.values() - resolved = self.resolve_ports(ports) - normalized = {get_nic_hwaddr(port): port for port in resolved - if port not in ports} - normalized.update({port: port for port in resolved - if port in ports}) - if resolved: - return {bridge: normalized[port] for bridge, port in - portmap.iteritems() if port in normalized.keys()} - - return None - - -class PhyNICMTUContext(DataPortContext): - - def __call__(self): - ctxt = {} - mappings = super(PhyNICMTUContext, self).__call__() - if mappings and mappings.values(): - ports = mappings.values() - napi_settings = neutron_api_settings() - mtu = napi_settings.get('network_device_mtu') - if mtu: - ctxt["devs"] = '\\n'.join(ports) - ctxt['mtu'] = mtu - - return ctxt - - class QuantumGatewayContext(OSContextGenerator): def __call__(self): - api_settings = neutron_api_settings() + api_settings = NeutronAPIContext()() ctxt = { 'shared_secret': get_shared_secret(), 'local_ip': @@ -281,7 +164,7 @@ class QuantumGatewayContext(OSContextGenerator): ctxt['network_providers'] = ' '.join(providers) ctxt['vlan_ranges'] = vlan_ranges - net_dev_mtu = neutron_api_settings().get('network_device_mtu') + net_dev_mtu = api_settings['network_device_mtu'] if net_dev_mtu: ctxt['network_device_mtu'] = net_dev_mtu ctxt['veth_mtu'] = net_dev_mtu diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 6583b860..04beae1b 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -41,7 +41,12 @@ from charmhelpers.contrib.openstack.neutron import ( import charmhelpers.contrib.openstack.context as context from charmhelpers.contrib.openstack.context import ( - SyslogContext + SyslogContext, + NeutronAPIContext, + ExternalPortContext, + PhyNICMTUContext, + DataPortContext, + NetworkServiceContext, ) import charmhelpers.contrib.openstack.templating as templating from charmhelpers.contrib.openstack.neutron import headers_package @@ -50,13 +55,8 @@ from quantum_contexts import ( NEUTRON, QUANTUM, networking_name, QuantumGatewayContext, - NetworkServiceContext, L3AgentContext, - ExternalPortContext, - PhyNICMTUContext, - DataPortContext, remap_plugin, - neutron_api_settings, ) from charmhelpers.contrib.openstack.neutron import ( parse_bridge_mappings, @@ -221,7 +221,7 @@ def get_common_package(): def use_l3ha(): - return neutron_api_settings()['enable_l3ha'] + return NeutronAPIContext()()['enable_l3ha'] EXT_PORT_CONF = '/etc/init/ext-port.conf' PHY_NIC_MTU_CONF = '/etc/init/os-charm-phy-nic-mtu.conf' diff --git a/unit_tests/test_quantum_utils.py b/unit_tests/test_quantum_utils.py index 6be942a8..7fa472c6 100644 --- a/unit_tests/test_quantum_utils.py +++ b/unit_tests/test_quantum_utils.py @@ -46,7 +46,7 @@ TO_PATCH = [ 'lsb_release', 'mkdir', 'copy2', - 'neutron_api_settings', + 'NeutronAPIContext', ] @@ -144,7 +144,7 @@ class TestQuantumUtils(CharmTestCase): self.get_os_codename_install_source.return_value = 'juno' self.assertTrue('keepalived' in quantum_utils.get_packages()) - @patch('quantum_contexts.config') + @patch('charmhelpers.contrib.openstack.context.config') def test_configure_ovs_starts_service_if_required(self, mock_config): mock_config.side_effect = self.test_config.get self.config.return_value = 'ovs' @@ -157,7 +157,7 @@ class TestQuantumUtils(CharmTestCase): quantum_utils.configure_ovs() self.assertFalse(self.full_restart.called) - @patch('quantum_contexts.config') + @patch('charmhelpers.contrib.openstack.context.config') def test_configure_ovs_ovs_ext_port(self, mock_config): mock_config.side_effect = self.test_config.get self.config.side_effect = self.test_config.get @@ -173,7 +173,7 @@ class TestQuantumUtils(CharmTestCase): ]) self.add_bridge_port.assert_called_with('br-ex', 'eth0') - @patch('quantum_contexts.config') + @patch('charmhelpers.contrib.openstack.context.config') def test_configure_ovs_ovs_data_port(self, mock_config): mock_config.side_effect = self.test_config.get self.config.side_effect = self.test_config.get From 0cb03bd5313a29f556c7053d5251995f56b3813f Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 25 Mar 2015 07:58:50 +0000 Subject: [PATCH 70/75] Fix lint --- charm-helpers-hooks.yaml | 3 +-- hooks/quantum_contexts.py | 11 ----------- unit_tests/test_quantum_contexts.py | 2 +- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index 0b5106d1..84cc6c76 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -1,5 +1,4 @@ -#branch: lp:charm-helpers -branch: lp:~gnuoy/charm-helpers/neutron-shuffle +branch: lp:charm-helpers destination: hooks/charmhelpers include: - core diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index b6b1ef19..82f112e7 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -4,9 +4,6 @@ import uuid import socket from charmhelpers.core.hookenv import ( config, - relation_ids, - related_units, - relation_get, unit_get, cached ) @@ -15,8 +12,6 @@ from charmhelpers.fetch import ( ) from charmhelpers.contrib.openstack.context import ( OSContextGenerator, - context_complete, - NeutronPortContext, NeutronAPIContext, ) from charmhelpers.contrib.openstack.utils import ( @@ -29,14 +24,8 @@ from charmhelpers.contrib.network.ip import ( get_address_in_network, ) from charmhelpers.contrib.openstack.neutron import ( - parse_data_port_mappings, parse_vlan_range_mappings, ) -from charmhelpers.core.host import ( - get_nic_hwaddr, -) -from charmhelpers.core.strutils import bool_from_string -import copy DB_USER = "quantum" QUANTUM_DB = "quantum" diff --git a/unit_tests/test_quantum_contexts.py b/unit_tests/test_quantum_contexts.py index d92deb68..9ccc7d06 100644 --- a/unit_tests/test_quantum_contexts.py +++ b/unit_tests/test_quantum_contexts.py @@ -267,7 +267,7 @@ class TestQuantumGatewayContext(CharmTestCase): super(TestQuantumGatewayContext, self).setUp(quantum_contexts, TO_PATCH) self.config.side_effect = self.test_config.get - self.maxDiff = None + self.maxDiff = None @patch.object(quantum_contexts, 'get_shared_secret') @patch.object(quantum_contexts, 'get_host_ip') From 66c321149f0322fef91f52132b31d9b16ed9c355 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 25 Mar 2015 09:18:35 +0000 Subject: [PATCH 71/75] resync helpers --- .../templates/section-keystone-authtoken | 9 ++++++++ .../openstack/templates/section-rabbitmq-oslo | 22 +++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken create mode 100644 hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken new file mode 100644 index 00000000..2a37edd5 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken @@ -0,0 +1,9 @@ +{% if auth_host -%} +[keystone_authtoken] +identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/{{ auth_admin_prefix }} +auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }} +admin_tenant_name = {{ admin_tenant_name }} +admin_user = {{ admin_user }} +admin_password = {{ admin_password }} +signing_dir = {{ signing_dir }} +{% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo b/hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo new file mode 100644 index 00000000..b444c9c9 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo @@ -0,0 +1,22 @@ +{% if rabbitmq_host or rabbitmq_hosts -%} +[oslo_messaging_rabbit] +rabbit_userid = {{ rabbitmq_user }} +rabbit_virtual_host = {{ rabbitmq_virtual_host }} +rabbit_password = {{ rabbitmq_password }} +{% if rabbitmq_hosts -%} +rabbit_hosts = {{ rabbitmq_hosts }} +{% if rabbitmq_ha_queues -%} +rabbit_ha_queues = True +rabbit_durable_queues = False +{% endif -%} +{% else -%} +rabbit_host = {{ rabbitmq_host }} +{% endif -%} +{% if rabbit_ssl_port -%} +rabbit_use_ssl = True +rabbit_port = {{ rabbit_ssl_port }} +{% if rabbit_ssl_ca -%} +kombu_ssl_ca_certs = {{ rabbit_ssl_ca }} +{% endif -%} +{% endif -%} +{% endif -%} From 415d97a858cf663828b4cc0edbd1092757f678c6 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 25 Mar 2015 09:22:50 +0000 Subject: [PATCH 72/75] Switch to using charm-helper templates for rabbitmq --- templates/kilo/neutron.conf | 2 +- templates/kilo/nova.conf | 2 +- templates/parts/section-rabbitmq | 22 ---------------------- 3 files changed, 2 insertions(+), 24 deletions(-) delete mode 100644 templates/parts/section-rabbitmq diff --git a/templates/kilo/neutron.conf b/templates/kilo/neutron.conf index b6ae7361..e541f337 100644 --- a/templates/kilo/neutron.conf +++ b/templates/kilo/neutron.conf @@ -17,7 +17,7 @@ network_device_mtu = {{ network_device_mtu }} [agent] root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf -{% include "parts/section-rabbitmq" %} +{% include "section-rabbitmq-oslo" %} [oslo_concurrency] lock_path = /var/lock/neutron diff --git a/templates/kilo/nova.conf b/templates/kilo/nova.conf index 14a6e68c..11316956 100644 --- a/templates/kilo/nova.conf +++ b/templates/kilo/nova.conf @@ -24,7 +24,7 @@ admin_auth_url={{ service_protocol }}://{{ keystone_host }}:{{ service_port }}/v service_metadata_proxy=True metadata_proxy_shared_secret={{ shared_secret }} -{% include "parts/section-rabbitmq" %} +{% include "section-rabbitmq-oslo" %} [oslo_concurrency] lock_path=/var/lock/nova diff --git a/templates/parts/section-rabbitmq b/templates/parts/section-rabbitmq deleted file mode 100644 index df21178b..00000000 --- a/templates/parts/section-rabbitmq +++ /dev/null @@ -1,22 +0,0 @@ -{% if rabbitmq_host or rabbitmq_hosts -%} -[oslo_messaging_rabbit] -rabbit_userid = {{ rabbitmq_user }} -rabbit_virtual_host = {{ rabbitmq_virtual_host }} -rabbit_password = {{ rabbitmq_password }} -{% if rabbitmq_hosts -%} -rabbit_hosts = {{ rabbitmq_hosts }} -{% if rabbitmq_ha_queues -%} -rabbit_ha_queues = True -rabbit_durable_queues = False -{% endif -%} -{% else -%} -rabbit_host = {{ rabbitmq_host }} -{% endif -%} -{% if rabbit_ssl_port -%} -rabbit_use_ssl = True -rabbit_port = {{ rabbit_ssl_port }} -{% if rabbit_ssl_ca -%} -kombu_ssl_ca_certs = {{ rabbit_ssl_ca }} -{% endif -%} -{% endif -%} -{% endif -%} \ No newline at end of file From 8144f2b48f10c8830fb264cf922fb203d7813983 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 25 Mar 2015 10:12:04 +0000 Subject: [PATCH 73/75] Fix charm-helpers-hooks.yaml --- charm-helpers-hooks.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index e1644103..84cc6c76 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -1,5 +1,4 @@ -branch: lp:~gnuoy/charm-helpers/neutron-shuffle -#branch: lp:charm-helpers +branch: lp:charm-helpers destination: hooks/charmhelpers include: - core From 5bfe69f4d28f38d193c4d812300895a2f4229010 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 26 Mar 2015 10:49:09 +0000 Subject: [PATCH 74/75] Fix juno templates --- templates/juno/ml2_conf.ini | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/templates/juno/ml2_conf.ini b/templates/juno/ml2_conf.ini index 5354b4c2..4c1dfa9d 100644 --- a/templates/juno/ml2_conf.ini +++ b/templates/juno/ml2_conf.ini @@ -14,20 +14,23 @@ tunnel_id_ranges = 1:1000 vni_ranges = 1001:2000 [ml2_type_vlan] -network_vlan_ranges = physnet1:1000:2000 +network_vlan_ranges = {{ vlan_ranges }} [ml2_type_flat] -flat_networks = physnet1 +flat_networks = {{ network_providers }} [ovs] enable_tunneling = True local_ip = {{ local_ip }} -bridge_mappings = physnet1:br-data +bridge_mappings = {{ bridge_mappings }} [agent] tunnel_types = {{ overlay_network_type }} l2_population = {{ l2_population }} enable_distributed_routing = {{ enable_dvr }} +{% if veth_mtu -%} +veth_mtu = {{ veth_mtu }} +{% endif %} [securitygroup] firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver From 1cd6743aeec64ee561cdc5471fd9bb60331e095c Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 31 Mar 2015 08:48:28 +0100 Subject: [PATCH 75/75] Fix unit tests --- unit_tests/test_quantum_contexts.py | 31 +++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/unit_tests/test_quantum_contexts.py b/unit_tests/test_quantum_contexts.py index 2d37dae4..f5513c74 100644 --- a/unit_tests/test_quantum_contexts.py +++ b/unit_tests/test_quantum_contexts.py @@ -38,6 +38,15 @@ def patch_open(): yield mock_open, mock_file +class DummyNeutronAPIContext(): + + def __init__(self, return_value): + self.return_value = return_value + + def __call__(self): + return self.return_value + + class TestL3AgentContext(CharmTestCase): def setUp(self): @@ -45,7 +54,10 @@ class TestL3AgentContext(CharmTestCase): TO_PATCH) self.config.side_effect = self.test_config.get - def test_no_ext_netid(self): + @patch('quantum_contexts.NeutronAPIContext') + def test_no_ext_netid(self, _NeutronAPIContext): + _NeutronAPIContext.return_value = \ + DummyNeutronAPIContext(return_value={'enable_dvr': False}) self.test_config.set('run-internal-router', 'none') self.test_config.set('external-network-id', '') self.eligible_leader.return_value = False @@ -54,7 +66,10 @@ class TestL3AgentContext(CharmTestCase): 'handle_internal_only_router': False, 'plugin': 'ovs'}) - def test_hior_leader(self): + @patch('quantum_contexts.NeutronAPIContext') + def test_hior_leader(self, _NeutronAPIContext): + _NeutronAPIContext.return_value = \ + DummyNeutronAPIContext(return_value={'enable_dvr': False}) self.test_config.set('run-internal-router', 'leader') self.test_config.set('external-network-id', 'netid') self.eligible_leader.return_value = True @@ -64,7 +79,10 @@ class TestL3AgentContext(CharmTestCase): 'ext_net_id': 'netid', 'plugin': 'ovs'}) - def test_hior_all(self): + @patch('quantum_contexts.NeutronAPIContext') + def test_hior_all(self, _NeutronAPIContext): + _NeutronAPIContext.return_value = \ + DummyNeutronAPIContext(return_value={'enable_dvr': False}) self.test_config.set('run-internal-router', 'all') self.test_config.set('external-network-id', 'netid') self.eligible_leader.return_value = True @@ -74,9 +92,10 @@ class TestL3AgentContext(CharmTestCase): 'ext_net_id': 'netid', 'plugin': 'ovs'}) - @patch.object(quantum_contexts, 'neutron_api_settings') - def test_dvr(self, _napi_settings): - _napi_settings.return_value = {'enable_dvr': True} + @patch('quantum_contexts.NeutronAPIContext') + def test_dvr(self, _NeutronAPIContext): + _NeutronAPIContext.return_value = \ + DummyNeutronAPIContext(return_value={'enable_dvr': True}) self.assertEquals(quantum_contexts.L3AgentContext()()['agent_mode'], 'dvr_snat')