From 25318707e042996a431f94bf602651ca92eccce6 Mon Sep 17 00:00:00 2001 From: David Ames Date: Thu, 27 Apr 2017 11:25:36 -0700 Subject: [PATCH] Enable Zesty-Ocata Amulet Tests - Turn on Zesty-Ocata Amulet test definitions. - Standardize test-requirements.txt - Sync charm helpers for various fixes Change-Id: Ifd77b31379fa55daa90a0e4d86ee3638ad3d74d7 --- hooks/charmhelpers/contrib/network/ip.py | 49 ++++++--- .../contrib/openstack/amulet/utils.py | 2 +- .../charmhelpers/contrib/openstack/context.py | 103 ++++++++++++------ hooks/charmhelpers/core/host.py | 4 +- test-requirements.txt | 18 +-- .../contrib/openstack/amulet/utils.py | 2 +- tests/charmhelpers/core/host.py | 4 +- tests/gate-basic-zesty-ocata | 0 tox.ini | 5 + 9 files changed, 124 insertions(+), 63 deletions(-) mode change 100644 => 100755 tests/gate-basic-zesty-ocata diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 14c93aad..fc3f5e3e 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -239,6 +239,16 @@ def format_ipv6_addr(address): return None +def is_ipv6_disabled(): + try: + result = subprocess.check_output( + ['sysctl', 'net.ipv6.conf.all.disable_ipv6'], + stderr=subprocess.STDOUT) + return "net.ipv6.conf.all.disable_ipv6 = 1" in result + except subprocess.CalledProcessError: + return True + + def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, fatal=True, exc_list=None): """Return the assigned IP address for a given interface, if any. @@ -544,31 +554,38 @@ def assert_charm_supports_ipv6(): "versions less than Trusty 14.04") -def get_relation_ip(interface, config_override=None): - """Return this unit's IP for the given relation. +def get_relation_ip(interface, cidr_network=None): + """Return this unit's IP for the given interface. Allow for an arbitrary interface to use with network-get to select an IP. - Handle all address selection options including configuration parameter - override and IPv6. + Handle all address selection options including passed cidr network and + IPv6. - Usage: get_relation_ip('amqp', config_override='access-network') + Usage: get_relation_ip('amqp', cidr_network='10.0.0.0/8') @param interface: string name of the relation. - @param config_override: string name of the config option for network - override. Supports legacy network override configuration parameters. + @param cidr_network: string CIDR Network to select an address from. @raises Exception if prefer-ipv6 is configured but IPv6 unsupported. @returns IPv6 or IPv4 address """ + # Select the interface address first + # For possible use as a fallback bellow with get_address_in_network + try: + # Get the interface specific IP + address = network_get_primary_address(interface) + except NotImplementedError: + # If network-get is not available + address = get_host_ip(unit_get('private-address')) - fallback = get_host_ip(unit_get('private-address')) if config('prefer-ipv6'): + # Currently IPv6 has priority, eventually we want IPv6 to just be + # another network space. assert_charm_supports_ipv6() return get_ipv6_addr()[0] - elif config_override and config(config_override): - return get_address_in_network(config(config_override), - fallback) - else: - try: - return network_get_primary_address(interface) - except NotImplementedError: - return fallback + elif cidr_network: + # If a specific CIDR network is passed get the address from that + # network. + return get_address_in_network(cidr_network, address) + + # Return the interface address + return address diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index 346e6fea..bcef4cd0 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -547,7 +547,7 @@ class OpenStackAmuletUtils(AmuletUtils): """Create the specified instance.""" self.log.debug('Creating instance ' '({}|{}|{})'.format(instance_name, image_name, flavor)) - image = nova.images.find(name=image_name) + image = nova.glance.find_image(image_name) flavor = nova.flavors.find(name=flavor) instance = nova.servers.create(name=instance_name, image=image, flavor=flavor) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 3e055422..2adf2cb8 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -89,6 +89,7 @@ from charmhelpers.contrib.network.ip import ( format_ipv6_addr, is_address_in_network, is_bridge_member, + is_ipv6_disabled, ) from charmhelpers.contrib.openstack.utils import ( config_flags_parser, @@ -110,6 +111,7 @@ except ImportError: CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' ADDRESS_TYPES = ['admin', 'internal', 'public'] +HAPROXY_RUN_DIR = '/var/run/haproxy/' def ensure_packages(packages): @@ -535,6 +537,8 @@ class HAProxyContext(OSContextGenerator): """Provides half a context for the haproxy template, which describes all peers to be included in the cluster. Each charm needs to include its own context generator that describes the port mapping. + + :side effect: mkdir is called on HAPROXY_RUN_DIR """ interfaces = ['cluster'] @@ -542,6 +546,8 @@ class HAProxyContext(OSContextGenerator): self.singlenode_mode = singlenode_mode def __call__(self): + if not os.path.isdir(HAPROXY_RUN_DIR): + mkdir(path=HAPROXY_RUN_DIR) if not relation_ids('cluster') and not self.singlenode_mode: return {} @@ -1226,31 +1232,50 @@ MAX_DEFAULT_WORKERS = 4 DEFAULT_MULTIPLIER = 2 +def _calculate_workers(): + ''' + Determine the number of worker processes based on the CPU + count of the unit containing the application. + + Workers will be limited to MAX_DEFAULT_WORKERS in + container environments where no worker-multipler configuration + option been set. + + @returns int: number of worker processes to use + ''' + multiplier = config('worker-multiplier') or DEFAULT_MULTIPLIER + count = int(_num_cpus() * multiplier) + if multiplier > 0 and count == 0: + count = 1 + + if config('worker-multiplier') is None and is_container(): + # NOTE(jamespage): Limit unconfigured worker-multiplier + # to MAX_DEFAULT_WORKERS to avoid insane + # worker configuration in LXD containers + # on large servers + # Reference: https://pad.lv/1665270 + count = min(count, MAX_DEFAULT_WORKERS) + + return count + + +def _num_cpus(): + ''' + Compatibility wrapper for calculating the number of CPU's + a unit has. + + @returns: int: number of CPU cores detected + ''' + try: + return psutil.cpu_count() + except AttributeError: + return psutil.NUM_CPUS + + class WorkerConfigContext(OSContextGenerator): - @property - def num_cpus(self): - # NOTE: use cpu_count if present (16.04 support) - if hasattr(psutil, 'cpu_count'): - return psutil.cpu_count() - else: - return psutil.NUM_CPUS - def __call__(self): - multiplier = config('worker-multiplier') or DEFAULT_MULTIPLIER - count = int(self.num_cpus * multiplier) - if multiplier > 0 and count == 0: - count = 1 - - if config('worker-multiplier') is None and is_container(): - # NOTE(jamespage): Limit unconfigured worker-multiplier - # to MAX_DEFAULT_WORKERS to avoid insane - # worker configuration in LXD containers - # on large servers - # Reference: https://pad.lv/1665270 - count = min(count, MAX_DEFAULT_WORKERS) - - ctxt = {"workers": count} + ctxt = {"workers": _calculate_workers()} return ctxt @@ -1258,7 +1283,7 @@ class WSGIWorkerConfigContext(WorkerConfigContext): def __init__(self, name=None, script=None, admin_script=None, public_script=None, process_weight=1.00, - admin_process_weight=0.75, public_process_weight=0.25): + admin_process_weight=0.25, public_process_weight=0.75): self.service_name = name self.user = name self.group = name @@ -1270,8 +1295,7 @@ class WSGIWorkerConfigContext(WorkerConfigContext): self.public_process_weight = public_process_weight def __call__(self): - multiplier = config('worker-multiplier') or 1 - total_processes = self.num_cpus * multiplier + total_processes = _calculate_workers() ctxt = { "service_name": self.service_name, "user": self.user, @@ -1602,7 +1626,7 @@ class MemcacheContext(OSContextGenerator): """Memcache context This context provides options for configuring a local memcache client and - server + server for both IPv4 and IPv6 """ def __init__(self, package=None): @@ -1620,13 +1644,24 @@ class MemcacheContext(OSContextGenerator): # Trusty version of memcached does not support ::1 as a listen # address so use host file entry instead release = lsb_release()['DISTRIB_CODENAME'].lower() - if CompareHostReleases(release) > 'trusty': - ctxt['memcache_server'] = '::1' + if is_ipv6_disabled(): + if CompareHostReleases(release) > 'trusty': + ctxt['memcache_server'] = '127.0.0.1' + else: + ctxt['memcache_server'] = 'localhost' + ctxt['memcache_server_formatted'] = '127.0.0.1' + ctxt['memcache_port'] = '11211' + ctxt['memcache_url'] = '{}:{}'.format( + ctxt['memcache_server_formatted'], + ctxt['memcache_port']) else: - ctxt['memcache_server'] = 'ip6-localhost' - ctxt['memcache_server_formatted'] = '[::1]' - ctxt['memcache_port'] = '11211' - ctxt['memcache_url'] = 'inet6:{}:{}'.format( - ctxt['memcache_server_formatted'], - ctxt['memcache_port']) + if CompareHostReleases(release) > 'trusty': + ctxt['memcache_server'] = '::1' + else: + ctxt['memcache_server'] = 'ip6-localhost' + ctxt['memcache_server_formatted'] = '[::1]' + ctxt['memcache_port'] = '11211' + ctxt['memcache_url'] = 'inet6:{}:{}'.format( + ctxt['memcache_server_formatted'], + ctxt['memcache_port']) return ctxt diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 0ee5cb9f..88e80a49 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -191,7 +191,7 @@ def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d", upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) sysv_file = os.path.join(initd_dir, service_name) if init_is_systemd(): - service('disable', service_name) + service('mask', service_name) elif os.path.exists(upstart_file): override_path = os.path.join( init_dir, '{}.override'.format(service_name)) @@ -224,7 +224,7 @@ def service_resume(service_name, init_dir="/etc/init", upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) sysv_file = os.path.join(initd_dir, service_name) if init_is_systemd(): - service('enable', service_name) + service('unmask', service_name) elif os.path.exists(upstart_file): override_path = os.path.join( init_dir, '{}.override'.format(service_name)) diff --git a/test-requirements.txt b/test-requirements.txt index 06972943..9edd4bbf 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,13 +11,17 @@ requests==2.6.0 # Liberty client lower constraints amulet>=1.14.3,<2.0 bundletester>=0.6.1,<1.0 -python-ceilometerclient>=1.5.0,<2.0 -python-cinderclient>=1.4.0,<2.0 -python-glanceclient>=1.1.0,<2.0 -python-heatclient>=0.8.0,<1.0 -python-novaclient>=2.30.1,<3.0 -python-openstackclient>=1.7.0,<2.0 -python-swiftclient>=2.6.0,<3.0 +python-ceilometerclient>=1.5.0 +python-cinderclient>=1.4.0 +python-glanceclient>=1.1.0 +python-heatclient>=0.8.0 +python-keystoneclient>=1.7.1 +python-neutronclient>=3.1.0 +python-novaclient>=2.30.1 +python-openstackclient>=1.7.0 +python-swiftclient>=2.6.0 pika>=0.10.0,<1.0 distro-info # END: Amulet OpenStack Charm Helper Requirements +# NOTE: workaround for 14.04 pip/tox +pytz diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index 346e6fea..bcef4cd0 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -547,7 +547,7 @@ class OpenStackAmuletUtils(AmuletUtils): """Create the specified instance.""" self.log.debug('Creating instance ' '({}|{}|{})'.format(instance_name, image_name, flavor)) - image = nova.images.find(name=image_name) + image = nova.glance.find_image(image_name) flavor = nova.flavors.find(name=flavor) instance = nova.servers.create(name=instance_name, image=image, flavor=flavor) diff --git a/tests/charmhelpers/core/host.py b/tests/charmhelpers/core/host.py index 0ee5cb9f..88e80a49 100644 --- a/tests/charmhelpers/core/host.py +++ b/tests/charmhelpers/core/host.py @@ -191,7 +191,7 @@ def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d", upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) sysv_file = os.path.join(initd_dir, service_name) if init_is_systemd(): - service('disable', service_name) + service('mask', service_name) elif os.path.exists(upstart_file): override_path = os.path.join( init_dir, '{}.override'.format(service_name)) @@ -224,7 +224,7 @@ def service_resume(service_name, init_dir="/etc/init", upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) sysv_file = os.path.join(initd_dir, service_name) if init_is_systemd(): - service('enable', service_name) + service('unmask', service_name) elif os.path.exists(upstart_file): override_path = os.path.join( init_dir, '{}.override'.format(service_name)) diff --git a/tests/gate-basic-zesty-ocata b/tests/gate-basic-zesty-ocata old mode 100644 new mode 100755 diff --git a/tox.ini b/tox.ini index 1610be31..7c2936e3 100644 --- a/tox.ini +++ b/tox.ini @@ -21,6 +21,11 @@ basepython = python2.7 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt +[testenv:py35] +basepython = python3.5 +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt + [testenv:pep8] basepython = python2.7 deps = -r{toxinidir}/requirements.txt