From 3354c23ffa5e13f7c521a828cb8363beeaec89b5 Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Fri, 16 Oct 2015 14:36:02 +0000 Subject: [PATCH 1/4] sync charmhelpers --- .../contrib/openstack/amulet/deployment.py | 40 +++++++++++++++++++ .../contrib/openstack/amulet/utils.py | 2 +- .../charmhelpers/contrib/openstack/context.py | 34 +++++++++++----- .../charmhelpers/contrib/openstack/neutron.py | 14 +++++++ hooks/charmhelpers/contrib/openstack/utils.py | 1 + hooks/charmhelpers/core/host.py | 13 +++++- .../contrib/openstack/amulet/deployment.py | 40 +++++++++++++++++++ 7 files changed, 133 insertions(+), 11 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 722bc645..f9304d45 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . +import re import six from collections import OrderedDict from charmhelpers.contrib.amulet.deployment import ( @@ -114,6 +115,45 @@ class OpenStackAmuletDeployment(AmuletDeployment): for service, config in six.iteritems(configs): self.d.configure(service, config) + def _auto_wait_for_status(self, message=None, exclude_services=None, + timeout=1800): + """Wait for all units to have a specific extended status, except + for any defined as excluded. Unless specified via message, any + status containing any case of 'ready' will be considered a match. + + Examples of message usage: + + Wait for all unit status to CONTAIN any case of 'ready' or 'ok': + message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE) + + Wait for all units to reach this status (exact match): + message = 'Unit is ready' + + Wait for all units to reach any one of these (exact match): + message = re.compile('Unit is ready|OK|Ready') + + Wait for at least one unit to reach this status (exact match): + message = {'ready'} + + See Amulet's sentry.wait_for_messages() for message usage detail. + https://github.com/juju/amulet/blob/master/amulet/sentry.py + + :param message: Expected status match + :param exclude_services: List of juju service names to ignore + :param timeout: Maximum time in seconds to wait for status match + :returns: None. Raises if timeout is hit. + """ + + if not message: + message = re.compile('.*ready.*', re.IGNORECASE) + + if not exclude_services: + exclude_services = [] + + services = list(set(self.d.services.keys()) - set(exclude_services)) + service_messages = {service: message for service in services} + self.d.sentry.wait_for_messages(service_messages, timeout=timeout) + def _get_openstack_release(self): """Get openstack release. diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index b1397419..2b3087ea 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -752,7 +752,7 @@ class OpenStackAmuletUtils(AmuletUtils): self.log.debug('SSL is enabled @{}:{} ' '({})'.format(host, port, unit_name)) return True - elif not port and not conf_ssl: + elif not conf_ssl: self.log.debug('SSL not enabled @{}:{} ' '({})'.format(host, port, unit_name)) return False diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 49c04de0..48216338 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -952,6 +952,19 @@ class NeutronContext(OSContextGenerator): 'config': config} return ovs_ctxt + def midonet_ctxt(self): + driver = neutron_plugin_attribute(self.plugin, 'driver', + self.network_manager) + midonet_config = neutron_plugin_attribute(self.plugin, 'config', + self.network_manager) + mido_ctxt = {'core_plugin': driver, + 'neutron_plugin': 'midonet', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': midonet_config} + + return mido_ctxt + def __call__(self): if self.network_manager not in ['quantum', 'neutron']: return {} @@ -973,6 +986,8 @@ class NeutronContext(OSContextGenerator): ctxt.update(self.nuage_ctxt()) elif self.plugin == 'plumgrid': ctxt.update(self.pg_ctxt()) + elif self.plugin == 'midonet': + ctxt.update(self.midonet_ctxt()) alchemy_flags = config('neutron-alchemy-flags') if alchemy_flags: @@ -1105,7 +1120,7 @@ class SubordinateConfigContext(OSContextGenerator): ctxt = { ... other context ... - 'subordinate_config': { + 'subordinate_configuration': { 'DEFAULT': { 'key1': 'value1', }, @@ -1146,22 +1161,23 @@ class SubordinateConfigContext(OSContextGenerator): try: sub_config = json.loads(sub_config) except: - log('Could not parse JSON from subordinate_config ' - 'setting from %s' % rid, level=ERROR) + log('Could not parse JSON from ' + 'subordinate_configuration setting from %s' + % rid, level=ERROR) continue for service in self.services: if service not in sub_config: - log('Found subordinate_config on %s but it contained' - 'nothing for %s service' % (rid, service), - level=INFO) + log('Found subordinate_configuration on %s but it ' + 'contained nothing for %s service' + % (rid, service), level=INFO) continue sub_config = sub_config[service] if self.config_file not in sub_config: - log('Found subordinate_config on %s but it contained' - 'nothing for %s' % (rid, self.config_file), - level=INFO) + log('Found subordinate_configuration on %s but it ' + 'contained nothing for %s' + % (rid, self.config_file), level=INFO) continue sub_config = sub_config[self.config_file] diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 2a59d86b..c54d63c7 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -209,6 +209,20 @@ def neutron_plugins(): 'server_packages': ['neutron-server', 'neutron-plugin-plumgrid'], 'server_services': ['neutron-server'] + }, + 'midonet': { + 'config': '/etc/neutron/plugins/midonet/midonet.ini', + 'driver': 'midonet.neutron.plugin.MidonetPluginV2', + 'contexts': [ + context.SharedDBContext(user=config('neutron-database-user'), + database=config('neutron-database'), + relation_prefix='neutron', + ssl_dir=NEUTRON_CONF_DIR)], + 'services': [], + 'packages': [[headers_package()] + determine_dkms_package()], + 'server_packages': ['neutron-server', + 'python-neutron-plugin-midonet'], + 'server_services': ['neutron-server'] } } if release >= 'icehouse': diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index eefcf08b..cf4e5547 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -121,6 +121,7 @@ SWIFT_CODENAMES = OrderedDict([ ('2.2.2', 'kilo'), ('2.3.0', 'liberty'), ('2.4.0', 'liberty'), + ('2.5.0', 'liberty'), ]) # >= Liberty version->codename mapping diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index cb3c527e..d6d2d312 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -566,7 +566,14 @@ def chdir(d): os.chdir(cur) -def chownr(path, owner, group, follow_links=True): +def chownr(path, owner, group, follow_links=True, chowntopdir=False): + """ + Recursively change user and group ownership of files and directories + in given path. Doesn't chown path itself by default, only its children. + + :param bool follow_links: Also Chown links if True + :param bool chowntopdir: Also chown path itself if True + """ uid = pwd.getpwnam(owner).pw_uid gid = grp.getgrnam(group).gr_gid if follow_links: @@ -574,6 +581,10 @@ def chownr(path, owner, group, follow_links=True): else: chown = os.lchown + if chowntopdir: + broken_symlink = os.path.lexists(path) and not os.path.exists(path) + if not broken_symlink: + chown(path, uid, gid) for root, dirs, files in os.walk(path): for name in dirs + files: full = os.path.join(root, name) diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 722bc645..f9304d45 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . +import re import six from collections import OrderedDict from charmhelpers.contrib.amulet.deployment import ( @@ -114,6 +115,45 @@ class OpenStackAmuletDeployment(AmuletDeployment): for service, config in six.iteritems(configs): self.d.configure(service, config) + def _auto_wait_for_status(self, message=None, exclude_services=None, + timeout=1800): + """Wait for all units to have a specific extended status, except + for any defined as excluded. Unless specified via message, any + status containing any case of 'ready' will be considered a match. + + Examples of message usage: + + Wait for all unit status to CONTAIN any case of 'ready' or 'ok': + message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE) + + Wait for all units to reach this status (exact match): + message = 'Unit is ready' + + Wait for all units to reach any one of these (exact match): + message = re.compile('Unit is ready|OK|Ready') + + Wait for at least one unit to reach this status (exact match): + message = {'ready'} + + See Amulet's sentry.wait_for_messages() for message usage detail. + https://github.com/juju/amulet/blob/master/amulet/sentry.py + + :param message: Expected status match + :param exclude_services: List of juju service names to ignore + :param timeout: Maximum time in seconds to wait for status match + :returns: None. Raises if timeout is hit. + """ + + if not message: + message = re.compile('.*ready.*', re.IGNORECASE) + + if not exclude_services: + exclude_services = [] + + services = list(set(self.d.services.keys()) - set(exclude_services)) + service_messages = {service: message for service in services} + self.d.sentry.wait_for_messages(service_messages, timeout=timeout) + def _get_openstack_release(self): """Get openstack release. From 235dacf835986699856b8248a29f92ce57bf01ba Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Fri, 16 Oct 2015 15:31:50 +0000 Subject: [PATCH 2/4] wait for workload status before testing; add service and relations to satisfy workload status. --- tests/basic_deployment.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index a1cb035d..7544eb92 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -1,5 +1,3 @@ -#!/usr/bin/python - import amulet import os import time @@ -47,7 +45,9 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): other_services = [{'name': 'mysql'}, {'name': 'rabbitmq-server'}, {'name': 'keystone'}, + {'name': 'glance'}, # satisfy workload status {'name': 'nova-cloud-controller'}, + {'name': 'nova-compute'}, # satisfy workload stat {'name': 'neutron-api'}] super(NeutronGatewayBasicDeployment, self)._add_services( @@ -68,7 +68,15 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): 'neutron-api:shared-db': 'mysql:shared-db', 'neutron-api:amqp': 'rabbitmq-server:amqp', 'neutron-api:neutron-api': 'nova-cloud-controller:neutron-api', - 'neutron-api:identity-service': 'keystone:identity-service' + 'neutron-api:identity-service': 'keystone:identity-service', + 'glance:identity-service': 'keystone:identity-service', + 'glance:shared-db': 'mysql:shared-db', + 'glance:amqp': 'rabbitmq-server:amqp', + 'nova-cloud-controller:cloud-compute': 'nova-compute:' + 'cloud-compute', + 'nova-compute:amqp': 'rabbitmq-server:amqp', + 'nova-compute:image-service': 'glance:image-service', + 'nova-cloud-controller:image-service': 'glance:image-service', } super(NeutronGatewayBasicDeployment, self)._add_relations(relations) From 4f8dcefed0ee721f5c2d91a686d6a808698aa4f2 Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Fri, 16 Oct 2015 16:36:52 +0000 Subject: [PATCH 3/4] fix relation check, remove varying unit data check --- tests/basic_deployment.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 7544eb92..ee9faaf9 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -432,13 +432,6 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): 'password': u.not_null } - if self._get_openstack_release() == self.precise_icehouse: - # Precise - expected['allowed_units'] = 'nova-cloud-controller/0 neutron-api/0' - else: - # Not Precise - expected['allowed_units'] = 'neutron-api/0' - ret = u.validate_relation_data(unit, relation, expected) if ret: message = u.relation_error('mysql shared-db', ret) From d008e851fbd01eb67cb7958a39daedc2796bcd7a Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Mon, 19 Oct 2015 14:50:54 +0000 Subject: [PATCH 4/4] enable wily test; add debug output --- tests/021-basic-wily-liberty | 0 tests/basic_deployment.py | 1 + 2 files changed, 1 insertion(+) mode change 100644 => 100755 tests/021-basic-wily-liberty diff --git a/tests/021-basic-wily-liberty b/tests/021-basic-wily-liberty old mode 100644 new mode 100755 diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index ee9faaf9..b2939a09 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -949,6 +949,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): self.neutron.create_network({'network': network}) networks = self.neutron.list_networks(name=net_name) + u.log.debug('Networks: {}'.format(networks)) net_len = len(networks['networks']) if net_len != 1: msg = "Expected 1 network, found {}".format(net_len)