Sync with charm-helpers

hooks/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -4,8 +4,11 @@ from charmhelpers.contrib.amulet.deployment import (
 
 
 class OpenStackAmuletDeployment(AmuletDeployment):
-    """This class inherits from AmuletDeployment and has additional support
+    """OpenStack amulet deployment.
-       that is specifically for use by OpenStack charms."""
+
+       This class inherits from AmuletDeployment and has additional support
+       that is specifically for use by OpenStack charms.
+       """
 
     def __init__(self, series=None, openstack=None, source=None):
         """Initialize the deployment environment."""
@@ -40,11 +43,14 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             self.d.configure(service, config)
 
     def _get_openstack_release(self):
-        """Return an integer representing the enum value of the openstack
+        """Get openstack release.
-           release."""
+
-        self.precise_essex, self.precise_folsom, self.precise_grizzly, \
+           Return an integer representing the enum value of the openstack
-            self.precise_havana, self.precise_icehouse, \
+           release.
-            self.trusty_icehouse = range(6)
+           """
+        (self.precise_essex, self.precise_folsom, self.precise_grizzly,
+         self.precise_havana, self.precise_icehouse,
+         self.trusty_icehouse) = range(6)
         releases = {
             ('precise', None): self.precise_essex,
             ('precise', 'cloud:precise-folsom'): self.precise_folsom,

hooks/charmhelpers/contrib/openstack/amulet/utils.py

@@ -16,8 +16,11 @@ ERROR = logging.ERROR
 
 
 class OpenStackAmuletUtils(AmuletUtils):
-    """This class inherits from AmuletUtils and has additional support
+    """OpenStack amulet utilities.
-       that is specifically for use by OpenStack charms."""
+
+       This class inherits from AmuletUtils and has additional support
+       that is specifically for use by OpenStack charms.
+       """
 
     def __init__(self, log_level=ERROR):
         """Initialize the deployment environment."""
@@ -25,13 +28,17 @@ class OpenStackAmuletUtils(AmuletUtils):
 
     def validate_endpoint_data(self, endpoints, admin_port, internal_port,
                                public_port, expected):
-        """Validate actual endpoint data vs expected endpoint data. The ports
+        """Validate endpoint data.
-           are used to find the matching endpoint."""
+
+           Validate actual endpoint data vs expected endpoint data. The ports
+           are used to find the matching endpoint.
+           """
         found = False
         for ep in endpoints:
             self.log.debug('endpoint: {}'.format(repr(ep)))
-            if admin_port in ep.adminurl and internal_port in ep.internalurl \
+            if (admin_port in ep.adminurl and
-               and public_port in ep.publicurl:
+                    internal_port in ep.internalurl and
+                    public_port in ep.publicurl):
                 found = True
                 actual = {'id': ep.id,
                           'region': ep.region,
@@ -47,8 +54,11 @@ class OpenStackAmuletUtils(AmuletUtils):
             return 'endpoint not found'
 
     def validate_svc_catalog_endpoint_data(self, expected, actual):
-        """Validate a list of actual service catalog endpoints vs a list of
+        """Validate service catalog endpoint data.
-           expected service catalog endpoints."""
+
+           Validate a list of actual service catalog endpoints vs a list of
+           expected service catalog endpoints.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for k, v in expected.iteritems():
             if k in actual:
@@ -60,8 +70,11 @@ class OpenStackAmuletUtils(AmuletUtils):
         return ret
 
     def validate_tenant_data(self, expected, actual):
-        """Validate a list of actual tenant data vs list of expected tenant
+        """Validate tenant data.
-           data."""
+
+           Validate a list of actual tenant data vs list of expected tenant
+           data.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -78,8 +91,11 @@ class OpenStackAmuletUtils(AmuletUtils):
         return ret
 
     def validate_role_data(self, expected, actual):
-        """Validate a list of actual role data vs a list of expected role
+        """Validate role data.
-           data."""
+
+           Validate a list of actual role data vs a list of expected role
+           data.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -95,8 +111,11 @@ class OpenStackAmuletUtils(AmuletUtils):
         return ret
 
     def validate_user_data(self, expected, actual):
-        """Validate a list of actual user data vs a list of expected user
+        """Validate user data.
-           data."""
+
+           Validate a list of actual user data vs a list of expected user
+           data.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -114,21 +133,24 @@ class OpenStackAmuletUtils(AmuletUtils):
         return ret
 
     def validate_flavor_data(self, expected, actual):
-        """Validate a list of actual flavors vs a list of expected flavors."""
+        """Validate flavor data.
+
+           Validate a list of actual flavors vs a list of expected flavors.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         act = [a.name for a in actual]
         return self._validate_list_data(expected, act)
 
     def tenant_exists(self, keystone, tenant):
-        """Return True if tenant exists"""
+        """Return True if tenant exists."""
         return tenant in [t.name for t in keystone.tenants.list()]
 
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
                                     tenant):
         """Authenticates admin user with the keystone admin endpoint."""
-        service_ip = \
+        unit = keystone_sentry
-            keystone_sentry.relation('shared-db',
+        service_ip = unit.relation('shared-db',
-                                     'mysql:shared-db')['private-address']
+                                   'mysql:shared-db')['private-address']
         ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8'))
         return keystone_client.Client(username=user, password=password,
                                       tenant_name=tenant, auth_url=ep)
@@ -177,12 +199,40 @@ class OpenStackAmuletUtils(AmuletUtils):
             image = glance.images.create(name=image_name, is_public=True,
                                          disk_format='qcow2',
                                          container_format='bare', data=f)
+        count = 1
+        status = image.status
+        while status != 'active' and count < 10:
+            time.sleep(3)
+            image = glance.images.get(image.id)
+            status = image.status
+            self.log.debug('image status: {}'.format(status))
+            count += 1
+
+        if status != 'active':
+            self.log.error('image creation timed out')
+            return None
+
         return image
 
     def delete_image(self, glance, image):
         """Delete the specified image."""
+        num_before = len(list(glance.images.list()))
         glance.images.delete(image)
 
+        count = 1
+        num_after = len(list(glance.images.list()))
+        while num_after != (num_before - 1) and count < 10:
+            time.sleep(3)
+            num_after = len(list(glance.images.list()))
+            self.log.debug('number of images: {}'.format(num_after))
+            count += 1
+
+        if num_after != (num_before - 1):
+            self.log.error('image deletion timed out')
+            return False
+
+        return True
+
     def create_instance(self, nova, image_name, instance_name, flavor):
         """Create the specified instance."""
         image = nova.images.find(name=image_name)
@@ -199,11 +249,27 @@ class OpenStackAmuletUtils(AmuletUtils):
             self.log.debug('instance status: {}'.format(status))
             count += 1
 
-        if status == 'BUILD':
+        if status != 'ACTIVE':
+            self.log.error('instance creation timed out')
             return None
 
         return instance
 
     def delete_instance(self, nova, instance):
         """Delete the specified instance."""
+        num_before = len(list(nova.servers.list()))
         nova.servers.delete(instance)
+
+        count = 1
+        num_after = len(list(nova.servers.list()))
+        while num_after != (num_before - 1) and count < 10:
+            time.sleep(3)
+            num_after = len(list(nova.servers.list()))
+            self.log.debug('number of instances: {}'.format(num_after))
+            count += 1
+
+        if num_after != (num_before - 1):
+            self.log.error('instance deletion timed out')
+            return False
+
+        return True

tests/charmhelpers/contrib/amulet/deployment.py

@@ -1,9 +1,14 @@
 import amulet
 
+import os
+
 
 class AmuletDeployment(object):
-    """This class provides generic Amulet deployment and test runner
+    """Amulet deployment.
-       methods."""
+
+       This class provides generic Amulet deployment and test runner
+       methods.
+       """
 
     def __init__(self, series=None):
         """Initialize the deployment environment."""
@@ -16,11 +21,19 @@ class AmuletDeployment(object):
             self.d = amulet.Deployment()
 
     def _add_services(self, this_service, other_services):
-        """Add services to the deployment where this_service is the local charm
+        """Add services.
+
+           Add services to the deployment where this_service is the local charm
            that we're focused on testing and other_services are the other
-           charms that come from the charm store."""
+           charms that come from the charm store.
+           """
         name, units = range(2)
-        self.this_service = this_service[name]
+
+        if this_service[name] != os.path.basename(os.getcwd()):
+            s = this_service[name]
+            msg = "The charm's root directory name needs to be {}".format(s)
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
         self.d.add(this_service[name], units=this_service[units])
 
         for svc in other_services:
@@ -45,10 +58,10 @@ class AmuletDeployment(object):
         """Deploy environment and wait for all hooks to finish executing."""
         try:
             self.d.setup()
-            self.d.sentry.wait()
+            self.d.sentry.wait(timeout=900)
         except amulet.helpers.TimeoutError:
             amulet.raise_status(amulet.FAIL, msg="Deployment timed out")
-        except:
+        except Exception:
             raise
 
     def run_tests(self):

tests/charmhelpers/contrib/amulet/utils.py

@@ -3,12 +3,15 @@ import io
 import logging
 import re
 import sys
-from time import sleep
+import time
 
 
 class AmuletUtils(object):
-    """This class provides common utility functions that are used by Amulet
+    """Amulet utilities.
-       tests."""
+
+       This class provides common utility functions that are used by Amulet
+       tests.
+       """
 
     def __init__(self, log_level=logging.ERROR):
         self.log = self.get_logger(level=log_level)
@@ -17,8 +20,8 @@ class AmuletUtils(object):
         """Get a logger object that will log to stdout."""
         log = logging
         logger = log.getLogger(name)
-        fmt = \
+        fmt = log.Formatter("%(asctime)s %(funcName)s "
-            log.Formatter("%(asctime)s %(funcName)s %(levelname)s: %(message)s")
+                            "%(levelname)s: %(message)s")
 
         handler = log.StreamHandler(stream=sys.stdout)
         handler.setLevel(level)
@@ -38,7 +41,7 @@ class AmuletUtils(object):
     def valid_url(self, url):
         p = re.compile(
             r'^(?:http|ftp)s?://'
-            r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # flake8: noqa
+            r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'  # noqa
             r'localhost|'
             r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'
             r'(?::\d+)?'
@@ -50,8 +53,11 @@ class AmuletUtils(object):
             return False
 
     def validate_services(self, commands):
-        """Verify the specified services are running on the corresponding
+        """Validate services.
-           service units."""
+
+           Verify the specified services are running on the corresponding
+           service units.
+           """
         for k, v in commands.iteritems():
             for cmd in v:
                 output, code = k.run(cmd)
@@ -66,9 +72,13 @@ class AmuletUtils(object):
         config.readfp(io.StringIO(file_contents))
         return config
 
-    def validate_config_data(self, sentry_unit, config_file, section, expected):
+    def validate_config_data(self, sentry_unit, config_file, section,
-        """Verify that the specified section of the config file contains
+                             expected):
-           the expected option key:value pairs."""
+        """Validate config file data.
+
+           Verify that the specified section of the config file contains
+           the expected option key:value pairs.
+           """
         config = self._get_config(sentry_unit, config_file)
 
         if section != 'DEFAULT' and not config.has_section(section):
@@ -78,20 +88,23 @@ class AmuletUtils(object):
             if not config.has_option(section, k):
                 return "section [{}] is missing option {}".format(section, k)
             if config.get(section, k) != expected[k]:
-                return "section [{}] {}:{} != expected {}:{}".format(section,
+                return "section [{}] {}:{} != expected {}:{}".format(
-                       k, config.get(section, k), k, expected[k])
+                       section, k, config.get(section, k), k, expected[k])
         return None
 
     def _validate_dict_data(self, expected, actual):
-        """Compare expected dictionary data vs actual dictionary data.
+        """Validate dictionary data.
+
+           Compare expected dictionary data vs actual dictionary data.
            The values in the 'expected' dictionary can be strings, bools, ints,
            longs, or can be a function that evaluate a variable and returns a
-           bool."""
+           bool.
+           """
         for k, v in expected.iteritems():
             if k in actual:
-                if isinstance(v, basestring) or \
+                if (isinstance(v, basestring) or
-                   isinstance(v, bool) or \
+                        isinstance(v, bool) or
-                   isinstance(v, (int, long)):
+                        isinstance(v, (int, long))):
                     if v != actual[k]:
                         return "{}:{}".format(k, actual[k])
                 elif not v(actual[k]):
@@ -114,7 +127,7 @@ class AmuletUtils(object):
         return None
 
     def not_null(self, string):
-        if string != None:
+        if string is not None:
             return True
         else:
             return False
@@ -128,9 +141,12 @@ class AmuletUtils(object):
         return sentry_unit.directory_stat(directory)['mtime']
 
     def _get_proc_start_time(self, sentry_unit, service, pgrep_full=False):
-        """Determine start time of the process based on the last modification
+        """Get process' start time.
+
+           Determine start time of the process based on the last modification
            time of the /proc/pid directory. If pgrep_full is True, the process
-           name is matched against the full command line."""
+           name is matched against the full command line.
+           """
         if pgrep_full:
             cmd = 'pgrep -o -f {}'.format(service)
         else:
@@ -139,13 +155,16 @@ class AmuletUtils(object):
         return self._get_dir_mtime(sentry_unit, proc_dir)
 
     def service_restarted(self, sentry_unit, service, filename,
-                          pgrep_full=False):
+                          pgrep_full=False, sleep_time=20):
-        """Compare a service's start time vs a file's last modification time
+        """Check if service was restarted.
+
+           Compare a service's start time vs a file's last modification time
            (such as a config file for that service) to determine if the service
-           has been restarted."""
+           has been restarted.
-        sleep(10)
+           """
-        if self._get_proc_start_time(sentry_unit, service, pgrep_full) >= \
+        time.sleep(sleep_time)
-           self._get_file_mtime(sentry_unit, filename):
+        if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >=
+                self._get_file_mtime(sentry_unit, filename)):
             return True
         else:
             return False

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -4,8 +4,11 @@ from charmhelpers.contrib.amulet.deployment import (
 
 
 class OpenStackAmuletDeployment(AmuletDeployment):
-    """This class inherits from AmuletDeployment and has additional support
+    """OpenStack amulet deployment.
-       that is specifically for use by OpenStack charms."""
+
+       This class inherits from AmuletDeployment and has additional support
+       that is specifically for use by OpenStack charms.
+       """
 
     def __init__(self, series=None, openstack=None, source=None):
         """Initialize the deployment environment."""
@@ -40,11 +43,14 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             self.d.configure(service, config)
 
     def _get_openstack_release(self):
-        """Return an integer representing the enum value of the openstack
+        """Get openstack release.
-           release."""
+
-        self.precise_essex, self.precise_folsom, self.precise_grizzly, \
+           Return an integer representing the enum value of the openstack
-            self.precise_havana, self.precise_icehouse, \
+           release.
-            self.trusty_icehouse = range(6)
+           """
+        (self.precise_essex, self.precise_folsom, self.precise_grizzly,
+         self.precise_havana, self.precise_icehouse,
+         self.trusty_icehouse) = range(6)
         releases = {
             ('precise', None): self.precise_essex,
             ('precise', 'cloud:precise-folsom'): self.precise_folsom,

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -16,8 +16,11 @@ ERROR = logging.ERROR
 
 
 class OpenStackAmuletUtils(AmuletUtils):
-    """This class inherits from AmuletUtils and has additional support
+    """OpenStack amulet utilities.
-       that is specifically for use by OpenStack charms."""
+
+       This class inherits from AmuletUtils and has additional support
+       that is specifically for use by OpenStack charms.
+       """
 
     def __init__(self, log_level=ERROR):
         """Initialize the deployment environment."""
@@ -25,13 +28,17 @@ class OpenStackAmuletUtils(AmuletUtils):
 
     def validate_endpoint_data(self, endpoints, admin_port, internal_port,
                                public_port, expected):
-        """Validate actual endpoint data vs expected endpoint data. The ports
+        """Validate endpoint data.
-           are used to find the matching endpoint."""
+
+           Validate actual endpoint data vs expected endpoint data. The ports
+           are used to find the matching endpoint.
+           """
         found = False
         for ep in endpoints:
             self.log.debug('endpoint: {}'.format(repr(ep)))
-            if admin_port in ep.adminurl and internal_port in ep.internalurl \
+            if (admin_port in ep.adminurl and
-               and public_port in ep.publicurl:
+                    internal_port in ep.internalurl and
+                    public_port in ep.publicurl):
                 found = True
                 actual = {'id': ep.id,
                           'region': ep.region,
@@ -47,8 +54,11 @@ class OpenStackAmuletUtils(AmuletUtils):
             return 'endpoint not found'
 
     def validate_svc_catalog_endpoint_data(self, expected, actual):
-        """Validate a list of actual service catalog endpoints vs a list of
+        """Validate service catalog endpoint data.
-           expected service catalog endpoints."""
+
+           Validate a list of actual service catalog endpoints vs a list of
+           expected service catalog endpoints.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for k, v in expected.iteritems():
             if k in actual:
@@ -60,8 +70,11 @@ class OpenStackAmuletUtils(AmuletUtils):
         return ret
 
     def validate_tenant_data(self, expected, actual):
-        """Validate a list of actual tenant data vs list of expected tenant
+        """Validate tenant data.
-           data."""
+
+           Validate a list of actual tenant data vs list of expected tenant
+           data.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -78,8 +91,11 @@ class OpenStackAmuletUtils(AmuletUtils):
         return ret
 
     def validate_role_data(self, expected, actual):
-        """Validate a list of actual role data vs a list of expected role
+        """Validate role data.
-           data."""
+
+           Validate a list of actual role data vs a list of expected role
+           data.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -95,8 +111,11 @@ class OpenStackAmuletUtils(AmuletUtils):
         return ret
 
     def validate_user_data(self, expected, actual):
-        """Validate a list of actual user data vs a list of expected user
+        """Validate user data.
-           data."""
+
+           Validate a list of actual user data vs a list of expected user
+           data.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -114,21 +133,24 @@ class OpenStackAmuletUtils(AmuletUtils):
         return ret
 
     def validate_flavor_data(self, expected, actual):
-        """Validate a list of actual flavors vs a list of expected flavors."""
+        """Validate flavor data.
+
+           Validate a list of actual flavors vs a list of expected flavors.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         act = [a.name for a in actual]
         return self._validate_list_data(expected, act)
 
     def tenant_exists(self, keystone, tenant):
-        """Return True if tenant exists"""
+        """Return True if tenant exists."""
         return tenant in [t.name for t in keystone.tenants.list()]
 
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
                                     tenant):
         """Authenticates admin user with the keystone admin endpoint."""
-        service_ip = \
+        unit = keystone_sentry
-            keystone_sentry.relation('shared-db',
+        service_ip = unit.relation('shared-db',
-                                     'mysql:shared-db')['private-address']
+                                   'mysql:shared-db')['private-address']
         ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8'))
         return keystone_client.Client(username=user, password=password,
                                       tenant_name=tenant, auth_url=ep)
@@ -177,12 +199,40 @@ class OpenStackAmuletUtils(AmuletUtils):
             image = glance.images.create(name=image_name, is_public=True,
                                          disk_format='qcow2',
                                          container_format='bare', data=f)
+        count = 1
+        status = image.status
+        while status != 'active' and count < 10:
+            time.sleep(3)
+            image = glance.images.get(image.id)
+            status = image.status
+            self.log.debug('image status: {}'.format(status))
+            count += 1
+
+        if status != 'active':
+            self.log.error('image creation timed out')
+            return None
+
         return image
 
     def delete_image(self, glance, image):
         """Delete the specified image."""
+        num_before = len(list(glance.images.list()))
         glance.images.delete(image)
 
+        count = 1
+        num_after = len(list(glance.images.list()))
+        while num_after != (num_before - 1) and count < 10:
+            time.sleep(3)
+            num_after = len(list(glance.images.list()))
+            self.log.debug('number of images: {}'.format(num_after))
+            count += 1
+
+        if num_after != (num_before - 1):
+            self.log.error('image deletion timed out')
+            return False
+
+        return True
+
     def create_instance(self, nova, image_name, instance_name, flavor):
         """Create the specified instance."""
         image = nova.images.find(name=image_name)
@@ -199,11 +249,27 @@ class OpenStackAmuletUtils(AmuletUtils):
             self.log.debug('instance status: {}'.format(status))
             count += 1
 
-        if status == 'BUILD':
+        if status != 'ACTIVE':
+            self.log.error('instance creation timed out')
             return None
 
         return instance
 
     def delete_instance(self, nova, instance):
         """Delete the specified instance."""
+        num_before = len(list(nova.servers.list()))
         nova.servers.delete(instance)
+
+        count = 1
+        num_after = len(list(nova.servers.list()))
+        while num_after != (num_before - 1) and count < 10:
+            time.sleep(3)
+            num_after = len(list(nova.servers.list()))
+            self.log.debug('number of instances: {}'.format(num_after))
+            count += 1
+
+        if num_after != (num_before - 1):
+            self.log.error('instance deletion timed out')
+            return False
+
+        return True