[1chb1n, r=gnuoy] Liberty amulet tests and amulet fixes

tests/020-basic-trusty-liberty

@@ -0,0 +1,12 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic nova cloud controller deployment on
+   trusty-liberty."""
+
+from basic_deployment import NovaCCBasicDeployment
+
+if __name__ == '__main__':
+    deployment = NovaCCBasicDeployment(series='trusty',
+                                       openstack='cloud:trusty-liberty',
+                                       source='cloud:trusty-updates/liberty')
+    deployment.run_tests()

tests/021-basic-wily-liberty

@@ -0,0 +1,10 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic nova cloud controller deployment on
+   wily-liberty."""
+
+from basic_deployment import NovaCCBasicDeployment
+
+if __name__ == '__main__':
+    deployment = NovaCCBasicDeployment(series='wily')
+    deployment.run_tests()

tests/README

@@ -1,6 +1,16 @@
 This directory provides Amulet tests that focus on verification of Nova Cloud
 Controller deployments.
 
+test_* methods are called in lexical sort order, although each individual test
+should be idempotent, and expected to pass regardless of run order.
+
+Test name convention to ensure desired test order:
+    1xx service and endpoint checks
+    2xx relation checks
+    3xx config checks
+    4xx functional checks
+    9xx restarts and other final checks
+
 In order to run tests, you'll need charm-tools installed (in addition to
 juju, of course):
     sudo add-apt-repository ppa:juju/stable

tests/basic_deployment.py

@@ -1,5 +1,3 @@
-#!/usr/bin/python
-
 import amulet
 import os
 import yaml
@@ -10,8 +8,8 @@ from charmhelpers.contrib.openstack.amulet.deployment import (
 
 from charmhelpers.contrib.openstack.amulet.utils import (
     OpenStackAmuletUtils,
-    DEBUG, # flake8: noqa
+    DEBUG,
-    ERROR
+    # ERROR
 )
 
 # Use DEBUG to turn on debug logging
@@ -24,12 +22,18 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
     def __init__(self, series=None, openstack=None, source=None, git=False,
                  stable=False):
         """Deploy the entire test environment."""
-        super(NovaCCBasicDeployment, self).__init__(series, openstack, source, stable)
+        super(NovaCCBasicDeployment, self).__init__(series, openstack,
+                                                    source, stable)
         self.git = git
         self._add_services()
         self._add_relations()
         self._configure_services()
         self._deploy()
+
+        u.log.info('Waiting on extended status checks...')
+        exclude_services = ['mysql']
+        self._auto_wait_for_status(exclude_services=exclude_services)
+
         self._initialize_tests()
 
     def _add_services(self):
@@ -40,27 +44,31 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
            compatible with the local charm (e.g. stable or next).
            """
         this_service = {'name': 'nova-cloud-controller'}
-        other_services = [{'name': 'mysql'}, {'name': 'rabbitmq-server'},
+        other_services = [{'name': 'mysql'},
+                          {'name': 'rabbitmq-server'},
                           {'name': 'nova-compute', 'units': 2},
-                          {'name': 'keystone'}, {'name': 'glance'}]
+                          {'name': 'keystone'},
+                          {'name': 'glance'}]
         super(NovaCCBasicDeployment, self)._add_services(this_service,
                                                          other_services)
 
     def _add_relations(self):
         """Add all of the relations for the services."""
         relations = {
-          'nova-cloud-controller:shared-db': 'mysql:shared-db',
+            'nova-cloud-controller:shared-db': 'mysql:shared-db',
-          'nova-cloud-controller:identity-service': 'keystone:identity-service',
+            'nova-cloud-controller:identity-service': 'keystone:'
-          'nova-cloud-controller:amqp': 'rabbitmq-server:amqp',
+                                                      'identity-service',
-          'nova-cloud-controller:cloud-compute': 'nova-compute:cloud-compute',
+            'nova-cloud-controller:amqp': 'rabbitmq-server:amqp',
-          'nova-cloud-controller:image-service': 'glance:image-service',
+            'nova-cloud-controller:cloud-compute': 'nova-compute:'
-          'nova-compute:image-service': 'glance:image-service',
+                                                   'cloud-compute',
-          'nova-compute:shared-db': 'mysql:shared-db',
+            'nova-cloud-controller:image-service': 'glance:image-service',
-          'nova-compute:amqp': 'rabbitmq-server:amqp',
+            'nova-compute:image-service': 'glance:image-service',
-          'keystone:shared-db': 'mysql:shared-db',
+            'nova-compute:shared-db': 'mysql:shared-db',
-          'glance:identity-service': 'keystone:identity-service',
+            'nova-compute:amqp': 'rabbitmq-server:amqp',
-          'glance:shared-db': 'mysql:shared-db',
+            'keystone:shared-db': 'mysql:shared-db',
-          'glance:amqp': 'rabbitmq-server:amqp'
+            'glance:identity-service': 'keystone:identity-service',
+            'glance:shared-db': 'mysql:shared-db',
+            'glance:amqp': 'rabbitmq-server:amqp'
         }
         super(NovaCCBasicDeployment, self)._add_relations(relations)
 
@@ -98,16 +106,24 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
                 'http_proxy': amulet_http_proxy,
                 'https_proxy': amulet_http_proxy,
             }
-            nova_cc_config['openstack-origin-git'] = yaml.dump(openstack_origin_git)
+
-            nova_config['openstack-origin-git'] = yaml.dump(openstack_origin_git)
+            nova_cc_config['openstack-origin-git'] = \
+                yaml.dump(openstack_origin_git)
+
+            nova_config['openstack-origin-git'] = \
+                yaml.dump(openstack_origin_git)
 
         # Add some rate-limiting options to the charm. These will noop before
         # icehouse.
-        nova_cc_config['api-rate-limit-rules'] = "( POST, '*', .*, 9999, MINUTE );"
+        nova_cc_config['api-rate-limit-rules'] = \
+            "( POST, '*', .*, 9999, MINUTE );"
+
         keystone_config = {'admin-password': 'openstack',
                            'admin-token': 'ubuntutesting'}
+
         configs = {'nova-cloud-controller': nova_cc_config,
                    'keystone': keystone_config, 'nova-compute': nova_config}
+
         super(NovaCCBasicDeployment, self)._configure_services(configs)
 
     def _initialize_tests(self):
@@ -120,6 +136,11 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
         self.nova_compute_sentry = self.d.sentry.unit['nova-compute/0']
         self.glance_sentry = self.d.sentry.unit['glance/0']
 
+        u.log.debug('openstack release val: {}'.format(
+            self._get_openstack_release()))
+        u.log.debug('openstack release str: {}'.format(
+            self._get_openstack_release_string()))
+
         # Authenticate admin with keystone
         self.keystone = u.authenticate_keystone_admin(self.keystone_sentry,
                                                       user='admin',
@@ -155,32 +176,33 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
                                                   password='password',
                                                   tenant=self.demo_tenant)
 
-    def test_services(self):
+    def test_100_services(self):
         """Verify the expected services are running on the corresponding
            service units."""
-        commands = {
+        u.log.debug('Checking system services on units...')
-            self.mysql_sentry: ['status mysql'],
+        services = {
-            self.rabbitmq_sentry: ['sudo service rabbitmq-server status'],
+            self.mysql_sentry: ['mysql'],
-            self.nova_cc_sentry: ['status nova-api-ec2',
+            self.rabbitmq_sentry: ['rabbitmq-server'],
-                                  'status nova-api-os-compute',
+            self.nova_cc_sentry: ['nova-api-ec2',
-                                  'status nova-objectstore',
+                                  'nova-api-os-compute',
-                                  'status nova-cert',
+                                  'nova-conductor',
-                                  'status nova-scheduler'],
+                                  'nova-objectstore',
-            self.nova_compute_sentry: ['status nova-compute',
+                                  'nova-cert',
-                                       'status nova-network',
+                                  'nova-scheduler'],
-                                       'status nova-api'],
+            self.nova_compute_sentry: ['nova-compute',
-            self.keystone_sentry: ['status keystone'],
+                                       'nova-network',
-            self.glance_sentry: ['status glance-registry', 'status glance-api']
+                                       'nova-api'],
+            self.keystone_sentry: ['keystone'],
+            self.glance_sentry: ['glance-registry', 'glance-api']
         }
-        if self._get_openstack_release() >= self.precise_grizzly:
-            commands[self.nova_cc_sentry] = ['status nova-conductor']
 
-        ret = u.validate_services(commands)
+        ret = u.validate_services_by_name(services)
         if ret:
             amulet.raise_status(amulet.FAIL, msg=ret)
 
-    def test_service_catalog(self):
+    def test_102_service_catalog(self):
         """Verify that the service catalog endpoint data is valid."""
+        u.log.debug('Checking keystone service catalog...')
         endpoint_vol = {'adminURL': u.valid_url,
                         'region': 'RegionOne',
                         'publicURL': u.valid_url,
@@ -189,30 +211,38 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
                        'region': 'RegionOne',
                        'publicURL': u.valid_url,
                        'internalURL': u.valid_url}
+
         if self._get_openstack_release() >= self.precise_folsom:
             endpoint_vol['id'] = u.not_null
             endpoint_id['id'] = u.not_null
+
         if self._get_openstack_release() >= self.trusty_kilo:
             expected = {'compute': [endpoint_vol], 'identity': [endpoint_id]}
         else:
             expected = {'s3': [endpoint_vol], 'compute': [endpoint_vol],
                         'ec2': [endpoint_vol], 'identity': [endpoint_id]}
+
         actual = self.keystone_demo.service_catalog.get_endpoints()
 
         ret = u.validate_svc_catalog_endpoint_data(expected, actual)
         if ret:
             amulet.raise_status(amulet.FAIL, msg=ret)
 
-    def test_openstack_compute_api_endpoint(self):
+    def test_104_openstack_compute_api_endpoint(self):
         """Verify the openstack compute api (osapi) endpoint data."""
+        u.log.debug('Checking compute endpoint data...')
+
         endpoints = self.keystone.endpoints.list()
         admin_port = internal_port = public_port = '8774'
-        expected = {'id': u.not_null,
+
-                    'region': 'RegionOne',
+        expected = {
-                    'adminurl': u.valid_url,
+            'id': u.not_null,
-                    'internalurl': u.valid_url,
+            'region': 'RegionOne',
-                    'publicurl': u.valid_url,
+            'adminurl': u.valid_url,
-                    'service_id': u.not_null}
+            'internalurl': u.valid_url,
+            'publicurl': u.valid_url,
+            'service_id': u.not_null
+        }
 
         ret = u.validate_endpoint_data(endpoints, admin_port, internal_port,
                                        public_port, expected)
@@ -220,19 +250,23 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = 'osapi endpoint: {}'.format(ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_ec2_api_endpoint(self):
+    def test_106_ec2_api_endpoint(self):
         """Verify the EC2 api endpoint data."""
         if self._get_openstack_release() >= self.trusty_kilo:
             return
 
+        u.log.debug('Checking ec2 endpoint data...')
         endpoints = self.keystone.endpoints.list()
         admin_port = internal_port = public_port = '8773'
-        expected = {'id': u.not_null,
+
-                    'region': 'RegionOne',
+        expected = {
-                    'adminurl': u.valid_url,
+            'id': u.not_null,
-                    'internalurl': u.valid_url,
+            'region': 'RegionOne',
-                    'publicurl': u.valid_url,
+            'adminurl': u.valid_url,
-                    'service_id': u.not_null}
+            'internalurl': u.valid_url,
+            'publicurl': u.valid_url,
+            'service_id': u.not_null
+        }
 
         ret = u.validate_endpoint_data(endpoints, admin_port, internal_port,
                                        public_port, expected)
@@ -240,19 +274,22 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = 'EC2 endpoint: {}'.format(ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_s3_api_endpoint(self):
+    def test_108_s3_api_endpoint(self):
         """Verify the S3 api endpoint data."""
         if self._get_openstack_release() >= self.trusty_kilo:
             return
 
+        u.log.debug('Checking s3 endpoint data...')
         endpoints = self.keystone.endpoints.list()
         admin_port = internal_port = public_port = '3333'
-        expected = {'id': u.not_null,
+        expected = {
-                    'region': 'RegionOne',
+            'id': u.not_null,
-                    'adminurl': u.valid_url,
+            'region': 'RegionOne',
-                    'internalurl': u.valid_url,
+            'adminurl': u.valid_url,
-                    'publicurl': u.valid_url,
+            'internalurl': u.valid_url,
-                    'service_id': u.not_null}
+            'publicurl': u.valid_url,
+            'service_id': u.not_null
+        }
 
         ret = u.validate_endpoint_data(endpoints, admin_port, internal_port,
                                        public_port, expected)
@@ -260,10 +297,12 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = 'S3 endpoint: {}'.format(ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_nova_cc_shared_db_relation(self):
+    def test_200_nova_cc_shared_db_relation(self):
         """Verify the nova-cc to mysql shared-db relation data"""
+        u.log.debug('Checking n-c-c:mysql db relation data...')
         unit = self.nova_cc_sentry
         relation = ['shared-db', 'mysql:shared-db']
+
         expected = {
             'private-address': u.valid_ip,
             'nova_database': 'nova',
@@ -276,8 +315,9 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('nova-cc shared-db', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_mysql_shared_db_relation(self):
+    def test_202_mysql_shared_db_relation(self):
         """Verify the mysql to nova-cc shared-db relation data"""
+        u.log.debug('Checking mysql:n-c-c db relation data...')
         unit = self.mysql_sentry
         relation = ['shared-db', 'nova-cloud-controller:shared-db']
         expected = {
@@ -291,8 +331,9 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('mysql shared-db', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_nova_cc_identity_service_relation(self):
+    def test_204_nova_cc_identity_service_relation(self):
         """Verify the nova-cc to keystone identity-service relation data"""
+        u.log.debug('Checking n-c-c:keystone identity relation data...')
         unit = self.nova_cc_sentry
         relation = ['identity-service', 'keystone:identity-service']
         expected = {
@@ -320,8 +361,9 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('nova-cc identity-service', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_keystone_identity_service_relation(self):
+    def test_206_keystone_identity_service_relation(self):
         """Verify the keystone to nova-cc identity-service relation data"""
+        u.log.debug('Checking keystone:n-c-c identity relation data...')
         unit = self.keystone_sentry
         relation = ['identity-service',
                     'nova-cloud-controller:identity-service']
@@ -347,8 +389,9 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('keystone identity-service', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_nova_cc_amqp_relation(self):
+    def test_208_nova_cc_amqp_relation(self):
         """Verify the nova-cc to rabbitmq-server amqp relation data"""
+        u.log.debug('Checking n-c-c:rmq amqp relation data...')
         unit = self.nova_cc_sentry
         relation = ['amqp', 'rabbitmq-server:amqp']
         expected = {
@@ -362,8 +405,9 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('nova-cc amqp', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_rabbitmq_amqp_relation(self):
+    def test_210_rabbitmq_amqp_relation(self):
         """Verify the rabbitmq-server to nova-cc amqp relation data"""
+        u.log.debug('Checking rmq:n-c-c amqp relation data...')
         unit = self.rabbitmq_sentry
         relation = ['amqp', 'nova-cloud-controller:amqp']
         expected = {
@@ -377,8 +421,11 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('rabbitmq amqp', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_nova_cc_cloud_compute_relation(self):
+    def test_212_nova_cc_cloud_compute_relation(self):
         """Verify the nova-cc to nova-compute cloud-compute relation data"""
+        u.log.debug('Checking n-c-c:nova-compute '
+                    'cloud-compute relation data...')
+
         unit = self.nova_cc_sentry
         relation = ['cloud-compute', 'nova-compute:cloud-compute']
         expected = {
@@ -396,8 +443,11 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('nova-cc cloud-compute', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_nova_cloud_compute_relation(self):
+    def test_214_nova_cloud_compute_relation(self):
         """Verify the nova-compute to nova-cc cloud-compute relation data"""
+        u.log.debug('Checking nova-compute:n-c-c '
+                    'cloud-compute relation data...')
+
         unit = self.nova_compute_sentry
         relation = ['cloud-compute', 'nova-cloud-controller:cloud-compute']
         expected = {
@@ -409,8 +459,9 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('nova-compute cloud-compute', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_nova_cc_image_service_relation(self):
+    def test_216_nova_cc_image_service_relation(self):
         """Verify the nova-cc to glance image-service relation data"""
+        u.log.debug('Checking n-c-c:glance image-service relation data...')
         unit = self.nova_cc_sentry
         relation = ['image-service', 'glance:image-service']
         expected = {
@@ -422,8 +473,9 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('nova-cc image-service', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_glance_image_service_relation(self):
+    def test_218_glance_image_service_relation(self):
         """Verify the glance to nova-cc image-service relation data"""
+        u.log.debug('Checking glance:n-c-c image-service relation data...')
         unit = self.glance_sentry
         relation = ['image-service', 'nova-cloud-controller:image-service']
         expected = {
@@ -436,72 +488,44 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('glance image-service', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_z_restart_on_config_change(self):
+    def test_300_nova_default_config(self):
-        """Verify that the specified services are restarted when the config
-           is changed.
-
-           Note(coreycb): The method name with the _z_ is a little odd
-           but it forces the test to run last.  It just makes things
-           easier because restarting services requires re-authorization.
-           """
-        # NOTE(coreycb): Skipping failing test on essex until resolved.
-        #                config-flags don't take effect on essex.
-        if self._get_openstack_release() == self.precise_essex:
-            u.log.error("Skipping failing test until resolved")
-            return
-
-        flags_set = 'quota_cores=20,quota_instances=40,quota_ram=102400'
-        flags_reset = 'quota_cores=10,quota_instances=20,quota_ram=51200'
-
-        services = ['nova-api-ec2', 'nova-api-os-compute', 'nova-objectstore',
-                    'nova-cert', 'nova-scheduler', 'nova-conductor']
-        self.d.configure('nova-cloud-controller', {'config-flags': flags_set})
-        pgrep_full = True
-
-        time = 20
-        conf = '/etc/nova/nova.conf'
-        for s in services:
-            if not u.service_restarted(self.nova_cc_sentry, s, conf,
-                                       pgrep_full=True, sleep_time=time):
-                self.d.configure('nova-cloud-controller',
-                                 {'config-flags': flags_reset})
-                msg = "service {} didn't restart after config change".format(s)
-                amulet.raise_status(amulet.FAIL, msg=msg)
-            time = 0
-
-        self.d.configure('nova-cloud-controller', {'config-flags': flags_reset})
-
-    def test_nova_default_config(self):
         """Verify the data in the nova config file's default section."""
         # NOTE(coreycb): Currently no way to test on essex because config file
         #                has no section headers.
         if self._get_openstack_release() == self.precise_essex:
             return
 
+        u.log.debug('Checking nova config file data...')
         unit = self.nova_cc_sentry
         conf = '/etc/nova/nova.conf'
-        rabbitmq_relation = self.rabbitmq_sentry.relation('amqp',
-                                                   'nova-cloud-controller:amqp')
-        glance_relation = self.glance_sentry.relation('image-service',
-                                          'nova-cloud-controller:image-service')
-        keystone_ep = self.keystone_demo.service_catalog.url_for(\
-                                                      service_type='identity',
-                                                      endpoint_type='publicURL')
-        keystone_ec2 = "{}/ec2tokens".format(keystone_ep)
 
-        keystone_relation = self.keystone_sentry.relation('identity-service',
+        rmq_ncc_rel = self.rabbitmq_sentry.relation(
-                                       'nova-cloud-controller:identity-service')
+            'amqp', 'nova-cloud-controller:amqp')
-        keystone_uri = "http://{}:{}/".format(keystone_relation['service_host'],
+
-                                              keystone_relation['service_port'])
+        gl_ncc_rel = self.glance_sentry.relation(
-        identity_uri = "{}://{}:{}/".format(keystone_relation['auth_protocol'],
+            'image-service', 'nova-cloud-controller:image-service')
-                                            keystone_relation['service_host'],
+
-                                            keystone_relation['auth_port'])
+        ks_ep = self.keystone_demo.service_catalog.url_for(
+            service_type='identity', endpoint_type='publicURL')
+
+        ks_ec2 = "{}/ec2tokens".format(ks_ep)
+
+        ks_ncc_rel = self.keystone_sentry.relation(
+            'identity-service', 'nova-cloud-controller:identity-service')
+
+        ks_uri = "http://{}:{}/".format(ks_ncc_rel['service_host'],
+                                        ks_ncc_rel['service_port'])
+
+        id_uri = "{}://{}:{}/".format(ks_ncc_rel['auth_protocol'],
+                                      ks_ncc_rel['service_host'],
+                                      ks_ncc_rel['auth_port'])
+
+        db_ncc_rel = self.mysql_sentry.relation(
+            'shared-db', 'nova-cloud-controller:shared-db')
 
-        mysql_relation = self.mysql_sentry.relation('shared-db',
-                                              'nova-cloud-controller:shared-db')
         db_uri = "mysql://{}:{}@{}/{}".format('nova',
-                                              mysql_relation['nova_password'],
+                                              db_ncc_rel['nova_password'],
-                                              mysql_relation['db_host'],
+                                              db_ncc_rel['db_host'],
                                               'nova')
 
         expected = {
@@ -523,7 +547,7 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
                 'enabled_apis': 'ec2,osapi_compute,metadata',
                 'auth_strategy': 'keystone',
                 'compute_driver': 'libvirt.LibvirtDriver',
-                'keystone_ec2_url': keystone_ec2,
+                'keystone_ec2_url': ks_ec2,
                 'network_manager': 'nova.network.manager.FlatDHCPManager',
                 's3_listen_port': '3323',
                 'osapi_compute_listen_port': '8764',
@@ -542,13 +566,13 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
                 }
                 keystone_authtoken = {
                     'keystone_authtoken': {
-                        'auth_uri': keystone_uri,
+                        'auth_uri': ks_uri,
-                        'auth_host': keystone_relation['service_host'],
+                        'auth_host': ks_ncc_rel['service_host'],
-                        'auth_port': keystone_relation['auth_port'],
+                        'auth_port': ks_ncc_rel['auth_port'],
-                        'auth_protocol': keystone_relation['auth_protocol'],
+                        'auth_protocol': ks_ncc_rel['auth_protocol'],
-                        'admin_tenant_name': keystone_relation['service_tenant'],
+                        'admin_tenant_name': ks_ncc_rel['service_tenant'],
-                        'admin_user': keystone_relation['service_username'],
+                        'admin_user': ks_ncc_rel['service_username'],
-                        'admin_password': keystone_relation['service_password'],
+                        'admin_password': ks_ncc_rel['service_password'],
                     }
                 }
                 expected.update(database)
@@ -558,9 +582,9 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             expected[d]['compute_driver'] = 'libvirt.LibvirtDriver'
             expected[d]['rabbit_userid'] = 'nova'
             expected[d]['rabbit_virtual_host'] = 'openstack'
-            expected[d]['rabbit_password'] = rabbitmq_relation['password']
+            expected[d]['rabbit_password'] = rmq_ncc_rel['password']
-            expected[d]['rabbit_host'] = rabbitmq_relation['hostname']
+            expected[d]['rabbit_host'] = rmq_ncc_rel['hostname']
-            expected[d]['glance_api_servers'] = glance_relation['glance-api-server']
+            expected[d]['glance_api_servers'] = gl_ncc_rel['glance-api-server']
 
         else:
             database = {
@@ -571,16 +595,16 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             }
             glance = {
                 'glance': {
-                    'api_servers': glance_relation['glance-api-server'],
+                    'api_servers': gl_ncc_rel['glance-api-server'],
                 }
             }
             keystone_authtoken = {
                 'keystone_authtoken': {
-                    'identity_uri': identity_uri,
+                    'identity_uri': id_uri,
-                    'auth_uri': keystone_uri,
+                    'auth_uri': ks_uri,
-                    'admin_tenant_name': keystone_relation['service_tenant'],
+                    'admin_tenant_name': ks_ncc_rel['service_tenant'],
-                    'admin_user': keystone_relation['service_username'],
+                    'admin_user': ks_ncc_rel['service_username'],
-                    'admin_password': keystone_relation['service_password'],
+                    'admin_password': ks_ncc_rel['service_password'],
                     'signing_dir': '/var/cache/nova',
                 }
             }
@@ -598,8 +622,8 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
                 'oslo_messaging_rabbit': {
                     'rabbit_userid': 'nova',
                     'rabbit_virtual_host': 'openstack',
-                    'rabbit_password': rabbitmq_relation['password'],
+                    'rabbit_password': rmq_ncc_rel['password'],
-                    'rabbit_host': rabbitmq_relation['hostname'],
+                    'rabbit_host': rmq_ncc_rel['hostname'],
                 }
             }
             oslo_concurrency = {
@@ -621,15 +645,41 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
                 message = "nova config error: {}".format(ret)
                 amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_image_instance_create(self):
+    def test_302_api_rate_limiting_is_enabled_for_icehouse_or_more(self):
+        """
+        The API rate limiting is enabled for icehouse or more. Otherwise the
+        api-paste.ini file is left untouched.
+        """
+        u.log.debug('Checking api-paste config file data...')
+
+        unit = self.nova_cc_sentry
+        conf = '/etc/nova/api-paste.ini'
+        section = "filter:ratelimit"
+        factory = ("nova.api.openstack.compute.limits:RateLimitingMiddleware"
+                   ".factory")
+
+        if self._get_openstack_release() >= self.precise_icehouse:
+            expected = {"paste.filter_factory": factory,
+                        "limits": "( POST, '*', .*, 9999, MINUTE );"}
+        else:
+            expected = {"paste.filter_factory": factory}
+
+        ret = u.validate_config_data(unit, conf, section, expected)
+        if ret:
+            message = "api paste config error: {}".format(ret)
+            amulet.raise_status(amulet.FAIL, msg=message)
+
+    def test_400_image_instance_create(self):
         """Create an image/instance, verify they exist, and delete them."""
         # NOTE(coreycb): Skipping failing test on essex until resolved. essex
-        #                nova API calls are getting "Malformed request url (HTTP
+        #                nova API calls are getting "Malformed request url
-        #                400)".
+        #                (HTTP 400)".
         if self._get_openstack_release() == self.precise_essex:
-            u.log.error("Skipping failing test until resolved")
+            u.log.error("Skipping test (due to Essex)")
             return
 
+        u.log.debug('Checking nova instance creation...')
+
         image = u.create_cirros_image(self.glance, "cirros-image")
         if not image:
             amulet.raise_status(amulet.FAIL, msg="Image create failed")
@@ -651,27 +701,56 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = "nova cirros instance does not exist"
             amulet.raise_status(amulet.FAIL, msg=message)
 
-        u.delete_image(self.glance, image)
+        u.delete_resource(self.glance.images, image.id,
-        u.delete_instance(self.nova_demo, instance)
+                          msg="glance image")
 
-    def test_api_rate_limiting_is_enabled_for_icehouse_or_more(self):
+        u.delete_resource(self.nova_demo.servers, instance.id,
-        """
+                          msg="nova instance")
-        The API rate limiting is enabled for icehouse or more. Otherwise the
-        api-paste.ini file is left untouched.
-        """
-        unit = self.nova_cc_sentry
-        conf = '/etc/nova/api-paste.ini'
-        section = "filter:ratelimit"
-        factory = ("nova.api.openstack.compute.limits:RateLimitingMiddleware"
-                   ".factory")
 
-        if self._get_openstack_release() >= self.precise_icehouse:
+    def test_900_restart_on_config_change(self):
-            expected = {"paste.filter_factory": factory,
+        """Verify that the specified services are restarted when the config
-                        "limits": "( POST, '*', .*, 9999, MINUTE );"}
+           is changed."""
-        else:
+        if self._get_openstack_release() == self.precise_essex:
-            expected = {"paste.filter_factory": factory}
+            u.log.error("Skipping test (due to Essex)")
+            return
 
-        ret = u.validate_config_data(unit, conf, section, expected)
+        u.log.info('Checking that conf files and system services respond '
-        if ret:
+                   'to a charm config change...')
-            message = "api paste config error: {}".format(ret)
+
-            amulet.raise_status(amulet.FAIL, msg=message)
+        sentry = self.nova_cc_sentry
+        juju_service = 'nova-cloud-controller'
+
+        # Process names, corresponding conf files
+        conf_file = '/etc/nova/nova.conf'
+        services = {
+            'nova-api-ec2': conf_file,
+            'nova-api-os-compute': conf_file,
+            'nova-objectstore': conf_file,
+            'nova-cert': conf_file,
+            'nova-scheduler': conf_file,
+            'nova-conductor': conf_file
+        }
+
+        # Expected default and alternate values
+        flags_default = 'quota_cores=20,quota_instances=40,quota_ram=102400'
+        flags_alt = 'quota_cores=10,quota_instances=20,quota_ram=51200'
+        set_default = {'config-flags': flags_default}
+        set_alternate = {'config-flags': flags_alt}
+
+        # Make config change, check for service restarts
+        u.log.debug('Making config change on {}...'.format(juju_service))
+        mtime = u.get_sentry_time(sentry)
+        self.d.configure(juju_service, set_alternate)
+
+        sleep_time = 60
+        for s, conf_file in services.iteritems():
+            u.log.debug("Checking that service restarted: {}".format(s))
+            if not u.validate_service_config_changed(sentry, mtime, s,
+                                                     conf_file,
+                                                     sleep_time=sleep_time):
+                self.d.configure(juju_service, set_default)
+                msg = "service {} didn't restart after config change".format(s)
+                amulet.raise_status(amulet.FAIL, msg=msg)
+            sleep_time = 0
+
+        self.d.configure(juju_service, set_default)

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import re
 import six
 from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
@@ -114,6 +115,45 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         for service, config in six.iteritems(configs):
             self.d.configure(service, config)
 
+    def _auto_wait_for_status(self, message=None, exclude_services=None,
+                              timeout=1800):
+        """Wait for all units to have a specific extended status, except
+        for any defined as excluded.  Unless specified via message, any
+        status containing any case of 'ready' will be considered a match.
+
+        Examples of message usage:
+
+          Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
+              message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
+
+          Wait for all units to reach this status (exact match):
+              message = 'Unit is ready'
+
+          Wait for all units to reach any one of these (exact match):
+              message = re.compile('Unit is ready|OK|Ready')
+
+          Wait for at least one unit to reach this status (exact match):
+              message = {'ready'}
+
+        See Amulet's sentry.wait_for_messages() for message usage detail.
+        https://github.com/juju/amulet/blob/master/amulet/sentry.py
+
+        :param message: Expected status match
+        :param exclude_services: List of juju service names to ignore
+        :param timeout: Maximum time in seconds to wait for status match
+        :returns: None.  Raises if timeout is hit.
+        """
+
+        if not message:
+            message = re.compile('.*ready.*', re.IGNORECASE)
+
+        if not exclude_services:
+            exclude_services = []
+
+        services = list(set(self.d.services.keys()) - set(exclude_services))
+        service_messages = {service: message for service in services}
+        self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+
     def _get_openstack_release(self):
         """Get openstack release.