From e891107b9d4e797c3d054d392fe26f39ac9c07c0 Mon Sep 17 00:00:00 2001
From: Ryan Beisner <ryan.beisner@canonical.com>
Date: Wed, 7 Oct 2015 17:39:14 +0000
Subject: [PATCH 01/10] sync tests/charmhelpers

---
 tests/charmhelpers/contrib/openstack/amulet/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py
index b1397419..2b3087ea 100644
--- a/tests/charmhelpers/contrib/openstack/amulet/utils.py
+++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py
@@ -752,7 +752,7 @@ class OpenStackAmuletUtils(AmuletUtils):
             self.log.debug('SSL is enabled  @{}:{} '
                            '({})'.format(host, port, unit_name))
             return True
-        elif not port and not conf_ssl:
+        elif not conf_ssl:
             self.log.debug('SSL not enabled @{}:{} '
                            '({})'.format(host, port, unit_name))
             return False

From 3e90a323fb6a5915ce87372c2cff70dc6f65197e Mon Sep 17 00:00:00 2001
From: Ryan Beisner <ryan.beisner@canonical.com>
Date: Wed, 7 Oct 2015 17:59:55 +0000
Subject: [PATCH 02/10] clean up really old lint

---
 tests/basic_deployment.py | 183 ++++++++++++++++++++++----------------
 1 file changed, 108 insertions(+), 75 deletions(-)

diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index fb43161c..1708bcc0 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -1,5 +1,3 @@
-#!/usr/bin/python
-
 import amulet
 import os
 import yaml
@@ -10,8 +8,8 @@ from charmhelpers.contrib.openstack.amulet.deployment import (
 
 from charmhelpers.contrib.openstack.amulet.utils import (
     OpenStackAmuletUtils,
-    DEBUG, # flake8: noqa
-    ERROR
+    DEBUG,
+    # ERROR
 )
 
 # Use DEBUG to turn on debug logging
@@ -24,7 +22,8 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
     def __init__(self, series=None, openstack=None, source=None, git=False,
                  stable=False):
         """Deploy the entire test environment."""
-        super(NovaCCBasicDeployment, self).__init__(series, openstack, source, stable)
+        super(NovaCCBasicDeployment, self).__init__(series, openstack,
+                                                    source, stable)
         self.git = git
         self._add_services()
         self._add_relations()
@@ -40,27 +39,31 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
            compatible with the local charm (e.g. stable or next).
            """
         this_service = {'name': 'nova-cloud-controller'}
-        other_services = [{'name': 'mysql'}, {'name': 'rabbitmq-server'},
+        other_services = [{'name': 'mysql'},
+                          {'name': 'rabbitmq-server'},
                           {'name': 'nova-compute', 'units': 2},
-                          {'name': 'keystone'}, {'name': 'glance'}]
+                          {'name': 'keystone'},
+                          {'name': 'glance'}]
         super(NovaCCBasicDeployment, self)._add_services(this_service,
                                                          other_services)
 
     def _add_relations(self):
         """Add all of the relations for the services."""
         relations = {
-          'nova-cloud-controller:shared-db': 'mysql:shared-db',
-          'nova-cloud-controller:identity-service': 'keystone:identity-service',
-          'nova-cloud-controller:amqp': 'rabbitmq-server:amqp',
-          'nova-cloud-controller:cloud-compute': 'nova-compute:cloud-compute',
-          'nova-cloud-controller:image-service': 'glance:image-service',
-          'nova-compute:image-service': 'glance:image-service',
-          'nova-compute:shared-db': 'mysql:shared-db',
-          'nova-compute:amqp': 'rabbitmq-server:amqp',
-          'keystone:shared-db': 'mysql:shared-db',
-          'glance:identity-service': 'keystone:identity-service',
-          'glance:shared-db': 'mysql:shared-db',
-          'glance:amqp': 'rabbitmq-server:amqp'
+            'nova-cloud-controller:shared-db': 'mysql:shared-db',
+            'nova-cloud-controller:identity-service': 'keystone:'
+                                                      'identity-service',
+            'nova-cloud-controller:amqp': 'rabbitmq-server:amqp',
+            'nova-cloud-controller:cloud-compute': 'nova-compute:'
+                                                   'cloud-compute',
+            'nova-cloud-controller:image-service': 'glance:image-service',
+            'nova-compute:image-service': 'glance:image-service',
+            'nova-compute:shared-db': 'mysql:shared-db',
+            'nova-compute:amqp': 'rabbitmq-server:amqp',
+            'keystone:shared-db': 'mysql:shared-db',
+            'glance:identity-service': 'keystone:identity-service',
+            'glance:shared-db': 'mysql:shared-db',
+            'glance:amqp': 'rabbitmq-server:amqp'
         }
         super(NovaCCBasicDeployment, self)._add_relations(relations)
 
@@ -98,16 +101,24 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
                 'http_proxy': amulet_http_proxy,
                 'https_proxy': amulet_http_proxy,
             }
-            nova_cc_config['openstack-origin-git'] = yaml.dump(openstack_origin_git)
-            nova_config['openstack-origin-git'] = yaml.dump(openstack_origin_git)
+
+            nova_cc_config['openstack-origin-git'] = \
+                yaml.dump(openstack_origin_git)
+
+            nova_config['openstack-origin-git'] = \
+                yaml.dump(openstack_origin_git)
 
         # Add some rate-limiting options to the charm. These will noop before
         # icehouse.
-        nova_cc_config['api-rate-limit-rules'] = "( POST, '*', .*, 9999, MINUTE );"
+        nova_cc_config['api-rate-limit-rules'] = \
+            "( POST, '*', .*, 9999, MINUTE );"
+
         keystone_config = {'admin-password': 'openstack',
                            'admin-token': 'ubuntutesting'}
+
         configs = {'nova-cloud-controller': nova_cc_config,
                    'keystone': keystone_config, 'nova-compute': nova_config}
+
         super(NovaCCBasicDeployment, self)._configure_services(configs)
 
     def _initialize_tests(self):
@@ -207,12 +218,15 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
         """Verify the openstack compute api (osapi) endpoint data."""
         endpoints = self.keystone.endpoints.list()
         admin_port = internal_port = public_port = '8774'
-        expected = {'id': u.not_null,
-                    'region': 'RegionOne',
-                    'adminurl': u.valid_url,
-                    'internalurl': u.valid_url,
-                    'publicurl': u.valid_url,
-                    'service_id': u.not_null}
+
+        expected = {
+            'id': u.not_null,
+            'region': 'RegionOne',
+            'adminurl': u.valid_url,
+            'internalurl': u.valid_url,
+            'publicurl': u.valid_url,
+            'service_id': u.not_null
+        }
 
         ret = u.validate_endpoint_data(endpoints, admin_port, internal_port,
                                        public_port, expected)
@@ -227,12 +241,15 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
 
         endpoints = self.keystone.endpoints.list()
         admin_port = internal_port = public_port = '8773'
-        expected = {'id': u.not_null,
-                    'region': 'RegionOne',
-                    'adminurl': u.valid_url,
-                    'internalurl': u.valid_url,
-                    'publicurl': u.valid_url,
-                    'service_id': u.not_null}
+
+        expected = {
+            'id': u.not_null,
+            'region': 'RegionOne',
+            'adminurl': u.valid_url,
+            'internalurl': u.valid_url,
+            'publicurl': u.valid_url,
+            'service_id': u.not_null
+        }
 
         ret = u.validate_endpoint_data(endpoints, admin_port, internal_port,
                                        public_port, expected)
@@ -247,12 +264,14 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
 
         endpoints = self.keystone.endpoints.list()
         admin_port = internal_port = public_port = '3333'
-        expected = {'id': u.not_null,
-                    'region': 'RegionOne',
-                    'adminurl': u.valid_url,
-                    'internalurl': u.valid_url,
-                    'publicurl': u.valid_url,
-                    'service_id': u.not_null}
+        expected = {
+            'id': u.not_null,
+            'region': 'RegionOne',
+            'adminurl': u.valid_url,
+            'internalurl': u.valid_url,
+            'publicurl': u.valid_url,
+            'service_id': u.not_null
+        }
 
         ret = u.validate_endpoint_data(endpoints, admin_port, internal_port,
                                        public_port, expected)
@@ -264,6 +283,7 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
         """Verify the nova-cc to mysql shared-db relation data"""
         unit = self.nova_cc_sentry
         relation = ['shared-db', 'mysql:shared-db']
+
         expected = {
             'private-address': u.valid_ip,
             'nova_database': 'nova',
@@ -453,10 +473,16 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
         flags_set = 'quota_cores=20,quota_instances=40,quota_ram=102400'
         flags_reset = 'quota_cores=10,quota_instances=20,quota_ram=51200'
 
-        services = ['nova-api-ec2', 'nova-api-os-compute', 'nova-objectstore',
-                    'nova-cert', 'nova-scheduler', 'nova-conductor']
+        services = [
+            'nova-api-ec2',
+            'nova-api-os-compute',
+            'nova-objectstore',
+            'nova-cert',
+            'nova-scheduler',
+            'nova-conductor'
+        ]
+
         self.d.configure('nova-cloud-controller', {'config-flags': flags_set})
-        pgrep_full = True
 
         time = 20
         conf = '/etc/nova/nova.conf'
@@ -469,7 +495,8 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
                 amulet.raise_status(amulet.FAIL, msg=msg)
             time = 0
 
-        self.d.configure('nova-cloud-controller', {'config-flags': flags_reset})
+        self.d.configure('nova-cloud-controller',
+                         {'config-flags': flags_reset})
 
     def test_nova_default_config(self):
         """Verify the data in the nova config file's default section."""
@@ -480,28 +507,34 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
 
         unit = self.nova_cc_sentry
         conf = '/etc/nova/nova.conf'
-        rabbitmq_relation = self.rabbitmq_sentry.relation('amqp',
-                                                   'nova-cloud-controller:amqp')
-        glance_relation = self.glance_sentry.relation('image-service',
-                                          'nova-cloud-controller:image-service')
-        keystone_ep = self.keystone_demo.service_catalog.url_for(\
-                                                      service_type='identity',
-                                                      endpoint_type='publicURL')
+
+        rabbitmq_relation = self.rabbitmq_sentry.relation(
+            'amqp', 'nova-cloud-controller:amqp')
+
+        gl_ncc_rel = self.glance_sentry.relation(
+            'image-service', 'nova-cloud-controller:image-service')
+
+        keystone_ep = self.keystone_demo.service_catalog.url_for(
+            service_type='identity', endpoint_type='publicURL')
+
         keystone_ec2 = "{}/ec2tokens".format(keystone_ep)
 
-        keystone_relation = self.keystone_sentry.relation('identity-service',
-                                       'nova-cloud-controller:identity-service')
-        keystone_uri = "http://{}:{}/".format(keystone_relation['service_host'],
-                                              keystone_relation['service_port'])
-        identity_uri = "{}://{}:{}/".format(keystone_relation['auth_protocol'],
-                                            keystone_relation['service_host'],
-                                            keystone_relation['auth_port'])
+        ks_ncc_rel = self.keystone_sentry.relation(
+            'identity-service', 'nova-cloud-controller:identity-service')
+
+        keystone_uri = "http://{}:{}/".format(ks_ncc_rel['service_host'],
+                                              ks_ncc_rel['service_port'])
+
+        identity_uri = "{}://{}:{}/".format(ks_ncc_rel['auth_protocol'],
+                                            ks_ncc_rel['service_host'],
+                                            ks_ncc_rel['auth_port'])
+
+        db_ncc_rel = self.mysql_sentry.relation(
+            'shared-db', 'nova-cloud-controller:shared-db')
 
-        mysql_relation = self.mysql_sentry.relation('shared-db',
-                                              'nova-cloud-controller:shared-db')
         db_uri = "mysql://{}:{}@{}/{}".format('nova',
-                                              mysql_relation['nova_password'],
-                                              mysql_relation['db_host'],
+                                              db_ncc_rel['nova_password'],
+                                              db_ncc_rel['db_host'],
                                               'nova')
 
         expected = {
@@ -543,12 +576,12 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
                 keystone_authtoken = {
                     'keystone_authtoken': {
                         'auth_uri': keystone_uri,
-                        'auth_host': keystone_relation['service_host'],
-                        'auth_port': keystone_relation['auth_port'],
-                        'auth_protocol': keystone_relation['auth_protocol'],
-                        'admin_tenant_name': keystone_relation['service_tenant'],
-                        'admin_user': keystone_relation['service_username'],
-                        'admin_password': keystone_relation['service_password'],
+                        'auth_host': ks_ncc_rel['service_host'],
+                        'auth_port': ks_ncc_rel['auth_port'],
+                        'auth_protocol': ks_ncc_rel['auth_protocol'],
+                        'admin_tenant_name': ks_ncc_rel['service_tenant'],
+                        'admin_user': ks_ncc_rel['service_username'],
+                        'admin_password': ks_ncc_rel['service_password'],
                     }
                 }
                 expected.update(database)
@@ -560,7 +593,7 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             expected[d]['rabbit_virtual_host'] = 'openstack'
             expected[d]['rabbit_password'] = rabbitmq_relation['password']
             expected[d]['rabbit_host'] = rabbitmq_relation['hostname']
-            expected[d]['glance_api_servers'] = glance_relation['glance-api-server']
+            expected[d]['glance_api_servers'] = gl_ncc_rel['glance-api-server']
 
         else:
             database = {
@@ -571,16 +604,16 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             }
             glance = {
                 'glance': {
-                    'api_servers': glance_relation['glance-api-server'],
+                    'api_servers': gl_ncc_rel['glance-api-server'],
                 }
             }
             keystone_authtoken = {
                 'keystone_authtoken': {
                     'identity_uri': identity_uri,
                     'auth_uri': keystone_uri,
-                    'admin_tenant_name': keystone_relation['service_tenant'],
-                    'admin_user': keystone_relation['service_username'],
-                    'admin_password': keystone_relation['service_password'],
+                    'admin_tenant_name': ks_ncc_rel['service_tenant'],
+                    'admin_user': ks_ncc_rel['service_username'],
+                    'admin_password': ks_ncc_rel['service_password'],
                     'signing_dir': '/var/cache/nova',
                 }
             }
@@ -624,8 +657,8 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
     def test_image_instance_create(self):
         """Create an image/instance, verify they exist, and delete them."""
         # NOTE(coreycb): Skipping failing test on essex until resolved. essex
-        #                nova API calls are getting "Malformed request url (HTTP
-        #                400)".
+        #                nova API calls are getting "Malformed request url
+        #                (HTTP 400)".
         if self._get_openstack_release() == self.precise_essex:
             u.log.error("Skipping failing test until resolved")
             return

From b76c87f18118ea8d1855f96e1bd0655bd86164d5 Mon Sep 17 00:00:00 2001
From: Ryan Beisner <ryan.beisner@canonical.com>
Date: Wed, 7 Oct 2015 18:03:38 +0000
Subject: [PATCH 03/10] prep liberty amulet tests, update readme

---
 tests/019-basic-vivid-kilo     |  0
 tests/020-basic-trusty-liberty | 12 ++++++++++++
 tests/021-basic-wily-liberty   | 10 ++++++++++
 tests/README                   | 10 ++++++++++
 4 files changed, 32 insertions(+)
 mode change 100644 => 100755 tests/019-basic-vivid-kilo
 create mode 100755 tests/020-basic-trusty-liberty
 create mode 100755 tests/021-basic-wily-liberty

diff --git a/tests/019-basic-vivid-kilo b/tests/019-basic-vivid-kilo
old mode 100644
new mode 100755
diff --git a/tests/020-basic-trusty-liberty b/tests/020-basic-trusty-liberty
new file mode 100755
index 00000000..1ff0565e
--- /dev/null
+++ b/tests/020-basic-trusty-liberty
@@ -0,0 +1,12 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic nova cloud controller deployment on
+   trusty-liberty."""
+
+from basic_deployment import NovaCCBasicDeployment
+
+if __name__ == '__main__':
+    deployment = NovaCCBasicDeployment(series='trusty',
+                                       openstack='cloud:trusty-liberty',
+                                       source='cloud:trusty-updates/liberty')
+    deployment.run_tests()
diff --git a/tests/021-basic-wily-liberty b/tests/021-basic-wily-liberty
new file mode 100755
index 00000000..9178ba08
--- /dev/null
+++ b/tests/021-basic-wily-liberty
@@ -0,0 +1,10 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic nova cloud controller deployment on
+   wily-liberty."""
+
+from basic_deployment import NovaCCBasicDeployment
+
+if __name__ == '__main__':
+    deployment = NovaCCBasicDeployment(series='wily')
+    deployment.run_tests()
diff --git a/tests/README b/tests/README
index 0666bf93..9de85852 100644
--- a/tests/README
+++ b/tests/README
@@ -1,6 +1,16 @@
 This directory provides Amulet tests that focus on verification of Nova Cloud
 Controller deployments.
 
+test_* methods are called in lexical sort order, although each individual test
+should be idempotent, and expected to pass regardless of run order.
+
+Test name convention to ensure desired test order:
+    1xx service and endpoint checks
+    2xx relation checks
+    3xx config checks
+    4xx functional checks
+    9xx restarts and other final checks
+
 In order to run tests, you'll need charm-tools installed (in addition to
 juju, of course):
     sudo add-apt-repository ppa:juju/stable

From f60578ba3112367d7c1465d1b7c5b70e9d20813b Mon Sep 17 00:00:00 2001
From: Ryan Beisner <ryan.beisner@canonical.com>
Date: Wed, 7 Oct 2015 18:42:13 +0000
Subject: [PATCH 04/10] add debug logging to tests

---
 tests/basic_deployment.py | 185 ++++++++++++++++++++++----------------
 1 file changed, 107 insertions(+), 78 deletions(-)

diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index 1708bcc0..fe55a49c 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -166,9 +166,10 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
                                                   password='password',
                                                   tenant=self.demo_tenant)
 
-    def test_services(self):
+    def test_100_services(self):
         """Verify the expected services are running on the corresponding
            service units."""
+        u.log.debug('Checking system services on units...')
         commands = {
             self.mysql_sentry: ['status mysql'],
             self.rabbitmq_sentry: ['sudo service rabbitmq-server status'],
@@ -190,8 +191,9 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
         if ret:
             amulet.raise_status(amulet.FAIL, msg=ret)
 
-    def test_service_catalog(self):
+    def test_102_service_catalog(self):
         """Verify that the service catalog endpoint data is valid."""
+        u.log.debug('Checking keystone service catalog...')
         endpoint_vol = {'adminURL': u.valid_url,
                         'region': 'RegionOne',
                         'publicURL': u.valid_url,
@@ -214,8 +216,10 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
         if ret:
             amulet.raise_status(amulet.FAIL, msg=ret)
 
-    def test_openstack_compute_api_endpoint(self):
+    def test_104_openstack_compute_api_endpoint(self):
         """Verify the openstack compute api (osapi) endpoint data."""
+        u.log.debug('Checking compute endpoint data...')
+
         endpoints = self.keystone.endpoints.list()
         admin_port = internal_port = public_port = '8774'
 
@@ -234,11 +238,12 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = 'osapi endpoint: {}'.format(ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_ec2_api_endpoint(self):
+    def test_106_ec2_api_endpoint(self):
         """Verify the EC2 api endpoint data."""
         if self._get_openstack_release() >= self.trusty_kilo:
             return
 
+        u.log.debug('Checking ec2 endpoint data...')
         endpoints = self.keystone.endpoints.list()
         admin_port = internal_port = public_port = '8773'
 
@@ -257,11 +262,12 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = 'EC2 endpoint: {}'.format(ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_s3_api_endpoint(self):
+    def test_108_s3_api_endpoint(self):
         """Verify the S3 api endpoint data."""
         if self._get_openstack_release() >= self.trusty_kilo:
             return
 
+        u.log.debug('Checking s3 endpoint data...')
         endpoints = self.keystone.endpoints.list()
         admin_port = internal_port = public_port = '3333'
         expected = {
@@ -279,8 +285,9 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = 'S3 endpoint: {}'.format(ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_nova_cc_shared_db_relation(self):
+    def test_200_nova_cc_shared_db_relation(self):
         """Verify the nova-cc to mysql shared-db relation data"""
+        u.log.debug('Checking n-c-c:mysql db relation data...')
         unit = self.nova_cc_sentry
         relation = ['shared-db', 'mysql:shared-db']
 
@@ -296,8 +303,9 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('nova-cc shared-db', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_mysql_shared_db_relation(self):
+    def test_202_mysql_shared_db_relation(self):
         """Verify the mysql to nova-cc shared-db relation data"""
+        u.log.debug('Checking mysql:n-c-c db relation data...')
         unit = self.mysql_sentry
         relation = ['shared-db', 'nova-cloud-controller:shared-db']
         expected = {
@@ -311,8 +319,9 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('mysql shared-db', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_nova_cc_identity_service_relation(self):
+    def test_204_nova_cc_identity_service_relation(self):
         """Verify the nova-cc to keystone identity-service relation data"""
+        u.log.debug('Checking n-c-c:keystone identity relation data...')
         unit = self.nova_cc_sentry
         relation = ['identity-service', 'keystone:identity-service']
         expected = {
@@ -340,8 +349,9 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('nova-cc identity-service', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_keystone_identity_service_relation(self):
+    def test_206_keystone_identity_service_relation(self):
         """Verify the keystone to nova-cc identity-service relation data"""
+        u.log.debug('Checking keystone:n-c-c identity relation data...')
         unit = self.keystone_sentry
         relation = ['identity-service',
                     'nova-cloud-controller:identity-service']
@@ -367,8 +377,9 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('keystone identity-service', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_nova_cc_amqp_relation(self):
+    def test_208_nova_cc_amqp_relation(self):
         """Verify the nova-cc to rabbitmq-server amqp relation data"""
+        u.log.debug('Checking n-c-c:rmq amqp relation data...')
         unit = self.nova_cc_sentry
         relation = ['amqp', 'rabbitmq-server:amqp']
         expected = {
@@ -382,8 +393,9 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('nova-cc amqp', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_rabbitmq_amqp_relation(self):
+    def test_210_rabbitmq_amqp_relation(self):
         """Verify the rabbitmq-server to nova-cc amqp relation data"""
+        u.log.debug('Checking rmq:n-c-c amqp relation data...')
         unit = self.rabbitmq_sentry
         relation = ['amqp', 'nova-cloud-controller:amqp']
         expected = {
@@ -397,8 +409,11 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('rabbitmq amqp', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_nova_cc_cloud_compute_relation(self):
+    def test_212_nova_cc_cloud_compute_relation(self):
         """Verify the nova-cc to nova-compute cloud-compute relation data"""
+        u.log.debug('Checking n-c-c:nova-compute '
+                    'cloud-compute relation data...')
+
         unit = self.nova_cc_sentry
         relation = ['cloud-compute', 'nova-compute:cloud-compute']
         expected = {
@@ -416,8 +431,11 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('nova-cc cloud-compute', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_nova_cloud_compute_relation(self):
+    def test_214_nova_cloud_compute_relation(self):
         """Verify the nova-compute to nova-cc cloud-compute relation data"""
+        u.log.debug('Checking nova-compute:n-c-c '
+                    'cloud-compute relation data...')
+
         unit = self.nova_compute_sentry
         relation = ['cloud-compute', 'nova-cloud-controller:cloud-compute']
         expected = {
@@ -429,8 +447,9 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('nova-compute cloud-compute', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_nova_cc_image_service_relation(self):
+    def test_216_nova_cc_image_service_relation(self):
         """Verify the nova-cc to glance image-service relation data"""
+        u.log.debug('Checking n-c-c:glance image-service relation data...')
         unit = self.nova_cc_sentry
         relation = ['image-service', 'glance:image-service']
         expected = {
@@ -442,8 +461,9 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('nova-cc image-service', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_glance_image_service_relation(self):
+    def test_218_glance_image_service_relation(self):
         """Verify the glance to nova-cc image-service relation data"""
+        u.log.debug('Checking glance:n-c-c image-service relation data...')
         unit = self.glance_sentry
         relation = ['image-service', 'nova-cloud-controller:image-service']
         expected = {
@@ -456,55 +476,14 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('glance image-service', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_z_restart_on_config_change(self):
-        """Verify that the specified services are restarted when the config
-           is changed.
-
-           Note(coreycb): The method name with the _z_ is a little odd
-           but it forces the test to run last.  It just makes things
-           easier because restarting services requires re-authorization.
-           """
-        # NOTE(coreycb): Skipping failing test on essex until resolved.
-        #                config-flags don't take effect on essex.
-        if self._get_openstack_release() == self.precise_essex:
-            u.log.error("Skipping failing test until resolved")
-            return
-
-        flags_set = 'quota_cores=20,quota_instances=40,quota_ram=102400'
-        flags_reset = 'quota_cores=10,quota_instances=20,quota_ram=51200'
-
-        services = [
-            'nova-api-ec2',
-            'nova-api-os-compute',
-            'nova-objectstore',
-            'nova-cert',
-            'nova-scheduler',
-            'nova-conductor'
-        ]
-
-        self.d.configure('nova-cloud-controller', {'config-flags': flags_set})
-
-        time = 20
-        conf = '/etc/nova/nova.conf'
-        for s in services:
-            if not u.service_restarted(self.nova_cc_sentry, s, conf,
-                                       pgrep_full=True, sleep_time=time):
-                self.d.configure('nova-cloud-controller',
-                                 {'config-flags': flags_reset})
-                msg = "service {} didn't restart after config change".format(s)
-                amulet.raise_status(amulet.FAIL, msg=msg)
-            time = 0
-
-        self.d.configure('nova-cloud-controller',
-                         {'config-flags': flags_reset})
-
-    def test_nova_default_config(self):
+    def test_300_nova_default_config(self):
         """Verify the data in the nova config file's default section."""
         # NOTE(coreycb): Currently no way to test on essex because config file
         #                has no section headers.
         if self._get_openstack_release() == self.precise_essex:
             return
 
+        u.log.debug('Checking nova config file data...')
         unit = self.nova_cc_sentry
         conf = '/etc/nova/nova.conf'
 
@@ -654,7 +633,35 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
                 message = "nova config error: {}".format(ret)
                 amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_image_instance_create(self):
+    def test_400_api_checks(self):
+        u.log.debug('Checking basic api functionality...')
+        pass
+
+    def test_302_api_rate_limiting_is_enabled_for_icehouse_or_more(self):
+        """
+        The API rate limiting is enabled for icehouse or more. Otherwise the
+        api-paste.ini file is left untouched.
+        """
+        u.log.debug('Checking api-paste config file data...')
+
+        unit = self.nova_cc_sentry
+        conf = '/etc/nova/api-paste.ini'
+        section = "filter:ratelimit"
+        factory = ("nova.api.openstack.compute.limits:RateLimitingMiddleware"
+                   ".factory")
+
+        if self._get_openstack_release() >= self.precise_icehouse:
+            expected = {"paste.filter_factory": factory,
+                        "limits": "( POST, '*', .*, 9999, MINUTE );"}
+        else:
+            expected = {"paste.filter_factory": factory}
+
+        ret = u.validate_config_data(unit, conf, section, expected)
+        if ret:
+            message = "api paste config error: {}".format(ret)
+            amulet.raise_status(amulet.FAIL, msg=message)
+
+    def test_402_image_instance_create(self):
         """Create an image/instance, verify they exist, and delete them."""
         # NOTE(coreycb): Skipping failing test on essex until resolved. essex
         #                nova API calls are getting "Malformed request url
@@ -663,6 +670,8 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             u.log.error("Skipping failing test until resolved")
             return
 
+        u.log.debug('Checking nova instance creation...')
+
         image = u.create_cirros_image(self.glance, "cirros-image")
         if not image:
             amulet.raise_status(amulet.FAIL, msg="Image create failed")
@@ -687,24 +696,44 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
         u.delete_image(self.glance, image)
         u.delete_instance(self.nova_demo, instance)
 
-    def test_api_rate_limiting_is_enabled_for_icehouse_or_more(self):
-        """
-        The API rate limiting is enabled for icehouse or more. Otherwise the
-        api-paste.ini file is left untouched.
-        """
-        unit = self.nova_cc_sentry
-        conf = '/etc/nova/api-paste.ini'
-        section = "filter:ratelimit"
-        factory = ("nova.api.openstack.compute.limits:RateLimitingMiddleware"
-                   ".factory")
+    def test_900_restart_on_config_change(self):
+        """Verify that the specified services are restarted when the config
+           is changed.
 
-        if self._get_openstack_release() >= self.precise_icehouse:
-            expected = {"paste.filter_factory": factory,
-                        "limits": "( POST, '*', .*, 9999, MINUTE );"}
-        else:
-            expected = {"paste.filter_factory": factory}
+           Note(coreycb): The method name with the _z_ is a little odd
+           but it forces the test to run last.  It just makes things
+           easier because restarting services requires re-authorization.
+           """
+        # NOTE(coreycb): Skipping failing test on essex until resolved.
+        #                config-flags don't take effect on essex.
+        if self._get_openstack_release() == self.precise_essex:
+            u.log.error("Skipping failing test until resolved")
+            return
 
-        ret = u.validate_config_data(unit, conf, section, expected)
-        if ret:
-            message = "api paste config error: {}".format(ret)
-            amulet.raise_status(amulet.FAIL, msg=message)
+        flags_set = 'quota_cores=20,quota_instances=40,quota_ram=102400'
+        flags_reset = 'quota_cores=10,quota_instances=20,quota_ram=51200'
+
+        services = [
+            'nova-api-ec2',
+            'nova-api-os-compute',
+            'nova-objectstore',
+            'nova-cert',
+            'nova-scheduler',
+            'nova-conductor'
+        ]
+
+        self.d.configure('nova-cloud-controller', {'config-flags': flags_set})
+
+        time = 20
+        conf = '/etc/nova/nova.conf'
+        for s in services:
+            if not u.service_restarted(self.nova_cc_sentry, s, conf,
+                                       pgrep_full=True, sleep_time=time):
+                self.d.configure('nova-cloud-controller',
+                                 {'config-flags': flags_reset})
+                msg = "service {} didn't restart after config change".format(s)
+                amulet.raise_status(amulet.FAIL, msg=msg)
+            time = 0
+
+        self.d.configure('nova-cloud-controller',
+                         {'config-flags': flags_reset})

From 9360af4c41ab99cf7e9ef018ceeb51174928c848 Mon Sep 17 00:00:00 2001
From: Ryan Beisner <ryan.beisner@canonical.com>
Date: Thu, 8 Oct 2015 13:35:50 +0000
Subject: [PATCH 05/10] add api check/wait

---
 tests/basic_deployment.py | 202 +++++++++++++++++++++++++-------------
 1 file changed, 135 insertions(+), 67 deletions(-)

diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index fe55a49c..ffcad6c8 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -1,5 +1,6 @@
 import amulet
 import os
+import time
 import yaml
 
 from charmhelpers.contrib.openstack.amulet.deployment import (
@@ -30,6 +31,7 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
         self._configure_services()
         self._deploy()
         self._initialize_tests()
+        self._expect_method_success(self._api_checks, 'API check(s)')
 
     def _add_services(self):
         """Add services
@@ -166,28 +168,89 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
                                                   password='password',
                                                   tenant=self.demo_tenant)
 
+    def _api_checks(self, something=None):
+        """Check basic api functionality for this deployment topology."""
+        u.log.debug('Checking APIs... {}'.format(something))
+
+        # glance api returns a generator whereas other apis return a list
+        assert list(self.glance.images.list()) == []
+        assert self.nova_demo.servers.list() == []
+        assert self.nova_demo.flavors.list() != []
+        assert self.nova_demo.floating_ips.list() == []
+        assert self.nova_demo.keypairs.list() == []
+        assert self.keystone.services.list() != []
+
+    # Will curate this as a local helper in n-c-c and n-c, validate and
+    # adjust as local helpers until ready to propose @ charmhelpers:
+    def _expect_method_success(self, my_method, msg="method check(s)",
+                               my_kwargs=None, min_success=10,
+                               max_wait=300, interval=5):
+        """Call a method/function with exception trapping and retry
+        thresholds.  Expect a minimum number of successful consecutive calls
+        within a maximum total time.  Useful to confirm that the method
+        starts to pass, and continues to pass until the threshold is met,
+        such as waiting for api services to become functional.
+
+        :param my_method: method or function to call
+        :param msg: description of what type of thing is being exercised
+        :param min_success: minimum consecutive successful calls
+        :param max_wait: maximum wait in seconds before raising
+        :param interval: time in seconds to wait between retries
+        :returns: the method's return value if successful, raises otherwise
+        """
+        if not my_kwargs:
+            my_kwargs = {}
+
+        successful_calls = time_elapsed = 0
+        time_start = time.time()
+        while successful_calls < min_success and time_elapsed <= max_wait:
+            time_elapsed = time.time() - time_start
+            try:
+                my_return = my_method(**my_kwargs)
+                successful_calls += 1
+                u.log.debug('Consecutive successful {}: {} ({} required)'
+                            '...'.format(msg, successful_calls,
+                                         min_success))
+            except Exception as e:
+                successful_calls = 0
+                u.log.debug('{} failed after {}s, retrying until {}s...'
+                            '\n{}'.format(msg, time_elapsed, max_wait, e))
+            time.sleep(interval)
+
+        if successful_calls < min_success:
+            raise_msg = ('{} not satisfied with {} successful checks '
+                         '({} consecutive required within '
+                         '{}s)'.format(msg, successful_calls,
+                                       min_success, max_wait))
+            amulet.raise_status(amulet.FAIL, msg=raise_msg)
+        else:
+            u.log.debug('OK')
+            return my_return
+
     def test_100_services(self):
         """Verify the expected services are running on the corresponding
            service units."""
         u.log.debug('Checking system services on units...')
-        commands = {
-            self.mysql_sentry: ['status mysql'],
-            self.rabbitmq_sentry: ['sudo service rabbitmq-server status'],
-            self.nova_cc_sentry: ['status nova-api-ec2',
-                                  'status nova-api-os-compute',
-                                  'status nova-objectstore',
-                                  'status nova-cert',
-                                  'status nova-scheduler'],
-            self.nova_compute_sentry: ['status nova-compute',
-                                       'status nova-network',
-                                       'status nova-api'],
-            self.keystone_sentry: ['status keystone'],
-            self.glance_sentry: ['status glance-registry', 'status glance-api']
+        services = {
+            self.mysql_sentry: ['mysql'],
+            self.rabbitmq_sentry: ['rabbitmq-server'],
+            self.nova_cc_sentry: ['nova-api-ec2',
+                                  'nova-api-os-compute',
+                                  'nova-objectstore',
+                                  'nova-cert',
+                                  'nova-scheduler'],
+            self.nova_compute_sentry: ['nova-compute',
+                                       'nova-network',
+                                       'nova-api'],
+            self.keystone_sentry: ['keystone'],
+            self.glance_sentry: ['glance-registry', 'glance-api']
         }
-        if self._get_openstack_release() >= self.precise_grizzly:
-            commands[self.nova_cc_sentry] = ['status nova-conductor']
 
-        ret = u.validate_services(commands)
+# Erroneous?  Inverted comparison?   Was this for essex?
+#        if self._get_openstack_release() >= self.precise_grizzly:
+#            commands[self.nova_cc_sentry] = ['nova-conductor']
+
+        ret = u.validate_services_by_name(services)
         if ret:
             amulet.raise_status(amulet.FAIL, msg=ret)
 
@@ -202,14 +265,17 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
                        'region': 'RegionOne',
                        'publicURL': u.valid_url,
                        'internalURL': u.valid_url}
+
         if self._get_openstack_release() >= self.precise_folsom:
             endpoint_vol['id'] = u.not_null
             endpoint_id['id'] = u.not_null
+
         if self._get_openstack_release() >= self.trusty_kilo:
             expected = {'compute': [endpoint_vol], 'identity': [endpoint_id]}
         else:
             expected = {'s3': [endpoint_vol], 'compute': [endpoint_vol],
                         'ec2': [endpoint_vol], 'identity': [endpoint_id]}
+
         actual = self.keystone_demo.service_catalog.get_endpoints()
 
         ret = u.validate_svc_catalog_endpoint_data(expected, actual)
@@ -487,26 +553,26 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
         unit = self.nova_cc_sentry
         conf = '/etc/nova/nova.conf'
 
-        rabbitmq_relation = self.rabbitmq_sentry.relation(
+        rmq_ncc_rel = self.rabbitmq_sentry.relation(
             'amqp', 'nova-cloud-controller:amqp')
 
         gl_ncc_rel = self.glance_sentry.relation(
             'image-service', 'nova-cloud-controller:image-service')
 
-        keystone_ep = self.keystone_demo.service_catalog.url_for(
+        ks_ep = self.keystone_demo.service_catalog.url_for(
             service_type='identity', endpoint_type='publicURL')
 
-        keystone_ec2 = "{}/ec2tokens".format(keystone_ep)
+        ks_ec2 = "{}/ec2tokens".format(ks_ep)
 
         ks_ncc_rel = self.keystone_sentry.relation(
             'identity-service', 'nova-cloud-controller:identity-service')
 
-        keystone_uri = "http://{}:{}/".format(ks_ncc_rel['service_host'],
-                                              ks_ncc_rel['service_port'])
+        ks_uri = "http://{}:{}/".format(ks_ncc_rel['service_host'],
+                                        ks_ncc_rel['service_port'])
 
-        identity_uri = "{}://{}:{}/".format(ks_ncc_rel['auth_protocol'],
-                                            ks_ncc_rel['service_host'],
-                                            ks_ncc_rel['auth_port'])
+        id_uri = "{}://{}:{}/".format(ks_ncc_rel['auth_protocol'],
+                                      ks_ncc_rel['service_host'],
+                                      ks_ncc_rel['auth_port'])
 
         db_ncc_rel = self.mysql_sentry.relation(
             'shared-db', 'nova-cloud-controller:shared-db')
@@ -535,7 +601,7 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
                 'enabled_apis': 'ec2,osapi_compute,metadata',
                 'auth_strategy': 'keystone',
                 'compute_driver': 'libvirt.LibvirtDriver',
-                'keystone_ec2_url': keystone_ec2,
+                'keystone_ec2_url': ks_ec2,
                 'network_manager': 'nova.network.manager.FlatDHCPManager',
                 's3_listen_port': '3323',
                 'osapi_compute_listen_port': '8764',
@@ -554,7 +620,7 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
                 }
                 keystone_authtoken = {
                     'keystone_authtoken': {
-                        'auth_uri': keystone_uri,
+                        'auth_uri': ks_uri,
                         'auth_host': ks_ncc_rel['service_host'],
                         'auth_port': ks_ncc_rel['auth_port'],
                         'auth_protocol': ks_ncc_rel['auth_protocol'],
@@ -570,8 +636,8 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             expected[d]['compute_driver'] = 'libvirt.LibvirtDriver'
             expected[d]['rabbit_userid'] = 'nova'
             expected[d]['rabbit_virtual_host'] = 'openstack'
-            expected[d]['rabbit_password'] = rabbitmq_relation['password']
-            expected[d]['rabbit_host'] = rabbitmq_relation['hostname']
+            expected[d]['rabbit_password'] = rmq_ncc_rel['password']
+            expected[d]['rabbit_host'] = rmq_ncc_rel['hostname']
             expected[d]['glance_api_servers'] = gl_ncc_rel['glance-api-server']
 
         else:
@@ -588,8 +654,8 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             }
             keystone_authtoken = {
                 'keystone_authtoken': {
-                    'identity_uri': identity_uri,
-                    'auth_uri': keystone_uri,
+                    'identity_uri': id_uri,
+                    'auth_uri': ks_uri,
                     'admin_tenant_name': ks_ncc_rel['service_tenant'],
                     'admin_user': ks_ncc_rel['service_username'],
                     'admin_password': ks_ncc_rel['service_password'],
@@ -610,8 +676,8 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
                 'oslo_messaging_rabbit': {
                     'rabbit_userid': 'nova',
                     'rabbit_virtual_host': 'openstack',
-                    'rabbit_password': rabbitmq_relation['password'],
-                    'rabbit_host': rabbitmq_relation['hostname'],
+                    'rabbit_password': rmq_ncc_rel['password'],
+                    'rabbit_host': rmq_ncc_rel['hostname'],
                 }
             }
             oslo_concurrency = {
@@ -633,10 +699,6 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
                 message = "nova config error: {}".format(ret)
                 amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_400_api_checks(self):
-        u.log.debug('Checking basic api functionality...')
-        pass
-
     def test_302_api_rate_limiting_is_enabled_for_icehouse_or_more(self):
         """
         The API rate limiting is enabled for icehouse or more. Otherwise the
@@ -667,7 +729,7 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
         #                nova API calls are getting "Malformed request url
         #                (HTTP 400)".
         if self._get_openstack_release() == self.precise_essex:
-            u.log.error("Skipping failing test until resolved")
+            u.log.error("Skipping test (due to Essex)")
             return
 
         u.log.debug('Checking nova instance creation...')
@@ -698,42 +760,48 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
 
     def test_900_restart_on_config_change(self):
         """Verify that the specified services are restarted when the config
-           is changed.
-
-           Note(coreycb): The method name with the _z_ is a little odd
-           but it forces the test to run last.  It just makes things
-           easier because restarting services requires re-authorization.
-           """
-        # NOTE(coreycb): Skipping failing test on essex until resolved.
-        #                config-flags don't take effect on essex.
+           is changed."""
         if self._get_openstack_release() == self.precise_essex:
-            u.log.error("Skipping failing test until resolved")
+            u.log.error("Skipping test (due to Essex)")
             return
 
-        flags_set = 'quota_cores=20,quota_instances=40,quota_ram=102400'
-        flags_reset = 'quota_cores=10,quota_instances=20,quota_ram=51200'
+        u.log.info('Checking that conf files and system services respond '
+                   'to a charm config change...')
 
-        services = [
-            'nova-api-ec2',
-            'nova-api-os-compute',
-            'nova-objectstore',
-            'nova-cert',
-            'nova-scheduler',
-            'nova-conductor'
-        ]
+        sentry = self.nova_cc_sentry
+        juju_service = 'nova-cloud-controller'
 
-        self.d.configure('nova-cloud-controller', {'config-flags': flags_set})
+        # Process names, corresponding conf files
+        conf_file = '/etc/nova/nova.conf'
+        services = {
+            'nova-api-ec2': conf_file,
+            'nova-api-os-compute': conf_file,
+            'nova-objectstore': conf_file,
+            'nova-cert': conf_file,
+            'nova-scheduler': conf_file,
+            'nova-conductor': conf_file
+        }
 
-        time = 20
-        conf = '/etc/nova/nova.conf'
-        for s in services:
-            if not u.service_restarted(self.nova_cc_sentry, s, conf,
-                                       pgrep_full=True, sleep_time=time):
-                self.d.configure('nova-cloud-controller',
-                                 {'config-flags': flags_reset})
+        # Expected default and alternate values
+        flags_default = 'quota_cores=20,quota_instances=40,quota_ram=102400'
+        flags_alt = 'quota_cores=10,quota_instances=20,quota_ram=51200'
+        set_default = {'config-flags': flags_default}
+        set_alternate = {'config-flags': flags_alt}
+
+        # Make config change, check for service restarts
+        u.log.debug('Making config change on {}...'.format(juju_service))
+        mtime = u.get_sentry_time(sentry)
+        self.d.configure(juju_service, set_alternate)
+
+        sleep_time = 60
+        for s, conf_file in services.iteritems():
+            u.log.debug("Checking that service restarted: {}".format(s))
+            if not u.validate_service_config_changed(sentry, mtime, s,
+                                                     conf_file,
+                                                     sleep_time=sleep_time):
+                self.d.configure(juju_service, set_default)
                 msg = "service {} didn't restart after config change".format(s)
                 amulet.raise_status(amulet.FAIL, msg=msg)
-            time = 0
+            sleep_time = 0
 
-        self.d.configure('nova-cloud-controller',
-                         {'config-flags': flags_reset})
+        self.d.configure(juju_service, set_default)

From 59ab56f8587b7227547dfedba84736604bd003d7 Mon Sep 17 00:00:00 2001
From: Ryan Beisner <ryan.beisner@canonical.com>
Date: Wed, 14 Oct 2015 02:27:48 +0000
Subject: [PATCH 06/10] disable failing essex amulet test

---
 tests/010-basic-precise-essex | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100755 => 100644 tests/010-basic-precise-essex

diff --git a/tests/010-basic-precise-essex b/tests/010-basic-precise-essex
old mode 100755
new mode 100644

From 498f049ee055bf2dc76c442476abaf159a5c7a5f Mon Sep 17 00:00:00 2001
From: Ryan Beisner <ryan.beisner@canonical.com>
Date: Wed, 14 Oct 2015 02:33:13 +0000
Subject: [PATCH 07/10] expect nova-conductor service

---
 tests/basic_deployment.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index ffcad6c8..6ff35b43 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -180,7 +180,7 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
         assert self.nova_demo.keypairs.list() == []
         assert self.keystone.services.list() != []
 
-    # Will curate this as a local helper in n-c-c and n-c, validate and
+    # NOTE(beisner):  Will curate this as a local helper, validate and
     # adjust as local helpers until ready to propose @ charmhelpers:
     def _expect_method_success(self, my_method, msg="method check(s)",
                                my_kwargs=None, min_success=10,
@@ -236,6 +236,7 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             self.rabbitmq_sentry: ['rabbitmq-server'],
             self.nova_cc_sentry: ['nova-api-ec2',
                                   'nova-api-os-compute',
+                                  'nova-conductor',
                                   'nova-objectstore',
                                   'nova-cert',
                                   'nova-scheduler'],
@@ -246,10 +247,6 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             self.glance_sentry: ['glance-registry', 'glance-api']
         }
 
-# Erroneous?  Inverted comparison?   Was this for essex?
-#        if self._get_openstack_release() >= self.precise_grizzly:
-#            commands[self.nova_cc_sentry] = ['nova-conductor']
-
         ret = u.validate_services_by_name(services)
         if ret:
             amulet.raise_status(amulet.FAIL, msg=ret)

From a80619a28989fcbd38ba5f77ff072ac9398ea9dc Mon Sep 17 00:00:00 2001
From: Ryan Beisner <ryan.beisner@canonical.com>
Date: Thu, 15 Oct 2015 16:57:48 +0000
Subject: [PATCH 08/10] wait for unit extended status message to determine when
 the deployment is ready for testing

---
 tests/basic_deployment.py                     | 66 ++-----------------
 .../contrib/openstack/amulet/deployment.py    | 40 +++++++++++
 2 files changed, 45 insertions(+), 61 deletions(-)

diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index 6ff35b43..cfb94249 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -1,6 +1,5 @@
 import amulet
 import os
-import time
 import yaml
 
 from charmhelpers.contrib.openstack.amulet.deployment import (
@@ -30,8 +29,12 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
         self._add_relations()
         self._configure_services()
         self._deploy()
+
+        u.log.info('Waiting on extended status checks...')
+        exclude_services = ['mysql']
+        self._auto_wait_for_status(exclude_services=exclude_services)
+
         self._initialize_tests()
-        self._expect_method_success(self._api_checks, 'API check(s)')
 
     def _add_services(self):
         """Add services
@@ -168,65 +171,6 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
                                                   password='password',
                                                   tenant=self.demo_tenant)
 
-    def _api_checks(self, something=None):
-        """Check basic api functionality for this deployment topology."""
-        u.log.debug('Checking APIs... {}'.format(something))
-
-        # glance api returns a generator whereas other apis return a list
-        assert list(self.glance.images.list()) == []
-        assert self.nova_demo.servers.list() == []
-        assert self.nova_demo.flavors.list() != []
-        assert self.nova_demo.floating_ips.list() == []
-        assert self.nova_demo.keypairs.list() == []
-        assert self.keystone.services.list() != []
-
-    # NOTE(beisner):  Will curate this as a local helper, validate and
-    # adjust as local helpers until ready to propose @ charmhelpers:
-    def _expect_method_success(self, my_method, msg="method check(s)",
-                               my_kwargs=None, min_success=10,
-                               max_wait=300, interval=5):
-        """Call a method/function with exception trapping and retry
-        thresholds.  Expect a minimum number of successful consecutive calls
-        within a maximum total time.  Useful to confirm that the method
-        starts to pass, and continues to pass until the threshold is met,
-        such as waiting for api services to become functional.
-
-        :param my_method: method or function to call
-        :param msg: description of what type of thing is being exercised
-        :param min_success: minimum consecutive successful calls
-        :param max_wait: maximum wait in seconds before raising
-        :param interval: time in seconds to wait between retries
-        :returns: the method's return value if successful, raises otherwise
-        """
-        if not my_kwargs:
-            my_kwargs = {}
-
-        successful_calls = time_elapsed = 0
-        time_start = time.time()
-        while successful_calls < min_success and time_elapsed <= max_wait:
-            time_elapsed = time.time() - time_start
-            try:
-                my_return = my_method(**my_kwargs)
-                successful_calls += 1
-                u.log.debug('Consecutive successful {}: {} ({} required)'
-                            '...'.format(msg, successful_calls,
-                                         min_success))
-            except Exception as e:
-                successful_calls = 0
-                u.log.debug('{} failed after {}s, retrying until {}s...'
-                            '\n{}'.format(msg, time_elapsed, max_wait, e))
-            time.sleep(interval)
-
-        if successful_calls < min_success:
-            raise_msg = ('{} not satisfied with {} successful checks '
-                         '({} consecutive required within '
-                         '{}s)'.format(msg, successful_calls,
-                                       min_success, max_wait))
-            amulet.raise_status(amulet.FAIL, msg=raise_msg)
-        else:
-            u.log.debug('OK')
-            return my_return
-
     def test_100_services(self):
         """Verify the expected services are running on the corresponding
            service units."""
diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
index 722bc645..dd81e4db 100644
--- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import re
 import six
 from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
@@ -114,6 +115,45 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         for service, config in six.iteritems(configs):
             self.d.configure(service, config)
 
+    def _auto_wait_for_status(self, message=None, exclude_services=None,
+                              timeout=1800):
+        """Wait for all units to have a specific extended status, except
+        for any defined as excluded.  Unless specified via message, any
+        status containing any case of 'ready' will be considered a match.
+
+        Examples of message usage:
+
+          Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
+              message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
+
+          Wait for all units to reach this status (exact match):
+              message = 'Unit is ready'
+
+          Wait for all units to reach any one of these (exact match):
+              message = re.compile('Unit is ready|OK|Ready')
+
+          Wait for at least one unit to reach this status (exact match):
+              message = {'ready'}
+
+        See Amulet's sentry.wait_for_messages() for message usage detail.
+        https://github.com/juju/amulet/blob/master/amulet/sentry.py
+
+        :param message: Expected status match
+        :param exclude_services: List of juju service names to ignore
+        :param timeout: Maximum time in seconds to wait for status match
+        :returns: None.  Raises if timeout is hit.
+        """
+
+        if not message:
+            message = re.compile('.*ready.*', re.IGNORECASE)
+
+        if not exclude_services:
+            exclude_services = []
+
+        services = list(set(self.d.services.keys()) - set(exclude_services))
+        service_messages = {service: message for service in services}
+        self.d.sentry.wait_for_messages(service_messages, timeout=900)
+
     def _get_openstack_release(self):
         """Get openstack release.
 

From 0f8976fb18151f60b5a5936f21154cd902650d34 Mon Sep 17 00:00:00 2001
From: Ryan Beisner <ryan.beisner@canonical.com>
Date: Thu, 15 Oct 2015 20:35:43 +0000
Subject: [PATCH 09/10] resync tests/charmhelpers

---
 tests/charmhelpers/contrib/openstack/amulet/deployment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
index dd81e4db..f9304d45 100644
--- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -152,7 +152,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
 
         services = list(set(self.d.services.keys()) - set(exclude_services))
         service_messages = {service: message for service in services}
-        self.d.sentry.wait_for_messages(service_messages, timeout=900)
+        self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
 
     def _get_openstack_release(self):
         """Get openstack release.

From 027dc7c9e02c0daccc9da01d548b4e02c30aa01e Mon Sep 17 00:00:00 2001
From: Ryan Beisner <ryan.beisner@canonical.com>
Date: Fri, 16 Oct 2015 05:41:09 +0000
Subject: [PATCH 10/10] stop using deprecated delete helpers

---
 tests/basic_deployment.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index cfb94249..29b79403 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -136,6 +136,11 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
         self.nova_compute_sentry = self.d.sentry.unit['nova-compute/0']
         self.glance_sentry = self.d.sentry.unit['glance/0']
 
+        u.log.debug('openstack release val: {}'.format(
+            self._get_openstack_release()))
+        u.log.debug('openstack release str: {}'.format(
+            self._get_openstack_release_string()))
+
         # Authenticate admin with keystone
         self.keystone = u.authenticate_keystone_admin(self.keystone_sentry,
                                                       user='admin',
@@ -664,7 +669,7 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = "api paste config error: {}".format(ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_402_image_instance_create(self):
+    def test_400_image_instance_create(self):
         """Create an image/instance, verify they exist, and delete them."""
         # NOTE(coreycb): Skipping failing test on essex until resolved. essex
         #                nova API calls are getting "Malformed request url
@@ -696,8 +701,11 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             message = "nova cirros instance does not exist"
             amulet.raise_status(amulet.FAIL, msg=message)
 
-        u.delete_image(self.glance, image)
-        u.delete_instance(self.nova_demo, instance)
+        u.delete_resource(self.glance.images, image.id,
+                          msg="glance image")
+
+        u.delete_resource(self.nova_demo.servers, instance.id,
+                          msg="nova instance")
 
     def test_900_restart_on_config_change(self):
         """Verify that the specified services are restarted when the config