Merge keystone/next into change to get ready for final merge

Makefile

@@ -13,6 +13,7 @@ test:
 
 functional_test:
 	@echo Starting Amulet tests...
+	@tests/setup/00-setup
 	@juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 2700
 
 bin/charm_helpers_sync.py:

charm-helpers-hooks.yaml

@@ -11,7 +11,7 @@ include:
         - cluster
     - contrib.python
     - contrib.unison
-    - payload.execd
+    - payload
     - contrib.peerstorage
     - contrib.network.ip
     - contrib.python.packages

charmhelpers/contrib/openstack/neutron.py

@@ -237,12 +237,14 @@ def neutron_plugins():
         plugins['midonet']['driver'] = (
             'neutron.plugins.midonet.plugin.MidonetPluginV2')
     if release >= 'liberty':
-        plugins['midonet']['driver'] = (
+        midonet_origin = config('midonet-origin')
-            'midonet.neutron.plugin_v1.MidonetPluginV2')
+        if midonet_origin is not None and midonet_origin[4:5] == '1':
-        plugins['midonet']['server_packages'].remove(
+            plugins['midonet']['driver'] = (
-            'python-neutron-plugin-midonet')
+                'midonet.neutron.plugin_v1.MidonetPluginV2')
-        plugins['midonet']['server_packages'].append(
+            plugins['midonet']['server_packages'].remove(
-            'python-networking-midonet')
+                'python-neutron-plugin-midonet')
+            plugins['midonet']['server_packages'].append(
+                'python-networking-midonet')
     return plugins
 
 

charmhelpers/payload/archive.py

@@ -0,0 +1,73 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import tarfile
+import zipfile
+from charmhelpers.core import (
+    host,
+    hookenv,
+)
+
+
+class ArchiveError(Exception):
+    pass
+
+
+def get_archive_handler(archive_name):
+    if os.path.isfile(archive_name):
+        if tarfile.is_tarfile(archive_name):
+            return extract_tarfile
+        elif zipfile.is_zipfile(archive_name):
+            return extract_zipfile
+    else:
+        # look at the file name
+        for ext in ('.tar', '.tar.gz', '.tgz', 'tar.bz2', '.tbz2', '.tbz'):
+            if archive_name.endswith(ext):
+                return extract_tarfile
+        for ext in ('.zip', '.jar'):
+            if archive_name.endswith(ext):
+                return extract_zipfile
+
+
+def archive_dest_default(archive_name):
+    archive_file = os.path.basename(archive_name)
+    return os.path.join(hookenv.charm_dir(), "archives", archive_file)
+
+
+def extract(archive_name, destpath=None):
+    handler = get_archive_handler(archive_name)
+    if handler:
+        if not destpath:
+            destpath = archive_dest_default(archive_name)
+        if not os.path.isdir(destpath):
+            host.mkdir(destpath)
+        handler(archive_name, destpath)
+        return destpath
+    else:
+        raise ArchiveError("No handler for archive")
+
+
+def extract_tarfile(archive_name, destpath):
+    "Unpack a tar archive, optionally compressed"
+    archive = tarfile.open(archive_name)
+    archive.extractall(destpath)
+
+
+def extract_zipfile(archive_name, destpath):
+    "Unpack a zip file"
+    archive = zipfile.ZipFile(archive_name)
+    archive.extractall(destpath)

config.yaml

@@ -1,11 +1,11 @@
 options:
   debug:
-    default: "false"
+    type: boolean
-    type: string
+    default: False
     description: Enable verbose logging.
   verbose:
-    default: "false"
+    type: boolean
-    type: string
+    default: False
     description: Enable debug logging.
   use-syslog:
     type: boolean

hooks/keystone_context.py

@@ -189,7 +189,7 @@ class KeystoneContext(context.OSContextGenerator):
     def __call__(self):
         from keystone_utils import (
             api_port, set_admin_token, endpoint_url, resolve_address,
-            PUBLIC, ADMIN, PKI_CERTS_DIR, SSH_USER, ensure_permissions,
+            PUBLIC, ADMIN, PKI_CERTS_DIR, ensure_pki_cert_paths,
         )
         ctxt = {}
         ctxt['token'] = set_admin_token(config('admin-token'))
@@ -198,10 +198,8 @@ class KeystoneContext(context.OSContextGenerator):
         ctxt['public_port'] = determine_api_port(api_port('keystone-public'),
                                                  singlenode_mode=True)
 
-        debug = config('debug')
+        ctxt['debug'] = config('debug')
-        ctxt['debug'] = debug and bool_from_string(debug)
+        ctxt['verbose'] = config('verbose')
-        verbose = config('verbose')
-        ctxt['verbose'] = verbose and bool_from_string(verbose)
         ctxt['token_expiration'] = config('token-expiration')
 
         ctxt['identity_backend'] = config('identity-backend')
@@ -219,32 +217,16 @@ class KeystoneContext(context.OSContextGenerator):
 
         enable_pki = config('enable-pki')
         if enable_pki and bool_from_string(enable_pki):
-            ctxt['signing'] = True
+            log("Enabling PKI", level=DEBUG)
             ctxt['token_provider'] = 'pki'
 
-        if 'token_provider' in ctxt:
+        ensure_pki_cert_paths()
-            log("Configuring PKI token cert paths", level=DEBUG)
+        certs = os.path.join(PKI_CERTS_DIR, 'certs')
-            certs = os.path.join(PKI_CERTS_DIR, 'certs')
+        privates = os.path.join(PKI_CERTS_DIR, 'privates')
-            privates = os.path.join(PKI_CERTS_DIR, 'privates')
+        ctxt.update({'certfile': os.path.join(certs, 'signing_cert.pem'),
-            for path in [PKI_CERTS_DIR, certs, privates]:
+                     'keyfile': os.path.join(privates, 'signing_key.pem'),
-                perms = 0o755
+                     'ca_certs': os.path.join(certs, 'ca.pem'),
-                if not os.path.isdir(path):
+                     'ca_key': os.path.join(certs, 'ca_key.pem')})
-                    mkdir(path=path, owner=SSH_USER, group='keystone',
-                          perms=perms)
-                else:
-                    # Ensure accessible by ssh user and group (for sync).
-                    ensure_permissions(path, user=SSH_USER,
-                                       group='keystone', perms=perms)
-
-            signing_paths = {'certfile': os.path.join(certs,
-                                                      'signing_cert.pem'),
-                             'keyfile': os.path.join(privates,
-                                                     'signing_key.pem'),
-                             'ca_certs': os.path.join(certs, 'ca.pem'),
-                             'ca_key': os.path.join(certs, 'ca_key.pem')}
-
-            for key, val in signing_paths.iteritems():
-                ctxt[key] = val
 
         # Base endpoint URL's which are used in keystone responses
         # to unauthenticated requests to redirect clients to the
@@ -255,6 +237,7 @@ class KeystoneContext(context.OSContextGenerator):
         ctxt['admin_endpoint'] = endpoint_url(
             resolve_address(ADMIN),
             api_port('keystone-admin')).rstrip('v2.0')
+
         return ctxt
 
 
@@ -263,7 +246,7 @@ class KeystoneLoggingContext(context.OSContextGenerator):
     def __call__(self):
         ctxt = {}
         debug = config('debug')
-        if debug and bool_from_string(debug):
+        if debug:
             ctxt['root_level'] = 'DEBUG'
 
         return ctxt

hooks/keystone_hooks.py

@@ -74,13 +74,15 @@ from keystone_utils import (
     clear_ssl_synced_units,
     is_db_initialised,
     update_certs_if_available,
-    is_pki_enabled,
     ensure_ssl_dir,
     ensure_pki_dir_permissions,
     ensure_permissions,
     force_ssl_sync,
     filter_null,
     ensure_ssl_dirs,
+    ensure_pki_cert_paths,
+    is_service_present,
+    delete_service_entry,
     assess_status,
 )
 
@@ -175,8 +177,7 @@ def config_changed_postupgrade():
     update_nrpe_config()
     CONFIGS.write_all()
 
-    if is_pki_enabled():
+    initialise_pki()
-        initialise_pki()
 
     update_all_identity_relation_units()
 
@@ -192,11 +193,14 @@ def config_changed_postupgrade():
 
 @synchronize_ca_if_changed(fatal=True)
 def initialise_pki():
-    """Create certs and keys required for PKI token signing.
+    """Create certs and keys required for token signing.
+
+    Used for PKI and signing token revocation list.
 
     NOTE: keystone.conf [signing] section must be up-to-date prior to
           executing this.
     """
+    ensure_pki_cert_paths()
     if not peer_units() or is_ssl_cert_master():
         log("Ensuring PKI token certs created", level=DEBUG)
         cmd = ['keystone-manage', 'pki_setup', '--keystone-user', 'keystone',
@@ -337,6 +341,8 @@ def identity_changed(relation_id=None, remote_unit=None):
             return
 
         add_service_to_keystone(relation_id, remote_unit)
+        if is_service_present('neutron', 'network'):
+            delete_service_entry('quantum', 'network')
         settings = relation_get(rid=relation_id, unit=remote_unit)
         service = settings.get('service', None)
         if service:
@@ -377,44 +383,36 @@ def send_ssl_sync_request():
     Note the we do nothing if the setting is already applied.
     """
     unit = local_unit().replace('/', '-')
-    count = 0
+    # Start with core config (e.g. used for signing revoked token list)
+    ssl_config = 0b1
 
     use_https = config('use-https')
     if use_https and bool_from_string(use_https):
-        count += 1
+        ssl_config ^= 0b10
 
     https_service_endpoints = config('https-service-endpoints')
     if (https_service_endpoints and
             bool_from_string(https_service_endpoints)):
-        count += 2
+        ssl_config ^= 0b100
 
     enable_pki = config('enable-pki')
     if enable_pki and bool_from_string(enable_pki):
-        count += 3
+        ssl_config ^= 0b1000
 
     key = 'ssl-sync-required-%s' % (unit)
-    settings = {key: count}
+    settings = {key: ssl_config}
 
-    # If all ssl is disabled ensure this is set to 0 so that cluster hook runs
+    prev = 0b0
-    # and endpoints are updated.
-    if not count:
-        log("Setting %s=%s" % (key, count), level=DEBUG)
-        for rid in relation_ids('cluster'):
-            relation_set(relation_id=rid, relation_settings=settings)
-
-        return
-
-    prev = 0
     rid = None
     for rid in relation_ids('cluster'):
         for unit in related_units(rid):
-            _prev = relation_get(rid=rid, unit=unit, attribute=key) or 0
+            _prev = relation_get(rid=rid, unit=unit, attribute=key) or 0b0
             if _prev and _prev > prev:
-                prev = _prev
+                prev = bin(_prev)
 
-    if rid and prev < count:
+    if rid and prev ^ ssl_config:
         clear_ssl_synced_units()
-        log("Setting %s=%s" % (key, count), level=DEBUG)
+        log("Setting %s=%s" % (key, bin(ssl_config)), level=DEBUG)
         relation_set(relation_id=rid, relation_settings=settings)
 
 
@@ -459,8 +457,7 @@ def cluster_changed():
 
     check_peer_actions()
 
-    if is_pki_enabled():
+    initialise_pki()
-        initialise_pki()
 
     # Figure out if we need to mandate a sync
     units = get_ssl_sync_request_units()

hooks/keystone_utils.py

@@ -251,6 +251,10 @@ valid_services = {
         "type": "network",
         "desc": "Quantum Networking Service"
     },
+    "neutron": {
+        "type": "network",
+        "desc": "Neutron Networking Service"
+    },
     "oxygen": {
         "type": "oxygen",
         "desc": "Oxygen Cloud Image Service"
@@ -282,7 +286,15 @@ valid_services = {
     "ironic": {
         "type": "baremetal",
         "desc": "Ironic bare metal provisioning service"
-    }
+    },
+    "designate": {
+        "type": "dns",
+        "desc": "Designate DNS service"
+    },
+    "astara": {
+        "type": "astara",
+        "desc": "Astara Network Orchestration Service",
+    },
 }
 
 # The interface is said to be satisfied if anyone of the interfaces in the
@@ -493,6 +505,25 @@ def get_admin_token():
     error_out('Could not find admin_token line in %s' % KEYSTONE_CONF)
 
 
+def is_service_present(service_name, service_type):
+    import manager
+    manager = manager.KeystoneManager(endpoint=get_local_endpoint(),
+                                      token=get_admin_token())
+    service_id = manager.resolve_service_id(service_name, service_type)
+    return service_id is not None
+
+
+def delete_service_entry(service_name, service_type):
+    """ Delete a service from keystone"""
+    import manager
+    manager = manager.KeystoneManager(endpoint=get_local_endpoint(),
+                                      token=get_admin_token())
+    service_id = manager.resolve_service_id(service_name, service_type)
+    if service_id:
+        manager.api.services.delete(service_id)
+        log("Deleted service entry '%s'" % service_name, level=DEBUG)
+
+
 def create_service_entry(service_name, service_type, service_desc, owner=None):
     """ Add a new service entry to keystone if one does not already exist """
     import manager
@@ -970,20 +1001,6 @@ def is_ssl_cert_master(votes=None):
     return False
 
 
-def is_ssl_enabled():
-    use_https = config('use-https')
-    https_service_endpoints = config('https-service-endpoints')
-    if ((use_https and bool_from_string(use_https)) or
-            (https_service_endpoints and
-                bool_from_string(https_service_endpoints)) or
-            is_pki_enabled()):
-        log("SSL/HTTPS is enabled", level=DEBUG)
-        return True
-
-    log("SSL/HTTPS is NOT enabled", level=DEBUG)
-    return False
-
-
 def get_ssl_cert_master_votes():
     """Returns a list of unique votes."""
     votes = []
@@ -1004,10 +1021,6 @@ def ensure_ssl_cert_master():
     Normally the cluster leader will take control but we allow for this to be
     ignored since this could be called before the cluster is ready.
     """
-    # Don't do anything if we are not in ssl/https mode
-    if not is_ssl_enabled():
-        return False
-
     master_override = False
     elect = is_elected_leader(CLUSTER_RES)
 
@@ -1067,6 +1080,23 @@ def is_pki_enabled():
     return False
 
 
+def ensure_pki_cert_paths():
+    certs = os.path.join(PKI_CERTS_DIR, 'certs')
+    privates = os.path.join(PKI_CERTS_DIR, 'privates')
+    not_exists = [p for p in [PKI_CERTS_DIR, certs, privates]
+                  if not os.path.exists(p)]
+    if not_exists:
+        log("Configuring token signing cert paths", level=DEBUG)
+        perms = 0o755
+        for path in not_exists:
+            if not os.path.isdir(path):
+                mkdir(path=path, owner=SSH_USER, group='keystone', perms=perms)
+            else:
+                # Ensure accessible by ssh user and group (for sync).
+                ensure_permissions(path, user=SSH_USER, group='keystone',
+                                   perms=perms)
+
+
 def ensure_pki_dir_permissions():
     # Ensure accessible by unison user and group (for sync).
     ensure_permissions(PKI_CERTS_DIR, user=SSH_USER, group='keystone',
@@ -1138,10 +1168,10 @@ def synchronize_ca(fatal=False):
         peer_service_actions['restart'].append('apache2')
         peer_actions.append('update-ca-certificates')
 
-    if is_pki_enabled():
+    # NOTE: certs needed for token signing e.g. pki and revocation list query.
-        log("Syncing token certs", level=DEBUG)
+    log("Syncing token certs", level=DEBUG)
-        paths_to_sync.append(PKI_CERTS_DIR)
+    paths_to_sync.append(PKI_CERTS_DIR)
-        peer_actions.append('ensure-pki-permissions')
+    peer_actions.append('ensure-pki-permissions')
 
     if not paths_to_sync:
         log("Nothing to sync - skipping", level=DEBUG)

hooks/manager.py

@@ -28,12 +28,16 @@ class KeystoneManager(object):
             if name == u['name']:
                 return u['id']
 
-    def resolve_service_id(self, name):
+    def resolve_service_id(self, name, service_type=None):
         """Find the service_id of a given service"""
         services = [s._info for s in self.api.services.list()]
         for s in services:
-            if name == s['name']:
+            if service_type:
-                return s['id']
+                if name == s['name'] and service_type == s['type']:
+                    return s['id']
+            else:
+                if name == s['name']:
+                    return s['id']
 
     def resolve_service_id_by_type(self, type):
         """Find the service_id of a given service"""

templates/kilo/keystone.conf

@@ -70,8 +70,6 @@ driver = keystone.assignment.backends.{{ assignment_backend }}.Assignment
 
 [oauth1]
 
-[signing]
-
 [auth]
 methods = external,password,token,oauth1
 password = keystone.auth.plugins.password.Password

tests/019-basic-trusty-mitaka

@@ -0,0 +1,11 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic keystone deployment on trusty-mitaka."""
+
+from basic_deployment import KeystoneBasicDeployment
+
+if __name__ == '__main__':
+    deployment = KeystoneBasicDeployment(series='trusty',
+                                         openstack='cloud:trusty-mitaka',
+                                         source='cloud:trusty-updates/mitaka')
+    deployment.run_tests()

tests/021-basic-xenial-mitaka

@@ -1,9 +1,9 @@
 #!/usr/bin/python
 
-"""Amulet tests on a basic keystone deployment on vivid-kilo."""
+"""Amulet tests on a basic keystone deployment on xenial-mitaka."""
 
 from basic_deployment import KeystoneBasicDeployment
 
 if __name__ == '__main__':
-    deployment = KeystoneBasicDeployment(series='vivid')
+    deployment = KeystoneBasicDeployment(series='xenial')
     deployment.run_tests()

tests/README

@@ -1,62 +1,113 @@
-This directory provides Amulet tests that focus on verification of Keystone
+This directory provides Amulet tests to verify basic deployment functionality
-deployments.
+from the perspective of this charm, its requirements and its features, as
+exercised in a subset of the full OpenStack deployment test bundle topology.
 
-test_* methods are called in lexical sort order.
+Reference:  lp:openstack-charm-testing for full test bundles.
 
-Test name convention to ensure desired test order:
+A single topology and configuration is defined and deployed, once for each of
+the defined Ubuntu:OpenStack release combos.  The ongoing goal is for this
+charm to always possess tests and combo definitions for all currently-supported
+release combinations of U:OS.
+
+test_* methods are called in lexical sort order, as with most runners.  However,
+each individual test method should be idempotent and expected to pass regardless
+of run order or Ubuntu:OpenStack combo.  When writing or modifying tests,
+ensure that every individual test is not dependent on another test_ method.
+
+Test naming convention, purely for code organization purposes:
     1xx service and endpoint checks
     2xx relation checks
     3xx config checks
     4xx functional checks
-    9xx restarts and other final checks
+    9xx restarts, config changes, actions and other final checks
 
-In order to run tests, you'll need charm-tools installed (in addition to
+In order to run tests, charm-tools and juju must be installed:
-juju, of course):
     sudo add-apt-repository ppa:juju/stable
     sudo apt-get update
-    sudo apt-get install charm-tools
+    sudo apt-get install charm-tools juju juju-deployer amulet
 
-If you use a web proxy server to access the web, you'll need to set the
+Alternatively, tests may be exercised with proposed or development versions
-AMULET_HTTP_PROXY environment variable to the http URL of the proxy server.
+of juju and related tools:
+
+    # juju proposed version
+    sudo add-apt-repository ppa:juju/proposed
+    sudo apt-get update
+    sudo apt-get install charm-tools juju juju-deployer
+
+    # juju development version
+    sudo add-apt-repository ppa:juju/devel
+    sudo apt-get update
+    sudo apt-get install charm-tools juju juju-deployer
+
+Some tests may need to download files. If a web proxy server is required in
+the environment, the AMULET_HTTP_PROXY environment variable must be set and
+passed into the juju test command.  This is unrelated to juju's http proxy
+settings or behavior.
 
 The following examples demonstrate different ways that tests can be executed.
 All examples are run from the charm's root directory.
 
-  * To run all tests (starting with 00-setup):
+  * To run all +x tests in the tests directory:
 
-      make test
+      bzr branch lp:charms/trusty/foo
+      cd foo
+      make functional_test
 
-  * To run a specific test module (or modules):
+  * To run the tests against a specific release combo as defined in tests/:
 
-      juju test -v -p AMULET_HTTP_PROXY 15-basic-trusty-icehouse
+      bzr branch lp:charms/trusty/foo
+      cd foo
+      juju test -v -p AMULET_HTTP_PROXY 015-basic-trusty-icehouse
 
-  * To run a specific test module (or modules), and keep the environment
+  * To run tests and keep the juju environment deployed after a failure:
-    deployed after a failure:
 
-      juju test --set-e -v -p AMULET_HTTP_PROXY 15-basic-trusty-icehouse
+      bzr branch lp:charms/trusty/foo
+      cd foo
+      juju test --set-e -v -p AMULET_HTTP_PROXY 015-basic-trusty-icehouse
 
   * To re-run a test module against an already deployed environment (one
     that was deployed by a previous call to 'juju test --set-e'):
 
-      ./tests/15-basic-trusty-icehouse
+      ./tests/015-basic-trusty-icehouse
 
-For debugging and test development purposes, all code should be idempotent.
+  * Even with --set-e, `juju test` will tear down the deployment when all
-In other words, the code should have the ability to be re-run without changing
+    tests pass. The following work flow may be more effective when
-the results beyond the initial run.  This enables editing and re-running of a
+    iterating on test writing.
-test module against an already deployed environment, as described above.
 
-Manual debugging tips:
+      bzr branch lp:charms/trusty/foo
+      cd foo
+      ./tests/setup/00-setup
+      juju bootstrap
+      ./tests/015-basic-trusty-icehouse
+      # make some changes, run tests again
+      ./tests/015-basic-trusty-icehouse
+      # make some changes, run tests again
+      ./tests/015-basic-trusty-icehouse
 
-  * Set the following env vars before using the OpenStack CLI as admin:
+  * There may be test definitions in the tests/ dir which are not set +x
-      export OS_AUTH_URL=http://`juju-deployer -f keystone 2>&1 | tail -n 1`:5000/v2.0
+    executable.  This is generally true for deprecated releases, or for
-      export OS_TENANT_NAME=admin
+    upcoming releases which are not yet validated and enabled.  To enable
+    and run these tests:
+      bzr branch lp:charms/trusty/foo
+      cd foo
+      ls tests
+      chmod +x tests/017-basic-trusty-kilo
+      ./tests/setup/00-setup
+      juju bootstrap
+      ./tests/017-basic-trusty-kilo
+
+
+Additional notes:
+
+  * Use DEBUG to turn on debug logging, use ERROR otherwise.
+      u = OpenStackAmuletUtils(ERROR)
+      u = OpenStackAmuletUtils(DEBUG)
+
+  * To interact with the deployed environment:
       export OS_USERNAME=admin
       export OS_PASSWORD=openstack
+      export OS_TENANT_NAME=admin
       export OS_REGION_NAME=RegionOne
-
+      export OS_AUTH_URL=${OS_AUTH_PROTOCOL:-http}://`juju-deployer -e trusty -f keystone`:5000/v2.0
-  * Set the following env vars before using the OpenStack CLI as demoUser:
+      keystone user-list
-      export OS_AUTH_URL=http://`juju-deployer -f keystone 2>&1 | tail -n 1`:5000/v2.0
+      glance image-list
-      export OS_TENANT_NAME=demoTenant
-      export OS_USERNAME=demoUser
-      export OS_PASSWORD=password
-      export OS_REGION_NAME=RegionOne

tests/basic_deployment.py

@@ -6,7 +6,6 @@ Basic keystone amulet functional tests.
 
 import amulet
 import os
-import time
 import yaml
 
 from charmhelpers.contrib.openstack.amulet.deployment import (
@@ -36,6 +35,11 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment):
         self._add_relations()
         self._configure_services()
         self._deploy()
+
+        u.log.info('Waiting on extended status checks...')
+        exclude_services = ['mysql']
+        self._auto_wait_for_status(exclude_services=exclude_services)
+
         self._initialize_tests()
 
     def _assert_services(self, should_run):
@@ -52,6 +56,7 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment):
            """
         this_service = {'name': 'keystone'}
         other_services = [{'name': 'mysql'},
+                          {'name': 'rabbitmq-server'},  # satisfy wrkload stat
                           {'name': 'cinder'}]
         super(KeystoneBasicDeployment, self)._add_services(this_service,
                                                            other_services)
@@ -59,6 +64,8 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment):
     def _add_relations(self):
         """Add all of the relations for the services."""
         relations = {'keystone:shared-db': 'mysql:shared-db',
+                     'cinder:shared-db': 'mysql:shared-db',
+                     'cinder:amqp': 'rabbitmq-server:amqp',
                      'cinder:identity-service': 'keystone:identity-service'}
         super(KeystoneBasicDeployment, self)._add_relations(relations)
 
@@ -113,9 +120,6 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment):
         u.log.debug('openstack release str: {}'.format(
             self._get_openstack_release_string()))
 
-        # Let things settle a bit before moving forward
-        time.sleep(30)
-
         # Authenticate keystone admin
         self.keystone = u.authenticate_keystone_admin(self.keystone_sentry,
                                                       user='admin',

tests/tests.yaml

@@ -1,5 +1,5 @@
 bootstrap: true
-reset: true
+reset: false
 virtualenv: true
 makefile:
     - lint
@@ -9,6 +9,7 @@ sources:
 packages:
     - amulet
     - distro-info-data
+    - python-ceilometerclient
     - python-cinderclient
     - python-distro-info
     - python-glanceclient

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = lint,py27
+envlist = pep8,py27
 skipsdist = True
 
 [testenv]
@@ -14,7 +14,7 @@ basepython = python2.7
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 
-[testenv:lint]
+[testenv:pep8]
 basepython = python2.7
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt

unit_tests/test_keystone_contexts.py

@@ -26,11 +26,9 @@ class TestKeystoneContexts(CharmTestCase):
     @patch('keystone_utils.ensure_permissions')
     @patch('keystone_utils.determine_ports')
     @patch('keystone_utils.is_ssl_cert_master')
-    @patch('keystone_utils.is_ssl_enabled')
     @patch.object(context, 'log')
     def test_apache_ssl_context_ssl_not_master(self,
                                                mock_log,
-                                               mock_is_ssl_enabled,
                                                mock_is_ssl_cert_master,
                                                mock_determine_ports,
                                                mock_ensure_permissions,
@@ -38,7 +36,6 @@ class TestKeystoneContexts(CharmTestCase):
                                                mock_mkdir,
                                                mock_cert_provided_in_config):
         mock_cert_provided_in_config.return_value = False
-        mock_is_ssl_enabled.return_value = True
         mock_is_ssl_cert_master.return_value = False
 
         context.ApacheSSLContext().configure_cert('foo')
@@ -49,7 +46,6 @@ class TestKeystoneContexts(CharmTestCase):
 
     @patch('keystone_utils.determine_ports')
     @patch('keystone_utils.is_ssl_cert_master')
-    @patch('keystone_utils.is_ssl_enabled')
     @patch('charmhelpers.contrib.openstack.context.config')
     @patch('charmhelpers.contrib.openstack.context.is_clustered')
     @patch('charmhelpers.contrib.openstack.context.determine_apache_port')
@@ -62,10 +58,8 @@ class TestKeystoneContexts(CharmTestCase):
                                                 mock_determine_apache_port,
                                                 mock_is_clustered,
                                                 mock_config,
-                                                mock_is_ssl_enabled,
                                                 mock_is_ssl_cert_master,
                                                 mock_determine_ports):
-        mock_is_ssl_enabled.return_value = True
         mock_is_ssl_cert_master.return_value = True
         mock_https.return_value = True
         mock_unit_get.return_value = '1.2.3.4'
@@ -97,10 +91,11 @@ class TestKeystoneContexts(CharmTestCase):
     @patch('charmhelpers.contrib.openstack.context.related_units')
     @patch('charmhelpers.contrib.openstack.context.relation_get')
     @patch('charmhelpers.contrib.openstack.context.log')
+    @patch('charmhelpers.contrib.openstack.context.kv')
     @patch('__builtin__.open')
     def test_haproxy_context_service_enabled(
-        self, mock_open, mock_log, mock_relation_get, mock_related_units,
+        self, mock_open, mock_kv, mock_log, mock_relation_get,
-            mock_unit_get, mock_relation_ids, mock_config,
+            mock_related_units, mock_unit_get, mock_relation_ids, mock_config,
             mock_get_address_in_network, mock_get_netmask_for_address,
             mock_api_port):
         os.environ['JUJU_UNIT_NAME'] = 'keystone'
@@ -114,13 +109,11 @@ class TestKeystoneContexts(CharmTestCase):
         mock_get_netmask_for_address.return_value = '255.255.255.0'
         self.determine_apache_port.return_value = '34'
         mock_api_port.return_value = '12'
+        mock_kv().get.return_value = 'abcdefghijklmnopqrstuvwxyz123456'
 
         ctxt = context.HAProxyContext()
 
         self.maxDiff = None
-        # [ajkavangh] due to rev:514 the stat_port has changed format
-        # and stat_password (new) is dynamically generated.
-        _ctxt = ctxt()
         self.assertEquals(
             ctxt(),
             {'listen_ports': {'admin_port': '12',
@@ -128,7 +121,7 @@ class TestKeystoneContexts(CharmTestCase):
              'local_host': '127.0.0.1',
              'haproxy_host': '0.0.0.0',
              'stat_port': '8888',
-             'stat_password': _ctxt['stat_password'],
+             'stat_password': 'abcdefghijklmnopqrstuvwxyz123456',
              'service_ports': {'admin-port': ['12', '34'],
                                'public-port': ['12', '34']},
              'default_backend': '1.2.3.4',

unit_tests/test_keystone_hooks.py

@@ -71,6 +71,8 @@ TO_PATCH = [
     'get_netmask_for_address',
     'get_address_in_network',
     'git_install',
+    'is_service_present',
+    'delete_service_entry',
 ]
 
 
@@ -349,9 +351,9 @@ class KeystoneRelationTests(CharmTestCase):
     @patch('keystone_utils.ensure_ssl_cert_master')
     @patch('keystone_utils.ensure_ssl_dirs')
     @patch.object(hooks, 'ensure_permissions')
+    @patch.object(hooks, 'ensure_pki_cert_paths')
     @patch.object(hooks, 'ensure_pki_dir_permissions')
     @patch.object(hooks, 'ensure_ssl_dir')
-    @patch.object(hooks, 'is_pki_enabled')
     @patch.object(hooks, 'is_ssl_cert_master')
     @patch.object(hooks, 'send_ssl_sync_request')
     @patch.object(hooks, 'peer_units')
@@ -371,8 +373,8 @@ class KeystoneRelationTests(CharmTestCase):
                                               mock_peer_units,
                                               mock_send_ssl_sync_request,
                                               mock_is_ssl_cert_master,
-                                              mock_is_pki_enabled,
                                               mock_ensure_ssl_dir,
+                                              mock_ensure_pki_cert_paths,
                                               mock_ensure_permissions,
                                               mock_ensure_pki_dir_permissions,
                                               mock_ensure_ssl_dirs,
@@ -380,7 +382,6 @@ class KeystoneRelationTests(CharmTestCase):
                                               mock_log, git_requested,
                                               mock_is_db_initialised):
         git_requested.return_value = False
-        mock_is_pki_enabled.return_value = True
         mock_is_ssl_cert_master.return_value = True
         mock_is_db_initialised.return_value = True
         self.is_db_ready.return_value = True
@@ -414,9 +415,9 @@ class KeystoneRelationTests(CharmTestCase):
     @patch('keystone_utils.ensure_ssl_dirs')
     @patch.object(hooks, 'update_all_identity_relation_units')
     @patch.object(hooks, 'ensure_permissions')
+    @patch.object(hooks, 'ensure_pki_cert_paths')
     @patch.object(hooks, 'ensure_pki_dir_permissions')
     @patch.object(hooks, 'ensure_ssl_dir')
-    @patch.object(hooks, 'is_pki_enabled')
     @patch.object(hooks, 'peer_units')
     @patch.object(hooks, 'is_ssl_cert_master')
     @patch.object(hooks, 'cluster_joined')
@@ -431,16 +432,15 @@ class KeystoneRelationTests(CharmTestCase):
                                                   ensure_user, cluster_joined,
                                                   mock_is_ssl_cert_master,
                                                   mock_peer_units,
-                                                  mock_is_pki_enabled,
                                                   mock_ensure_ssl_dir,
                                                   mock_ensure_permissions,
+                                                  mock_ensure_pki_cert_paths,
                                                   mock_ensure_pki_permissions,
                                                   mock_update_all_id_rel_units,
                                                   ensure_ssl_dirs,
                                                   mock_ensure_ssl_cert_master,
                                                   mock_log, git_requested):
         git_requested.return_value = False
-        mock_is_pki_enabled.return_value = True
         mock_is_ssl_cert_master.return_value = True
         mock_peer_units.return_value = []
         self.openstack_upgrade_available.return_value = False
@@ -465,9 +465,9 @@ class KeystoneRelationTests(CharmTestCase):
     @patch('keystone_utils.ensure_ssl_cert_master')
     @patch('keystone_utils.ensure_ssl_dirs')
     @patch.object(hooks, 'ensure_permissions')
+    @patch.object(hooks, 'ensure_pki_cert_paths')
     @patch.object(hooks, 'ensure_pki_dir_permissions')
     @patch.object(hooks, 'ensure_ssl_dir')
-    @patch.object(hooks, 'is_pki_enabled')
     @patch.object(hooks, 'is_ssl_cert_master')
     @patch.object(hooks, 'send_ssl_sync_request')
     @patch.object(hooks, 'peer_units')
@@ -486,16 +486,15 @@ class KeystoneRelationTests(CharmTestCase):
                                                    mock_peer_units,
                                                    mock_send_ssl_sync_request,
                                                    mock_is_ssl_cert_master,
-                                                   mock_is_pki_enabled,
                                                    mock_ensure_ssl_dir,
                                                    mock_ensure_permissions,
+                                                   mock_ensure_pki_cert_paths,
                                                    mock_ensure_pki_permissions,
                                                    mock_ensure_ssl_dirs,
                                                    mock_ensure_ssl_cert_master,
                                                    mock_log, git_requested,
                                                    mock_is_db_initialised):
         git_requested.return_value = False
-        mock_is_pki_enabled.return_value = True
         mock_is_ssl_cert_master.return_value = True
         self.is_db_ready.return_value = True
         mock_is_db_initialised.return_value = True
@@ -525,12 +524,12 @@ class KeystoneRelationTests(CharmTestCase):
             remote_unit='unit/0')
         admin_relation_changed.assert_called_with('identity-service:0')
 
+    @patch.object(hooks, 'initialise_pki')
     @patch.object(hooks, 'git_install_requested')
     @patch.object(hooks, 'config_value_changed')
     @patch('keystone_utils.log')
     @patch('keystone_utils.ensure_ssl_cert_master')
     @patch.object(hooks, 'ensure_ssl_dir')
-    @patch.object(hooks, 'is_pki_enabled')
     @patch.object(hooks, 'send_ssl_sync_request')
     @patch.object(hooks, 'is_db_initialised')
     @patch.object(hooks, 'is_db_ready')
@@ -550,14 +549,13 @@ class KeystoneRelationTests(CharmTestCase):
                                         mock_is_db_ready,
                                         mock_is_db_initialised,
                                         mock_send_ssl_sync_request,
-                                        mock_is_pki_enabled,
                                         mock_ensure_ssl_dir,
                                         mock_ensure_ssl_cert_master,
                                         mock_log, config_val_changed,
-                                        git_requested):
+                                        git_requested,
+                                        mock_initialise_pki):
         git_requested.return_value = True
         mock_ensure_ssl_cert_master.return_value = False
-        mock_is_pki_enabled.return_value = False
         self.openstack_upgrade_available.return_value = False
         self.is_elected_leader.return_value = True
         mock_peer_units.return_value = []
@@ -584,12 +582,12 @@ class KeystoneRelationTests(CharmTestCase):
         self.assertFalse(self.openstack_upgrade_available.called)
         self.assertFalse(self.do_openstack_upgrade_reexec.called)
 
+    @patch.object(hooks, 'initialise_pki')
     @patch.object(hooks, 'is_db_initialised')
     @patch.object(hooks, 'git_install_requested')
     @patch.object(hooks, 'config_value_changed')
     @patch.object(hooks, 'ensure_ssl_dir')
     @patch.object(hooks, 'configure_https')
-    @patch.object(hooks, 'is_pki_enabled')
     @patch.object(hooks, 'is_ssl_cert_master')
     @patch.object(hooks, 'peer_units')
     @patch.object(unison, 'get_homedir')
@@ -600,13 +598,13 @@ class KeystoneRelationTests(CharmTestCase):
                                                           ensure_user,
                                                           get_home,
                                                           peer_units, is_ssl,
-                                                          is_pki, config_https,
+                                                          config_https,
                                                           ensure_ssl_dir,
                                                           config_value_changed,
                                                           git_requested,
-                                                          mock_db_init):
+                                                          mock_db_init,
+                                                          mock_initialise_pki):
         ensure_ssl_cert.return_value = False
-        is_pki.return_value = False
         peer_units.return_value = []
 
         git_requested.return_value = False
@@ -627,6 +625,7 @@ class KeystoneRelationTests(CharmTestCase):
                                      mock_log, mock_is_db_initialised):
         mock_is_db_initialised.return_value = True
         self.is_db_ready.return_value = True
+        self.is_service_present.return_value = True
         mock_ensure_ssl_cert_master.return_value = False
         hooks.identity_changed(
             relation_id='identity-service:0',
@@ -634,6 +633,28 @@ class KeystoneRelationTests(CharmTestCase):
         self.add_service_to_keystone.assert_called_with(
             'identity-service:0',
             'unit/0')
+        self.delete_service_entry.assert_called_with(
+            'quantum',
+            'network')
+
+    @patch.object(hooks, 'is_db_initialised')
+    @patch('keystone_utils.log')
+    @patch('keystone_utils.ensure_ssl_cert_master')
+    @patch.object(hooks, 'hashlib')
+    @patch.object(hooks, 'send_notifications')
+    def test_identity_changed_leader_no_neutron(self, mock_send_notifications,
+                                                mock_hashlib,
+                                                mock_ensure_ssl_cert_master,
+                                                mock_log,
+                                                mock_is_db_initialised):
+        mock_is_db_initialised.return_value = True
+        self.is_db_ready.return_value = True
+        self.is_service_present.return_value = False
+        mock_ensure_ssl_cert_master.return_value = False
+        hooks.identity_changed(
+            relation_id='identity-service:0',
+            remote_unit='unit/0')
+        self.assertFalse(self.delete_service_entry.called)
 
     @patch.object(hooks, 'local_unit')
     @patch('keystone_utils.log')
@@ -662,6 +683,7 @@ class KeystoneRelationTests(CharmTestCase):
             user=self.ssh_user, group='juju_keystone',
             peer_interface='cluster', ensure_local_user=True)
 
+    @patch.object(hooks, 'initialise_pki')
     @patch.object(hooks, 'update_all_identity_relation_units')
     @patch.object(hooks, 'get_ssl_sync_request_units')
     @patch.object(hooks, 'is_ssl_cert_master')
@@ -681,7 +703,8 @@ class KeystoneRelationTests(CharmTestCase):
                              mock_peer_units,
                              mock_is_ssl_cert_master,
                              mock_get_ssl_sync_request_units,
-                             mock_update_all_identity_relation_units):
+                             mock_update_all_identity_relation_units,
+                             mock_initialise_pki):
 
         relation_settings = {'foo_passwd': '123',
                              'identity-service:16_foo': 'bar'}

unit_tests/test_keystone_utils.py

@@ -479,22 +479,11 @@ class TestKeystoneUtils(CharmTestCase):
         self.assertTrue(utils.is_db_ready())
 
     @patch.object(utils, 'peer_units')
-    @patch.object(utils, 'is_ssl_enabled')
+    def test_ensure_ssl_cert_master_ssl_no_peers(self, mock_peer_units):
-    def test_ensure_ssl_cert_master_no_ssl(self, mock_is_ssl_enabled,
-                                           mock_peer_units):
-        mock_is_ssl_enabled.return_value = False
-        self.assertFalse(utils.ensure_ssl_cert_master())
-        self.assertFalse(self.relation_set.called)
-
-    @patch.object(utils, 'peer_units')
-    @patch.object(utils, 'is_ssl_enabled')
-    def test_ensure_ssl_cert_master_ssl_no_peers(self, mock_is_ssl_enabled,
-                                                 mock_peer_units):
         def mock_rel_get(unit=None, **kwargs):
             return None
 
         self.relation_get.side_effect = mock_rel_get
-        mock_is_ssl_enabled.return_value = True
         self.relation_ids.return_value = ['cluster:0']
         self.local_unit.return_value = 'unit/0'
         self.related_units.return_value = []
@@ -508,9 +497,7 @@ class TestKeystoneUtils(CharmTestCase):
                                              relation_settings=settings)
 
     @patch.object(utils, 'peer_units')
-    @patch.object(utils, 'is_ssl_enabled')
     def test_ensure_ssl_cert_master_ssl_master_no_peers(self,
-                                                        mock_is_ssl_enabled,
                                                         mock_peer_units):
         def mock_rel_get(unit=None, **kwargs):
             if unit == 'unit/0':
@@ -519,7 +506,6 @@ class TestKeystoneUtils(CharmTestCase):
             return None
 
         self.relation_get.side_effect = mock_rel_get
-        mock_is_ssl_enabled.return_value = True
         self.relation_ids.return_value = ['cluster:0']
         self.local_unit.return_value = 'unit/0'
         self.related_units.return_value = []
@@ -533,10 +519,7 @@ class TestKeystoneUtils(CharmTestCase):
                                              relation_settings=settings)
 
     @patch.object(utils, 'peer_units')
-    @patch.object(utils, 'is_ssl_enabled')
+    def test_ensure_ssl_cert_master_ssl_not_leader(self, mock_peer_units):
-    def test_ensure_ssl_cert_master_ssl_not_leader(self, mock_is_ssl_enabled,
-                                                   mock_peer_units):
-        mock_is_ssl_enabled.return_value = True
         self.relation_ids.return_value = ['cluster:0']
         self.local_unit.return_value = 'unit/0'
         mock_peer_units.return_value = ['unit/1']
@@ -546,9 +529,7 @@ class TestKeystoneUtils(CharmTestCase):
         self.assertFalse(self.relation_set.called)
 
     @patch.object(utils, 'peer_units')
-    @patch.object(utils, 'is_ssl_enabled')
     def test_ensure_ssl_cert_master_is_leader_new_peer(self,
-                                                       mock_is_ssl_enabled,
                                                        mock_peer_units):
         def mock_rel_get(unit=None, **kwargs):
             if unit == 'unit/0':
@@ -557,7 +538,6 @@ class TestKeystoneUtils(CharmTestCase):
             return 'unknown'
 
         self.relation_get.side_effect = mock_rel_get
-        mock_is_ssl_enabled.return_value = True
         self.relation_ids.return_value = ['cluster:0']
         self.local_unit.return_value = 'unit/0'
         mock_peer_units.return_value = ['unit/1']
@@ -570,9 +550,7 @@ class TestKeystoneUtils(CharmTestCase):
                                              relation_settings=settings)
 
     @patch.object(utils, 'peer_units')
-    @patch.object(utils, 'is_ssl_enabled')
     def test_ensure_ssl_cert_master_is_leader_no_new_peer(self,
-                                                          mock_is_ssl_enabled,
                                                           mock_peer_units):
         def mock_rel_get(unit=None, **kwargs):
             if unit == 'unit/0':
@@ -581,7 +559,6 @@ class TestKeystoneUtils(CharmTestCase):
             return 'unit/0'
 
         self.relation_get.side_effect = mock_rel_get
-        mock_is_ssl_enabled.return_value = True
         self.relation_ids.return_value = ['cluster:0']
         self.local_unit.return_value = 'unit/0'
         mock_peer_units.return_value = ['unit/1']
@@ -621,9 +598,7 @@ class TestKeystoneUtils(CharmTestCase):
         )
 
     @patch.object(utils, 'peer_units')
-    @patch.object(utils, 'is_ssl_enabled')
     def test_ensure_ssl_cert_master_is_leader_bad_votes(self,
-                                                        mock_is_ssl_enabled,
                                                         mock_peer_units):
         counter = {0: 0}
 
@@ -637,7 +612,6 @@ class TestKeystoneUtils(CharmTestCase):
             return ret
 
         self.relation_get.side_effect = mock_rel_get
-        mock_is_ssl_enabled.return_value = True
         self.relation_ids.return_value = ['cluster:0']
         self.local_unit.return_value = 'unit/0'
         mock_peer_units.return_value = ['unit/1']
@@ -730,6 +704,28 @@ class TestKeystoneUtils(CharmTestCase):
         self.assertEquals(render.call_args_list, expected)
         service_restart.assert_called_with('keystone')
 
+    @patch.object(manager, 'KeystoneManager')
+    def test_is_service_present(self, KeystoneManager):
+        mock_keystone = MagicMock()
+        mock_keystone.resolve_service_id.return_value = 'sid1'
+        KeystoneManager.return_value = mock_keystone
+        self.assertTrue(utils.is_service_present('bob', 'bill'))
+
+    @patch.object(manager, 'KeystoneManager')
+    def test_is_service_present_false(self, KeystoneManager):
+        mock_keystone = MagicMock()
+        mock_keystone.resolve_service_id.return_value = None
+        KeystoneManager.return_value = mock_keystone
+        self.assertFalse(utils.is_service_present('bob', 'bill'))
+
+    @patch.object(manager, 'KeystoneManager')
+    def test_delete_service_entry(self, KeystoneManager):
+        mock_keystone = MagicMock()
+        mock_keystone.resolve_service_id.return_value = 'sid1'
+        KeystoneManager.return_value = mock_keystone
+        utils.delete_service_entry('bob', 'bill')
+        mock_keystone.api.services.delete.assert_called_with('sid1')
+
     @patch.object(utils, 'HookData')
     @patch.object(utils, 'kv')
     def test_is_paused(self, kv, HookData):