Sync charm-helpers for Rocky series support

Change-Id: Ib15aa2d76e009415f525132492bcc3679750bfd9

hooks/charmhelpers/contrib/hahelpers/cluster.py

@@ -223,6 +223,11 @@ def https():
         return True
     if config_get('ssl_cert') and config_get('ssl_key'):
         return True
+    for r_id in relation_ids('certificates'):
+        for unit in relation_list(r_id):
+            ca = relation_get('ca', rid=r_id, unit=unit)
+            if ca:
+                return True
     for r_id in relation_ids('identity-service'):
         for unit in relation_list(r_id):
             # TODO - needs fixing for new helper as ssl_cert/key suffixes with CN

hooks/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -291,6 +291,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('zesty', None): self.zesty_ocata,
             ('artful', None): self.artful_pike,
             ('bionic', None): self.bionic_queens,
+            ('bionic', 'cloud:bionic-rocky'): self.bionic_rocky,
+            ('cosmic', None): self.cosmic_rocky,
         }
         return releases[(self.series, self.openstack)]
 
@@ -306,6 +308,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('zesty', 'ocata'),
             ('artful', 'pike'),
             ('bionic', 'queens'),
+            ('cosmic', 'rocky'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]

hooks/charmhelpers/contrib/openstack/amulet/utils.py

@@ -40,6 +40,7 @@ import novaclient
 import pika
 import swiftclient
 
+from charmhelpers.core.decorators import retry_on_exception
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
 )
@@ -55,7 +56,7 @@ OPENSTACK_RELEASES_PAIRS = [
     'trusty_mitaka', 'xenial_mitaka', 'xenial_newton',
     'yakkety_newton', 'xenial_ocata', 'zesty_ocata',
     'xenial_pike', 'artful_pike', 'xenial_queens',
-    'bionic_queens']
+    'bionic_queens', 'bionic_rocky', 'cosmic_rocky']
 
 
 class OpenStackAmuletUtils(AmuletUtils):
@@ -423,6 +424,7 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.debug('Checking if tenant exists ({})...'.format(tenant))
         return tenant in [t.name for t in keystone.tenants.list()]
 
+    @retry_on_exception(num_retries=5, base_delay=1)
     def keystone_wait_for_propagation(self, sentry_relation_pairs,
                                       api_version):
         """Iterate over list of sentry and relation tuples and verify that
@@ -542,7 +544,7 @@ class OpenStackAmuletUtils(AmuletUtils):
         return ep
 
     def get_default_keystone_session(self, keystone_sentry,
-                                     openstack_release=None):
+                                     openstack_release=None, api_version=2):
         """Return a keystone session object and client object assuming standard
            default settings
 
@@ -557,12 +559,12 @@ class OpenStackAmuletUtils(AmuletUtils):
                eyc
         """
         self.log.debug('Authenticating keystone admin...')
-        api_version = 2
-        client_class = keystone_client.Client
         # 11 => xenial_queens
-        if openstack_release and openstack_release >= 11:
+        if api_version == 3 or (openstack_release and openstack_release >= 11):
-            api_version = 3
             client_class = keystone_client_v3.Client
+            api_version = 3
+        else:
+            client_class = keystone_client.Client
         keystone_ip = keystone_sentry.info['public-address']
         session, auth = self.get_keystone_session(
             keystone_ip,

hooks/charmhelpers/contrib/openstack/context.py

@@ -190,8 +190,8 @@ class OSContextGenerator(object):
 class SharedDBContext(OSContextGenerator):
     interfaces = ['shared-db']
 
-    def __init__(self,
+    def __init__(self, database=None, user=None, relation_prefix=None,
-                 database=None, user=None, relation_prefix=None, ssl_dir=None):
+                 ssl_dir=None, relation_id=None):
         """Allows inspecting relation for settings prefixed with
         relation_prefix. This is useful for parsing access for multiple
         databases returned via the shared-db interface (eg, nova_password,
@@ -202,6 +202,7 @@ class SharedDBContext(OSContextGenerator):
         self.user = user
         self.ssl_dir = ssl_dir
         self.rel_name = self.interfaces[0]
+        self.relation_id = relation_id
 
     def __call__(self):
         self.database = self.database or config('database')
@@ -235,7 +236,12 @@ class SharedDBContext(OSContextGenerator):
         if self.relation_prefix:
             password_setting = self.relation_prefix + '_password'
 
-        for rid in relation_ids(self.interfaces[0]):
+        if self.relation_id:
+            rids = [self.relation_id]
+        else:
+            rids = relation_ids(self.interfaces[0])
+
+        for rid in rids:
             self.related = True
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
@@ -448,11 +454,13 @@ class IdentityCredentialsContext(IdentityServiceContext):
 
 class AMQPContext(OSContextGenerator):
 
-    def __init__(self, ssl_dir=None, rel_name='amqp', relation_prefix=None):
+    def __init__(self, ssl_dir=None, rel_name='amqp', relation_prefix=None,
+                 relation_id=None):
         self.ssl_dir = ssl_dir
         self.rel_name = rel_name
         self.relation_prefix = relation_prefix
         self.interfaces = [rel_name]
+        self.relation_id = relation_id
 
     def __call__(self):
         log('Generating template context for amqp', level=DEBUG)
@@ -473,7 +481,11 @@ class AMQPContext(OSContextGenerator):
             raise OSContextError
 
         ctxt = {}
-        for rid in relation_ids(self.rel_name):
+        if self.relation_id:
+            rids = [self.relation_id]
+        else:
+            rids = relation_ids(self.rel_name)
+        for rid in rids:
             ha_vip_only = False
             self.related = True
             transport_hosts = None
@@ -789,17 +801,18 @@ class ApacheSSLContext(OSContextGenerator):
         ssl_dir = os.path.join('/etc/apache2/ssl/', self.service_namespace)
         mkdir(path=ssl_dir)
         cert, key = get_cert(cn)
-        if cn:
+        if cert and key:
-            cert_filename = 'cert_{}'.format(cn)
+            if cn:
-            key_filename = 'key_{}'.format(cn)
+                cert_filename = 'cert_{}'.format(cn)
-        else:
+                key_filename = 'key_{}'.format(cn)
-            cert_filename = 'cert'
+            else:
-            key_filename = 'key'
+                cert_filename = 'cert'
+                key_filename = 'key'
 
-        write_file(path=os.path.join(ssl_dir, cert_filename),
+            write_file(path=os.path.join(ssl_dir, cert_filename),
-                   content=b64decode(cert), perms=0o640)
+                       content=b64decode(cert), perms=0o640)
-        write_file(path=os.path.join(ssl_dir, key_filename),
+            write_file(path=os.path.join(ssl_dir, key_filename),
-                   content=b64decode(key), perms=0o640)
+                       content=b64decode(key), perms=0o640)
 
     def configure_ca(self):
         ca_cert = get_ca_cert()
@@ -871,23 +884,31 @@ class ApacheSSLContext(OSContextGenerator):
         if not self.external_ports or not https():
             return {}
 
-        self.configure_ca()
+        use_keystone_ca = True
+        for rid in relation_ids('certificates'):
+            if related_units(rid):
+                use_keystone_ca = False
+
+        if use_keystone_ca:
+            self.configure_ca()
+
         self.enable_modules()
 
         ctxt = {'namespace': self.service_namespace,
                 'endpoints': [],
                 'ext_ports': []}
 
-        cns = self.canonical_names()
+        if use_keystone_ca:
-        if cns:
+            cns = self.canonical_names()
-            for cn in cns:
+            if cns:
-                self.configure_cert(cn)
+                for cn in cns:
-        else:
+                    self.configure_cert(cn)
-            # Expect cert/key provided in config (currently assumed that ca
+            else:
-            # uses ip for cn)
+                # Expect cert/key provided in config (currently assumed that ca
-            for net_type in (INTERNAL, ADMIN, PUBLIC):
+                # uses ip for cn)
-                cn = resolve_address(endpoint_type=net_type)
+                for net_type in (INTERNAL, ADMIN, PUBLIC):
-                self.configure_cert(cn)
+                    cn = resolve_address(endpoint_type=net_type)
+                    self.configure_cert(cn)
 
         addresses = self.get_network_addresses()
         for address, endpoint in addresses:

hooks/charmhelpers/contrib/openstack/ip.py

@@ -184,3 +184,13 @@ def resolve_address(endpoint_type=PUBLIC, override=True):
                          "clustered=%s)" % (net_type, clustered))
 
     return resolved_address
+
+
+def get_vip_in_network(network):
+    matching_vip = None
+    vips = config('vip')
+    if vips:
+        for vip in vips.split():
+            if is_address_in_network(network, vip):
+                matching_vip = vip
+    return matching_vip

hooks/charmhelpers/contrib/openstack/utils.py

@@ -133,6 +133,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('zesty', 'ocata'),
     ('artful', 'pike'),
     ('bionic', 'queens'),
+    ('cosmic', 'rocky'),
 ])
 
 
@@ -151,6 +152,7 @@ OPENSTACK_CODENAMES = OrderedDict([
     ('2017.1', 'ocata'),
     ('2017.2', 'pike'),
     ('2018.1', 'queens'),
+    ('2018.2', 'rocky'),
 ])
 
 # The ugly duckling - must list releases oldest to newest
@@ -183,6 +185,8 @@ SWIFT_CODENAMES = OrderedDict([
         ['2.13.0', '2.15.0']),
     ('queens',
         ['2.16.0', '2.17.0']),
+    ('rocky',
+        ['2.18.0']),
 ])
 
 # >= Liberty version->codename mapping

hooks/charmhelpers/core/hookenv.py

@@ -972,6 +972,13 @@ def application_version_set(version):
         log("Application Version: {}".format(version))
 
 
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def goal_state():
+    """Juju goal state values"""
+    cmd = ['goal-state', '--format=json']
+    return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+
+
 @translate_exc(from_exc=OSError, to_exc=NotImplementedError)
 def is_leader():
     """Does the current unit hold the juju leadership

hooks/charmhelpers/fetch/ubuntu.py

@@ -158,6 +158,14 @@ CLOUD_ARCHIVE_POCKETS = {
     'queens/proposed': 'xenial-proposed/queens',
     'xenial-queens/proposed': 'xenial-proposed/queens',
     'xenial-proposed/queens': 'xenial-proposed/queens',
+    # Rocky
+    'rocky': 'bionic-updates/rocky',
+    'bionic-rocky': 'bionic-updates/rocky',
+    'bionic-rocky/updates': 'bionic-updates/rocky',
+    'bionic-updates/rocky': 'bionic-updates/rocky',
+    'rocky/proposed': 'bionic-proposed/rocky',
+    'bionic-rocky/proposed': 'bionic-proposed/rocky',
+    'bionic-proposed/rocky': 'bionic-proposed/rocky',
 }
 
 

tests/charmhelpers/contrib/amulet/deployment.py

@@ -50,7 +50,8 @@ class AmuletDeployment(object):
             this_service['units'] = 1
 
         self.d.add(this_service['name'], units=this_service['units'],
-                   constraints=this_service.get('constraints'))
+                   constraints=this_service.get('constraints'),
+                   storage=this_service.get('storage'))
 
         for svc in other_services:
             if 'location' in svc:
@@ -64,7 +65,8 @@ class AmuletDeployment(object):
                 svc['units'] = 1
 
             self.d.add(svc['name'], charm=branch_location, units=svc['units'],
-                       constraints=svc.get('constraints'))
+                       constraints=svc.get('constraints'),
+                       storage=svc.get('storage'))
 
     def _add_relations(self, relations):
         """Add all of the relations for the services."""

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -291,6 +291,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('zesty', None): self.zesty_ocata,
             ('artful', None): self.artful_pike,
             ('bionic', None): self.bionic_queens,
+            ('bionic', 'cloud:bionic-rocky'): self.bionic_rocky,
+            ('cosmic', None): self.cosmic_rocky,
         }
         return releases[(self.series, self.openstack)]
 
@@ -306,6 +308,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('zesty', 'ocata'),
             ('artful', 'pike'),
             ('bionic', 'queens'),
+            ('cosmic', 'rocky'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -40,6 +40,7 @@ import novaclient
 import pika
 import swiftclient
 
+from charmhelpers.core.decorators import retry_on_exception
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
 )
@@ -55,7 +56,7 @@ OPENSTACK_RELEASES_PAIRS = [
     'trusty_mitaka', 'xenial_mitaka', 'xenial_newton',
     'yakkety_newton', 'xenial_ocata', 'zesty_ocata',
     'xenial_pike', 'artful_pike', 'xenial_queens',
-    'bionic_queens']
+    'bionic_queens', 'bionic_rocky', 'cosmic_rocky']
 
 
 class OpenStackAmuletUtils(AmuletUtils):
@@ -423,6 +424,7 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.debug('Checking if tenant exists ({})...'.format(tenant))
         return tenant in [t.name for t in keystone.tenants.list()]
 
+    @retry_on_exception(num_retries=5, base_delay=1)
     def keystone_wait_for_propagation(self, sentry_relation_pairs,
                                       api_version):
         """Iterate over list of sentry and relation tuples and verify that
@@ -542,7 +544,7 @@ class OpenStackAmuletUtils(AmuletUtils):
         return ep
 
     def get_default_keystone_session(self, keystone_sentry,
-                                     openstack_release=None):
+                                     openstack_release=None, api_version=2):
         """Return a keystone session object and client object assuming standard
            default settings
 
@@ -557,12 +559,12 @@ class OpenStackAmuletUtils(AmuletUtils):
                eyc
         """
         self.log.debug('Authenticating keystone admin...')
-        api_version = 2
-        client_class = keystone_client.Client
         # 11 => xenial_queens
-        if openstack_release and openstack_release >= 11:
+        if api_version == 3 or (openstack_release and openstack_release >= 11):
-            api_version = 3
             client_class = keystone_client_v3.Client
+            api_version = 3
+        else:
+            client_class = keystone_client.Client
         keystone_ip = keystone_sentry.info['public-address']
         session, auth = self.get_keystone_session(
             keystone_ip,

tests/charmhelpers/core/hookenv.py

@@ -972,6 +972,13 @@ def application_version_set(version):
         log("Application Version: {}".format(version))
 
 
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def goal_state():
+    """Juju goal state values"""
+    cmd = ['goal-state', '--format=json']
+    return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+
+
 @translate_exc(from_exc=OSError, to_exc=NotImplementedError)
 def is_leader():
     """Does the current unit hold the juju leadership