Merge "Enable focal testing and support"

charmhelpers/contrib/openstack/policyd.py

@@ -17,7 +17,6 @@ import contextlib
 import os
 import six
 import shutil
-import sys
 import yaml
 import zipfile
 
@@ -531,7 +530,7 @@ def clean_policyd_dir_for(service, keep_paths=None, user=None, group=None):
     hookenv.log("Cleaning path: {}".format(path), level=hookenv.DEBUG)
     if not os.path.exists(path):
         ch_host.mkdir(path, owner=_user, group=_group, perms=0o775)
-    _scanner = os.scandir if sys.version_info > (3, 4) else _py2_scandir
+    _scanner = os.scandir if hasattr(os, 'scandir') else _fallback_scandir
     for direntry in _scanner(path):
         # see if the path should be kept.
         if direntry.path in keep_paths:
@@ -560,23 +559,25 @@ def maybe_create_directory_for(path, user, group):
 
 
 @contextlib.contextmanager
-def _py2_scandir(path):
-    """provide a py2 implementation of os.scandir if this module ever gets used
-    in a py2 charm (unlikely).  uses os.listdir() to get the names in the path,
-    and then mocks the is_dir() function using os.path.isdir() to check for a
+def _fallback_scandir(path):
+    """Fallback os.scandir implementation.
+
+    provide a fallback implementation of os.scandir if this module ever gets
+    used in a py2 or py34 charm. Uses os.listdir() to get the names in the path,
+    and then mocks the is_dir() function using os.path.isdir() to check for
     directory.
 
     :param path: the path to list the directories for
     :type path: str
-    :returns: Generator that provides _P27Direntry objects
-    :rtype: ContextManager[_P27Direntry]
+    :returns: Generator that provides _FBDirectory objects
+    :rtype: ContextManager[_FBDirectory]
     """
     for f in os.listdir(path):
-        yield _P27Direntry(f)
+        yield _FBDirectory(f)
 
 
-class _P27Direntry(object):
-    """Mock a scandir Direntry object with enough to use in
+class _FBDirectory(object):
+    """Mock a scandir Directory object with enough to use in
     clean_policyd_dir_for
     """
 

charmhelpers/contrib/openstack/utils.py

@@ -278,7 +278,7 @@ PACKAGE_CODENAMES = {
         ('14', 'rocky'),
         ('15', 'stein'),
         ('16', 'train'),
-        ('17', 'ussuri'),
+        ('18', 'ussuri'),
     ]),
     'ceilometer-common': OrderedDict([
         ('5', 'liberty'),
@@ -326,7 +326,7 @@ PACKAGE_CODENAMES = {
         ('14', 'rocky'),
         ('15', 'stein'),
         ('16', 'train'),
-        ('17', 'ussuri'),
+        ('18', 'ussuri'),
     ]),
 }
 
@@ -555,9 +555,8 @@ def reset_os_release():
     _os_rel = None
 
 
-def os_release(package, base=None, reset_cache=False):
-    '''
-    Returns OpenStack release codename from a cached global.
+def os_release(package, base=None, reset_cache=False, source_key=None):
+    """Returns OpenStack release codename from a cached global.
 
     If reset_cache then unset the cached os_release version and return the
     freshly determined version.
@@ -565,7 +564,20 @@ def os_release(package, base=None, reset_cache=False):
     If the codename can not be determined from either an installed package or
     the installation source, the earliest release supported by the charm should
     be returned.
-    '''
+
+    :param package: Name of package to determine release from
+    :type package: str
+    :param base: Fallback codename if endavours to determine from package fail
+    :type base: Optional[str]
+    :param reset_cache: Reset any cached codename value
+    :type reset_cache: bool
+    :param source_key: Name of source configuration option
+                       (default: 'openstack-origin')
+    :type source_key: Optional[str]
+    :returns: OpenStack release codename
+    :rtype: str
+    """
+    source_key = source_key or 'openstack-origin'
     if not base:
         base = UBUNTU_OPENSTACK_RELEASE[lsb_release()['DISTRIB_CODENAME']]
     global _os_rel
@@ -575,7 +587,7 @@ def os_release(package, base=None, reset_cache=False):
         return _os_rel
     _os_rel = (
         get_os_codename_package(package, fatal=False) or
-        get_os_codename_install_source(config('openstack-origin')) or
+        get_os_codename_install_source(config(source_key)) or
         base)
     return _os_rel
 
@@ -658,6 +670,93 @@ def config_value_changed(option):
         return current != saved
 
 
+def get_endpoint_key(service_name, relation_id, unit_name):
+    """Return the key used to refer to an ep changed notification from a unit.
+
+    :param service_name: Service name eg nova, neutron, placement etc
+    :type service_name: str
+    :param relation_id: The id of the relation the unit is on.
+    :type relation_id: str
+    :param unit_name: The name of the unit publishing the notification.
+    :type unit_name: str
+    :returns: The key used to refer to an ep changed notification from a unit
+    :rtype: str
+    """
+    return '{}-{}-{}'.format(
+        service_name,
+        relation_id.replace(':', '_'),
+        unit_name.replace('/', '_'))
+
+
+def get_endpoint_notifications(service_names, rel_name='identity-service'):
+    """Return all notifications for the given services.
+
+    :param service_names: List of service name.
+    :type service_name: List
+    :param rel_name: Name of the relation to query
+    :type rel_name: str
+    :returns: A dict containing the source of the notification and its nonce.
+    :rtype: Dict[str, str]
+    """
+    notifications = {}
+    for rid in relation_ids(rel_name):
+        for unit in related_units(relid=rid):
+            ep_changed_json = relation_get(
+                rid=rid,
+                unit=unit,
+                attribute='ep_changed')
+            if ep_changed_json:
+                ep_changed = json.loads(ep_changed_json)
+                for service in service_names:
+                    if ep_changed.get(service):
+                        key = get_endpoint_key(service, rid, unit)
+                        notifications[key] = ep_changed[service]
+    return notifications
+
+
+def endpoint_changed(service_name, rel_name='identity-service'):
+    """Whether a new notification has been recieved for an endpoint.
+
+    :param service_name: Service name eg nova, neutron, placement etc
+    :type service_name: str
+    :param rel_name: Name of the relation to query
+    :type rel_name: str
+    :returns: Whether endpoint has changed
+    :rtype: bool
+    """
+    changed = False
+    with unitdata.HookData()() as t:
+        db = t[0]
+        notifications = get_endpoint_notifications(
+            [service_name],
+            rel_name=rel_name)
+        for key, nonce in notifications.items():
+            if db.get(key) != nonce:
+                juju_log(('New endpoint change notification found: '
+                          '{}={}').format(key, nonce),
+                         'INFO')
+                changed = True
+                break
+    return changed
+
+
+def save_endpoint_changed_triggers(service_names, rel_name='identity-service'):
+    """Save the enpoint triggers in  db so it can be tracked if they changed.
+
+    :param service_names: List of service name.
+    :type service_name: List
+    :param rel_name: Name of the relation to query
+    :type rel_name: str
+    """
+    with unitdata.HookData()() as t:
+        db = t[0]
+        notifications = get_endpoint_notifications(
+            service_names,
+            rel_name=rel_name)
+        for key, nonce in notifications.items():
+            db.set(key, nonce)
+
+
 def save_script_rc(script_path="scripts/scriptrc", **env_vars):
     """
     Write an rc file in the charm-delivered directory containing

charmhelpers/contrib/openstack/vaultlocker.py

@@ -37,7 +37,19 @@ class VaultKVContext(context.OSContextGenerator):
         )
 
     def __call__(self):
-        import hvac
+        try:
+            import hvac
+        except ImportError:
+            # BUG: #1862085 - if the relation is made to vault, but the
+            # 'encrypt' option is not made, then the charm errors with an
+            # import warning.  This catches that, logs a warning, and returns
+            # with an empty context.
+            hookenv.log("VaultKVContext: trying to use hvac pythong module "
+                        "but it's not available.  Is secrets-stroage relation "
+                        "made, but encrypt option not set?",
+                        level=hookenv.WARNING)
+            # return an emptry context on hvac import error
+            return {}
         ctxt = {}
         # NOTE(hopem): see https://bugs.launchpad.net/charm-helpers/+bug/1849323
         db = unitdata.kv()

charmhelpers/contrib/storage/linux/ceph.py

@@ -1042,7 +1042,7 @@ def filesystem_mounted(fs):
 def make_filesystem(blk_device, fstype='ext4', timeout=10):
     """Make a new filesystem on the specified block device."""
     count = 0
-    e_noent = os.errno.ENOENT
+    e_noent = errno.ENOENT
     while not os.path.exists(blk_device):
         if count >= timeout:
             log('Gave up waiting on block device %s' % blk_device,

charmhelpers/core/host_factory/ubuntu.py

@@ -25,6 +25,7 @@ UBUNTU_RELEASES = (
     'cosmic',
     'disco',
     'eoan',
+    'focal'
 )
 
 

charmhelpers/osplatform.py

@@ -1,4 +1,5 @@
 import platform
+import os
 
 
 def get_platform():
@@ -9,9 +10,13 @@ def get_platform():
     This string is used to decide which platform module should be imported.
     """
     # linux_distribution is deprecated and will be removed in Python 3.7
-    # Warings *not* disabled, as we certainly need to fix this.
-    tuple_platform = platform.linux_distribution()
-    current_platform = tuple_platform[0]
+    # Warnings *not* disabled, as we certainly need to fix this.
+    if hasattr(platform, 'linux_distribution'):
+        tuple_platform = platform.linux_distribution()
+        current_platform = tuple_platform[0]
+    else:
+        current_platform = _get_platform_from_fs()
+
     if "Ubuntu" in current_platform:
         return "ubuntu"
     elif "CentOS" in current_platform:
@@ -26,3 +31,16 @@ def get_platform():
     else:
         raise RuntimeError("This module is not supported on {}."
                            .format(current_platform))
+
+
+def _get_platform_from_fs():
+    """Get Platform from /etc/os-release."""
+    with open(os.path.join(os.sep, 'etc', 'os-release')) as fin:
+        content = dict(
+            line.split('=', 1)
+            for line in fin.read().splitlines()
+            if '=' in line
+        )
+    for k, v in content.items():
+        content[k] = v.strip('"')
+    return content["NAME"]

metadata.yaml

@@ -11,6 +11,7 @@ series:
   - xenial
   - bionic
   - eoan
+  - focal
   - trusty
 extra-bindings:
   public:

tests/bundles/bionic-train.yaml

@@ -122,4 +122,3 @@ applications:
       block-device: None
       glance-api-version: 2
       overwrite: "true"
-      #ephemeral-unmount: /mnt

tests/bundles/bionic-ussuri.yaml

@@ -0,0 +1,125 @@
+series: bionic
+relations:
+- - nova-compute:image-service
+  - glance:image-service
+- - nova-compute:amqp
+  - rabbitmq-server:amqp
+- - nova-cloud-controller:shared-db
+  - percona-cluster:shared-db
+- - nova-cloud-controller:identity-service
+  - keystone:identity-service
+- - nova-cloud-controller:amqp
+  - rabbitmq-server:amqp
+- - nova-cloud-controller:cloud-compute
+  - nova-compute:cloud-compute
+- - nova-cloud-controller:image-service
+  - glance:image-service
+- - placement:shared-db
+  - percona-cluster:shared-db
+- - placement:identity-service
+  - keystone:identity-service
+- - placement:placement
+  - nova-cloud-controller:placement
+- - keystone:shared-db
+  - percona-cluster:shared-db
+- - glance:identity-service
+  - keystone:identity-service
+- - glance:shared-db
+  - percona-cluster:shared-db
+- - glance:amqp
+  - rabbitmq-server:amqp
+- - neutron-gateway:amqp
+  - rabbitmq-server:amqp
+- - nova-cloud-controller:quantum-network-service
+  - neutron-gateway:quantum-network-service
+- - neutron-api:shared-db
+  - percona-cluster:shared-db
+- - neutron-api:amqp
+  - rabbitmq-server:amqp
+- - neutron-api:neutron-api
+  - nova-cloud-controller:neutron-api
+- - neutron-api:identity-service
+  - keystone:identity-service
+- - nova-compute:neutron-plugin
+  - neutron-openvswitch:neutron-plugin
+- - rabbitmq-server:amqp
+  - neutron-openvswitch:amqp
+- - cinder:shared-db
+  - percona-cluster:shared-db
+- - cinder:identity-service
+  - keystone:identity-service
+- - cinder:amqp
+  - rabbitmq-server:amqp
+- - cinder:image-service
+  - glance:image-service
+applications:
+  rabbitmq-server:
+    charm: cs:~openstack-charmers-next/rabbitmq-server
+    num_units: 1
+  percona-cluster:
+    charm: cs:~openstack-charmers-next/percona-cluster
+    num_units: 1
+    options:
+      max-connections: 1000
+      innodb-buffer-pool-size: 256M
+  nova-cloud-controller:
+    charm: cs:~openstack-charmers-next/nova-cloud-controller
+    num_units: 1
+    options:
+      openstack-origin: cloud:bionic-ussuri
+      network-manager: Neutron
+      debug: true
+  placement:
+    charm: cs:~openstack-charmers-next/placement
+    num_units: 1
+    constraints: mem=1G
+    options:
+      openstack-origin: cloud:bionic-ussuri
+  neutron-api:
+    charm: cs:~openstack-charmers-next/neutron-api
+    num_units: 1
+    options:
+      openstack-origin: cloud:bionic-ussuri
+      flat-network-providers: physnet1
+      neutron-security-groups: true
+      manage-neutron-plugin-legacy-mode: true
+  keystone:
+    charm: cs:~openstack-charmers-next/keystone
+    num_units: 1
+    options:
+      openstack-origin: cloud:bionic-ussuri
+  neutron-gateway:
+    charm: cs:~openstack-charmers-next/neutron-gateway
+    num_units: 1
+    options:
+      openstack-origin: cloud:bionic-ussuri
+      bridge-mappings: physnet1:br-ex
+  glance:
+    charm: cs:~openstack-charmers-next/glance
+    num_units: 1
+    options:
+      openstack-origin: cloud:bionic-ussuri
+  neutron-openvswitch:
+    charm: cs:~openstack-charmers-next/neutron-openvswitch
+  nova-compute:
+    charm: cs:~openstack-charmers-next/nova-compute
+    num_units: 1
+    constraints: mem=4G cores=4
+    options:
+      openstack-origin: cloud:bionic-ussuri
+      config-flags: auto_assign_floating_ip=False
+      enable-live-migration: false
+      aa-profile-mode: enforce
+      ephemeral-device: /dev/vdb
+      ephemeral-unmount: /mnt
+      debug: true
+  cinder:
+    series: bionic
+    num_units: 1
+    storage:
+      block-devices: '40G'
+    options:
+      openstack-origin: cloud:bionic-ussuri
+      block-device: None
+      glance-api-version: 2
+      overwrite: "true"

tests/bundles/focal-ussuri.yaml

@@ -0,0 +1,235 @@
+variables:
+  openstack-origin: &openstack-origin distro
+
+series: focal
+
+comment:
+- 'machines section to decide order of deployment. database sooner = faster'
+machines:
+  '0':
+    constraints: mem=3072M
+  '1':
+    constraints: mem=3072M
+  '2':
+    constraints: mem=3072M
+  '3':
+  '4':
+  '5':
+  '6':
+  '7':
+  '8':
+  '9':
+  '10':
+  '11':
+
+applications:
+
+  nova-cloud-controller-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+  placement-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+  keystone-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+  glance-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+  neutron-api-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+  cinder-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+
+  mysql-innodb-cluster:
+    charm: cs:~openstack-charmers-next/mysql-innodb-cluster
+    num_units: 3
+    options:
+      source: *openstack-origin
+    to:
+      - '0'
+      - '1'
+      - '2'
+
+  rabbitmq-server:
+    charm: cs:~openstack-charmers-next/rabbitmq-server
+    num_units: 1
+    options:
+      source: *openstack-origin
+    to:
+      - '3'
+
+  nova-cloud-controller:
+    charm: cs:~openstack-charmers-next/nova-cloud-controller
+    num_units: 1
+    options:
+      network-manager: Neutron
+      debug: true
+      openstack-origin: *openstack-origin
+    to:
+      - '4'
+
+  placement:
+    charm: cs:~openstack-charmers-next/placement
+    num_units: 1
+    constraints: mem=1G
+    options:
+      openstack-origin: *openstack-origin
+    to:
+     - '5'
+
+  neutron-api:
+    charm: cs:~openstack-charmers-next/neutron-api
+    num_units: 1
+    options:
+      flat-network-providers: physnet1
+      neutron-security-groups: true
+      manage-neutron-plugin-legacy-mode: true
+      openstack-origin: *openstack-origin
+    to:
+      - '6'
+
+  keystone:
+    charm: cs:~openstack-charmers-next/keystone
+    num_units: 1
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '7'
+
+  neutron-gateway:
+    charm: cs:~openstack-charmers-next/neutron-gateway
+    num_units: 1
+    options:
+      bridge-mappings: physnet1:br-ex
+      openstack-origin: *openstack-origin
+    to:
+      - '8'
+
+  glance:
+    charm: cs:~openstack-charmers-next/glance
+    num_units: 1
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '9'
+
+  neutron-openvswitch:
+    charm: cs:~openstack-charmers-next/neutron-openvswitch
+
+  nova-compute:
+    charm: cs:~openstack-charmers-next/nova-compute
+    num_units: 1
+    constraints: mem=4G cores=4
+    storage:
+      ephemeral-device: '10G'
+    options:
+      config-flags: auto_assign_floating_ip=False
+      enable-live-migration: false
+      aa-profile-mode: enforce
+      #ephemeral-device: /dev/vdb
+      #ephemeral-unmount: /mnt
+      debug: true
+      openstack-origin: *openstack-origin
+    to:
+      - '10'
+
+  cinder:
+    num_units: 1
+    storage:
+      block-devices: '40G'
+    options:
+      openstack-origin: *openstack-origin
+      glance-api-version: 2
+      block-device: None
+      overwrite: "true"
+    to:
+      - '11'
+
+relations:
+
+  - - 'nova-compute:image-service'
+    - 'glance:image-service'
+
+  - - 'nova-compute:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'nova-cloud-controller:shared-db'
+    - 'nova-cloud-controller-mysql-router:shared-db'
+  - - 'nova-cloud-controller-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'nova-cloud-controller:identity-service'
+    - 'keystone:identity-service'
+
+  - - 'nova-cloud-controller:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'nova-cloud-controller:cloud-compute'
+    - 'nova-compute:cloud-compute'
+
+  - - 'nova-cloud-controller:image-service'
+    - 'glance:image-service'
+
+  - - 'placement:shared-db'
+    - 'placement-mysql-router:shared-db'
+  - - 'placement-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'placement:identity-service'
+    - 'keystone:identity-service'
+
+  - - 'placement:placement'
+    - 'nova-cloud-controller:placement'
+
+  - - 'keystone:shared-db'
+    - 'keystone-mysql-router:shared-db'
+  - - 'keystone-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'glance:identity-service'
+    - 'keystone:identity-service'
+
+  - - 'glance:shared-db'
+    - 'glance-mysql-router:shared-db'
+  - - 'glance-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'glance:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'neutron-gateway:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'nova-cloud-controller:quantum-network-service'
+    - 'neutron-gateway:quantum-network-service'
+
+  - - 'neutron-api:shared-db'
+    - 'neutron-api-mysql-router:shared-db'
+  - - 'neutron-api-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'neutron-api:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'neutron-api:neutron-api'
+    - 'nova-cloud-controller:neutron-api'
+
+  - - 'neutron-api:identity-service'
+    - 'keystone:identity-service'
+
+  - - 'nova-compute:neutron-plugin'
+    - 'neutron-openvswitch:neutron-plugin'
+
+  - - 'rabbitmq-server:amqp'
+    - 'neutron-openvswitch:amqp'
+
+  - - 'cinder:shared-db'
+    - 'cinder-mysql-router:shared-db'
+  - - 'cinder-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'cinder:identity-service'
+    - 'keystone:identity-service'
+
+  - - 'cinder:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'cinder:image-service'
+    - 'glance:image-service'

tests/tests.yaml

@@ -2,6 +2,8 @@ charm_name: cinder
 smoke_bundles:
 - bionic-train
 gate_bundles:
+- focal-ussuri
+- bionic-ussuri
 - bionic-train
 - bionic-stein
 - bionic-rocky
@@ -12,6 +14,7 @@ gate_bundles:
 - xenial-mitaka
 - trusty-mitaka
 dev_bundles:
+- focal-ussuri
 configure:
 - zaza.openstack.charm_tests.glance.setup.add_cirros_image
 - zaza.openstack.charm_tests.glance.setup.add_lts_image
@@ -26,3 +29,5 @@ tests:
 tests_options:
   policyd:
     service: cinder
+  force_deploy:
+    - focal-ussuri