Add 2023.2 Bobcat support

* sync charm-helpers to classic charms
* change openstack-origin/source default to bobcat
* add mantic to metadata series
* align testing with bobcat
* add new bobcat bundles
* add bobcat bundles to tests.yaml
* add bobcat tests to osci.yaml
* update build-on and run-on bases
* drop kinetic

Change-Id: I272c90848169e142a8c032f581b0d7cecebc299f

charmcraft.yaml

@@ -28,9 +28,9 @@ bases:
       - name: ubuntu
         channel: "22.04"
         architectures: [amd64, s390x, ppc64el, arm64]
-      - name: ubuntu
-        channel: "22.10"
-        architectures: [amd64, s390x, ppc64el, arm64]
       - name: ubuntu
         channel: "23.04"
         architectures: [amd64, s390x, ppc64el, arm64]
+      - name: ubuntu
+        channel: "23.10"
+        architectures: [amd64, s390x, ppc64el, arm64]

hooks/charmhelpers/contrib/hahelpers/cluster.py

@@ -221,6 +221,13 @@ def https():
         return True
     if config_get('ssl_cert') and config_get('ssl_key'):
         return True
+    # Local import to avoid ciruclar dependency.
+    import charmhelpers.contrib.openstack.cert_utils as cert_utils
+    if (
+        cert_utils.get_certificate_request() and not
+        cert_utils.get_requests_for_local_unit("certificates")
+    ):
+        return False
     for r_id in relation_ids('certificates'):
         for unit in relation_list(r_id):
             ca = relation_get('ca', rid=r_id, unit=unit)

hooks/charmhelpers/contrib/openstack/cert_utils.py

@@ -409,6 +409,9 @@ def get_requests_for_local_unit(relation_name=None):
     relation_name = relation_name or 'certificates'
     bundles = []
     for rid in relation_ids(relation_name):
+        sent = relation_get(rid=rid, unit=local_unit())
+        legacy_keys = ['certificate_name', 'common_name']
+        is_legacy_request = set(sent).intersection(legacy_keys)
         for unit in related_units(rid):
             data = relation_get(rid=rid, unit=unit)
             if data.get(raw_certs_key):
@@ -416,6 +419,14 @@ def get_requests_for_local_unit(relation_name=None):
                     'ca': data['ca'],
                     'chain': data.get('chain'),
                     'certs': json.loads(data[raw_certs_key])})
+            elif is_legacy_request:
+                bundles.append({
+                    'ca': data['ca'],
+                    'chain': data.get('chain'),
+                    'certs': {sent['common_name']:
+                              {'cert': data.get(local_name + '.server.cert'),
+                               'key': data.get(local_name + '.server.key')}}})
+
     return bundles
 
 

hooks/charmhelpers/contrib/openstack/context.py

@@ -1748,6 +1748,9 @@ class WSGIWorkerConfigContext(WorkerConfigContext):
 
     def __call__(self):
         total_processes = _calculate_workers()
+        enable_wsgi_rotation = config('wsgi-rotation')
+        if enable_wsgi_rotation is None:
+            enable_wsgi_rotation = True
         ctxt = {
             "service_name": self.service_name,
             "user": self.user,
@@ -1761,6 +1764,7 @@ class WSGIWorkerConfigContext(WorkerConfigContext):
             "public_processes": int(math.ceil(self.public_process_weight *
                                               total_processes)),
             "threads": 1,
+            "wsgi_rotation": enable_wsgi_rotation,
         }
         return ctxt
 

hooks/charmhelpers/contrib/openstack/deferred_events.py

@@ -127,7 +127,9 @@ def deferred_events():
     """
     events = []
     for defer_file in deferred_events_files():
-        events.append((defer_file, read_event_file(defer_file)))
+        event = read_event_file(defer_file)
+        if event.policy_requestor_name == hookenv.service_name():
+            events.append((defer_file, event))
     return events
 
 

hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken

@@ -12,6 +12,8 @@ signing_dir = {{ signing_dir }}
 {% if service_type -%}
 service_type = {{ service_type }}
 {% endif -%}
+{% if admin_role -%}
 service_token_roles = {{ admin_role }}
 service_token_roles_required = True
 {% endif -%}
+{% endif -%}

hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka

@@ -22,4 +22,8 @@ signing_dir = {{ signing_dir }}
 {% if use_memcache == true %}
 memcached_servers = {{ memcache_url }}
 {% endif -%}
+{% if admin_role -%}
+service_token_roles = {{ admin_role }}
+service_token_roles_required = True
+{% endif -%}
 {% endif -%}

hooks/charmhelpers/contrib/openstack/templates/section-service-user

@@ -3,8 +3,8 @@
 send_service_user_token = true
 auth_type = password
 auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
-project_domain_id = default
-user_domain_id = default
+project_domain_name = service_domain
+user_domain_name = service_domain
 project_name = {{ admin_tenant_name }}
 username = {{ admin_user }}
 password = {{ admin_password }}

hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf

@@ -12,6 +12,12 @@ Listen {{ admin_port }}
 Listen {{ public_port }}
 {% endif -%}
 
+{% if wsgi_rotation -%}
+WSGISocketRotation On
+{% else -%}
+WSGISocketRotation Off
+{% endif -%}
+
 {% if port -%}
 <VirtualHost *:{{ port }}>
     WSGIDaemonProcess {{ service_name }} processes={{ processes }} threads={{ threads }} user={{ user }} group={{ group }} \

hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-metadata.conf

@@ -12,6 +12,12 @@ Listen {{ admin_port }}
 Listen {{ public_port }}
 {% endif -%}
 
+{% if wsgi_rotation -%}
+WSGISocketRotation On
+{% else -%}
+WSGISocketRotation Off
+{% endif -%}
+
 {% if port -%}
 <VirtualHost *:{{ port }}>
     WSGIDaemonProcess {{ service_name }} processes={{ processes }} threads={{ threads }} user={{ user }} group={{ group }} \

hooks/charmhelpers/contrib/openstack/utils.py

@@ -160,6 +160,7 @@ OPENSTACK_CODENAMES = OrderedDict([
     ('2022.1', 'yoga'),
     ('2022.2', 'zed'),
     ('2023.1', 'antelope'),
+    ('2023.2', 'bobcat'),
 ])
 
 # The ugly duckling - must list releases oldest to newest
@@ -957,7 +958,7 @@ def os_requires_version(ostack_release, pkg):
     def wrap(f):
         @wraps(f)
         def wrapped_f(*args):
-            if os_release(pkg) < ostack_release:
+            if CompareOpenStackReleases(os_release(pkg)) < ostack_release:
                 raise Exception("This hook is not supported on releases"
                                 " before %s" % ostack_release)
             f(*args)

hooks/charmhelpers/contrib/storage/linux/ceph.py

@@ -28,7 +28,6 @@ import os
 import shutil
 import json
 import time
-import uuid
 
 from subprocess import (
     check_call,
@@ -1677,6 +1676,10 @@ class CephBrokerRq(object):
     The API is versioned and defaults to version 1.
     """
 
+    # The below hash is the result of running
+    # `hashlib.sha1('[]'.encode()).hexdigest()`
+    EMPTY_LIST_SHA = '97d170e1550eee4afc0af065b78cda302a97674c'
+
     def __init__(self, api_version=1, request_id=None, raw_request_data=None):
         """Initialize CephBrokerRq object.
 
@@ -1685,8 +1688,12 @@ class CephBrokerRq(object):
 
         :param api_version: API version for request (default: 1).
         :type api_version: Optional[int]
-        :param request_id: Unique identifier for request.
-                           (default: string representation of generated UUID)
+        :param request_id: Unique identifier for request. The identifier will
+                           be updated as ops are added or removed from the
+                           broker request. This ensures that Ceph will
+                           correctly process requests where operations are
+                           added after the initial request is processed.
+                           (default: sha1 of operations)
         :type request_id: Optional[str]
         :param raw_request_data: JSON-encoded string to build request from.
         :type raw_request_data: Optional[str]
@@ -1695,16 +1702,20 @@ class CephBrokerRq(object):
         if raw_request_data:
             request_data = json.loads(raw_request_data)
             self.api_version = request_data['api-version']
-            self.request_id = request_data['request-id']
             self.set_ops(request_data['ops'])
+            self.request_id = request_data['request-id']
         else:
             self.api_version = api_version
             if request_id:
                 self.request_id = request_id
             else:
-                self.request_id = str(uuid.uuid1())
+                self.request_id = CephBrokerRq.EMPTY_LIST_SHA
             self.ops = []
 
+    def _hash_ops(self):
+        """Return the sha1 of the requested Broker ops."""
+        return hashlib.sha1(json.dumps(self.ops, sort_keys=True).encode()).hexdigest()
+
     def add_op(self, op):
         """Add an op if it is not already in the list.
 
@@ -1713,6 +1724,7 @@ class CephBrokerRq(object):
         """
         if op not in self.ops:
             self.ops.append(op)
+            self.request_id = self._hash_ops()
 
     def add_op_request_access_to_group(self, name, namespace=None,
                                        permission=None, key_name=None,
@@ -1991,6 +2003,7 @@ class CephBrokerRq(object):
         to allow comparisons to ensure validity.
         """
         self.ops = ops
+        self.request_id = self._hash_ops()
 
     @property
     def request(self):

hooks/charmhelpers/core/host_factory/ubuntu.py

@@ -32,6 +32,7 @@ UBUNTU_RELEASES = (
     'jammy',
     'kinetic',
     'lunar',
+    'mantic',
 )
 
 

hooks/charmhelpers/fetch/ubuntu.py

@@ -238,6 +238,14 @@ CLOUD_ARCHIVE_POCKETS = {
     'antelope/proposed': 'jammy-proposed/antelope',
     'jammy-antelope/proposed': 'jammy-proposed/antelope',
     'jammy-proposed/antelope': 'jammy-proposed/antelope',
+    # bobcat
+    'bobcat': 'jammy-updates/bobcat',
+    'jammy-bobcat': 'jammy-updates/bobcat',
+    'jammy-bobcat/updates': 'jammy-updates/bobcat',
+    'jammy-updates/bobcat': 'jammy-updates/bobcat',
+    'bobcat/proposed': 'jammy-proposed/bobcat',
+    'jammy-bobcat/proposed': 'jammy-proposed/bobcat',
+    'jammy-proposed/bobcat': 'jammy-proposed/bobcat',
 
     # OVN
     'focal-ovn-22.03': 'focal-updates/ovn-22.03',
@@ -270,6 +278,7 @@ OPENSTACK_RELEASES = (
     'yoga',
     'zed',
     'antelope',
+    'bobcat',
 )
 
 
@@ -298,6 +307,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('jammy', 'yoga'),
     ('kinetic', 'zed'),
     ('lunar', 'antelope'),
+    ('mantic', 'bobcat'),
 ])
 
 
@@ -591,7 +601,7 @@ def _get_key_by_keyid(keyid):
     curl_cmd = ['curl', keyserver_url.format(keyid)]
     # use proxy server settings in order to retrieve the key
     return subprocess.check_output(curl_cmd,
-                                   env=env_proxy_settings(['https']))
+                                   env=env_proxy_settings(['https', 'no_proxy']))
 
 
 def _dearmor_gpg_key(key_asc):

hooks/charmhelpers/fetch/ubuntu_apt_pkg.py

@@ -122,13 +122,12 @@ class Cache(object):
         :raises: subprocess.CalledProcessError
         """
         pkgs = {}
-        cmd = ['dpkg-query', '--list']
+        cmd = [
+            'dpkg-query', '--show',
+            '--showformat',
+            r'${db:Status-Abbrev}\t${Package}\t${Version}\t${Architecture}\t${binary:Summary}\n'
+        ]
         cmd.extend(packages)
-        if locale.getlocale() == (None, None):
-            # subprocess calls out to locale.getpreferredencoding(False) to
-            # determine encoding.  Workaround for Trusty where the
-            # environment appears to not be set up correctly.
-            locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
         try:
             output = subprocess.check_output(cmd,
                                              stderr=subprocess.STDOUT,
@@ -140,24 +139,17 @@ class Cache(object):
             if cp.returncode != 1:
                 raise
             output = cp.output
-        headings = []
         for line in output.splitlines():
-            if line.startswith('||/'):
-                headings = line.split()
-                headings.pop(0)
+            # only process lines for successfully installed packages
+            if not (line.startswith('ii ') or line.startswith('hi ')):
                 continue
-            elif (line.startswith('|') or line.startswith('+') or
-                  line.startswith('dpkg-query:')):
-                continue
-            else:
-                data = line.split(None, 4)
-                status = data.pop(0)
-                if status not in ('ii', 'hi'):
-                    continue
-                pkg = {}
-                pkg.update({k.lower(): v for k, v in zip(headings, data)})
-                if 'name' in pkg:
-                    pkgs.update({pkg['name']: pkg})
+            status, name, version, arch, desc = line.split('\t', 4)
+            pkgs[name] = {
+                'name': name,
+                'version': version,
+                'architecture': arch,
+                'description': desc,
+            }
         return pkgs
 
     def _apt_cache_show(self, packages):

metadata.yaml

@@ -9,8 +9,8 @@ tags:
 - miscellaneous
 series:
 - jammy
-- kinetic
 - lunar
+- mantic
 subordinate: true
 provides:
   backup-backend:

tests/bundles/jammy-bobcat.yaml

@@ -0,0 +1,178 @@
+variables:
+  openstack-origin: &openstack-origin cloud:jammy-bobcat
+
+series: jammy
+
+comment:
+- 'machines section to decide order of deployment. database sooner = faster'
+machines:
+  '0':
+    constraints: mem=3072M
+  '1':
+    constraints: mem=3072M
+  '2':
+    constraints: mem=3072M
+  '3':
+  '4':
+  '5':
+  '6':
+  '7':
+  '8':
+  '9':
+  '10':
+  '11':
+  '12':
+  '13':
+
+applications:
+
+  keystone-mysql-router:
+    charm: ch:mysql-router
+    channel: latest/edge
+  cinder-mysql-router:
+    charm: ch:mysql-router
+    channel: latest/edge
+  glance-mysql-router:
+    charm: ch:mysql-router
+    channel: latest/edge
+
+  mysql-innodb-cluster:
+    charm: ch:mysql-innodb-cluster
+    num_units: 3
+    to:
+      - '0'
+      - '1'
+      - '2'
+    channel: latest/edge
+
+  keystone:
+    charm: ch:keystone
+    num_units: 1
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '3'
+    channel: latest/edge
+
+  rabbitmq-server:
+    charm: ch:rabbitmq-server
+    num_units: 1
+    to:
+      - '4'
+    channel: latest/edge
+
+  ceph-mon:
+    charm: ch:ceph-mon
+    num_units: 3
+    options:
+      monitor-count: '3'
+      source: *openstack-origin
+    to:
+      - '5'
+      - '6'
+      - '7'
+    channel: latest/edge
+
+  ceph-osd:
+    charm: ch:ceph-osd
+    num_units: 3
+    storage:
+      osd-devices: 'cinder,10G'
+    options:
+      osd-devices: '/dev/test-non-existent'
+      source: *openstack-origin
+    to:
+      - '8'
+      - '9'
+      - '10'
+    channel: latest/edge
+
+  cinder:
+    charm: ch:cinder
+    num_units: 1
+    options:
+      block-device: 'None'
+      glance-api-version: '2'
+      openstack-origin: *openstack-origin
+    to:
+      - '11'
+    channel: latest/edge
+
+  cinder-backup:
+    charm: ../../cinder-backup.charm
+    options:
+      ceph-osd-replication-count: 3
+
+  cinder-ceph:
+    charm: ch:cinder-ceph
+    options:
+      ceph-osd-replication-count: 3
+    channel: latest/edge
+
+  glance:
+    charm: ch:glance
+    num_units: 1
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '12'
+    channel: latest/edge
+
+  nova-compute:
+    charm: ch:nova-compute
+    num_units: 1
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '13'
+    channel: latest/edge
+
+relations:
+
+  - - 'cinder-backup:ceph'
+    - 'ceph-mon:client'
+
+  - - 'cinder-ceph:ceph'
+    - 'ceph-mon:client'
+
+  - - 'ceph-osd:mon'
+    - 'ceph-mon:osd'
+
+  - - 'cinder:storage-backend'
+    - 'cinder-ceph:storage-backend'
+
+  - - 'cinder:backup-backend'
+    - 'cinder-backup:backup-backend'
+
+  - - 'keystone:shared-db'
+    - 'keystone-mysql-router:shared-db'
+  - - 'keystone-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'cinder:shared-db'
+    - 'cinder-mysql-router:shared-db'
+  - - 'cinder-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'cinder:identity-service'
+    - 'keystone:identity-service'
+
+  - - 'cinder:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'glance:image-service'
+    - 'nova-compute:image-service'
+
+  - - 'glance:identity-service'
+    - 'keystone:identity-service'
+
+  - - 'glance:shared-db'
+    - 'glance-mysql-router:shared-db'
+  - - 'glance-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'nova-compute:ceph-access'
+    - 'cinder-ceph:ceph-access'
+
+  - - 'nova-compute:amqp'
+    - 'rabbitmq-server:amqp'

tests/bundles/mantic-bobcat.yaml

@@ -1,7 +1,7 @@
 variables:
-  openstack-origin: &openstack-origin cloud:jammy-zed
+  openstack-origin: &openstack-origin distro
 
-series: jammy
+series: mantic
 
 comment:
 - 'machines section to decide order of deployment. database sooner = faster'

tests/tests.yaml

@@ -1,14 +1,15 @@
 charm_name: cinder-backup
 
 smoke_bundles:
-  - jammy-zed
+  - jammy-antelope
 
 gate_bundles:
-  - jammy-zed
+  - jammy-antelope
 
 dev_bundles:
-  - jammy-antelope
+  - jammy-bobcat
   - lunar-antelope
+  - mantic-bobcat
 
 tests:
   - zaza.openstack.charm_tests.cinder_backup.tests.CinderBackupTest
@@ -16,3 +17,4 @@ tests:
 tests_options:
   force_deploy:
     - lunar-antelope
+    - mantic-bobcat