diff --git a/charmcraft.yaml b/charmcraft.yaml index c2d7417..f523182 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -28,9 +28,9 @@ bases: - name: ubuntu channel: "22.04" architectures: [amd64, s390x, ppc64el, arm64] - - name: ubuntu - channel: "22.10" - architectures: [amd64, s390x, ppc64el, arm64] - name: ubuntu channel: "23.04" architectures: [amd64, s390x, ppc64el, arm64] + - name: ubuntu + channel: "23.10" + architectures: [amd64, s390x, ppc64el, arm64] diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index ffda5fe..7b30925 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -221,6 +221,13 @@ def https(): return True if config_get('ssl_cert') and config_get('ssl_key'): return True + # Local import to avoid ciruclar dependency. + import charmhelpers.contrib.openstack.cert_utils as cert_utils + if ( + cert_utils.get_certificate_request() and not + cert_utils.get_requests_for_local_unit("certificates") + ): + return False for r_id in relation_ids('certificates'): for unit in relation_list(r_id): ca = relation_get('ca', rid=r_id, unit=unit) diff --git a/hooks/charmhelpers/contrib/openstack/cert_utils.py b/hooks/charmhelpers/contrib/openstack/cert_utils.py index 5c961c5..a25ca99 100644 --- a/hooks/charmhelpers/contrib/openstack/cert_utils.py +++ b/hooks/charmhelpers/contrib/openstack/cert_utils.py @@ -409,6 +409,9 @@ def get_requests_for_local_unit(relation_name=None): relation_name = relation_name or 'certificates' bundles = [] for rid in relation_ids(relation_name): + sent = relation_get(rid=rid, unit=local_unit()) + legacy_keys = ['certificate_name', 'common_name'] + is_legacy_request = set(sent).intersection(legacy_keys) for unit in related_units(rid): data = relation_get(rid=rid, unit=unit) if data.get(raw_certs_key): @@ -416,6 +419,14 @@ def get_requests_for_local_unit(relation_name=None): 'ca': data['ca'], 'chain': data.get('chain'), 'certs': json.loads(data[raw_certs_key])}) + elif is_legacy_request: + bundles.append({ + 'ca': data['ca'], + 'chain': data.get('chain'), + 'certs': {sent['common_name']: + {'cert': data.get(local_name + '.server.cert'), + 'key': data.get(local_name + '.server.key')}}}) + return bundles diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index d894b6a..24a13d0 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -1748,6 +1748,9 @@ class WSGIWorkerConfigContext(WorkerConfigContext): def __call__(self): total_processes = _calculate_workers() + enable_wsgi_rotation = config('wsgi-rotation') + if enable_wsgi_rotation is None: + enable_wsgi_rotation = True ctxt = { "service_name": self.service_name, "user": self.user, @@ -1761,6 +1764,7 @@ class WSGIWorkerConfigContext(WorkerConfigContext): "public_processes": int(math.ceil(self.public_process_weight * total_processes)), "threads": 1, + "wsgi_rotation": enable_wsgi_rotation, } return ctxt diff --git a/hooks/charmhelpers/contrib/openstack/deferred_events.py b/hooks/charmhelpers/contrib/openstack/deferred_events.py index 94eacf6..4c46e41 100644 --- a/hooks/charmhelpers/contrib/openstack/deferred_events.py +++ b/hooks/charmhelpers/contrib/openstack/deferred_events.py @@ -127,7 +127,9 @@ def deferred_events(): """ events = [] for defer_file in deferred_events_files(): - events.append((defer_file, read_event_file(defer_file))) + event = read_event_file(defer_file) + if event.policy_requestor_name == hookenv.service_name(): + events.append((defer_file, event)) return events diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken index dbad506..aef5edd 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken +++ b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken @@ -12,6 +12,8 @@ signing_dir = {{ signing_dir }} {% if service_type -%} service_type = {{ service_type }} {% endif -%} +{% if admin_role -%} service_token_roles = {{ admin_role }} service_token_roles_required = True {% endif -%} +{% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka index 14c25b4..31c21b4 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka +++ b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka @@ -22,4 +22,8 @@ signing_dir = {{ signing_dir }} {% if use_memcache == true %} memcached_servers = {{ memcache_url }} {% endif -%} +{% if admin_role -%} +service_token_roles = {{ admin_role }} +service_token_roles_required = True +{% endif -%} {% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-service-user b/hooks/charmhelpers/contrib/openstack/templates/section-service-user index c740cc2..ff45408 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/section-service-user +++ b/hooks/charmhelpers/contrib/openstack/templates/section-service-user @@ -3,8 +3,8 @@ send_service_user_token = true auth_type = password auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }} -project_domain_id = default -user_domain_id = default +project_domain_name = service_domain +user_domain_name = service_domain project_name = {{ admin_tenant_name }} username = {{ admin_user }} password = {{ admin_password }} diff --git a/hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf b/hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf index 6c4e37e..2cb735e 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf +++ b/hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf @@ -12,6 +12,12 @@ Listen {{ admin_port }} Listen {{ public_port }} {% endif -%} +{% if wsgi_rotation -%} +WSGISocketRotation On +{% else -%} +WSGISocketRotation Off +{% endif -%} + {% if port -%} WSGIDaemonProcess {{ service_name }} processes={{ processes }} threads={{ threads }} user={{ user }} group={{ group }} \ diff --git a/hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-metadata.conf b/hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-metadata.conf index 6c4e37e..2cb735e 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-metadata.conf +++ b/hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-metadata.conf @@ -12,6 +12,12 @@ Listen {{ admin_port }} Listen {{ public_port }} {% endif -%} +{% if wsgi_rotation -%} +WSGISocketRotation On +{% else -%} +WSGISocketRotation Off +{% endif -%} + {% if port -%} WSGIDaemonProcess {{ service_name }} processes={{ processes }} threads={{ threads }} user={{ user }} group={{ group }} \ diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 3d52eb1..e98be2c 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -160,6 +160,7 @@ OPENSTACK_CODENAMES = OrderedDict([ ('2022.1', 'yoga'), ('2022.2', 'zed'), ('2023.1', 'antelope'), + ('2023.2', 'bobcat'), ]) # The ugly duckling - must list releases oldest to newest @@ -957,7 +958,7 @@ def os_requires_version(ostack_release, pkg): def wrap(f): @wraps(f) def wrapped_f(*args): - if os_release(pkg) < ostack_release: + if CompareOpenStackReleases(os_release(pkg)) < ostack_release: raise Exception("This hook is not supported on releases" " before %s" % ostack_release) f(*args) diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 1b20b8f..2e1fc1b 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -28,7 +28,6 @@ import os import shutil import json import time -import uuid from subprocess import ( check_call, @@ -1677,6 +1676,10 @@ class CephBrokerRq(object): The API is versioned and defaults to version 1. """ + # The below hash is the result of running + # `hashlib.sha1('[]'.encode()).hexdigest()` + EMPTY_LIST_SHA = '97d170e1550eee4afc0af065b78cda302a97674c' + def __init__(self, api_version=1, request_id=None, raw_request_data=None): """Initialize CephBrokerRq object. @@ -1685,8 +1688,12 @@ class CephBrokerRq(object): :param api_version: API version for request (default: 1). :type api_version: Optional[int] - :param request_id: Unique identifier for request. - (default: string representation of generated UUID) + :param request_id: Unique identifier for request. The identifier will + be updated as ops are added or removed from the + broker request. This ensures that Ceph will + correctly process requests where operations are + added after the initial request is processed. + (default: sha1 of operations) :type request_id: Optional[str] :param raw_request_data: JSON-encoded string to build request from. :type raw_request_data: Optional[str] @@ -1695,16 +1702,20 @@ class CephBrokerRq(object): if raw_request_data: request_data = json.loads(raw_request_data) self.api_version = request_data['api-version'] - self.request_id = request_data['request-id'] self.set_ops(request_data['ops']) + self.request_id = request_data['request-id'] else: self.api_version = api_version if request_id: self.request_id = request_id else: - self.request_id = str(uuid.uuid1()) + self.request_id = CephBrokerRq.EMPTY_LIST_SHA self.ops = [] + def _hash_ops(self): + """Return the sha1 of the requested Broker ops.""" + return hashlib.sha1(json.dumps(self.ops, sort_keys=True).encode()).hexdigest() + def add_op(self, op): """Add an op if it is not already in the list. @@ -1713,6 +1724,7 @@ class CephBrokerRq(object): """ if op not in self.ops: self.ops.append(op) + self.request_id = self._hash_ops() def add_op_request_access_to_group(self, name, namespace=None, permission=None, key_name=None, @@ -1991,6 +2003,7 @@ class CephBrokerRq(object): to allow comparisons to ensure validity. """ self.ops = ops + self.request_id = self._hash_ops() @property def request(self): diff --git a/hooks/charmhelpers/core/host_factory/ubuntu.py b/hooks/charmhelpers/core/host_factory/ubuntu.py index a279d5b..732d76c 100644 --- a/hooks/charmhelpers/core/host_factory/ubuntu.py +++ b/hooks/charmhelpers/core/host_factory/ubuntu.py @@ -32,6 +32,7 @@ UBUNTU_RELEASES = ( 'jammy', 'kinetic', 'lunar', + 'mantic', ) diff --git a/hooks/charmhelpers/fetch/ubuntu.py b/hooks/charmhelpers/fetch/ubuntu.py index effc884..1be992c 100644 --- a/hooks/charmhelpers/fetch/ubuntu.py +++ b/hooks/charmhelpers/fetch/ubuntu.py @@ -238,6 +238,14 @@ CLOUD_ARCHIVE_POCKETS = { 'antelope/proposed': 'jammy-proposed/antelope', 'jammy-antelope/proposed': 'jammy-proposed/antelope', 'jammy-proposed/antelope': 'jammy-proposed/antelope', + # bobcat + 'bobcat': 'jammy-updates/bobcat', + 'jammy-bobcat': 'jammy-updates/bobcat', + 'jammy-bobcat/updates': 'jammy-updates/bobcat', + 'jammy-updates/bobcat': 'jammy-updates/bobcat', + 'bobcat/proposed': 'jammy-proposed/bobcat', + 'jammy-bobcat/proposed': 'jammy-proposed/bobcat', + 'jammy-proposed/bobcat': 'jammy-proposed/bobcat', # OVN 'focal-ovn-22.03': 'focal-updates/ovn-22.03', @@ -270,6 +278,7 @@ OPENSTACK_RELEASES = ( 'yoga', 'zed', 'antelope', + 'bobcat', ) @@ -298,6 +307,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('jammy', 'yoga'), ('kinetic', 'zed'), ('lunar', 'antelope'), + ('mantic', 'bobcat'), ]) @@ -591,7 +601,7 @@ def _get_key_by_keyid(keyid): curl_cmd = ['curl', keyserver_url.format(keyid)] # use proxy server settings in order to retrieve the key return subprocess.check_output(curl_cmd, - env=env_proxy_settings(['https'])) + env=env_proxy_settings(['https', 'no_proxy'])) def _dearmor_gpg_key(key_asc): diff --git a/hooks/charmhelpers/fetch/ubuntu_apt_pkg.py b/hooks/charmhelpers/fetch/ubuntu_apt_pkg.py index 6da355f..f4dde4a 100644 --- a/hooks/charmhelpers/fetch/ubuntu_apt_pkg.py +++ b/hooks/charmhelpers/fetch/ubuntu_apt_pkg.py @@ -122,13 +122,12 @@ class Cache(object): :raises: subprocess.CalledProcessError """ pkgs = {} - cmd = ['dpkg-query', '--list'] + cmd = [ + 'dpkg-query', '--show', + '--showformat', + r'${db:Status-Abbrev}\t${Package}\t${Version}\t${Architecture}\t${binary:Summary}\n' + ] cmd.extend(packages) - if locale.getlocale() == (None, None): - # subprocess calls out to locale.getpreferredencoding(False) to - # determine encoding. Workaround for Trusty where the - # environment appears to not be set up correctly. - locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') try: output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, @@ -140,24 +139,17 @@ class Cache(object): if cp.returncode != 1: raise output = cp.output - headings = [] for line in output.splitlines(): - if line.startswith('||/'): - headings = line.split() - headings.pop(0) + # only process lines for successfully installed packages + if not (line.startswith('ii ') or line.startswith('hi ')): continue - elif (line.startswith('|') or line.startswith('+') or - line.startswith('dpkg-query:')): - continue - else: - data = line.split(None, 4) - status = data.pop(0) - if status not in ('ii', 'hi'): - continue - pkg = {} - pkg.update({k.lower(): v for k, v in zip(headings, data)}) - if 'name' in pkg: - pkgs.update({pkg['name']: pkg}) + status, name, version, arch, desc = line.split('\t', 4) + pkgs[name] = { + 'name': name, + 'version': version, + 'architecture': arch, + 'description': desc, + } return pkgs def _apt_cache_show(self, packages): diff --git a/metadata.yaml b/metadata.yaml index eb53f4f..42e9858 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -9,8 +9,8 @@ tags: - miscellaneous series: - jammy -- kinetic - lunar +- mantic subordinate: true provides: backup-backend: diff --git a/tests/bundles/jammy-bobcat.yaml b/tests/bundles/jammy-bobcat.yaml new file mode 100644 index 0000000..e9e35e5 --- /dev/null +++ b/tests/bundles/jammy-bobcat.yaml @@ -0,0 +1,178 @@ +variables: + openstack-origin: &openstack-origin cloud:jammy-bobcat + +series: jammy + +comment: +- 'machines section to decide order of deployment. database sooner = faster' +machines: + '0': + constraints: mem=3072M + '1': + constraints: mem=3072M + '2': + constraints: mem=3072M + '3': + '4': + '5': + '6': + '7': + '8': + '9': + '10': + '11': + '12': + '13': + +applications: + + keystone-mysql-router: + charm: ch:mysql-router + channel: latest/edge + cinder-mysql-router: + charm: ch:mysql-router + channel: latest/edge + glance-mysql-router: + charm: ch:mysql-router + channel: latest/edge + + mysql-innodb-cluster: + charm: ch:mysql-innodb-cluster + num_units: 3 + to: + - '0' + - '1' + - '2' + channel: latest/edge + + keystone: + charm: ch:keystone + num_units: 1 + options: + openstack-origin: *openstack-origin + to: + - '3' + channel: latest/edge + + rabbitmq-server: + charm: ch:rabbitmq-server + num_units: 1 + to: + - '4' + channel: latest/edge + + ceph-mon: + charm: ch:ceph-mon + num_units: 3 + options: + monitor-count: '3' + source: *openstack-origin + to: + - '5' + - '6' + - '7' + channel: latest/edge + + ceph-osd: + charm: ch:ceph-osd + num_units: 3 + storage: + osd-devices: 'cinder,10G' + options: + osd-devices: '/dev/test-non-existent' + source: *openstack-origin + to: + - '8' + - '9' + - '10' + channel: latest/edge + + cinder: + charm: ch:cinder + num_units: 1 + options: + block-device: 'None' + glance-api-version: '2' + openstack-origin: *openstack-origin + to: + - '11' + channel: latest/edge + + cinder-backup: + charm: ../../cinder-backup.charm + options: + ceph-osd-replication-count: 3 + + cinder-ceph: + charm: ch:cinder-ceph + options: + ceph-osd-replication-count: 3 + channel: latest/edge + + glance: + charm: ch:glance + num_units: 1 + options: + openstack-origin: *openstack-origin + to: + - '12' + channel: latest/edge + + nova-compute: + charm: ch:nova-compute + num_units: 1 + options: + openstack-origin: *openstack-origin + to: + - '13' + channel: latest/edge + +relations: + + - - 'cinder-backup:ceph' + - 'ceph-mon:client' + + - - 'cinder-ceph:ceph' + - 'ceph-mon:client' + + - - 'ceph-osd:mon' + - 'ceph-mon:osd' + + - - 'cinder:storage-backend' + - 'cinder-ceph:storage-backend' + + - - 'cinder:backup-backend' + - 'cinder-backup:backup-backend' + + - - 'keystone:shared-db' + - 'keystone-mysql-router:shared-db' + - - 'keystone-mysql-router:db-router' + - 'mysql-innodb-cluster:db-router' + + - - 'cinder:shared-db' + - 'cinder-mysql-router:shared-db' + - - 'cinder-mysql-router:db-router' + - 'mysql-innodb-cluster:db-router' + + - - 'cinder:identity-service' + - 'keystone:identity-service' + + - - 'cinder:amqp' + - 'rabbitmq-server:amqp' + + - - 'glance:image-service' + - 'nova-compute:image-service' + + - - 'glance:identity-service' + - 'keystone:identity-service' + + - - 'glance:shared-db' + - 'glance-mysql-router:shared-db' + - - 'glance-mysql-router:db-router' + - 'mysql-innodb-cluster:db-router' + + - - 'nova-compute:ceph-access' + - 'cinder-ceph:ceph-access' + + - - 'nova-compute:amqp' + - 'rabbitmq-server:amqp' diff --git a/tests/bundles/mantic-bobcat.yaml b/tests/bundles/mantic-bobcat.yaml new file mode 100644 index 0000000..c794f09 --- /dev/null +++ b/tests/bundles/mantic-bobcat.yaml @@ -0,0 +1,180 @@ +variables: + openstack-origin: &openstack-origin distro + +series: mantic + +comment: +- 'machines section to decide order of deployment. database sooner = faster' +machines: + '0': + constraints: mem=3072M + '1': + constraints: mem=3072M + '2': + constraints: mem=3072M + '3': + '4': + '5': + '6': + '7': + '8': + '9': + '10': + '11': + '12': + '13': + +applications: + + keystone-mysql-router: + charm: ch:mysql-router + channel: latest/edge + cinder-mysql-router: + charm: ch:mysql-router + channel: latest/edge + glance-mysql-router: + charm: ch:mysql-router + channel: latest/edge + + mysql-innodb-cluster: + charm: ch:mysql-innodb-cluster + num_units: 3 + options: + source: *openstack-origin + to: + - '0' + - '1' + - '2' + channel: latest/edge + + keystone: + charm: ch:keystone + num_units: 1 + options: + openstack-origin: *openstack-origin + to: + - '3' + channel: latest/edge + + rabbitmq-server: + charm: ch:rabbitmq-server + num_units: 1 + to: + - '4' + channel: latest/edge + + ceph-mon: + charm: ch:ceph-mon + num_units: 3 + options: + monitor-count: '3' + source: *openstack-origin + to: + - '5' + - '6' + - '7' + channel: latest/edge + + ceph-osd: + charm: ch:ceph-osd + num_units: 3 + storage: + osd-devices: 'cinder,10G' + options: + osd-devices: '/dev/test-non-existent' + source: *openstack-origin + to: + - '8' + - '9' + - '10' + channel: latest/edge + + cinder: + charm: ch:cinder + num_units: 1 + options: + block-device: 'None' + glance-api-version: '2' + openstack-origin: *openstack-origin + to: + - '11' + channel: latest/edge + + cinder-backup: + charm: ../../cinder-backup.charm + options: + ceph-osd-replication-count: 3 + + cinder-ceph: + charm: ch:cinder-ceph + options: + ceph-osd-replication-count: 3 + channel: latest/edge + + glance: + charm: ch:glance + num_units: 1 + options: + openstack-origin: *openstack-origin + to: + - '12' + channel: latest/edge + + nova-compute: + charm: ch:nova-compute + num_units: 1 + options: + openstack-origin: *openstack-origin + to: + - '13' + channel: latest/edge + +relations: + + - - 'cinder-backup:ceph' + - 'ceph-mon:client' + + - - 'cinder-ceph:ceph' + - 'ceph-mon:client' + + - - 'ceph-osd:mon' + - 'ceph-mon:osd' + + - - 'cinder:storage-backend' + - 'cinder-ceph:storage-backend' + + - - 'cinder:backup-backend' + - 'cinder-backup:backup-backend' + + - - 'keystone:shared-db' + - 'keystone-mysql-router:shared-db' + - - 'keystone-mysql-router:db-router' + - 'mysql-innodb-cluster:db-router' + + - - 'cinder:shared-db' + - 'cinder-mysql-router:shared-db' + - - 'cinder-mysql-router:db-router' + - 'mysql-innodb-cluster:db-router' + + - - 'cinder:identity-service' + - 'keystone:identity-service' + + - - 'cinder:amqp' + - 'rabbitmq-server:amqp' + + - - 'glance:image-service' + - 'nova-compute:image-service' + + - - 'glance:identity-service' + - 'keystone:identity-service' + + - - 'glance:shared-db' + - 'glance-mysql-router:shared-db' + - - 'glance-mysql-router:db-router' + - 'mysql-innodb-cluster:db-router' + + - - 'nova-compute:ceph-access' + - 'cinder-ceph:ceph-access' + + - - 'nova-compute:amqp' + - 'rabbitmq-server:amqp' diff --git a/tests/tests.yaml b/tests/tests.yaml index 86596d2..6735238 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -8,7 +8,9 @@ gate_bundles: dev_bundles: - jammy-antelope + - jammy-bobcat - lunar-antelope + - mantic-bobcat tests: - zaza.openstack.charm_tests.cinder_backup.tests.CinderBackupTest @@ -16,3 +18,4 @@ tests: tests_options: force_deploy: - lunar-antelope + - mantic-bobcat