From b6b29ce894fd41b2e8428738665fbe178d646528 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 18 Jul 2023 16:47:18 -0400 Subject: [PATCH] Add 2023.2 Bobcat support * sync charm-helpers to classic charms * change openstack-origin/source default to bobcat * add mantic to metadata series * align testing with bobcat * add new bobcat bundles * add bobcat bundles to tests.yaml * add bobcat tests to osci.yaml and .zuul.yaml * update build-on and run-on bases Func-Test-Pr: https://github.com/openstack-charmers/zaza-openstack-tests/pull/1062 Change-Id: Ie2fdfae9a2304fd6cb55edd4c1b93f1fdbe4014d --- .zuul.yaml | 1 + charmcraft.yaml | 2 +- charmhelpers/contrib/hahelpers/cluster.py | 7 + .../contrib/openstack/deferred_events.py | 4 +- .../files/check_deferred_restarts.py | 128 ++++++++++++++++++ charmhelpers/contrib/openstack/utils.py | 3 +- charmhelpers/contrib/storage/linux/ceph.py | 23 +++- charmhelpers/core/host_factory/ubuntu.py | 1 + charmhelpers/fetch/ubuntu.py | 12 +- charmhelpers/fetch/ubuntu_apt_pkg.py | 36 ++--- metadata.yaml | 2 +- osci.yaml | 3 - tests/bundles/focal-yoga.yaml | 4 +- tests/bundles/jammy-bobcat.yaml | 57 ++++++++ tests/bundles/jammy-yoga.yaml | 4 +- .../{kinetic-zed.yaml => mantic-bobcat.yaml} | 6 +- tests/tests.yaml | 5 +- 17 files changed, 254 insertions(+), 44 deletions(-) create mode 100755 charmhelpers/contrib/openstack/files/check_deferred_restarts.py create mode 100644 tests/bundles/jammy-bobcat.yaml rename tests/bundles/{kinetic-zed.yaml => mantic-bobcat.yaml} (93%) diff --git a/.zuul.yaml b/.zuul.yaml index 7ffc71c..6997408 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,4 +1,5 @@ - project: templates: - openstack-python3-charm-yoga-jobs + - openstack-python3-charm-jobs - openstack-cover-jobs diff --git a/charmcraft.yaml b/charmcraft.yaml index 140c495..65b8911 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -34,5 +34,5 @@ bases: channel: "22.04" architectures: [amd64, s390x, ppc64el, arm64] - name: ubuntu - channel: "22.10" + channel: "23.10" architectures: [amd64, s390x, ppc64el, arm64] diff --git a/charmhelpers/contrib/hahelpers/cluster.py b/charmhelpers/contrib/hahelpers/cluster.py index ffda5fe..7b30925 100644 --- a/charmhelpers/contrib/hahelpers/cluster.py +++ b/charmhelpers/contrib/hahelpers/cluster.py @@ -221,6 +221,13 @@ def https(): return True if config_get('ssl_cert') and config_get('ssl_key'): return True + # Local import to avoid ciruclar dependency. + import charmhelpers.contrib.openstack.cert_utils as cert_utils + if ( + cert_utils.get_certificate_request() and not + cert_utils.get_requests_for_local_unit("certificates") + ): + return False for r_id in relation_ids('certificates'): for unit in relation_list(r_id): ca = relation_get('ca', rid=r_id, unit=unit) diff --git a/charmhelpers/contrib/openstack/deferred_events.py b/charmhelpers/contrib/openstack/deferred_events.py index 94eacf6..4c46e41 100644 --- a/charmhelpers/contrib/openstack/deferred_events.py +++ b/charmhelpers/contrib/openstack/deferred_events.py @@ -127,7 +127,9 @@ def deferred_events(): """ events = [] for defer_file in deferred_events_files(): - events.append((defer_file, read_event_file(defer_file))) + event = read_event_file(defer_file) + if event.policy_requestor_name == hookenv.service_name(): + events.append((defer_file, event)) return events diff --git a/charmhelpers/contrib/openstack/files/check_deferred_restarts.py b/charmhelpers/contrib/openstack/files/check_deferred_restarts.py new file mode 100755 index 0000000..5f392b3 --- /dev/null +++ b/charmhelpers/contrib/openstack/files/check_deferred_restarts.py @@ -0,0 +1,128 @@ +#!/usr/bin/python3 + +# Copyright 2014-2022 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Checks for services with deferred restarts. + +This Nagios check will parse /var/lib/policy-rd.d/ +to find any restarts that are currently deferred. +""" + +import argparse +import glob +import sys +import yaml + + +DEFERRED_EVENTS_DIR = '/var/lib/policy-rc.d' + + +def get_deferred_events(): + """Return a list of deferred events dicts from policy-rc.d files. + + Events are read from DEFERRED_EVENTS_DIR and are of the form: + { + action: restart, + policy_requestor_name: rabbitmq-server, + policy_requestor_type: charm, + reason: 'Pkg update', + service: rabbitmq-server, + time: 1614328743 + } + + :raises OSError: Raised in case of a system error while reading a policy file + :raises yaml.YAMLError: Raised if parsing a policy file fails + + :returns: List of deferred event dictionaries + :rtype: list + """ + deferred_events_files = glob.glob( + '{}/*.deferred'.format(DEFERRED_EVENTS_DIR)) + + deferred_events = [] + for event_file in deferred_events_files: + with open(event_file, 'r') as f: + event = yaml.safe_load(f) + deferred_events.append(event) + + return deferred_events + + +def get_deferred_restart_services(application=None): + """Returns a list of services with deferred restarts. + + :param str application: Name of the application that blocked the service restart. + If application is None, all services with deferred restarts + are returned. Services which are blocked by a non-charm + requestor are always returned. + + :raises OSError: Raised in case of a system error while reading a policy file + :raises yaml.YAMLError: Raised if parsing a policy file fails + + :returns: List of services with deferred restarts belonging to application. + :rtype: list + """ + + deferred_restart_events = filter( + lambda e: e['action'] == 'restart', get_deferred_events()) + + deferred_restart_services = set() + for restart_event in deferred_restart_events: + if application: + if ( + restart_event['policy_requestor_type'] != 'charm' or + restart_event['policy_requestor_type'] == 'charm' and + restart_event['policy_requestor_name'] == application + ): + deferred_restart_services.add(restart_event['service']) + else: + deferred_restart_services.add(restart_event['service']) + + return list(deferred_restart_services) + + +def main(): + """Check for services with deferred restarts.""" + parser = argparse.ArgumentParser( + description='Check for services with deferred restarts') + parser.add_argument( + '--application', help='Check services belonging to this application only') + + args = parser.parse_args() + + services = set(get_deferred_restart_services(args.application)) + + if len(services) == 0: + print('OK: No deferred service restarts.') + sys.exit(0) + else: + print( + 'CRITICAL: Restarts are deferred for services: {}.'.format(', '.join(services))) + sys.exit(1) + + +if __name__ == '__main__': + try: + main() + except OSError as e: + print('CRITICAL: A system error occurred: {} ({})'.format(e.errno, e.strerror)) + sys.exit(1) + except yaml.YAMLError as e: + print('CRITICAL: Failed to parse a policy file: {}'.format(str(e))) + sys.exit(1) + except Exception as e: + print('CRITICAL: An unknown error occurred: {}'.format(str(e))) + sys.exit(1) diff --git a/charmhelpers/contrib/openstack/utils.py b/charmhelpers/contrib/openstack/utils.py index 3d52eb1..e98be2c 100644 --- a/charmhelpers/contrib/openstack/utils.py +++ b/charmhelpers/contrib/openstack/utils.py @@ -160,6 +160,7 @@ OPENSTACK_CODENAMES = OrderedDict([ ('2022.1', 'yoga'), ('2022.2', 'zed'), ('2023.1', 'antelope'), + ('2023.2', 'bobcat'), ]) # The ugly duckling - must list releases oldest to newest @@ -957,7 +958,7 @@ def os_requires_version(ostack_release, pkg): def wrap(f): @wraps(f) def wrapped_f(*args): - if os_release(pkg) < ostack_release: + if CompareOpenStackReleases(os_release(pkg)) < ostack_release: raise Exception("This hook is not supported on releases" " before %s" % ostack_release) f(*args) diff --git a/charmhelpers/contrib/storage/linux/ceph.py b/charmhelpers/contrib/storage/linux/ceph.py index 1b20b8f..2e1fc1b 100644 --- a/charmhelpers/contrib/storage/linux/ceph.py +++ b/charmhelpers/contrib/storage/linux/ceph.py @@ -28,7 +28,6 @@ import os import shutil import json import time -import uuid from subprocess import ( check_call, @@ -1677,6 +1676,10 @@ class CephBrokerRq(object): The API is versioned and defaults to version 1. """ + # The below hash is the result of running + # `hashlib.sha1('[]'.encode()).hexdigest()` + EMPTY_LIST_SHA = '97d170e1550eee4afc0af065b78cda302a97674c' + def __init__(self, api_version=1, request_id=None, raw_request_data=None): """Initialize CephBrokerRq object. @@ -1685,8 +1688,12 @@ class CephBrokerRq(object): :param api_version: API version for request (default: 1). :type api_version: Optional[int] - :param request_id: Unique identifier for request. - (default: string representation of generated UUID) + :param request_id: Unique identifier for request. The identifier will + be updated as ops are added or removed from the + broker request. This ensures that Ceph will + correctly process requests where operations are + added after the initial request is processed. + (default: sha1 of operations) :type request_id: Optional[str] :param raw_request_data: JSON-encoded string to build request from. :type raw_request_data: Optional[str] @@ -1695,16 +1702,20 @@ class CephBrokerRq(object): if raw_request_data: request_data = json.loads(raw_request_data) self.api_version = request_data['api-version'] - self.request_id = request_data['request-id'] self.set_ops(request_data['ops']) + self.request_id = request_data['request-id'] else: self.api_version = api_version if request_id: self.request_id = request_id else: - self.request_id = str(uuid.uuid1()) + self.request_id = CephBrokerRq.EMPTY_LIST_SHA self.ops = [] + def _hash_ops(self): + """Return the sha1 of the requested Broker ops.""" + return hashlib.sha1(json.dumps(self.ops, sort_keys=True).encode()).hexdigest() + def add_op(self, op): """Add an op if it is not already in the list. @@ -1713,6 +1724,7 @@ class CephBrokerRq(object): """ if op not in self.ops: self.ops.append(op) + self.request_id = self._hash_ops() def add_op_request_access_to_group(self, name, namespace=None, permission=None, key_name=None, @@ -1991,6 +2003,7 @@ class CephBrokerRq(object): to allow comparisons to ensure validity. """ self.ops = ops + self.request_id = self._hash_ops() @property def request(self): diff --git a/charmhelpers/core/host_factory/ubuntu.py b/charmhelpers/core/host_factory/ubuntu.py index a279d5b..732d76c 100644 --- a/charmhelpers/core/host_factory/ubuntu.py +++ b/charmhelpers/core/host_factory/ubuntu.py @@ -32,6 +32,7 @@ UBUNTU_RELEASES = ( 'jammy', 'kinetic', 'lunar', + 'mantic', ) diff --git a/charmhelpers/fetch/ubuntu.py b/charmhelpers/fetch/ubuntu.py index effc884..1be992c 100644 --- a/charmhelpers/fetch/ubuntu.py +++ b/charmhelpers/fetch/ubuntu.py @@ -238,6 +238,14 @@ CLOUD_ARCHIVE_POCKETS = { 'antelope/proposed': 'jammy-proposed/antelope', 'jammy-antelope/proposed': 'jammy-proposed/antelope', 'jammy-proposed/antelope': 'jammy-proposed/antelope', + # bobcat + 'bobcat': 'jammy-updates/bobcat', + 'jammy-bobcat': 'jammy-updates/bobcat', + 'jammy-bobcat/updates': 'jammy-updates/bobcat', + 'jammy-updates/bobcat': 'jammy-updates/bobcat', + 'bobcat/proposed': 'jammy-proposed/bobcat', + 'jammy-bobcat/proposed': 'jammy-proposed/bobcat', + 'jammy-proposed/bobcat': 'jammy-proposed/bobcat', # OVN 'focal-ovn-22.03': 'focal-updates/ovn-22.03', @@ -270,6 +278,7 @@ OPENSTACK_RELEASES = ( 'yoga', 'zed', 'antelope', + 'bobcat', ) @@ -298,6 +307,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('jammy', 'yoga'), ('kinetic', 'zed'), ('lunar', 'antelope'), + ('mantic', 'bobcat'), ]) @@ -591,7 +601,7 @@ def _get_key_by_keyid(keyid): curl_cmd = ['curl', keyserver_url.format(keyid)] # use proxy server settings in order to retrieve the key return subprocess.check_output(curl_cmd, - env=env_proxy_settings(['https'])) + env=env_proxy_settings(['https', 'no_proxy'])) def _dearmor_gpg_key(key_asc): diff --git a/charmhelpers/fetch/ubuntu_apt_pkg.py b/charmhelpers/fetch/ubuntu_apt_pkg.py index 6da355f..f4dde4a 100644 --- a/charmhelpers/fetch/ubuntu_apt_pkg.py +++ b/charmhelpers/fetch/ubuntu_apt_pkg.py @@ -122,13 +122,12 @@ class Cache(object): :raises: subprocess.CalledProcessError """ pkgs = {} - cmd = ['dpkg-query', '--list'] + cmd = [ + 'dpkg-query', '--show', + '--showformat', + r'${db:Status-Abbrev}\t${Package}\t${Version}\t${Architecture}\t${binary:Summary}\n' + ] cmd.extend(packages) - if locale.getlocale() == (None, None): - # subprocess calls out to locale.getpreferredencoding(False) to - # determine encoding. Workaround for Trusty where the - # environment appears to not be set up correctly. - locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') try: output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, @@ -140,24 +139,17 @@ class Cache(object): if cp.returncode != 1: raise output = cp.output - headings = [] for line in output.splitlines(): - if line.startswith('||/'): - headings = line.split() - headings.pop(0) + # only process lines for successfully installed packages + if not (line.startswith('ii ') or line.startswith('hi ')): continue - elif (line.startswith('|') or line.startswith('+') or - line.startswith('dpkg-query:')): - continue - else: - data = line.split(None, 4) - status = data.pop(0) - if status not in ('ii', 'hi'): - continue - pkg = {} - pkg.update({k.lower(): v for k, v in zip(headings, data)}) - if 'name' in pkg: - pkgs.update({pkg['name']: pkg}) + status, name, version, arch, desc = line.split('\t', 4) + pkgs[name] = { + 'name': name, + 'version': version, + 'architecture': arch, + 'description': desc, + } return pkgs def _apt_cache_show(self, packages): diff --git a/metadata.yaml b/metadata.yaml index bfbbc6a..dda6189 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -9,7 +9,7 @@ tags: series: - focal - jammy -- kinetic +- mantic requires: juju-info: interface: juju-info diff --git a/osci.yaml b/osci.yaml index 825dbe6..221eb97 100644 --- a/osci.yaml +++ b/osci.yaml @@ -3,9 +3,6 @@ - charm-unit-jobs-py38 - charm-unit-jobs-py310 - charm-yoga-functional-jobs - check: - jobs: - - kinetic-zed vars: needs_charm_build: true charm_build_name: hacluster diff --git a/tests/bundles/focal-yoga.yaml b/tests/bundles/focal-yoga.yaml index 7e3affa..3072cc5 100644 --- a/tests/bundles/focal-yoga.yaml +++ b/tests/bundles/focal-yoga.yaml @@ -43,7 +43,7 @@ applications: - '5' channel: yoga/edge - hacluster: + keystone-hacluster: charm: ../../hacluster.charm subordinate-to: - keystone @@ -51,7 +51,7 @@ applications: relations: - - 'keystone:shared-db' - 'keystone-mysql-router:shared-db' - - - 'hacluster:ha' + - - 'keystone-hacluster:ha' - 'keystone:ha' - - "keystone-mysql-router:db-router" - "mysql-innodb-cluster:db-router" diff --git a/tests/bundles/jammy-bobcat.yaml b/tests/bundles/jammy-bobcat.yaml new file mode 100644 index 0000000..bda60f3 --- /dev/null +++ b/tests/bundles/jammy-bobcat.yaml @@ -0,0 +1,57 @@ +variables: + openstack-origin: &openstack-origin cloud:jammy-bobcat + +series: jammy + +machines: + '0': + constraints: mem=3072M + '1': + constraints: mem=3072M + '2': + constraints: mem=3072M + '3': + '4': + '5': + +applications: + + keystone-mysql-router: + charm: ch:mysql-router + channel: latest/edge + + mysql-innodb-cluster: + charm: ch:mysql-innodb-cluster + num_units: 3 + options: + source: *openstack-origin + to: + - '0' + - '1' + - '2' + channel: latest/edge + + keystone: + charm: ch:keystone + num_units: 3 + options: + token-expiration: 60 + openstack-origin: *openstack-origin + to: + - '3' + - '4' + - '5' + channel: yoga/edge + + keystone-hacluster: + charm: ../../hacluster.charm + subordinate-to: + - keystone + +relations: + - - 'keystone:shared-db' + - 'keystone-mysql-router:shared-db' + - - 'keystone-hacluster:ha' + - 'keystone:ha' + - - "keystone-mysql-router:db-router" + - "mysql-innodb-cluster:db-router" diff --git a/tests/bundles/jammy-yoga.yaml b/tests/bundles/jammy-yoga.yaml index b42c688..fcf761b 100644 --- a/tests/bundles/jammy-yoga.yaml +++ b/tests/bundles/jammy-yoga.yaml @@ -43,7 +43,7 @@ applications: - '5' channel: yoga/edge - hacluster: + keystone-hacluster: charm: ../../hacluster.charm subordinate-to: - keystone @@ -51,7 +51,7 @@ applications: relations: - - 'keystone:shared-db' - 'keystone-mysql-router:shared-db' - - - 'hacluster:ha' + - - 'keystone-hacluster:ha' - 'keystone:ha' - - "keystone-mysql-router:db-router" - "mysql-innodb-cluster:db-router" diff --git a/tests/bundles/kinetic-zed.yaml b/tests/bundles/mantic-bobcat.yaml similarity index 93% rename from tests/bundles/kinetic-zed.yaml rename to tests/bundles/mantic-bobcat.yaml index 37dd45e..a942a1f 100644 --- a/tests/bundles/kinetic-zed.yaml +++ b/tests/bundles/mantic-bobcat.yaml @@ -1,7 +1,7 @@ variables: openstack-origin: &openstack-origin distro -series: kinetic +series: mantic machines: '0': @@ -43,7 +43,7 @@ applications: - '5' channel: zed/edge - hacluster: + keystone-hacluster: charm: ../../hacluster.charm subordinate-to: - keystone @@ -51,7 +51,7 @@ applications: relations: - - 'keystone:shared-db' - 'keystone-mysql-router:shared-db' - - - 'hacluster:ha' + - - 'keystone-hacluster:ha' - 'keystone:ha' - - "keystone-mysql-router:db-router" - "mysql-innodb-cluster:db-router" diff --git a/tests/tests.yaml b/tests/tests.yaml index f766256..c48f99c 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -8,7 +8,8 @@ gate_bundles: dev_bundles: - jammy-yoga - - kinetic-zed + - jammy-bobcat + - mantic-bobcat configure: - zaza.openstack.charm_tests.keystone.setup.add_demo_user @@ -23,4 +24,4 @@ tests_options: hacluster-charm-name: hacluster force_deploy: - jammy-yoga - - kinetic-zed + - mantic-bobcat