From 29d03fc673452d42e235668ec66aae49b162c264 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 10 Jun 2022 20:14:47 +0000 Subject: [PATCH] Add Kinetic and Zed support * sync charm-helpers to classic charms * change openstack-origin/source default to zed * align testing with zed * add new zed bundles * add zed bundles to tests.yaml * add zed tests to osci.yaml and .zuul.yaml * update build-on and run-on bases * add bindep.txt for py310 * sync tox.ini and requirements.txt for ruamel * use charmcraft_channel 2.0/stable * drop reactive plugin overrides * move interface/layer env vars to charmcraft.yaml Change-Id: Idf4a6cd1e0888576f890b00aa5b343936900d6dd --- .zuul.yaml | 2 +- bindep.txt | 7 +- charmcraft.yaml | 5 +- charmhelpers/contrib/network/ip.py | 2 +- charmhelpers/contrib/openstack/context.py | 65 ++++++++--------- charmhelpers/contrib/openstack/utils.py | 6 +- charmhelpers/core/host.py | 6 +- charmhelpers/core/host_factory/ubuntu.py | 1 + charmhelpers/core/services/base.py | 3 +- charmhelpers/fetch/archiveurl.py | 31 ++++++-- charmhelpers/fetch/ubuntu.py | 10 +++ config.yaml | 2 +- metadata.yaml | 1 - osci.yaml | 5 +- requirements.txt | 7 +- test-requirements.txt | 16 +---- tests/bundles/focal-yoga.yaml | 72 ------------------- tests/bundles/jammy-yoga.yaml | 2 +- .../{focal-xena.yaml => jammy-zed.yaml} | 6 +- .../{impish-xena.yaml => kinetic-zed.yaml} | 4 +- tests/tests.yaml | 11 ++- tox.ini | 33 ++------- 22 files changed, 111 insertions(+), 186 deletions(-) delete mode 100644 tests/bundles/focal-yoga.yaml rename tests/bundles/{focal-xena.yaml => jammy-zed.yaml} (93%) rename tests/bundles/{impish-xena.yaml => kinetic-zed.yaml} (97%) diff --git a/.zuul.yaml b/.zuul.yaml index 7ffc71cb..23bf5f62 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,4 +1,4 @@ - project: templates: - - openstack-python3-charm-yoga-jobs + - openstack-python3-charm-zed-jobs - openstack-cover-jobs diff --git a/bindep.txt b/bindep.txt index ba2ccb4b..bdbe8d56 100644 --- a/bindep.txt +++ b/bindep.txt @@ -1,4 +1,3 @@ -libxml2-dev [platform:dpkg test] -libxslt1-dev [platform:dpkg test] -build-essential [platform:dpkg test] -zlib1g-dev [platform:dpkg test] +libffi-dev [platform:dpkg] +libxml2-dev [platform:dpkg] +libxslt1-dev [platform:dpkg] diff --git a/charmcraft.yaml b/charmcraft.yaml index b34d1442..79551bcd 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -25,13 +25,10 @@ parts: bases: - build-on: - name: ubuntu - channel: "20.04" + channel: "22.04" architectures: - amd64 run-on: - - name: ubuntu - channel: "20.04" - architectures: [amd64, s390x, ppc64el, arm64] - name: ubuntu channel: "22.04" architectures: [amd64, s390x, ppc64el, arm64] diff --git a/charmhelpers/contrib/network/ip.py b/charmhelpers/contrib/network/ip.py index de56584d..f8edf37a 100644 --- a/charmhelpers/contrib/network/ip.py +++ b/charmhelpers/contrib/network/ip.py @@ -467,7 +467,7 @@ def ns_query(address): try: answers = dns.resolver.query(address, rtype) - except dns.resolver.NXDOMAIN: + except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers): return None if answers: diff --git a/charmhelpers/contrib/openstack/context.py b/charmhelpers/contrib/openstack/context.py index 32c69ff7..970a657b 100644 --- a/charmhelpers/contrib/openstack/context.py +++ b/charmhelpers/contrib/openstack/context.py @@ -2560,14 +2560,18 @@ class OVSDPDKDeviceContext(OSContextGenerator): :rtype: List[int] """ cores = [] - ranges = cpulist.split(',') - for cpu_range in ranges: - if "-" in cpu_range: - cpu_min_max = cpu_range.split('-') - cores += range(int(cpu_min_max[0]), - int(cpu_min_max[1]) + 1) - else: - cores.append(int(cpu_range)) + if cpulist and re.match(r"^[0-9,\-^]*$", cpulist): + ranges = cpulist.split(',') + for cpu_range in ranges: + if "-" in cpu_range: + cpu_min_max = cpu_range.split('-') + cores += range(int(cpu_min_max[0]), + int(cpu_min_max[1]) + 1) + elif "^" in cpu_range: + cpu_rm = cpu_range.split('^') + cores.remove(int(cpu_rm[1])) + else: + cores.append(int(cpu_range)) return cores def _numa_node_cores(self): @@ -2586,36 +2590,32 @@ class OVSDPDKDeviceContext(OSContextGenerator): def cpu_mask(self): """Get hex formatted CPU mask - The mask is based on using the first config:dpdk-socket-cores cores of each NUMA node in the unit. :returns: hex formatted CPU mask :rtype: str """ - return self.cpu_masks()['dpdk_lcore_mask'] - - def cpu_masks(self): - """Get hex formatted CPU masks - - The mask is based on using the first config:dpdk-socket-cores - cores of each NUMA node in the unit, followed by the - next config:pmd-socket-cores - - :returns: Dict of hex formatted CPU masks - :rtype: Dict[str, str] - """ - num_lcores = config('dpdk-socket-cores') - pmd_cores = config('pmd-socket-cores') - lcore_mask = 0 - pmd_mask = 0 + num_cores = config('dpdk-socket-cores') + mask = 0 for cores in self._numa_node_cores().values(): - for core in cores[:num_lcores]: - lcore_mask = lcore_mask | 1 << core - for core in cores[num_lcores:][:pmd_cores]: - pmd_mask = pmd_mask | 1 << core - return { - 'pmd_cpu_mask': format(pmd_mask, '#04x'), - 'dpdk_lcore_mask': format(lcore_mask, '#04x')} + for core in cores[:num_cores]: + mask = mask | 1 << core + return format(mask, '#04x') + + @classmethod + def pmd_cpu_mask(cls): + """Get hex formatted pmd CPU mask + + The mask is based on config:pmd-cpu-set. + :returns: hex formatted CPU mask + :rtype: str + """ + mask = 0 + cpu_list = cls._parse_cpu_list(config('pmd-cpu-set')) + if cpu_list: + for core in cpu_list: + mask = mask | 1 << core + return format(mask, '#x') def socket_memory(self): """Formatted list of socket memory configuration per socket. @@ -2694,6 +2694,7 @@ class OVSDPDKDeviceContext(OSContextGenerator): ctxt['device_whitelist'] = self.device_whitelist() ctxt['socket_memory'] = self.socket_memory() ctxt['cpu_mask'] = self.cpu_mask() + ctxt['pmd_cpu_mask'] = self.pmd_cpu_mask() return ctxt diff --git a/charmhelpers/contrib/openstack/utils.py b/charmhelpers/contrib/openstack/utils.py index c8747c16..1fa2814a 100644 --- a/charmhelpers/contrib/openstack/utils.py +++ b/charmhelpers/contrib/openstack/utils.py @@ -158,6 +158,7 @@ OPENSTACK_CODENAMES = OrderedDict([ ('2021.1', 'wallaby'), ('2021.2', 'xena'), ('2022.1', 'yoga'), + ('2022.2', 'zed'), ]) # The ugly duckling - must list releases oldest to newest @@ -400,13 +401,16 @@ def get_os_codename_version(vers): error_out(e) -def get_os_version_codename(codename, version_map=OPENSTACK_CODENAMES): +def get_os_version_codename(codename, version_map=OPENSTACK_CODENAMES, + raise_exception=False): '''Determine OpenStack version number from codename.''' for k, v in version_map.items(): if v == codename: return k e = 'Could not derive OpenStack version for '\ 'codename: %s' % codename + if raise_exception: + raise ValueError(str(e)) error_out(e) diff --git a/charmhelpers/core/host.py b/charmhelpers/core/host.py index ad2cab46..ef6c8eca 100644 --- a/charmhelpers/core/host.py +++ b/charmhelpers/core/host.py @@ -277,7 +277,7 @@ def service_resume(service_name, init_dir="/etc/init", return started -def service(action, service_name, **kwargs): +def service(action, service_name=None, **kwargs): """Control a system service. :param action: the action to take on the service @@ -286,7 +286,9 @@ def service(action, service_name, **kwargs): the form of key=value. """ if init_is_systemd(service_name=service_name): - cmd = ['systemctl', action, service_name] + cmd = ['systemctl', action] + if service_name is not None: + cmd.append(service_name) else: cmd = ['service', service_name, action] for key, value in kwargs.items(): diff --git a/charmhelpers/core/host_factory/ubuntu.py b/charmhelpers/core/host_factory/ubuntu.py index 0906c5c0..cc2d89fe 100644 --- a/charmhelpers/core/host_factory/ubuntu.py +++ b/charmhelpers/core/host_factory/ubuntu.py @@ -30,6 +30,7 @@ UBUNTU_RELEASES = ( 'hirsute', 'impish', 'jammy', + 'kinetic', ) diff --git a/charmhelpers/core/services/base.py b/charmhelpers/core/services/base.py index 7c37c65c..8d217b59 100644 --- a/charmhelpers/core/services/base.py +++ b/charmhelpers/core/services/base.py @@ -15,7 +15,8 @@ import os import json import inspect -from collections import Iterable, OrderedDict +from collections import OrderedDict +from collections.abc import Iterable from charmhelpers.core import host from charmhelpers.core import hookenv diff --git a/charmhelpers/fetch/archiveurl.py b/charmhelpers/fetch/archiveurl.py index 2cb2e88b..0e35c901 100644 --- a/charmhelpers/fetch/archiveurl.py +++ b/charmhelpers/fetch/archiveurl.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import contextlib import os import hashlib import re @@ -24,11 +25,15 @@ from charmhelpers.payload.archive import ( get_archive_handler, extract, ) +from charmhelpers.core.hookenv import ( + env_proxy_settings, +) from charmhelpers.core.host import mkdir, check_hash from urllib.request import ( build_opener, install_opener, urlopen, urlretrieve, HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, + ProxyHandler ) from urllib.parse import urlparse, urlunparse, parse_qs from urllib.error import URLError @@ -50,6 +55,20 @@ def splitpasswd(user): return user, None +@contextlib.contextmanager +def proxy_env(): + """ + Creates a context which temporarily modifies the proxy settings in os.environ. + """ + restore = {**os.environ} # Copy the current os.environ + juju_proxies = env_proxy_settings() or {} + os.environ.update(**juju_proxies) # Insert or Update the os.environ + yield os.environ + for key in juju_proxies: + del os.environ[key] # remove any keys which were added or updated + os.environ.update(**restore) # restore any original values + + class ArchiveUrlFetchHandler(BaseFetchHandler): """ Handler to download archive files from arbitrary URLs. @@ -80,6 +99,7 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): # propagate all exceptions # URLError, OSError, etc proto, netloc, path, params, query, fragment = urlparse(source) + handlers = [] if proto in ('http', 'https'): auth, barehost = splituser(netloc) if auth is not None: @@ -89,10 +109,13 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): # Realm is set to None in add_password to force the username and password # to be used whatever the realm passman.add_password(None, source, username, password) - authhandler = HTTPBasicAuthHandler(passman) - opener = build_opener(authhandler) - install_opener(opener) - response = urlopen(source) + handlers.append(HTTPBasicAuthHandler(passman)) + + with proxy_env(): + handlers.append(ProxyHandler()) + opener = build_opener(*handlers) + install_opener(opener) + response = urlopen(source) try: with open(dest, 'wb') as dest_file: dest_file.write(response.read()) diff --git a/charmhelpers/fetch/ubuntu.py b/charmhelpers/fetch/ubuntu.py index e6f8a0ad..93b92765 100644 --- a/charmhelpers/fetch/ubuntu.py +++ b/charmhelpers/fetch/ubuntu.py @@ -222,6 +222,14 @@ CLOUD_ARCHIVE_POCKETS = { 'yoga/proposed': 'focal-proposed/yoga', 'focal-yoga/proposed': 'focal-proposed/yoga', 'focal-proposed/yoga': 'focal-proposed/yoga', + # Zed + 'zed': 'jammy-updates/zed', + 'jammy-zed': 'jammy-updates/zed', + 'jammy-zed/updates': 'jammy-updates/zed', + 'jammy-updates/zed': 'jammy-updates/zed', + 'zed/proposed': 'jammy-proposed/zed', + 'jammy-zed/proposed': 'jammy-proposed/zed', + 'jammy-proposed/zed': 'jammy-proposed/zed', } @@ -248,6 +256,7 @@ OPENSTACK_RELEASES = ( 'wallaby', 'xena', 'yoga', + 'zed', ) @@ -274,6 +283,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('hirsute', 'wallaby'), ('impish', 'xena'), ('jammy', 'yoga'), + ('kinetic', 'zed'), ]) diff --git a/config.yaml b/config.yaml index 07172ad0..bf67c133 100644 --- a/config.yaml +++ b/config.yaml @@ -18,7 +18,7 @@ options: Setting this to True will allow supporting services to log to syslog. openstack-origin: type: string - default: yoga + default: zed description: | Repository from which to install. May be one of the following: distro (default), ppa:somecustom/ppa, a deb url sources entry, diff --git a/metadata.yaml b/metadata.yaml index 5695c553..7e62f871 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -10,7 +10,6 @@ tags: - identity - misc series: -- focal - jammy extra-bindings: public: diff --git a/osci.yaml b/osci.yaml index 444f7334..8da12c62 100644 --- a/osci.yaml +++ b/osci.yaml @@ -1,10 +1,9 @@ - project: templates: - - charm-unit-jobs-py38 - charm-unit-jobs-py310 - - charm-xena-functional-jobs - - charm-yoga-functional-jobs + - charm-zed-functional-jobs vars: needs_charm_build: true charm_build_name: keystone build_type: charmcraft + charmcraft_channel: 2.0/stable diff --git a/requirements.txt b/requirements.txt index 196ff38f..3b1cb7b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,9 +11,6 @@ pbr==5.6.0 simplejson>=2.2.0 netifaces>=0.10.4 -# Build requirements -cffi==1.14.6; python_version < '3.6' # cffi 1.15.0 drops support for py35. - # NOTE: newer versions of cryptography require a Rust compiler to build, # see # * https://github.com/openstack-charmers/zaza/issues/421 @@ -27,8 +24,6 @@ netaddr>0.7.16,<0.8.0 Jinja2>=2.6 # BSD License (3 clause) six>=1.9.0 -# dnspython 2.0.0 dropped py3.5 support -dnspython<2.0.0; python_version < '3.6' -dnspython; python_version >= '3.6' +dnspython psutil>=1.1.1,<2.0.0 diff --git a/test-requirements.txt b/test-requirements.txt index 0aabe171..4ef87dc5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,6 @@ # all of its own requirements and if it doesn't, fix it there. # pyparsing<3.0.0 # aodhclient is pinned in zaza and needs pyparsing < 3.0.0, but cffi also needs it, so pin here. -cffi==1.14.6; python_version < '3.6' # cffi 1.15.0 drops support for py35. setuptools<50.0.0 # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85 requests>=2.18.4 @@ -19,25 +18,12 @@ stestr>=2.2.0 # https://github.com/mtreinish/stestr/issues/145 cliff<3.0.0 -# Dependencies of stestr. Newer versions use keywords that didn't exist in -# python 3.5 yet (e.g. "ModuleNotFoundError") -importlib-metadata<3.0.0; python_version < '3.6' -importlib-resources<3.0.0; python_version < '3.6' - -# Some Zuul nodes sometimes pull newer versions of these dependencies which -# dropped support for python 3.5: -osprofiler<2.7.0;python_version<'3.6' -stevedore<1.31.0;python_version<'3.6' -debtcollector<1.22.0;python_version<'3.6' -oslo.utils<=3.41.0;python_version<'3.6' - coverage>=4.5.2 pyudev # for ceph-* charm unit tests (need to fix the ceph-* charm unit tests/mocking) git+https://github.com/openstack-charmers/zaza.git#egg=zaza git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack # Needed for charm-glance: -git+https://opendev.org/openstack/tempest.git#egg=tempest;python_version>='3.6' -tempest<24.0.0;python_version<'3.6' +git+https://opendev.org/openstack/tempest.git#egg=tempest croniter # needed for charm-rabbitmq-server unit tests diff --git a/tests/bundles/focal-yoga.yaml b/tests/bundles/focal-yoga.yaml deleted file mode 100644 index bee89f38..00000000 --- a/tests/bundles/focal-yoga.yaml +++ /dev/null @@ -1,72 +0,0 @@ -variables: - openstack-origin: &openstack-origin cloud:focal-yoga - -series: focal - -comment: -- 'machines section to decide order of deployment. database sooner = faster' -machines: - '0': - constraints: mem=3072M - '1': - constraints: mem=3072M - '2': - constraints: mem=3072M - '3': - '4': - '5': - '6': - -applications: - - glance-mysql-router: - charm: ch:mysql-router - channel: latest/edge - keystone-mysql-router: - charm: ch:mysql-router - channel: latest/edge - - mysql-innodb-cluster: - charm: ch:mysql-innodb-cluster - num_units: 3 - options: - source: *openstack-origin - to: - - '0' - - '1' - - '2' - channel: latest/edge - - keystone: - charm: ../../keystone.charm - num_units: 3 - options: - openstack-origin: *openstack-origin - token-provider: 'fernet' - token-expiration: 300 - to: - - '3' - - '4' - - '5' - - glance: - charm: ch:glance - num_units: 1 - options: - openstack-origin: *openstack-origin - to: - - '6' - channel: yoga/edge - -relations: - - - - 'keystone:shared-db' - - 'keystone-mysql-router:shared-db' - - - 'glance:shared-db' - - 'glance-mysql-router:shared-db' - - - 'glance:identity-service' - - 'keystone:identity-service' - - - 'glance-mysql-router:db-router' - - 'mysql-innodb-cluster:db-router' - - - 'keystone-mysql-router:db-router' - - 'mysql-innodb-cluster:db-router' diff --git a/tests/bundles/jammy-yoga.yaml b/tests/bundles/jammy-yoga.yaml index 5849cbb5..c7bbf1b3 100644 --- a/tests/bundles/jammy-yoga.yaml +++ b/tests/bundles/jammy-yoga.yaml @@ -56,7 +56,7 @@ applications: openstack-origin: *openstack-origin to: - '6' - channel: yoga/edge + channel: latest/edge relations: diff --git a/tests/bundles/focal-xena.yaml b/tests/bundles/jammy-zed.yaml similarity index 93% rename from tests/bundles/focal-xena.yaml rename to tests/bundles/jammy-zed.yaml index debf56da..a362f388 100644 --- a/tests/bundles/focal-xena.yaml +++ b/tests/bundles/jammy-zed.yaml @@ -1,7 +1,7 @@ variables: - openstack-origin: &openstack-origin cloud:focal-xena + openstack-origin: &openstack-origin cloud:jammy-zed -series: focal +series: jammy comment: - 'machines section to decide order of deployment. database sooner = faster' @@ -56,7 +56,7 @@ applications: openstack-origin: *openstack-origin to: - '6' - channel: yoga/edge + channel: latest/edge relations: diff --git a/tests/bundles/impish-xena.yaml b/tests/bundles/kinetic-zed.yaml similarity index 97% rename from tests/bundles/impish-xena.yaml rename to tests/bundles/kinetic-zed.yaml index a28cad7c..780f4024 100644 --- a/tests/bundles/impish-xena.yaml +++ b/tests/bundles/kinetic-zed.yaml @@ -1,7 +1,7 @@ variables: openstack-origin: &openstack-origin distro -series: impish +series: kinetic comment: - 'machines section to decide order of deployment. database sooner = faster' @@ -56,7 +56,7 @@ applications: openstack-origin: *openstack-origin to: - '6' - channel: yoga/edge + channel: latest/edge relations: diff --git a/tests/tests.yaml b/tests/tests.yaml index 15b31552..24a04fe5 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -1,15 +1,15 @@ charm_name: keystone smoke_bundles: -- focal-xena +- jammy-yoga gate_bundles: -- focal-xena -- impish-xena +- jammy-yoga dev_bundles: -- focal-yoga - jammy-yoga +- jammy-zed +- kinetic-zed comment: | the glance configure job validates operation of identity-service relation. @@ -40,8 +40,7 @@ tests_options: policyd: service: keystone force_deploy: - - impish-xena - - jammy-yoga + - kinetic-zed target_deploy_status: vault: workload-status: blocked diff --git a/tox.ini b/tox.ini index c41077de..bddbd1f2 100644 --- a/tox.ini +++ b/tox.ini @@ -48,42 +48,23 @@ basepython = python3 deps = -r{toxinidir}/build-requirements.txt commands = charmcraft clean - charmcraft -v build + charmcraft -v pack {toxinidir}/rename.sh +[testenv:py310] +basepython = python3.10 +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt + [testenv:py3] basepython = python3 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -[testenv:py36] -basepython = python3.6 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -commands = stestr run --slowest {posargs} - -[testenv:py38] -basepython = python3.8 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -commands = stestr run --slowest {posargs} - -[testenv:py39] -basepython = python3.9 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -commands = stestr run --slowest {posargs} - -[testenv:py310] -basepython = python3.10 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -commands = stestr run --slowest {posargs} - [testenv:pep8] basepython = python3 deps = flake8==3.9.2 - charm-tools==2.8.3 + git+https://github.com/juju/charm-tools.git commands = flake8 {posargs} hooks unit_tests tests actions lib files charm-proof