From 70b67e0c845983cbb17ffa865018d03f3bc801af Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Mon, 18 May 2020 14:46:24 +0200 Subject: [PATCH] Sync helpers for 20.05 Change-Id: Idb46b1a924314a126367bfada8c7851f0b4ea21f --- Makefile | 2 +- hooks/charmhelpers/contrib/network/ovs/ovn.py | 11 +++-- .../contrib/storage/linux/ceph.py | 40 +++++++++++++--- requirements.txt | 12 +++-- test-requirements.txt | 22 +++++---- tox.ini | 46 +++++++++++-------- 6 files changed, 90 insertions(+), 43 deletions(-) diff --git a/Makefile b/Makefile index a7f6dee1..99fa2efc 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ #!/usr/bin/make -PYTHON := /usr/bin/env python +PYTHON := /usr/bin/env python3 lint: @tox -e pep8 diff --git a/hooks/charmhelpers/contrib/network/ovs/ovn.py b/hooks/charmhelpers/contrib/network/ovs/ovn.py index 7a9de1e7..2075f11a 100644 --- a/hooks/charmhelpers/contrib/network/ovs/ovn.py +++ b/hooks/charmhelpers/contrib/network/ovs/ovn.py @@ -140,7 +140,7 @@ class OVNClusterStatus(object): return self.leader == 'self' -def cluster_status(target, schema=None, use_ovs_appctl=False): +def cluster_status(target, schema=None, use_ovs_appctl=False, rundir=None): """Retrieve status information from clustered OVSDB. :param target: Usually one of 'ovsdb-server', 'ovnnb_db', 'ovnsb_db', can @@ -151,6 +151,8 @@ def cluster_status(target, schema=None, use_ovs_appctl=False): :param use_ovs_appctl: The ``ovn-appctl`` command appeared in OVN 20.03, set this to True to use ``ovs-appctl`` instead. :type use_ovs_appctl: bool + :param rundir: Override path to sockets + :type rundir: Optional[str] :returns: cluster status data object :rtype: OVNClusterStatus :raises: subprocess.CalledProcessError, KeyError, RuntimeError @@ -164,8 +166,9 @@ def cluster_status(target, schema=None, use_ovs_appctl=False): status = {} k = '' - for line in ovn_appctl(target, 'cluster/status', - schema or schema_map[target], + for line in ovn_appctl(target, + ('cluster/status', schema or schema_map[target]), + rundir=rundir, use_ovs_appctl=use_ovs_appctl).splitlines(): if k and line.startswith(' '): # there is no key which means this is a instance of a multi-line/ @@ -222,7 +225,7 @@ def is_northd_active(): :rtype: bool """ try: - for line in ovn_appctl('ovn-northd', 'status').splitlines(): + for line in ovn_appctl('ovn-northd', ('status',)).splitlines(): if line.startswith('Status:') and 'active' in line: return True except subprocess.CalledProcessError: diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index eb31b782..95a0d82a 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -92,6 +92,7 @@ DEFAULT_PGS_PER_OSD_TARGET = 100 DEFAULT_POOL_WEIGHT = 10.0 LEGACY_PG_COUNT = 200 DEFAULT_MINIMUM_PGS = 2 +AUTOSCALER_DEFAULT_PGS = 32 class OsdPostUpgradeError(Exception): @@ -399,16 +400,28 @@ class ReplicatedPool(Pool): def create(self): if not pool_exists(self.service, self.name): + nautilus_or_later = cmp_pkgrevno('ceph-common', '14.2.0') >= 0 # Create it - cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', - self.name, str(self.pg_num)] + if nautilus_or_later: + cmd = [ + 'ceph', '--id', self.service, 'osd', 'pool', 'create', + '--pg-num-min={}'.format( + min(AUTOSCALER_DEFAULT_PGS, self.pg_num) + ), + self.name, str(self.pg_num) + ] + else: + cmd = [ + 'ceph', '--id', self.service, 'osd', 'pool', 'create', + self.name, str(self.pg_num) + ] + try: check_call(cmd) # Set the pool replica size update_pool(client=self.service, pool=self.name, settings={'size': str(self.replicas)}) - nautilus_or_later = cmp_pkgrevno('ceph-common', '14.2.0') >= 0 if nautilus_or_later: # Ensure we set the expected pool ratio update_pool(client=self.service, @@ -466,10 +479,24 @@ class ErasurePool(Pool): k = int(erasure_profile['k']) m = int(erasure_profile['m']) pgs = self.get_pgs(k + m, self.percent_data) + nautilus_or_later = cmp_pkgrevno('ceph-common', '14.2.0') >= 0 # Create it - cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', - self.name, str(pgs), str(pgs), - 'erasure', self.erasure_code_profile] + if nautilus_or_later: + cmd = [ + 'ceph', '--id', self.service, 'osd', 'pool', 'create', + '--pg-num-min={}'.format( + min(AUTOSCALER_DEFAULT_PGS, pgs) + ), + self.name, str(pgs), str(pgs), + 'erasure', self.erasure_code_profile + ] + else: + cmd = [ + 'ceph', '--id', self.service, 'osd', 'pool', 'create', + self.name, str(pgs), str(pgs), + 'erasure', self.erasure_code_profile + ] + try: check_call(cmd) try: @@ -478,7 +505,6 @@ class ErasurePool(Pool): name=self.app_name) except CalledProcessError: log('Could not set app name for pool {}'.format(self.name, level=WARNING)) - nautilus_or_later = cmp_pkgrevno('ceph-common', '14.2.0') >= 0 if nautilus_or_later: # Ensure we set the expected pool ratio update_pool(client=self.service, diff --git a/requirements.txt b/requirements.txt index b8fec1e2..343beed1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,12 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. +# This file is managed centrally by release-tools and should not be modified +# within individual charm repos. See the 'global' dir contents for available +# choices of *requirements.txt files for OpenStack Charms: +# https://github.com/openstack-charmers/release-tools +# +# TODO: Distill the func test requirements from the lint/unit test +# requirements. They are intertwined. Also, Zaza itself should specify +# all of its own requirements and if it doesn't, fix it there. +# pbr>=1.8.0,<1.9.0 simplejson>=2.2.0 netifaces>=0.10.4 diff --git a/test-requirements.txt b/test-requirements.txt index bf98624f..7d9c2587 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,14 +1,18 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. +# This file is managed centrally by release-tools and should not be modified +# within individual charm repos. See the 'global' dir contents for available +# choices of *requirements.txt files for OpenStack Charms: +# https://github.com/openstack-charmers/release-tools +# +# TODO: Distill the func test requirements from the lint/unit test +# requirements. They are intertwined. Also, Zaza itself should specify +# all of its own requirements and if it doesn't, fix it there. +# charm-tools>=2.4.4 -coverage>=3.6 +requests>=2.18.4 mock>=1.2 flake8>=2.2.4,<=2.4.1 stestr>=2.2.0 -requests>=2.18.4 -# NOTE: workaround for 14.04 pip/tox -pytz -pyudev # for ceph-* charm unit tests (not mocked?) -git+https://github.com/openstack-charmers/zaza.git#egg=zaza +coverage>=4.5.2 +pyudev # for ceph-* charm unit tests (need to fix the ceph-* charm unit tests/mocking) +git+https://github.com/openstack-charmers/zaza.git#egg=zaza;python_version>='3.0' git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack diff --git a/tox.ini b/tox.ini index efc8fa0c..b835733a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,12 @@ -# Classic charm: ./tox.ini +# Classic charm (with zaza): ./tox.ini # This file is managed centrally by release-tools and should not be modified # within individual charm repos. See the 'global' dir contents for available # choices of tox.ini for OpenStack Charms: # https://github.com/openstack-charmers/release-tools +# +# TODO: Distill the func test requirements from the lint/unit test +# requirements. They are intertwined. Also, Zaza itself should specify +# all of its own requirements and if it doesn't, fix it there. [tox] envlist = pep8,py3 skipsdist = True @@ -20,11 +24,7 @@ install_command = commands = stestr run --slowest {posargs} whitelist_externals = juju passenv = HOME TERM CS_* OS_* TEST_* - -[testenv:py3] -basepython = python3 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt +deps = -r{toxinidir}/test-requirements.txt [testenv:py35] basepython = python3.5 @@ -41,6 +41,16 @@ basepython = python3.7 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt +[testenv:py38] +basepython = python3.8 +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt + +[testenv:py3] +basepython = python3 +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt + [testenv:pep8] basepython = python3 deps = -r{toxinidir}/requirements.txt @@ -80,33 +90,31 @@ omit = basepython = python3 commands = {posargs} -[testenv:func] -# Charm Functional Test -# Run all bundles (expected to always pass) +[testenv:func-noop] +basepython = python3 +commands = + functest-run-suite --help + +[testenv:func] basepython = python3 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt commands = functest-run-suite --keep-model [testenv:func-smoke] -# Charm Functional Test -# Run a specific bundle as a smoke test (expected to always pass) basepython = python3 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt commands = functest-run-suite --keep-model --smoke [testenv:func-dev] -# Charm Functional Test -# Run all development test targets (may not always pass!) basepython = python3 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt commands = functest-run-suite --keep-model --dev +[testenv:func-target] +basepython = python3 +commands = + functest-run-suite --keep-model --bundle {posargs} + [flake8] ignore = E402,E226 exclude = */charmhelpers