From 3c2b7d363b83f85f0ea668aca854b1306f72acf7 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 19 May 2022 11:36:51 +0100 Subject: [PATCH] Migrate charm charmhub for stable/victoria - start from stable/victoria branch - Do charmcraft, osci, zuul changes - Modify libs to point to stable/victoria for: - charmhelpers - charms.openstack (as needed) - zaza & zaza.openstack.tests - Remove unsupported test bundles - migration bundles to charmhub.io (ch: prefix + channels) - Build on 20.04 amd64 - Run on 20.04 amd64, s390x, ppc64el, arm64 Change-Id: I702abb787c38049a8d73b50655239b39bf39f219 --- .gitignore | 1 + .zuul.yaml | 3 +- build-requirements.txt | 7 ++ charmcraft.yaml | 31 +++++++-- osci.yaml | 4 +- rename.sh | 13 ++++ requirements.txt | 1 + src/build.lock | 12 +--- src/test-requirements.txt | 8 ++- src/tests/bundles/focal-ussuri.yaml | 20 ++++-- src/tests/bundles/focal-victoria.yaml | 16 +++-- src/tests/bundles/groovy-victoria.yaml | 93 -------------------------- src/tests/tests.yaml | 3 +- src/wheelhouse.txt | 4 +- test-requirements.txt | 3 +- tox.ini | 45 ++++++++++--- 16 files changed, 124 insertions(+), 140 deletions(-) create mode 100644 build-requirements.txt create mode 100755 rename.sh delete mode 100644 src/tests/bundles/groovy-victoria.yaml diff --git a/.gitignore b/.gitignore index a28e19e..c6ee379 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ trusty .testrepository .stestr __pycache__ +*.charm diff --git a/.zuul.yaml b/.zuul.yaml index b3037e9..1382d5e 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,5 +1,4 @@ - project: templates: - - python35-charm-jobs - - openstack-python3-ussuri-jobs + - openstack-python3-victoria-jobs - openstack-cover-jobs diff --git a/build-requirements.txt b/build-requirements.txt new file mode 100644 index 0000000..b6d2452 --- /dev/null +++ b/build-requirements.txt @@ -0,0 +1,7 @@ +# NOTES(lourot): +# * We don't install charmcraft via pip anymore because it anyway spins up a +# container and scp the system's charmcraft snap inside it. So the charmcraft +# snap is necessary on the system anyway. +# * `tox -e build` successfully validated with charmcraft 1.2.1 + +cffi==1.14.6; python_version < '3.6' # cffi 1.15.0 drops support for py35. diff --git a/charmcraft.yaml b/charmcraft.yaml index a850351..dc055bd 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -2,12 +2,29 @@ type: charm parts: charm: - source: src/ - plugin: reactive - build-snaps: [charm] + build-packages: + - tox + - git + - python3-dev + override-build: | + apt-get install ca-certificates -y + tox -e build-reactive + override-stage: | + echo "Copying charm to staging area: $CHARMCRAFT_STAGE" + NAME=$(ls $CHARMCRAFT_PART_BUILD/build/builds) + cp -r $CHARMCRAFT_PART_BUILD/build/builds/$NAME/* $CHARMCRAFT_STAGE/ + override-prime: | + # For some reason, the normal priming chokes on the fact that there's a + # hooks directory. + cp -r $CHARMCRAFT_STAGE/* . bases: - - name: ubuntu - channel: "20.04" - architectures: - - amd64 + - build-on: + - name: ubuntu + channel: "20.04" + architectures: + - amd64 + run-on: + - name: ubuntu + channel: "20.04" + architectures: [amd64, s390x, ppc64el, arm64] diff --git a/osci.yaml b/osci.yaml index 015937e..8384041 100644 --- a/osci.yaml +++ b/osci.yaml @@ -1,9 +1,9 @@ - project: templates: - charm-unit-jobs-py38 - - charm-unit-jobs-py36 + - charm-focal-ussuri-functional-jobs - charm-victoria-functional-jobs - - charm-ussuri-functional-jobs vars: needs_charm_build: true charm_build_name: manila-generic + build_type: charmcraft diff --git a/rename.sh b/rename.sh new file mode 100755 index 0000000..d0c35c9 --- /dev/null +++ b/rename.sh @@ -0,0 +1,13 @@ +#!/bin/bash +charm=$(grep "charm_build_name" osci.yaml | awk '{print $2}') +echo "renaming ${charm}_*.charm to ${charm}.charm" +echo -n "pwd: " +pwd +ls -al +echo "Removing bad downloaded charm maybe?" +if [[ -e "${charm}.charm" ]]; +then + rm "${charm}.charm" +fi +echo "Renaming charm here." +mv ${charm}_*.charm ${charm}.charm diff --git a/requirements.txt b/requirements.txt index a68620f..e182e8a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ setuptools<50.0.0 # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85 # Build requirements +pyparsing<3.0.0 # cffi needs pyparsing < 3.0.0. cffi==1.14.6; python_version < '3.6' # cffi 1.15.0 drops support for py35. charm-tools==2.8.3 diff --git a/src/build.lock b/src/build.lock index f15219c..b395640 100644 --- a/src/build.lock +++ b/src/build.lock @@ -108,17 +108,11 @@ "vcs": null, "version": "1.1.1" }, - { - "type": "python_module", - "package": "Tempita", - "vcs": null, - "version": "0.4" - }, { "type": "python_module", "package": "charms.openstack", "url": "git+https://github.com/openstack/charms.openstack.git", - "branch": "refs/heads/stable/21.10", + "branch": "stable/victoria", "version": "10627ee5f991c268f174d6d100e218a0e1867af1", "vcs": "git" }, @@ -138,8 +132,8 @@ "type": "python_module", "package": "charmhelpers", "url": "git+https://github.com/juju/charm-helpers.git", - "branch": "refs/heads/stable/21.10", - "version": "26efcd0da51d580f68ead2ca13c38f58766f8a14", + "branch": "stable/victoria", + "version": "df2e33ed634200de1d02d7e5f1baaf80f22a6315", "vcs": "git" }, { diff --git a/src/test-requirements.txt b/src/test-requirements.txt index bd0c1a8..2e352ba 100644 --- a/src/test-requirements.txt +++ b/src/test-requirements.txt @@ -5,5 +5,9 @@ # # Functional Test Requirements (let Zaza's dependencies solve all dependencies here!) -git+https://github.com/openstack-charmers/zaza.git@stable/21.10#egg=zaza -git+https://github.com/openstack-charmers/zaza-openstack-tests.git@stable/21.10#egg=zaza.openstack +git+https://github.com/openstack-charmers/zaza.git@stable/victoria#egg=zaza +git+https://github.com/openstack-charmers/zaza-openstack-tests.git@stable/victoria#egg=zaza.openstack + +# Need tox to be available from tox... inception yes, but its a workaround for now +tox + diff --git a/src/tests/bundles/focal-ussuri.yaml b/src/tests/bundles/focal-ussuri.yaml index fa12d06..6363e78 100644 --- a/src/tests/bundles/focal-ussuri.yaml +++ b/src/tests/bundles/focal-ussuri.yaml @@ -1,3 +1,5 @@ +local_overlay_enabled: False + variables: openstack-origin: &openstack-origin distro @@ -20,10 +22,10 @@ services: manila-mysql-router: charm: ch:mysql-router - channel: 8.0.19/edge + channel: 8.0/edge keystone-mysql-router: charm: ch:mysql-router - channel: 8.0.19/edge + channel: 8.0/edge mysql-innodb-cluster: charm: ch:mysql-innodb-cluster @@ -34,7 +36,7 @@ services: - '0' - '1' - '2' - channel: 8.0.19/edge + channel: 8.0/edge manila: charm: ch:manila @@ -44,10 +46,10 @@ services: openstack-origin: *openstack-origin to: - '3' - channel: ussuri/edge + channel: victoria/edge manila-generic: - charm: ../../../manila-generic + charm: ../../../manila-generic.charm options: driver-handles-share-servers: False @@ -67,7 +69,10 @@ services: openstack-origin: *openstack-origin to: - '5' - channel: ussuri/edge + channel: victoria/edge + + nrpe: + charm: cs:nrpe relations: @@ -89,3 +94,6 @@ relations: - 'keystone-mysql-router:shared-db' - - 'keystone-mysql-router:db-router' - 'mysql-innodb-cluster:db-router' + + - - 'nrpe:nrpe-external-master' + - 'manila:nrpe-external-master' diff --git a/src/tests/bundles/focal-victoria.yaml b/src/tests/bundles/focal-victoria.yaml index b77808c..510638c 100644 --- a/src/tests/bundles/focal-victoria.yaml +++ b/src/tests/bundles/focal-victoria.yaml @@ -1,3 +1,5 @@ +local_overlay_enabled: False + variables: openstack-origin: &openstack-origin cloud:focal-victoria @@ -20,10 +22,10 @@ services: manila-mysql-router: charm: ch:mysql-router - channel: 8.0.19/edge + channel: 8.0/edge keystone-mysql-router: charm: ch:mysql-router - channel: 8.0.19/edge + channel: 8.0/edge mysql-innodb-cluster: charm: ch:mysql-innodb-cluster @@ -34,7 +36,7 @@ services: - '0' - '1' - '2' - channel: 8.0.19/edge + channel: 8.0/edge manila: charm: ch:manila @@ -47,7 +49,7 @@ services: channel: victoria/edge manila-generic: - charm: ../../../manila-generic + charm: ../../../manila-generic.charm options: driver-handles-share-servers: False @@ -69,6 +71,9 @@ services: - '5' channel: victoria/edge + nrpe: + charm: cs:nrpe + relations: - - 'manila:shared-db' @@ -89,3 +94,6 @@ relations: - 'keystone-mysql-router:shared-db' - - 'keystone-mysql-router:db-router' - 'mysql-innodb-cluster:db-router' + + - - 'nrpe:nrpe-external-master' + - 'manila:nrpe-external-master' diff --git a/src/tests/bundles/groovy-victoria.yaml b/src/tests/bundles/groovy-victoria.yaml deleted file mode 100644 index 94e2f65..0000000 --- a/src/tests/bundles/groovy-victoria.yaml +++ /dev/null @@ -1,93 +0,0 @@ -variables: - openstack-origin: &openstack-origin distro - -series: groovy - -comment: -- 'machines section to decide order of deployment. database sooner = faster' -machines: - '0': - constraints: mem=3072M - '1': - constraints: mem=3072M - '2': - constraints: mem=3072M - '3': - '4': - '5': - -services: - - manila-mysql-router: - charm: ch:mysql-router - channel: 8.0.19/edge - keystone-mysql-router: - charm: ch:mysql-router - channel: 8.0.19/edge - - mysql-innodb-cluster: - charm: ch:mysql-innodb-cluster - num_units: 3 - options: - source: *openstack-origin - to: - - '0' - - '1' - - '2' - channel: 8.0.19/edge - series: focal - - manila: - charm: ch:manila - num_units: 1 - options: - default-share-backend: generic - openstack-origin: *openstack-origin - to: - - '3' - channel: victoria/edge - - manila-generic: - charm: ../../../manila-generic - options: - driver-handles-share-servers: False - - rabbitmq-server: - charm: ch:rabbitmq-server - num_units: 1 - options: - source: *openstack-origin - to: - - '4' - channel: 3.8/edge - series: focal - - keystone: - charm: ch:keystone - num_units: 1 - options: - openstack-origin: *openstack-origin - to: - - '5' - channel: victoria/edge - -relations: - - - - 'manila:shared-db' - - 'manila-mysql-router:shared-db' - - - 'manila-mysql-router:db-router' - - 'mysql-innodb-cluster:db-router' - - - - 'manila:manila-plugin' - - 'manila-generic' - - - - 'manila' - - 'rabbitmq-server' - - - - 'manila' - - 'keystone' - - - - 'keystone:shared-db' - - 'keystone-mysql-router:shared-db' - - - 'keystone-mysql-router:db-router' - - 'mysql-innodb-cluster:db-router' diff --git a/src/tests/tests.yaml b/src/tests/tests.yaml index 6910183..9092ad3 100644 --- a/src/tests/tests.yaml +++ b/src/tests/tests.yaml @@ -7,8 +7,7 @@ gate_bundles: smoke_bundles: - focal-victoria -dev_bundles: - - groovy-victoria +dev_bundles: [] target_deploy_status: {} diff --git a/src/wheelhouse.txt b/src/wheelhouse.txt index f7c2efa..b91601c 100644 --- a/src/wheelhouse.txt +++ b/src/wheelhouse.txt @@ -1,4 +1,4 @@ -git+https://github.com/openstack/charms.openstack.git@stable/21.10#egg=charms.openstack +git+https://github.com/openstack/charms.openstack.git@stable/victoria#egg=charms.openstack -git+https://github.com/juju/charm-helpers.git@stable/21.10#egg=charmhelpers +git+https://github.com/juju/charm-helpers@stable/victoria#egg=charmhelpers diff --git a/test-requirements.txt b/test-requirements.txt index 4afbb28..6b1cd71 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,6 +3,7 @@ # choices of *requirements.txt files for OpenStack Charms: # https://github.com/openstack-charmers/release-tools # +pyparsing<3.0.0 # cffi needs pyparsing < 3.0.0. cffi==1.14.6; python_version < '3.6' # cffi 1.15.0 drops support for py35. setuptools<50.0.0 # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85 @@ -34,7 +35,7 @@ mock>=1.2; python_version >= '3.6' nose>=1.3.7 coverage>=3.6 -git+https://github.com/openstack/charms.openstack.git@stable/21.10#egg=charms.openstack +git+https://github.com/openstack/charms.openstack.git@stable/victoria#egg=charms.openstack # # Revisit for removal / mock improvement: # diff --git a/tox.ini b/tox.ini index 22159df..e22fe48 100644 --- a/tox.ini +++ b/tox.ini @@ -37,11 +37,24 @@ setenv = VIRTUAL_ENV={envdir} passenv = http_proxy https_proxy INTERFACE_PATH LAYER_PATH JUJU_REPOSITORY install_command = {toxinidir}/pip.sh install {opts} {packages} +allowlist_externals = + charmcraft + bash + tox + rename.sh deps = -r{toxinidir}/requirements.txt [testenv:build] basepython = python3 +deps = -r{toxinidir}/build-requirements.txt +commands = + charmcraft clean + charmcraft -v build + {toxinidir}/rename.sh + +[testenv:build-reactive] +basepython = python3 commands = charm-build --log-level DEBUG --use-lock-file-branches -o {toxinidir}/build/builds src {posargs} @@ -55,32 +68,44 @@ basepython = python3 deps = -r{toxinidir}/test-requirements.txt commands = stestr run --slowest {posargs} -[testenv:py35] -basepython = python3.5 -deps = -r{toxinidir}/test-requirements.txt -commands = stestr run --slowest {posargs} - [testenv:py36] basepython = python3.6 deps = -r{toxinidir}/test-requirements.txt commands = stestr run --slowest {posargs} -[testenv:py37] -basepython = python3.7 -deps = -r{toxinidir}/test-requirements.txt -commands = stestr run --slowest {posargs} - [testenv:py38] basepython = python3.8 deps = -r{toxinidir}/test-requirements.txt commands = stestr run --slowest {posargs} +[testenv:py39] +basepython = python3.9 +deps = -r{toxinidir}/test-requirements.txt +commands = stestr run --slowest {posargs} + +[testenv:py310] +basepython = python3.10 +deps = -r{toxinidir}/test-requirements.txt +commands = stestr run --slowest {posargs} + [testenv:pep8] basepython = python3 deps = flake8==3.9.2 charm-tools==2.8.3 commands = flake8 {posargs} src unit_tests +[testenv:func-target] +# Hack to get functional tests working in the charmcraft +# world. We should fix this. +basepython = python3 +passenv = HOME TERM CS_* OS_* TEST_* +deps = -r{toxinidir}/src/test-requirements.txt +changedir = {toxinidir}/src +commands = + bash -c "if [ ! -f ../*.charm ]; then echo 'Charm does not exist. Run tox -e build'; exit 1; fi" + tox --version + tox -e func-target {posargs} + [testenv:cover] # Technique based heavily upon # https://github.com/openstack/nova/blob/master/tox.ini