Add xena bundles

- add non-voting focal-xena bundle
- add non-voting impish-xena bundle
- rebuild to pick up charm-helpers changes
- update tox/pip.sh to ensure setuptools<50.0.0
- sync *requirements.txt in order to pin cffi

Change-Id: Ic3c4a6d382d734dab5821c35eda4ba2d8e77ce53
Co-authored-by: Aurelien Lourot <aurelien.lourot@canonical.com>

osci.yaml

@@ -9,8 +9,15 @@
       # Wallaby is currently broken because of a change in the
       # default token-type requirements in manila's policy.json
       # that the charm / tests need to learn to handle
-      # - focal-wallaby
+      - focal-wallaby:
+          voting: false
+      - focal-xena:
+          voting: false
       - groovy-victoria
+      - hirsute-wallaby:
+          voting: false
+      - impish-xena:
+          voting: false
     vars:
       needs_charm_build: true
       charm_build_name: manila-ganesha

pip.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# This file is managed centrally by release-tools and should not be modified
+# within individual charm repos.  See the 'global' dir contents for available
+# choices of tox.ini for OpenStack Charms:
+#     https://github.com/openstack-charmers/release-tools
+#
+# setuptools 58.0 dropped the support for use_2to3=true which is needed to
+# install blessings (an indirect dependency of charm-tools).
+#
+# More details on the beahvior of tox and virtualenv creation can be found at
+# https://github.com/tox-dev/tox/issues/448
+#
+# This script is wrapper to force the use of the pinned versions early in the
+# process when the virtualenv was created and upgraded before installing the
+# depedencies declared in the target.
+pip install 'pip<20.3' 'setuptools<50.0.0'
+pip "$@"

rebuild

@@ -2,4 +2,4 @@
 # when dependencies of the charm change,
 # but nothing in the charm needs to.
 # simply change the uuid to something new
-fa14a3e8-b3eb-11eb-9a1d-a73843f31142
+53d1cce6-1178-11ec-9722-97d8d41c4f76

requirements.txt

@@ -9,12 +9,9 @@
 setuptools<50.0.0  # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85
 
 # Build requirements
+cffi==1.14.6; python_version < '3.6'  # cffi 1.15.0 drops support for py35.
 charm-tools==2.8.3
 
-# Workaround until https://github.com/juju/charm-tools/pull/589 gets
-# published
-keyring<21
-
 simplejson
 
 # Newer versions use keywords that didn't exist in python 3.5 yet (e.g.

src/tests/bundles/focal-xena.yaml

@@ -0,0 +1,309 @@
+variables:
+  openstack-origin: &openstack-origin cloud:focal-xena
+
+series: focal
+
+comment:
+- 'machines section to decide order of deployment. database sooner = faster'
+machines:
+  '0':
+    constraints: mem=3072M
+  '1':
+    constraints: mem=3072M
+  '2':
+    constraints: mem=3072M
+  '3':
+  '4':
+  '5':
+  '6':
+  '7':
+  '8':
+  '9':
+  '10':
+  '11':
+  '12':
+  '13':
+  '14':
+  '15':
+  '16':
+    constraints: mem=8G
+  '17':
+    constraints: mem=8G
+  '18':
+  '19':
+  '20':
+  '21':
+  '22':
+
+services:
+
+  manila-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+  manila-ganesha-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+  keystone-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+  neutron-api-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+  nova-cloud-controller-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+  glance-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+  placement-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+
+  mysql-innodb-cluster:
+    charm: cs:~openstack-charmers-next/mysql-innodb-cluster
+    num_units: 3
+    options:
+      source: *openstack-origin
+    to:
+      - '0'
+      - '1'
+      - '2'
+
+  manila-ganesha-az1:
+    num_units: 3
+    charm: ../../../manila-ganesha
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '3'
+      - '4'
+      - '5'
+
+  ceph-mon:
+    charm: cs:~openstack-charmers-next/ceph-mon
+    num_units: 3
+    options:
+      source: *openstack-origin
+    to:
+      - '6'
+      - '7'
+
+  ceph-osd:
+    charm: cs:~openstack-charmers-next/ceph-osd
+    num_units: 3
+    options:
+      source: *openstack-origin
+    storage:
+      osd-devices: 'cinder,10G'
+    to:
+      - '8'
+      - '9'
+      - '10'
+
+  ceph-fs:
+    charm: cs:~openstack-charmers-next/ceph-fs
+    num_units: 2
+    options:
+      source: *openstack-origin
+    to:
+      - '11'
+      - '12'
+
+  manila:
+    charm: cs:~openstack-charmers-next/manila
+    num_units: 1
+    options:
+      default-share-backend: cephfsnfs1
+      share-protocols: NFS
+      openstack-origin: *openstack-origin
+    to:
+      - '13'
+
+  nova-cloud-controller:
+    charm: cs:~openstack-charmers-next/nova-cloud-controller
+    num_units: 1
+    options:
+      network-manager: Neutron
+      openstack-origin: *openstack-origin
+    to:
+      - '14'
+
+  placement:
+    charm: cs:~openstack-charmers-next/placement
+    num_units: 1
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '15'
+
+  nova-compute:
+    charm: cs:~openstack-charmers-next/nova-compute
+    num_units: 2
+    options:
+      config-flags: default_ephemeral_format=ext4
+      enable-live-migration: true
+      enable-resize: true
+      migration-auth-type: ssh
+      openstack-origin: *openstack-origin
+    to:
+      - '16'
+      - '17'
+
+  glance:
+    charm: cs:~openstack-charmers-next/glance
+    num_units: 1
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '18'
+
+  neutron-api:
+    charm: cs:~openstack-charmers-next/neutron-api
+    num_units: 1
+    options:
+      manage-neutron-plugin-legacy-mode: true
+      neutron-plugin: ovs
+      flat-network-providers: physnet1
+      neutron-security-groups: true
+      openstack-origin: *openstack-origin
+    to:
+      - '19'
+
+  neutron-openvswitch:
+    charm: cs:~openstack-charmers-next/neutron-openvswitch
+
+  neutron-gateway:
+    charm: cs:~openstack-charmers-next/neutron-gateway
+    num_units: 1
+    options:
+      bridge-mappings: physnet1:br-ex
+      openstack-origin: *openstack-origin
+    to:
+      - '20'
+
+  rabbitmq-server:
+    charm: cs:~openstack-charmers-next/rabbitmq-server
+    num_units: 1
+    options:
+      source: *openstack-origin
+    to:
+      - '21'
+
+  keystone:
+    charm: cs:~openstack-charmers-next/keystone
+    num_units: 1
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '22'
+
+relations:
+
+  - - 'ceph-mon'
+    - 'ceph-osd'
+
+  - - 'ceph-mon'
+    - 'ceph-fs'
+
+  - - 'ceph-mon'
+    - 'manila-ganesha-az1'
+
+  - - 'manila:shared-db'
+    - 'manila-mysql-router:shared-db'
+  - - 'manila-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'manila-ganesha-az1'
+    - 'rabbitmq-server'
+
+  - - 'manila-ganesha-az1'
+    - 'keystone'
+
+  - - 'manila'
+    - 'manila-ganesha-az1'
+
+  - - 'manila-ganesha-az1:shared-db'
+    - 'manila-ganesha-mysql-router:shared-db'
+  - - 'manila-ganesha-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'manila'
+    - 'rabbitmq-server'
+
+  - - 'manila'
+    - 'keystone'
+
+  - - 'keystone:shared-db'
+    - 'keystone-mysql-router:shared-db'
+  - - 'keystone-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'neutron-api:shared-db'
+    - 'neutron-api-mysql-router:shared-db'
+  - - 'neutron-api-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'neutron-api:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'neutron-api:neutron-api'
+    - 'nova-cloud-controller:neutron-api'
+
+  - - 'placement:placement'
+    - 'nova-cloud-controller:placement'
+
+  - - 'placement:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'placement:shared-db'
+    - 'placement-mysql-router:shared-db'
+  - - 'placement-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'placement:identity-service'
+    - 'keystone:identity-service'
+
+  - - 'neutron-api:neutron-plugin-api'
+    - 'neutron-gateway:neutron-plugin-api'
+
+  - - 'neutron-api:identity-service'
+    - 'keystone:identity-service'
+
+  - - 'nova-compute:neutron-plugin'
+    - 'neutron-openvswitch:neutron-plugin'
+
+  - - 'nova-cloud-controller:shared-db'
+    - 'nova-cloud-controller-mysql-router:shared-db'
+  - - 'nova-cloud-controller-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'neutron-gateway:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'nova-cloud-controller:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'nova-compute:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'neutron-openvswitch:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'nova-cloud-controller:identity-service'
+    - 'keystone:identity-service'
+
+  - - 'nova-cloud-controller:cloud-compute'
+    - 'nova-compute:cloud-compute'
+
+  - - 'glance:identity-service'
+    - 'keystone:identity-service'
+
+  - - 'glance:shared-db'
+    - 'glance-mysql-router:shared-db'
+  - - 'glance-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'glance:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'nova-compute:image-service'
+    - 'glance:image-service'
+
+  - - 'nova-cloud-controller:image-service'
+    - 'glance:image-service'
+
+  - - 'nova-cloud-controller:quantum-network-service'
+    - 'neutron-gateway:quantum-network-service'

src/tests/bundles/impish-xena.yaml

@@ -0,0 +1,311 @@
+variables:
+  openstack-origin: &openstack-origin distro
+
+series: impish
+
+comment:
+- 'machines section to decide order of deployment. database sooner = faster'
+machines:
+  '0':
+    constraints: mem=3072M
+  '1':
+    constraints: mem=3072M
+  '2':
+    constraints: mem=3072M
+  '3':
+  '4':
+  '5':
+  '6':
+  '7':
+  '8':
+  '9':
+  '10':
+  '11':
+  '12':
+  '13':
+  '14':
+  '15':
+  '16':
+  '17':
+    constraints: mem=8G
+  '18':
+    constraints: mem=8G
+  '19':
+  '20':
+  '21':
+  '22':
+  '23':
+
+services:
+
+  manila-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+  manila-ganesha-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+  keystone-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+  neutron-api-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+  nova-cloud-controller-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+  glance-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+  placement-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+
+  mysql-innodb-cluster:
+    charm: cs:~openstack-charmers-next/mysql-innodb-cluster
+    num_units: 3
+    options:
+      source: *openstack-origin
+    to:
+      - '0'
+      - '1'
+      - '2'
+
+  manila-ganesha-az1:
+    num_units: 3
+    charm: ../../../manila-ganesha
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '3'
+      - '4'
+      - '5'
+
+  ceph-mon:
+    charm: cs:~openstack-charmers-next/ceph-mon
+    num_units: 3
+    options:
+      source: *openstack-origin
+    to:
+      - '6'
+      - '7'
+      - '8'
+
+  ceph-osd:
+    charm: cs:~openstack-charmers-next/ceph-osd
+    num_units: 3
+    options:
+      source: *openstack-origin
+    storage:
+      osd-devices: 'cinder,10G'
+    to:
+      - '9'
+      - '10'
+      - '11'
+
+  ceph-fs:
+    charm: cs:~openstack-charmers-next/ceph-fs
+    num_units: 2
+    options:
+      source: *openstack-origin
+    to:
+      - '12'
+      - '13'
+
+  manila:
+    charm: cs:~openstack-charmers-next/manila
+    num_units: 1
+    options:
+      default-share-backend: cephfsnfs1
+      share-protocols: NFS
+      openstack-origin: *openstack-origin
+    to:
+      - '14'
+
+  nova-cloud-controller:
+    charm: cs:~openstack-charmers-next/nova-cloud-controller
+    num_units: 1
+    options:
+      network-manager: Neutron
+      openstack-origin: *openstack-origin
+    to:
+      - '15'
+
+  placement:
+    charm: cs:~openstack-charmers-next/placement
+    num_units: 1
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '16'
+
+  nova-compute:
+    charm: cs:~openstack-charmers-next/nova-compute
+    num_units: 2
+    options:
+      config-flags: default_ephemeral_format=ext4
+      enable-live-migration: true
+      enable-resize: true
+      migration-auth-type: ssh
+      openstack-origin: *openstack-origin
+    to:
+      - '17'
+      - '18'
+
+  glance:
+    charm: cs:~openstack-charmers-next/glance
+    num_units: 1
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '19'
+
+  neutron-api:
+    charm: cs:~openstack-charmers-next/neutron-api
+    num_units: 1
+    options:
+      manage-neutron-plugin-legacy-mode: true
+      neutron-plugin: ovs
+      flat-network-providers: physnet1
+      neutron-security-groups: true
+      openstack-origin: *openstack-origin
+    to:
+      - '20'
+
+  neutron-openvswitch:
+    charm: cs:~openstack-charmers-next/neutron-openvswitch
+
+  neutron-gateway:
+    charm: cs:~openstack-charmers-next/neutron-gateway
+    num_units: 1
+    options:
+      bridge-mappings: physnet1:br-ex
+      openstack-origin: *openstack-origin
+    to:
+      - '21'
+
+  rabbitmq-server:
+    charm: cs:~openstack-charmers-next/rabbitmq-server
+    num_units: 1
+    options:
+      source: *openstack-origin
+    to:
+      - '22'
+
+  keystone:
+    charm: cs:~openstack-charmers-next/keystone
+    num_units: 1
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '23'
+
+relations:
+
+  - - 'ceph-mon'
+    - 'ceph-osd'
+
+  - - 'ceph-mon'
+    - 'ceph-fs'
+
+  - - 'ceph-mon'
+    - 'manila-ganesha-az1'
+
+  - - 'manila:shared-db'
+    - 'manila-mysql-router:shared-db'
+  - - 'manila-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'manila-ganesha-az1'
+    - 'rabbitmq-server'
+
+  - - 'manila-ganesha-az1'
+    - 'keystone'
+
+  - - 'manila'
+    - 'manila-ganesha-az1'
+
+  - - 'manila-ganesha-az1:shared-db'
+    - 'manila-ganesha-mysql-router:shared-db'
+  - - 'manila-ganesha-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'manila'
+    - 'rabbitmq-server'
+
+  - - 'manila'
+    - 'keystone'
+
+  - - 'keystone:shared-db'
+    - 'keystone-mysql-router:shared-db'
+  - - 'keystone-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'neutron-api:shared-db'
+    - 'neutron-api-mysql-router:shared-db'
+  - - 'neutron-api-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'neutron-api:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'neutron-api:neutron-api'
+    - 'nova-cloud-controller:neutron-api'
+
+  - - 'placement:placement'
+    - 'nova-cloud-controller:placement'
+
+  - - 'placement:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'placement:shared-db'
+    - 'placement-mysql-router:shared-db'
+  - - 'placement-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'placement:identity-service'
+    - 'keystone:identity-service'
+
+  - - 'neutron-api:neutron-plugin-api'
+    - 'neutron-gateway:neutron-plugin-api'
+
+  - - 'neutron-api:identity-service'
+    - 'keystone:identity-service'
+
+  - - 'nova-compute:neutron-plugin'
+    - 'neutron-openvswitch:neutron-plugin'
+
+  - - 'nova-cloud-controller:shared-db'
+    - 'nova-cloud-controller-mysql-router:shared-db'
+  - - 'nova-cloud-controller-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'neutron-gateway:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'nova-cloud-controller:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'nova-compute:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'neutron-openvswitch:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'nova-cloud-controller:identity-service'
+    - 'keystone:identity-service'
+
+  - - 'nova-cloud-controller:cloud-compute'
+    - 'nova-compute:cloud-compute'
+
+  - - 'glance:identity-service'
+    - 'keystone:identity-service'
+
+  - - 'glance:shared-db'
+    - 'glance-mysql-router:shared-db'
+  - - 'glance-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'glance:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'nova-compute:image-service'
+    - 'glance:image-service'
+
+  - - 'nova-cloud-controller:image-service'
+    - 'glance:image-service'
+
+  - - 'nova-cloud-controller:quantum-network-service'
+    - 'neutron-gateway:quantum-network-service'

src/tests/tests.yaml

@@ -1,6 +1,10 @@
 charm_name: manila-ganesha
 dev_bundles:
+- focal-wallaby
+- focal-xena
 - groovy-victoria
+- hirsute-wallaby
+- impish-xena
 gate_bundles:
 - focal-ussuri
 - focal-victoria
@@ -21,3 +25,4 @@ configure:
 tests_options:
   force_deploy:
     - groovy-victoria
+    - impish-xena

src/tox.ini

@@ -22,12 +22,12 @@ skip_missing_interpreters = False
 requires = pip < 20.3
            virtualenv < 20.0
 # NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci
-minversion = 3.2.0
+minversion = 3.18.0
 
 [testenv]
 setenv = VIRTUAL_ENV={envdir}
          PYTHONHASHSEED=0
-whitelist_externals = juju
+allowlist_externals = juju
 passenv = HOME TERM CS_* OS_* TEST_*
 deps = -r{toxinidir}/test-requirements.txt
 install_command =

test-requirements.txt

@@ -3,9 +3,8 @@
 # choices of *requirements.txt files for OpenStack Charms:
 #     https://github.com/openstack-charmers/release-tools
 #
+cffi==1.14.6; python_version < '3.6'  # cffi 1.15.0 drops support for py35.
 setuptools<50.0.0  # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85
-# Lint and unit test requirements
-flake8>=2.2.4
 
 stestr>=2.2.0
 
@@ -38,10 +37,16 @@ coverage>=3.6
 git+https://github.com/openstack/charms.openstack.git#egg=charms.openstack
 #
 # Revisit for removal / mock improvement:
+#
+# NOTE(lourot): newer versions of cryptography require a Rust compiler to build,
+# see
+# * https://github.com/openstack-charmers/zaza/issues/421
+# * https://mail.python.org/pipermail/cryptography-dev/2021-January/001003.html
+#
 netifaces        # vault
 psycopg2-binary  # vault
 tenacity         # vault
-pbr              # vault
+pbr==5.6.0       # vault
-cryptography     # vault, keystone-saml-mellon
+cryptography<3.4 # vault, keystone-saml-mellon
 lxml             # keystone-saml-mellon
 hvac             # vault, barbican-vault

tox.ini

@@ -11,6 +11,21 @@ envlist = pep8,py3
 sitepackages = False
 # NOTE: Avoid false positives by not skipping missing interpreters.
 skip_missing_interpreters = False
+# NOTES:
+# * We avoid the new dependency resolver by pinning pip < 20.3, see
+#   https://github.com/pypa/pip/issues/9187
+# * Pinning dependencies requires tox >= 3.2.0, see
+#   https://tox.readthedocs.io/en/latest/config.html#conf-requires
+# * It is also necessary to pin virtualenv as a newer virtualenv would still
+#   lead to fetching the latest pip in the func* tox targets, see
+#   https://stackoverflow.com/a/38133283
+requires =
+  pip < 20.3
+  virtualenv < 20.0
+  setuptools<50.0.0
+
+# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci
+minversion = 3.18.0
 
 [testenv]
 setenv = VIRTUAL_ENV={envdir}
@@ -21,7 +36,7 @@ setenv = VIRTUAL_ENV={envdir}
          JUJU_REPOSITORY={toxinidir}/build
 passenv = http_proxy https_proxy INTERFACE_PATH LAYER_PATH JUJU_REPOSITORY
 install_command =
-  pip install {opts} {packages}
+  {toxinidir}/pip.sh install {opts} {packages}
 deps =
     -r{toxinidir}/requirements.txt
 
@@ -62,7 +77,8 @@ commands = stestr run --slowest {posargs}
 
 [testenv:pep8]
 basepython = python3
-deps = -r{toxinidir}/test-requirements.txt
+deps = flake8==3.9.2
+       charm-tools==2.8.3
 commands = flake8 {posargs} src unit_tests
 
 [testenv:cover]