Add functional tests

Also migrate test bundles to charmhub

func-test-pr: https://github.com/openstack-charmers/zaza-openstack-tests/pull/700
Change-Id: I001b79506ed623f39b61b62db1f70f3829bb4bbb
Co-authored-by: Aurelien Lourot <aurelien.lourot@canonical.com>

build-requirements.txt

@@ -1 +1,7 @@
-git+https://github.com/canonical/charmcraft.git@0.10.2#egg=charmcraft
+# NOTES(lourot):
+# * We don't install charmcraft via pip anymore because it anyway spins up a
+#   container and scp the system's charmcraft snap inside it. So the charmcraft
+#   snap is necessary on the system anyway.
+# * `tox -e build` successfully validated with charmcraft 1.2.1
+
+cffi==1.14.6; python_version < '3.6'  # cffi 1.15.0 drops support for py35.

charmcraft.yaml

@@ -0,0 +1,35 @@
+type: charm
+
+parts:
+  charm:
+    after:
+      - update-certificates
+    charm-python-packages:
+      # NOTE(lourot): see
+      # * https://github.com/canonical/charmcraft/issues/551
+      - setuptools
+    build-packages:
+      - git
+
+  update-certificates:
+    plugin: nil
+    # See https://github.com/canonical/charmcraft/issues/658
+    override-build: |
+      apt update
+      apt install -y ca-certificates
+      update-ca-certificates
+
+bases:
+  - build-on:
+      - name: ubuntu
+        channel: "20.04"
+        architectures:
+          - amd64
+          - s390x
+          - ppc64el
+          - arm64
+    run-on:
+      - name: ubuntu
+        channel: "20.04"
+      - name: ubuntu
+        channel: "21.10"

osci.yaml

@@ -1,9 +1,8 @@
 - project:
     templates:
-      - charm-unit-jobs
+      - charm-unit-jobs-py38
-    check:
+      - charm-unit-jobs-py39
-      jobs:
-        - focal-ussuri
     vars:
       needs_charm_build: true
       charm_build_name: cinder-solidfire
+      build_type: charmcraft

rename.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+charm=$(grep "charm_build_name" osci.yaml | awk '{print $2}')
+echo "renaming ${charm}_*.charm to ${charm}.charm"
+echo -n "pwd: "
+pwd
+ls -al
+echo "Removing bad downloaded charm maybe?"
+if [[ -e "${charm}.charm" ]];
+then
+    rm "${charm}.charm"
+fi
+echo "Renaming charm here."
+mv ${charm}_*.charm ${charm}.charm

src/test-requirements.txt

@@ -1,9 +0,0 @@
-# This file is managed centrally by release-tools and should not be modified
-# within individual charm repos.  See the 'global' dir contents for available
-# choices of *requirements.txt files for OpenStack Charms:
-#     https://github.com/openstack-charmers/release-tools
-#
-
-# Functional Test Requirements (let Zaza's dependencies solve all dependencies here!)
-git+https://github.com/openstack-charmers/zaza.git#egg=zaza
-git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack

src/tox.ini

@@ -1,61 +0,0 @@
-# Source charm (with zaza): ./src/tox.ini
-# This file is managed centrally by release-tools and should not be modified
-# within individual charm repos.  See the 'global' dir contents for available
-# choices of tox.ini for OpenStack Charms:
-#     https://github.com/openstack-charmers/release-tools
-
-[tox]
-envlist = pep8
-skipsdist = True
-# NOTE: Avoid build/test env pollution by not enabling sitepackages.
-sitepackages = False
-# NOTE: Avoid false positives by not skipping missing interpreters.
-skip_missing_interpreters = False
-# NOTES:
-# * We avoid the new dependency resolver by pinning pip < 20.3, see
-#   https://github.com/pypa/pip/issues/9187
-# * Pinning dependencies requires tox >= 3.2.0, see
-#   https://tox.readthedocs.io/en/latest/config.html#conf-requires
-# * It is also necessary to pin virtualenv as a newer virtualenv would still
-#   lead to fetching the latest pip in the func* tox targets, see
-#   https://stackoverflow.com/a/38133283
-requires = pip < 20.3
-           virtualenv < 20.0
-# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci
-minversion = 3.18.0
-
-[testenv]
-setenv = VIRTUAL_ENV={envdir}
-         PYTHONHASHSEED=0
-allowlist_externals = juju
-passenv = HOME TERM CS_* OS_* TEST_*
-deps = -r{toxinidir}/test-requirements.txt
-install_command =
-  pip install {opts} {packages}
-
-[testenv:pep8]
-basepython = python3
-commands = charm-proof
-
-[testenv:func-noop]
-basepython = python3
-commands =
-    functest-run-suite --help
-
-[testenv:func]
-basepython = python3
-commands =
-    functest-run-suite --keep-model
-
-[testenv:func-smoke]
-basepython = python3
-commands =
-    functest-run-suite --keep-model --smoke
-
-[testenv:func-target]
-basepython = python3
-commands =
-    functest-run-suite --keep-model --bundle {posargs}
-
-[testenv:venv]
-commands = {posargs}

test-requirements.txt

@@ -4,7 +4,7 @@
 charm-tools>=2.4.4
 coverage>=3.6
 mock>=1.2
-flake8>=4.0.1
+flake8>=4.0.1; python_version >= '3.6'
 stestr>=2.2.0
 requests>=2.18.4
 psutil
@@ -14,3 +14,4 @@ git+https://github.com/openstack-charmers/zaza.git#egg=zaza
 git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack
 pytz    # workaround for 14.04 pip/tox
 pyudev  # for ceph-* charm unit tests (not mocked?)
+cffi==1.14.6; python_version < '3.6'  # cffi 1.15.0 drops support for py35.

tests/README.md

@@ -0,0 +1,18 @@
+# Overview
+
+This directory provides Zaza test definitions and bundles to verify basic
+deployment functionality from the perspective of this charm, its requirements
+and its features, as exercised in a subset of the full OpenStack deployment
+test bundle topology.
+
+Run the smoke tests with:
+
+```bash
+cd ../
+tox -e build
+tox -e func-smoke
+```
+
+For full details on functional testing of OpenStack charms please refer to
+the [functional testing](https://docs.openstack.org/charm-guide/latest/reference/testing.html#functional-testing)
+section of the OpenStack Charm Guide.

tests/bundles/focal-ussuri.yaml

@@ -18,7 +18,8 @@ machines:
 
 applications:
   mysql-innodb-cluster:
-    charm: cs:~openstack-charmers-next/mysql-innodb-cluster
+    charm: ch:mysql-innodb-cluster
+    channel: latest/edge
     num_units: 3
     options:
       source: *openstack-origin
@@ -27,23 +28,27 @@ applications:
       - '1'
       - '2'
   rabbitmq-server:
-    charm: cs:~openstack-charmers-next/rabbitmq-server
+    charm: ch:rabbitmq-server
+    channel: latest/edge
     num_units: 1
     options:
       source: *openstack-origin
     to:
       - '3'
   keystone:
-    charm: cs:~openstack-charmers/keystone
+    charm: ch:keystone
+    channel: yoga/edge
     options:
       openstack-origin: *openstack-origin
     num_units: 1
     to:
       - '4'
   keystone-mysql-router:
-    charm: cs:~openstack-charmers-next/mysql-router
+    charm: ch:mysql-router
+    channel: latest/edge
   cinder:
-    charm: cs:~openstack-charmers-next/cinder
+    charm: ch:cinder
+    channel: yoga/edge
     num_units: 1
     storage:
       block-devices: '40G'
@@ -55,14 +60,11 @@ applications:
       - '5'
 
   cinder-mysql-router:
-    charm: cs:~openstack-charmers-next/mysql-router
+    charm: ch:mysql-router
+    channel: latest/edge
 
   cinder-solidfire:
-    charm: ../../cinder-solidfire
+    charm: ../../cinder-solidfire.charm
-    options:
-      san-ip: 10.0.0.5
-      san-login: username
-      san-password: password
 
 relations:
   - [ keystone:shared-db, keystone-mysql-router:shared-db ]

tests/tests.py

@@ -0,0 +1,17 @@
+from zaza.openstack.charm_tests.cinder_backend.tests import CinderBackendTest
+from os import environ
+
+
+class CinderSolidfireTest(CinderBackendTest):
+    """Encapsulate cinder-solidfire tests."""
+
+    backend_name = 'cinder-solidfire'
+
+    expected_config_content = {
+        'cinder-solidfire': {
+            'volume_driver':
+            ['cinder.volume.drivers.solidfire.SolidFireDriver'],
+            'san_ip': environ['TEST_SOLIDFIRE_SAN_IP'],
+            'san_login': environ['TEST_SOLIDFIRE_SAN_USERNAME'],
+            'san_password': environ['TEST_SOLIDFIRE_SAN_PASSWORD']
+        }}

tests/tests.yaml

@@ -1,6 +1,6 @@
 charm_name: cinder-solidfire
 tests:
-  - tests.tests_cinder_solidfire.CinderSolidfireTest
+  - tests.tests.CinderSolidfireTest
 configure:
   - zaza.openstack.charm_tests.keystone.setup.add_demo_user
 gate_bundles:

tests/tests_cinder_solidfire.py

@@ -1,71 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright 2019 Canonical Ltd.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Encapsulate cinder-solidfire testing."""
-
-import logging
-import uuid
-
-import zaza.model
-import zaza.openstack.charm_tests.test_utils as test_utils
-import zaza.openstack.utilities.openstack as openstack_utils
-
-
-class CinderSolidfireTest(test_utils.OpenStackBaseTest):
-    """Encapsulate Solidfire tests."""
-
-    @classmethod
-    def setUpClass(cls):
-        """Run class setup for running tests."""
-        super(CinderSolidfireTest, cls).setUpClass()
-        cls.keystone_session = openstack_utils.get_overcloud_keystone_session()
-        cls.model_name = zaza.model.get_juju_model()
-        cls.cinder_client = openstack_utils.get_cinder_session_client(
-            cls.keystone_session)
-
-    def test_cinder_config(self):
-        logging.info('solidfire')
-        expected_contents = {
-            'cinder-solidfire': {
-                'iscsi_helper': ['tgtadm'],
-                'volume_dd_blocksize': ['512']}}
-
-        zaza.model.run_on_leader(
-            'cinder',
-            'sudo cp /etc/cinder/cinder.conf /tmp/',
-            model_name=self.model_name)
-        zaza.model.block_until_oslo_config_entries_match(
-            'cinder',
-            '/tmp/cinder.conf',
-            expected_contents,
-            model_name=self.model_name,
-            timeout=2)
-
-    # Disabled while there is no hardware Solidfire appliance
-    def _disabled_test_create_volume(self):
-        test_vol_name = "zaza{}".format(uuid.uuid1().fields[0])
-        vol_new = self.cinder_client.volumes.create(
-            name=test_vol_name,
-            size=2)
-        openstack_utils.resource_reaches_status(
-            self.cinder_client.volumes,
-            vol_new.id,
-            expected_status='available')
-        test_vol = self.cinder_client.volumes.find(name=test_vol_name)
-        self.assertEqual(
-            getattr(test_vol, 'os-vol-host-attr:host').split('#')[0],
-            'cinder@cinder-solidfire')
-        self.cinder_client.volumes.delete(vol_new)

tox.ini

@@ -15,8 +15,12 @@ skip_missing_interpreters = False
 # * It is also necessary to pin virtualenv as a newer virtualenv would still
 #   lead to fetching the latest pip in the func* tox targets, see
 #   https://stackoverflow.com/a/38133283
+# * It is necessary to declare setuptools as a dependency otherwise tox will
+#   fail very early at not being able to load it. The version pinning is in
+#   line with `pip.sh`.
 requires = pip < 20.3
            virtualenv < 20.0
+           setuptools < 50.0.0
 # NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci
 minversion = 3.2.0
 
@@ -29,8 +33,9 @@ install_command =
 commands = stestr run --slowest {posargs}
 whitelist_externals =
     git
-    add-to-archive.py
     bash
+    charmcraft
+    rename.sh
 passenv = HOME TERM CS_* OS_* TEST_*
 deps = -r{toxinidir}/test-requirements.txt
 
@@ -54,6 +59,11 @@ basepython = python3.8
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 
+[testenv:py39]
+basepython = python3.9
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+
 [testenv:py3]
 basepython = python3
 deps = -r{toxinidir}/requirements.txt
@@ -100,8 +110,15 @@ commands = {posargs}
 [testenv:build]
 basepython = python3
 deps = -r{toxinidir}/build-requirements.txt
+# NOTE(lourot): charmcraft 1.0.0 used to generate
+# cinder-solidfire.charm, which is the behaviour expected by OSCI.
+# However charmcraft 1.2.1 now generates
+# cinder-solidfire_ubuntu-20.04-amd64.charm instead. In order to keep
+# the old behaviour we rename the file at the en
 commands =
-    charmcraft build
+    charmcraft clean
+    charmcraft -v build
+    {toxinidir}/rename.sh
 
 [testenv:func-noop]
 basepython = python3