From 5e40ba7d811ffd19f3c9c2a7a5f7fe55a8342771 Mon Sep 17 00:00:00 2001
From: Samuel Allan <samuel.allan@canonical.com>
Date: Wed, 21 Feb 2024 12:35:59 +1030
Subject: [PATCH] Add func tests for tempest-k8s charm

This is to verify that the tempest-k8s charm deploys successfully
(reaches active/idle, the tempest environment was set up successfully),
and doesn't adversely impact the rest of the sunbeam deployment.

Func tests with this charm have been observed to be unstable
(unexpected errors running certain tests),
so for now only minimal tests are added,
and it has been implemented in its own new test bundle.
This new test bundle is only triggered on changes in tempest-k8s code,
so it shouldn't affect the CI for other charms if it becomes unstable.

We still need to add more tests and debug the issues we were seeing,
but in the interests of time, we aim to implement these tests
and debug the issues in follow up patches.

Change-Id: I548eb07ed1eb9e67c46f62a16d36bfaec83d7040
---
 .gitignore                                    |   3 +
 CONTRIBUTING.md                               |  38 ++++
 README.md                                     |   5 +
 .../charm_tests/tempest_k8s/__init__.py       |   0
 .../sunbeam/charm_tests/tempest_k8s/tests.py  |  37 ++++
 tests/tempest/smoke.yaml.j2                   | 209 ++++++++++++++++++
 tests/tempest/tests.yaml                      |  58 +++++
 zuul.d/jobs.yaml                              |  43 ++++
 zuul.d/zuul.yaml                              |   2 +
 9 files changed, 395 insertions(+)
 create mode 100644 CONTRIBUTING.md
 create mode 100644 tests/local/zaza/sunbeam/charm_tests/tempest_k8s/__init__.py
 create mode 100644 tests/local/zaza/sunbeam/charm_tests/tempest_k8s/tests.py
 create mode 100644 tests/tempest/smoke.yaml.j2
 create mode 100644 tests/tempest/tests.yaml

diff --git a/.gitignore b/.gitignore
index 86942a32..403f2bb6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,6 @@ charms/*/.jujuignore
 charms/*/.stestr.conf
 charms/*/lib/
 charms/*/src/templates/parts/
+
+# artefacts from functional tests
+tempest.log
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 00000000..1493be98
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,38 @@
+# Contributing
+
+## Using functional tests
+
+The functional tests can be run locally.
+You will need microk8s configured, and juju bootstrapped on it.
+See ./zuul.d/zuul.yaml variables for the current supported versions
+of microk8s and juju as used for functional tests in CI.
+
+Ensure microk8s is the active controller on juju,
+and run the tox target, with the appropriate zaza options
+to select the test config you want to run:
+
+```
+tox -e func -- --smoke --test-directory=tests/core
+```
+
+See the tox.ini file for specifically what is being run.
+
+After running this, the juju model and the tox virtual environment will remain,
+so you can activate the virtual environment,
+and directly run zaza commands to re-run tests for debugging, etc.
+
+For example:
+
+```
+source .tox/func/bin/activate
+
+# fix the path to pick up the local test modules
+export PYTHONPATH="tests/local:$PYTHONPATH"
+
+# run a zaza command directly
+functest-test -m zaza-1537f907eca2 -t zaza.sunbeam.charm_tests.tempest_k8s.tests.TempestK8sTest --test-directory tests/core
+```
+
+Note that you can run `juju models` to see the generated name for the zaza model used.
+The `PYTHONPATH` must also be updated to pick up the tests from `./tests/local/*`,
+which is where the zaza tests are specifically for sunbeam-charms.
diff --git a/README.md b/README.md
index 2a136058..e9ccf47d 100644
--- a/README.md
+++ b/README.md
@@ -2,3 +2,8 @@
 
 This repository contains the core charms for the Sunbeam deployment and associated
 charm libraries.
+
+## Development docs
+
+See the CONTRIBUTING.md in the root of the repository and individual charm directories
+for more information.
diff --git a/tests/local/zaza/sunbeam/charm_tests/tempest_k8s/__init__.py b/tests/local/zaza/sunbeam/charm_tests/tempest_k8s/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/local/zaza/sunbeam/charm_tests/tempest_k8s/tests.py b/tests/local/zaza/sunbeam/charm_tests/tempest_k8s/tests.py
new file mode 100644
index 00000000..dfff4bb9
--- /dev/null
+++ b/tests/local/zaza/sunbeam/charm_tests/tempest_k8s/tests.py
@@ -0,0 +1,37 @@
+# Copyright (c) 2024 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from zaza import model
+from zaza.openstack.charm_tests import test_utils
+
+
+class TempestK8sTest(test_utils.BaseCharmTest):
+    """Charm tests for tempest-k8s."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for running tests."""
+        super(TempestK8sTest, cls).setUpClass(
+            application_name="tempest"
+        )
+
+    def test_get_lists(self):
+        """Verify that the get-lists action returns list names as expected."""
+        action = model.run_action_on_leader(
+            self.application_name, "get-lists"
+        )
+        lists = action.data["results"]["stdout"].splitlines()
+        self.assertIn("readonly-quick", lists)
+        self.assertIn("refstack-2022.11", lists)
diff --git a/tests/tempest/smoke.yaml.j2 b/tests/tempest/smoke.yaml.j2
new file mode 100644
index 00000000..eea67158
--- /dev/null
+++ b/tests/tempest/smoke.yaml.j2
@@ -0,0 +1,209 @@
+bundle: kubernetes
+
+applications:
+  traefik:
+    charm: ch:traefik-k8s
+    channel: 1.0/candidate
+    base: ubuntu@20.04
+    scale: 1
+    trust: true
+    options:
+      kubernetes-service-annotations: metallb.universe.tf/address-pool=public
+  mysql:
+    charm: ch:mysql-k8s
+    channel: 8.0/stable
+    base: ubuntu@22.04
+    scale: 1
+    trust: true
+    constraints: cpu-power=1000 mem=1G
+  tls-operator:
+    charm: self-signed-certificates
+    channel: latest/beta
+    base: ubuntu@22.04
+    scale: 1
+    options:
+      ca-common-name: internal-ca
+  rabbitmq:
+    charm: ch:rabbitmq-k8s
+    channel: 3.12/edge
+    base: ubuntu@22.04
+    scale: 1
+    trust: true
+    options:
+      minimum-replicas: 1
+  ovn-central:
+    {% if ovn_central_k8s is defined and ovn_central_k8s is sameas true -%}
+    charm: ../../../ovn-central-k8s.charm
+    {% else -%}
+    charm: ch:ovn-central-k8s
+    channel: 23.03/edge
+    {% endif -%}
+    base: ubuntu@22.04
+    scale: 1
+    trust: true
+    resources:
+      ovn-sb-db-server-image: ghcr.io/canonical/ovn-consolidated:23.09
+      ovn-nb-db-server-image: ghcr.io/canonical/ovn-consolidated:23.09
+      ovn-northd-image: ghcr.io/canonical/ovn-consolidated:23.09
+  ovn-relay:
+    {% if ovn_relay_k8s is defined and ovn_relay_k8s is sameas true -%}
+    charm: ../../../ovn-relay-k8s.charm
+    {% else -%}
+    charm: ch:ovn-relay-k8s
+    channel: 23.03/edge
+    {% endif -%}
+    base: ubuntu@22.04
+    scale: 1
+    trust: true
+    resources:
+      ovn-sb-db-server-image: ghcr.io/canonical/ovn-consolidated:23.09
+  keystone:
+    {% if keystone_k8s is defined and keystone_k8s is sameas true -%}
+    charm: ../../../keystone-k8s.charm
+    {% else -%}
+    charm: ch:keystone-k8s
+    channel: 2023.2/edge
+    {% endif -%}
+    base: ubuntu@22.04
+    scale: 1
+    trust: true
+    options:
+      admin-role: admin
+    storage:
+      fernet-keys: 5M
+      credential-keys: 5M
+    resources:
+      keystone-image: ghcr.io/canonical/keystone:2023.2
+  glance:
+    {% if glance_k8s is defined and glance_k8s is sameas true -%}
+    charm: ../../../glance-k8s.charm
+    {% else -%}
+    charm: ch:glance-k8s
+    channel: 2023.2/edge
+    {% endif -%}
+    base: ubuntu@22.04
+    scale: 1
+    trust: true
+    storage:
+      local-repository: 5G
+    resources:
+      glance-api-image: ghcr.io/canonical/glance-api:2023.2
+  nova:
+    {% if nova_k8s is defined and nova_k8s is sameas true -%}
+    charm: ../../../nova-k8s.charm
+    {% else -%}
+    charm: ch:nova-k8s
+    channel: 2023.2/edge
+    {% endif -%}
+    base: ubuntu@22.04
+    scale: 1
+    trust: true
+    resources:
+      nova-api-image: ghcr.io/canonical/nova-consolidated:2023.2
+      nova-scheduler-image: ghcr.io/canonical/nova-consolidated:2023.2
+      nova-conductor-image: ghcr.io/canonical/nova-consolidated:2023.2
+  placement:
+    {% if placement_k8s is defined and placement_k8s is sameas true -%}
+    charm: ../../../placement-k8s.charm
+    {% else -%}
+    charm: ch:placement-k8s
+    channel: 2023.2/edge
+    {% endif -%}
+    base: ubuntu@22.04
+    scale: 1
+    trust: true
+    resources:
+      placement-api-image: ghcr.io/canonical/placement-api:2023.2
+  neutron:
+    {% if neutron_k8s is defined and neutron_k8s is sameas true -%}
+    charm: ../../../neutron-k8s.charm
+    {% else -%}
+    charm: ch:neutron-k8s
+    channel: 2023.2/edge
+    {% endif -%}
+    base: ubuntu@22.04
+    scale: 1
+    trust: true
+    options:
+      debug: true
+    resources:
+      neutron-server-image: ghcr.io/canonical/neutron-server:2023.2
+  tempest:
+    {% if tempest_k8s is defined and tempest_k8s is sameas true -%}
+    charm: ../../../tempest-k8s.charm
+    {% else -%}
+    charm: ch:tempest-k8s
+    channel: 2023.2/edge
+    {% endif -%}
+    base: ubuntu@22.04
+    scale: 1
+    trust: true
+    resources:
+      tempest-image: ghcr.io/canonical/tempest:2023.2
+
+relations:
+- - tls-operator:certificates
+  - ovn-central:certificates
+
+- - tls-operator:certificates
+  - ovn-relay:certificates
+- - ovn-relay:ovsdb-cms
+  - ovn-central:ovsdb-cms
+
+- - mysql:database
+  - keystone:database
+- - traefik:ingress
+  - keystone:ingress-public
+
+- - mysql:database
+  - glance:database
+- - keystone:identity-service
+  - glance:identity-service
+- - rabbitmq:amqp
+  - glance:amqp
+- - traefik:ingress
+  - glance:ingress-public
+- - keystone:send-ca-cert
+  - glance:receive-ca-cert
+
+- - mysql:database
+  - nova:database
+- - mysql:database
+  - nova:api-database
+- - mysql:database
+  - nova:cell-database
+- - rabbitmq:amqp
+  - nova:amqp
+- - keystone:identity-service
+  - nova:identity-service
+- - traefik:ingress
+  - nova:ingress-public
+- - keystone:send-ca-cert
+  - nova:receive-ca-cert
+
+- - mysql:database
+  - placement:database
+- - keystone:identity-service
+  - placement:identity-service
+- - traefik:ingress
+  - placement:ingress-public
+- - keystone:send-ca-cert
+  - placement:receive-ca-cert
+
+- - mysql:database
+  - neutron:database
+- - rabbitmq:amqp
+  - neutron:amqp
+- - keystone:identity-service
+  - neutron:identity-service
+- - traefik:ingress
+  - neutron:ingress-public
+- - tls-operator:certificates
+  - neutron:certificates
+- - neutron:ovsdb-cms
+  - ovn-central:ovsdb-cms
+- - keystone:send-ca-cert
+  - neutron:receive-ca-cert
+
+- - tempest:identity-ops
+  - keystone:identity-ops
diff --git a/tests/tempest/tests.yaml b/tests/tempest/tests.yaml
new file mode 100644
index 00000000..7dadf221
--- /dev/null
+++ b/tests/tempest/tests.yaml
@@ -0,0 +1,58 @@
+# to clear up any confusion:
+# - tempest-k8s / tempest charm here is the charm that is part of the validation plugin, that runs tempest against a sunbeam deployment
+# - tempest keys and tempest tests mentioned here are for an external tempest running tests against this test deployment
+gate_bundles:
+  - smoke
+smoke_bundles:
+  - smoke
+configure:
+  - zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints
+  - zaza.openstack.charm_tests.keystone.setup.add_tempest_roles
+  - zaza.openstack.charm_tests.nova.setup.create_flavors
+  - zaza.openstack.charm_tests.nova.setup.manage_ssh_key
+tests:
+  - zaza.sunbeam.charm_tests.tempest_k8s.tests.TempestK8sTest
+tests_options:
+  trust:
+    - smoke
+  ignore_hard_deploy_errors:
+    - smoke
+
+target_deploy_status:
+  traefik:
+    workload-status: active
+    workload-status-message-regex: '^$'
+  mysql:
+    workload-status: active
+    workload-status-message-regex: '^.*$'
+  tls-operator:
+    workload-status: active
+    workload-status-message-regex: '^$'
+  rabbitmq:
+    workload-status: active
+    workload-status-message-regex: '^$'
+  ovn-central:
+    workload-status: active
+    workload-status-message-regex: '^$'
+  ovn-relay:
+    workload-status: active
+    workload-status-message-regex: '^$'
+  keystone:
+    workload-status: active
+    workload-status-message-regex: '^$'
+  glance:
+    workload-status: active
+    workload-status-message-regex: '^$'
+  nova:
+    workload-status: active
+    workload-status-message-regex: '^$'
+  placement:
+    workload-status: active
+    workload-status-message-regex: '^$'
+  neutron:
+    workload-status: active
+    workload-status-message-regex: '^$'
+  # the tempest-k8s charm
+  tempest:
+    workload-status: active
+    workload-status-message-regex: '^$'
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index 02f69ea0..fcabd4d5 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -343,6 +343,49 @@
         - charm-build-ovn-relay-k8s
       # test_dir relative to project src dir
       test_dir: tests/core
+# This test set may be unstable, and more tests need to be added for the tempest-k8s charm,
+# so it is only trigger by changes in the tempest-k8s charm and associated tests for now.
+# It has been copied from the tests/core charm bundle.
+- job:
+    name: func-test-tempest
+    description: |
+      Zaza tests for the tempest-k8s charm.
+    timeout: 3600
+    run: playbooks/zaza-func-test.yaml
+    post-run: playbooks/collect-run-data.yaml
+    dependencies:
+      - name: charm-build-keystone-k8s
+        soft: true
+      - name: charm-build-glance-k8s
+        soft: true
+      - name: charm-build-nova-k8s
+        soft: true
+      - name: charm-build-placement-k8s
+        soft: true
+      - name: charm-build-neutron-k8s
+        soft: true
+      - name: charm-build-ovn-central-k8s
+        soft: true
+      - name: charm-build-ovn-relay-k8s
+        soft: true
+      - name: charm-build-tempest-k8s
+        soft: true
+    files:
+      - charms/tempest-k8s/*
+      - tests/local/zaza/sunbeam/charm_tests/tempest_k8s/*
+    vars:
+      # Artifacts will be downloaded from below charm jobs
+      charm_jobs:
+        - charm-build-keystone-k8s
+        - charm-build-glance-k8s
+        - charm-build-nova-k8s
+        - charm-build-placement-k8s
+        - charm-build-neutron-k8s
+        - charm-build-ovn-central-k8s
+        - charm-build-ovn-relay-k8s
+        - charm-build-tempest-k8s
+      # test_dir relative to project src dir
+      test_dir: tests/tempest
 - job:
     name: func-test-ceph
     description: |
diff --git a/zuul.d/zuul.yaml b/zuul.d/zuul.yaml
index 5fef23fb..8545e466 100644
--- a/zuul.d/zuul.yaml
+++ b/zuul.d/zuul.yaml
@@ -16,6 +16,8 @@
             nodeset: ubuntu-jammy
         - func-test-machine:
             nodeset: ubuntu-jammy
+        - func-test-tempest:
+            nodeset: ubuntu-jammy
     vars:
       juju_channel: 3.4/stable
       juju_classic_mode: false