Adding functional/integration tests

This change adds ansible config to run the charm build and
tests steps to the upstream gate. The testsing involves
setting up a microk8s deployment and the running zaza
to deploy and test using microk8s. The functional testing
on the deployed cloud is currently tempest smoke tests.

Change includes some unit test fixes needed after changes to
ops and ops_sunbeam

Change-Id: I9a6ea70df0ceb2580bb806dc43f6afa66a449cd6

charms/keystone-k8s/.zuul.yaml

@@ -2,3 +2,21 @@
     templates:
       - openstack-python3-charm-yoga-jobs
       - openstack-cover-jobs
+    check:
+      jobs:
+        - build-charm
+        - func-tests
+    vars:
+      charm_build_name: keystone-k8s
+
+- job:
+    name: build-charm
+    run: playbooks/build-charm.yaml
+    provides: charm
+    timeout: 3600
+- job:
+    name: func-tests
+    run: playbooks/run-tests.yaml
+    requires: charm
+    dependencies:
+      - name: build-charm

charms/keystone-k8s/osci.yaml

@@ -1,8 +1,10 @@
 - project:
     templates:
-      - charm-unit-jobs-py38
-      - charm-unit-jobs-py310
       - charm-publish-jobs
+    check:
+      jobs:
+        - build-charm
+        - func-tests
     vars:
       needs_charm_build: true
       charm_build_name: keystone-k8s
@@ -10,3 +12,24 @@
       publish_charm: true
       charmcraft_channel: 2.0/stable
       publish_channel: latest/edge
+# Canonical CI also reads the zuul.yaml and the jobs defined in there. As a
+# result Canonical CI attempts to run these jobs again despite them having
+# already run in the OpenDev CI. The job definitions below overide the
+# ones from the zuul.yaml and replace the `run` directive with a noop.
+# Originally the setup of microk8s was in a seperate `pre-run` file.
+# However this caused problems as you cannot override the `pre-run`
+# job only append to it. To get around this the microk8s setup, the
+# test run and cleanup steps were all put into the `run` step. When
+# these job definitions move out of the charm these should be split
+# apart again.
+- job:
+    name: build-charm
+    run: playbooks/osci-noop.yaml
+    provides: charm
+    timeout: 3600
+- job:
+    name: func-tests
+    run: playbooks/osci-noop.yaml
+    requires: charm
+    dependencies:
+      - name: build-charm

charms/keystone-k8s/playbooks/build-charm.yaml

@@ -0,0 +1,75 @@
+- hosts: all
+  tasks:
+
+    - name: lxd apt packages are not present
+      apt:
+        name:
+          - lxd
+          - lxd-client
+        state: absent
+        purge: yes
+      become: true
+
+    - name: snapd is installed
+      apt:
+        name: snapd
+      become: true
+
+    - name: lxd snap is installed
+      snap:
+        name: lxd
+        channel: latest/stable
+      become: true
+
+    - name: lxd is initialised
+      command: lxd init --auto
+      become: true
+
+    - name: current user is in lxd group
+      user:
+        name: "{{ ansible_user }}"
+        groups: lxd
+        append: true
+      become: true
+
+    - name: reset ssh connection to apply permissions from new group
+      meta: reset_connection
+
+    - name: charmcraft is installed
+      snap:
+        name: charmcraft
+        channel: latest/stable
+        classic: true
+      become: true
+
+    - name: charm is packed
+      command:
+        cmd: charmcraft --verbosity trace pack
+        chdir: "{{ zuul.project.src_dir }}"
+
+    - name: local charm is named consistently
+      command:
+        cmd: ./rename.sh
+        chdir: "{{ zuul.project.src_dir }}"
+
+    - name: Compress built charm
+      archive:
+        dest: "{{ zuul.project.src_dir }}/{{ charm_build_name }}.charm.gz"
+        path: "{{ zuul.project.src_dir }}/{{ charm_build_name }}.charm"
+        format: gz
+
+    - name: built charm is available in the zuul log root for auto artifact upload
+      fetch:
+        src: "{{ zuul.project.src_dir }}/{{ charm_build_name }}.charm.gz"
+        dest: "{{ zuul.executor.log_root }}/"
+        flat: true
+      become: true
+
+    - zuul_return:
+        data:
+          zuul:
+            artifacts:
+              - name: charm
+                url: "{{ charm_build_name }}.charm.gz"
+                metadata:
+                  type: charm

charms/keystone-k8s/playbooks/osci-noop.yaml

@@ -0,0 +1,5 @@
+- hosts: all
+  tasks:
+    - name: noop job
+      debug:
+        msg: "Skipping job as it has run upstream"

charms/keystone-k8s/playbooks/run-tests.yaml

@@ -0,0 +1,125 @@
+- hosts: all
+  tasks:
+# Start Microk8s setup
+    - name: snapd is installed
+      apt:
+        name: snapd
+      become: true
+
+    - name: microk8s is installed
+      snap:
+        name: microk8s
+        classic: true
+      become: true
+
+    - name: current user is in microk8s group
+      user:
+        name: "{{ ansible_user }}"
+        groups: microk8s
+        append: true
+      become: true
+
+    - name: reset ssh connection to apply permissions from new group
+      meta: reset_connection
+
+    - name: microk8s is started
+      command:
+        cmd: microk8s start
+
+    - name: microk8s is running and ready
+      command:
+        cmd: microk8s status --wait-ready
+      register: res
+      failed_when: '"is running" not in res.stdout'
+
+    - name: microk8s dns addon is enabled
+      command:
+        cmd: microk8s enable dns
+      register: res
+      changed_when: '"already enabled" not in res.stdout'
+
+    - name: microk8s hostpath storage addon is enabled
+      command:
+        cmd: microk8s enable hostpath-storage
+      register: res
+      changed_when: '"already enabled" not in res.stdout'
+
+    - name: microk8s metallb addon is enabled
+      command:
+        # ip range is an arbitrary choice; may need to be changed later
+        cmd: microk8s enable metallb:10.170.0.1-10.170.0.100
+      register: res
+      changed_when: '"already enabled" not in res.stdout'
+
+    - name: microk8s addons are ready
+      command:
+        cmd: microk8s status --format short
+      register: res
+      retries: 18
+      delay: 10  # 18 * 10 = 3 minutes
+      until: >
+        "core/dns: enabled" in res.stdout and
+        "core/hostpath-storage: enabled" in res.stdout and
+        "core/metallb: enabled" in res.stdout
+      changed_when: res.attempts > 1
+
+    - name: juju is installed
+      snap:
+        name: juju
+        classic: true
+      become: true
+
+    - name: juju is bootstrapped on microk8s
+      command:
+        cmd: juju bootstrap microk8s microk8s
+      register: res
+      changed_when: '"already exists" not in res.stderr'
+      failed_when: '"ERROR" in res.stderr and "already exists" not in res.stderr'
+
+    - name: current juju controller is microk8s
+      command:
+        cmd: juju switch microk8s
+      register: res
+      changed_when: '"no change" not in res.stderr'
+# End Microk8s setup
+# Start Run functional tests
+
+    - name: test runner packages are installed
+      apt:
+        name:
+          - tox
+          - jq
+          - gzip
+      become: true
+
+    - name: built charm is present locally (artefact from previous job)
+      include_role:
+        name: download-artifact
+      vars:
+        download_artifact_api: "https://zuul.opendev.org/api/tenant/{{ zuul.tenant }}"
+        download_artifact_type: charm
+        download_artifact_pipeline: check
+        download_artifact_job: build-charm
+        download_artifact_directory: "{{ zuul.project.src_dir }}"
+
+    - name: Gunzip built charm
+      command:
+        cmd: gunzip {{ charm_build_name }}.charm.gz
+        chdir: "{{ zuul.project.src_dir }}"
+
+    - name: run smoke tests
+      command:
+        cmd: tox -e func-smoke
+        chdir: "{{ zuul.project.src_dir }}"
+# End Run functional tests
+# Start Post run job
+
+    - name: debug output for debugging the functional test
+      shell: |
+        set -x
+        MODEL="$(juju models --format=json | jq -r '.models[]["short-name"]' | grep '^zaza-')"
+        juju switch $MODEL
+        juju status
+        juju debug-log --replay
+        exit 0
+# End Post run job

charms/keystone-k8s/test-requirements.txt

@@ -8,3 +8,7 @@ coverage
 mock
 flake8
 stestr
+git+https://github.com/openstack-charmers/zaza.git#egg=zaza
+git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack
+git+https://opendev.org/openstack/tempest.git#egg=tempest
+ops

charms/keystone-k8s/tests/bundles/xena.yaml

@@ -0,0 +1,73 @@
+bundle: kubernetes
+applications:
+
+  mysql:
+    charm: ch:mysql-k8s
+    channel: edge
+    scale: 1
+    trust: false
+
+  # Currently traefik is required for networking things.
+  # If this isn't present, the units will hang at "installing agent".
+  traefik:
+    charm: ch:traefik-k8s
+    channel: edge
+    scale: 1
+    trust: true
+
+  traefik-public:
+    charm: ch:traefik-k8s
+    channel: edge
+    scale: 1
+    trust: true
+    options:
+      kubernetes-service-annotations: metallb.universe.tf/address-pool=public
+
+  # required for glance
+  rabbitmq:
+    charm: ch:sunbeam-rabbitmq-operator
+    channel: edge
+    scale: 1
+    trust: true
+
+  keystone:
+    charm: ../../keystone-k8s.charm
+    scale: 1
+    trust: true
+    options:
+      admin-role: admin
+    storage:
+      fernet-keys: 5M
+      credential-keys: 5M
+    resources:
+      keystone-image: kolla/ubuntu-binary-keystone:xena
+      # NOTE: should be in sync with resources described in ../../metadata.yaml
+
+  glance:
+    charm: ch:glance-k8s
+    channel: edge
+    scale: 1
+    trust: true
+    storage:
+      local-repository: 5G
+
+relations:
+- - keystone:identity-service
+  - glance:identity-service
+- - rabbitmq:amqp
+  - glance:amqp
+
+- - traefik:ingress
+  - keystone:ingress-internal
+- - traefik-public:ingress
+  - keystone:ingress-public
+- - traefik:ingress
+  - glance:ingress-internal
+- - traefik-public:ingress
+  - glance:ingress-public
+
+- - mysql:database
+  - keystone:database
+
+- - mysql:database
+  - glance:database

charms/keystone-k8s/tests/config.yaml

@@ -0,0 +1 @@
+../config.yaml

charms/keystone-k8s/tests/tests.yaml

@@ -0,0 +1,37 @@
+gate_bundles:
+  - xena
+smoke_bundles:
+  - xena
+configure:
+  - zaza.openstack.charm_tests.keystone.setup.add_tempest_roles
+tests:
+  - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
+tests_options:
+  trust:
+    - xena
+
+  tempest:
+    default:
+      smoke: True
+      exclude-list:
+        - "tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_register_upload_get_image_file"
+
+target_deploy_status:
+  traefik:
+    workload-status: active
+    workload-status-message-regex: '^$'
+  traefik-public:
+    workload-status: active
+    workload-status-message-regex: '^$'
+  rabbitmq:
+    workload-status: active
+    workload-status-message-regex: '^$'
+  glance:
+    workload-status: active
+    workload-status-message-regex: '^$'
+  keystone:
+    workload-status: active
+    workload-status-message-regex: '^$'
+  mysql:
+    workload-status: active
+    workload-status-message-regex: '^$'

charms/keystone-k8s/tests/unit/test_keystone_charm.py

@@ -198,10 +198,10 @@ class TestKeystoneOperatorCharm(test_utils.CharmTestCase):
     def test_leader_bootstraps(self):
         test_utils.add_complete_ingress_relation(self.harness)
         self.harness.set_leader()
-        rel_id = self.harness.add_relation('peers', 'keystone')
+        rel_id = self.harness.add_relation('peers', 'keystone-k8s')
         self.harness.add_relation_unit(
             rel_id,
-            'keystone/1')
+            'keystone-k8s/1')
         self.harness.container_pebble_ready('keystone')
         test_utils.add_db_relation_credentials(
             self.harness,
@@ -212,10 +212,10 @@ class TestKeystoneOperatorCharm(test_utils.CharmTestCase):
     def test_leader_rotate_fernet_keys(self):
         test_utils.add_complete_ingress_relation(self.harness)
         self.harness.set_leader()
-        rel_id = self.harness.add_relation('peers', 'keystone')
+        rel_id = self.harness.add_relation('peers', 'keystone-k8s')
         self.harness.add_relation_unit(
             rel_id,
-            'keystone/1')
+            'keystone-k8s/1')
         self.harness.container_pebble_ready('keystone')
         test_utils.add_db_relation_credentials(
             self.harness,
@@ -225,10 +225,10 @@ class TestKeystoneOperatorCharm(test_utils.CharmTestCase):
 
     def test_not_leader_rotate_fernet_keys(self):
         test_utils.add_complete_ingress_relation(self.harness)
-        rel_id = self.harness.add_relation('peers', 'keystone')
+        rel_id = self.harness.add_relation('peers', 'keystone-k8s')
         self.harness.add_relation_unit(
             rel_id,
-            'keystone/1')
+            'keystone-k8s/1')
         self.harness.container_pebble_ready('keystone')
         test_utils.add_db_relation_credentials(
             self.harness,
@@ -239,10 +239,10 @@ class TestKeystoneOperatorCharm(test_utils.CharmTestCase):
     def test_on_heartbeat(self):
         test_utils.add_complete_ingress_relation(self.harness)
         self.harness.set_leader()
-        rel_id = self.harness.add_relation('peers', 'keystone')
+        rel_id = self.harness.add_relation('peers', 'keystone-k8s')
         self.harness.add_relation_unit(
             rel_id,
-            'keystone/1')
+            'keystone-k8s/1')
         self.harness.container_pebble_ready('keystone')
         test_utils.add_db_relation_credentials(
             self.harness,
@@ -271,10 +271,10 @@ class TestKeystoneOperatorCharm(test_utils.CharmTestCase):
     def test_non_leader_no_bootstraps(self):
         test_utils.add_complete_ingress_relation(self.harness)
         self.harness.set_leader(False)
-        rel_id = self.harness.add_relation('peers', 'keystone')
+        rel_id = self.harness.add_relation('peers', 'keystone-k8s')
         self.harness.add_relation_unit(
             rel_id,
-            'keystone/1')
+            'keystone-k8s/1')
         self.harness.container_pebble_ready('keystone')
         test_utils.add_db_relation_credentials(
             self.harness,

charms/keystone-k8s/tox.ini

@@ -83,6 +83,31 @@ commands =
     coverage xml -o cover/coverage.xml
     coverage report
 
+[testenv:func-noop]
+basepython = python3
+commands =
+    functest-run-suite --help
+
+[testenv:func]
+basepython = python3
+commands =
+    functest-run-suite --keep-model
+
+[testenv:func-smoke]
+basepython = python3
+commands =
+    functest-run-suite --keep-model --smoke
+
+[testenv:func-dev]
+basepython = python3
+commands =
+    functest-run-suite --keep-model --dev
+
+[testenv:func-target]
+basepython = python3
+commands =
+    functest-run-suite --keep-model --bundle {posargs}
+
 [coverage:run]
 branch = True
 concurrency = multiprocessing