Add a k8s cluster for the k8s related FT in zuul

This patch add a kubernetes cluster for the kubernetes related
functional tests of the VNF LCM in the zuul environment.

There is no impact to the existing jobs because this patch only
add a new job, however we may need to watch the load on the Zuul
environment due to its parallel jobs.

A new node-set consists of four nodes;

  * Controller: Keystone, Nova, Neutron, Glance, Cinder, Octavia,
                MySQL, MQ, ETCD
  * Controller-tacker: Tacker, Tacker-conductor
  * Controller-k8s: kuryr-k8s, kuryr-CNI, k8s-api, kubelet
  * Compute: Nova-compute

All kubernetes resources are created on the controller-k8s node.

This patch includes the following changes:

  * Added a execution command for the functional test of related
    k8s for the VNF LCM in tox.ini.
  * Registered a vim of the `kubernetes` type by ansible. Also
    added related materials.
  * Moved the functional test files for k8s to other new directory.
  * Fixed a minor invalid definition in the definition file used
    for functional testing.

Change-Id: I1621b904450e94d6793b4c524de6785520f2e805

.zuul.yaml

@@ -32,6 +32,36 @@
           - compute1
           - compute2
 
+- nodeset:
+    name: openstack-k8s-4-nodes-focal
+    nodes:
+      - name: controller
+        label: ubuntu-focal
+      - name: controller-tacker
+        label: ubuntu-focal
+      - name: controller-k8s
+        label: ubuntu-focal
+      - name: compute1
+        label: ubuntu-focal
+    groups:
+      - name: compute
+        nodes:
+          - controller-k8s
+          - compute1
+      - name: subnode
+        nodes:
+          - controller-tacker
+          - controller-k8s
+          - compute1
+      - name: switch
+        nodes:
+          - controller
+      - name: peers
+        nodes:
+          - controller-tacker
+          - controller-k8s
+          - compute1
+
 - job:
     name: tacker-functional-devstack-multinode-legacy
     parent: devstack
@@ -232,6 +262,193 @@
                 base_url: http://127.0.0.1:9990/grant/v1/grants
         tox_envlist: dsvm-functional-sol-separated-nfvo
 
+- job:
+    name: tacker-functional-devstack-multinode-sol-kubernetes
+    parent: devstack
+    description: |
+      Multinodes job for SOL devstack-based kubernetes functional tests
+    nodeset: openstack-k8s-4-nodes-focal
+    pre-run: playbooks/devstack/pre.yaml
+    run: playbooks/devstack/run.yaml
+    post-run: playbooks/devstack/post.yaml
+    roles:
+      - zuul: openstack-infra/devstack
+    timeout: 10800
+    required-projects:
+      - openstack/barbican
+      - openstack/cinder
+      - openstack/devstack-gate
+      - openstack/devstack-plugin-container
+      - openstack/glance
+      - openstack/heat
+      - openstack/horizon
+      - openstack/keystone
+      - openstack/kuryr-kubernetes
+      - openstack/mistral
+      - openstack/neutron
+      - openstack/nova
+      - openstack/octavia
+      - openstack/placement
+      - openstack/python-barbicanclient
+      - openstack/python-blazarclient
+      - openstack/python-mistralclient
+      - openstack/python-octaviaclient
+      - openstack/python-tackerclient
+      - openstack/tacker
+      - openstack/tacker-horizon
+    host-vars:
+      controller:
+        devstack_plugins:
+          barbican: https://opendev.org/openstack/barbican
+          heat: https://opendev.org/openstack/heat
+          mistral: https://opendev.org/openstack/mistral
+          octavia: https://opendev.org/openstack/octavia
+        devstack_services:
+          base: false
+          c-api: true
+          c-bak: false
+          c-sch: true
+          c-vol: true
+          cinder: true
+          coredns: false
+          etcd3: true
+          g-api: true
+          g-reg: true
+          horizon: false
+          key: true
+          mysql: true
+          n-api-meta: true
+          n-api: true
+          n-cond: true
+          n-cpu: false
+          n-novnc: true
+          n-sch: true
+          neutron: true
+          o-api: true
+          o-cw: true
+          o-hk: true
+          o-hm: true
+          octavia: true
+          placement-api: true
+          placement-client: true
+          q-agt: true
+          q-dhcp: true
+          q-l3: true
+          q-meta: true
+          q-metering: true
+          q-qos: true
+          q-svc: true
+          rabbit: true
+          s-account: false
+          s-container: false
+          s-object: false
+          s-proxy: false
+          swift: false
+          tempest: false
+          tls-proxy: false
+        tox_install_siblings: false
+      controller-tacker:
+        devstack_local_conf: {}
+        devstack_services:
+          q-agt: true
+          tacker: true
+          tacker-conductor: true
+        devstack_plugins:
+          tacker: https://opendev.org/openstack/tacker
+        tox_envlist: dsvm-functional-sol-kubernetes
+      controller-k8s:
+        devstack_local_conf: {}
+        devstack_plugins:
+          devstack-plugin-container: https://opendev.org/openstack/devstack-plugin-container
+          kuryr-kubernetes: https://opendev.org/openstack/kuryr-kubernetes
+        devstack_services:
+          etcd3: false
+          kubelet: true
+          kubernetes-api: true
+          kubernetes-controller-manager: true
+          kubernetes-scheduler: true
+          kuryr-daemon: true
+          kuryr-kubernetes: true
+          octavia: false
+          q-agt: true
+        tox_install_siblings: false
+    group-vars:
+      subnode:
+        devstack_localrc:
+          CELLSV2_SETUP: singleconductor
+          DATABASE_TYPE: mysql
+          IS_ZUUL_FT: True
+          KEYSTONE_SERVICE_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}"
+          KURYR_FORCE_IMAGE_BUILD: true
+          KURYR_K8S_API_PORT: 8080
+          KURYR_K8S_API_URL: "http://{{ hostvars['controller-k8s']['nodepool']['private_ipv4'] }}:${KURYR_K8S_API_PORT}"
+          KURYR_K8S_CLOUD_PROVIDER: false
+          KURYR_K8S_CONTAINERIZED_DEPLOYMENT: false
+          KURYR_K8S_MULTI_WORKER_TESTS: false
+          KURYR_NEUTRON_DEFAULT_SUBNETPOOL_ID: shared-default-subnetpool-v4
+          MYSQL_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}"
+          OCTAVIA_AMP_IMAGE_FILE: "/tmp/test-only-amphora-x64-haproxy-ubuntu-bionic.qcow2"
+          OCTAVIA_AMP_IMAGE_NAME: "test-only-amphora-x64-haproxy-ubuntu-bionic"
+          OCTAVIA_AMP_IMAGE_SIZE: 3
+          OVS_BRIDGE_MAPPINGS: public:br-ex,mgmtphysnet0:br-infra
+          PHYSICAL_NETWORK: mgmtphysnet0
+          Q_DVR_MODE: dvr
+          TACKER_HOST: "{{ hostvars['controller-tacker']['nodepool']['private_ipv4'] }}"
+          TACKER_MODE: standalone
+          USE_PYTHON3: true
+        devstack_services:
+          dstat: false
+          horizon: false
+          n-api-meta: false
+          n-api: false
+          n-cauth: false
+          n-cond: false
+          n-cpu: true
+          n-novnc: false
+          n-obj: false
+          n-sch: false
+          q-agt: true
+          tls-proxy: false
+    vars:
+      devstack_localrc:
+        CELLSV2_SETUP: singleconductor
+        DATABASE_TYPE: mysql
+        ETCD_USE_RAMDISK: true
+        IS_ZUUL_FT: True
+        KEYSTONE_SERVICE_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}"
+        L2_AGENT_EXTENSIONS: qos
+        MYSQL_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}"
+        OCTAVIA_AMP_IMAGE_FILE: "/tmp/test-only-amphora-x64-haproxy-ubuntu-bionic.qcow2"
+        OCTAVIA_AMP_IMAGE_NAME: "test-only-amphora-x64-haproxy-ubuntu-bionic"
+        OCTAVIA_AMP_IMAGE_SIZE: 3
+        OVS_BRIDGE_MAPPINGS: public:br-ex,mgmtphysnet0:br-infra
+        PHYSICAL_NETWORK: mgmtphysnet0
+        Q_DVR_MODE: dvr
+        Q_ML2_PLUGIN_EXT_DRIVERS: port_security,qos
+        Q_SERVICE_PLUGIN_CLASSES: router,neutron.services.metering.metering_plugin.MeteringPlugin,neutron.services.qos.qos_plugin.QoSPlugin,qos
+        TACKER_HOST: "{{ hostvars['controller-tacker']['nodepool']['private_ipv4'] }}"
+        TACKER_MODE: standalone
+        USE_PYTHON3: true
+      devstack_local_conf:
+        post-config:
+          $NEUTRON_DHCP_CONF:
+            DEFAULT:
+              enable_isolated_metadata: True
+          $OCTAVIA_CONF:
+            controller_worker:
+              amp_active_retries: 9999
+      kuryr_k8s_api_url: "http://{{ hostvars['controller-k8s']['nodepool']['private_ipv4'] }}:8080"
+      test_matrix_configs: [neutron]
+      zuul_work_dir: src/opendev.org/openstack/tacker
+      zuul_copy_output:
+        '{{ devstack_log_dir }}/kubernetes': 'logs'
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/.*$
+      - ^releasenotes/.*$
+      - ^contrib/.*$
+    voting: false
+
 - project:
     templates:
       - check-requirements
@@ -245,3 +462,4 @@
         - tacker-functional-devstack-multinode-legacy
         - tacker-functional-devstack-multinode-sol
         - tacker-functional-devstack-multinode-sol-separated-nfvo
+        - tacker-functional-devstack-multinode-sol-kubernetes

roles/setup-default-vim/tasks/main.yaml

@@ -65,3 +65,81 @@
 
   when:
     - inventory_hostname == 'controller-tacker'
+
+- block:
+  - name: Copy tools/test-setup-k8s-vim.sh
+    copy:
+       remote_src=True
+       src={{ devstack_base_dir }}/tacker/tools/test-setup-k8s-vim.sh
+       dest={{ zuul_work_dir }}/tools/test-setup-k8s-vim.sh
+       mode=0755
+
+  - name: Copy test k8s vim file
+    copy:
+       remote_src=True
+       src={{ devstack_base_dir }}/tacker/tacker/tests/etc/samples/local-k8s-vim.yaml
+       dest={{ zuul_work_dir }}/tacker/tests/etc/samples/local-k8s-vim.yaml
+
+  - name: Check if project's tools/test-setup-k8s-vim.sh exists
+    stat:
+      path: "{{ zuul_work_dir }}/tools/test-setup-k8s-vim.sh"
+    register: p
+  - fail:
+      msg: >
+            {{ zuul_work_dir }}/tools/test-setup-k8s-vim.sh doesn't exists
+            or it doesn't have execute permission.
+    when:  p.stat.exists != True or p.stat.executable != True
+
+  - name: Get stackenv from devstack environment
+    slurp:
+      src: "{{ devstack_base_dir }}/devstack/.stackenv"
+    register: stackenv
+
+  - name: Set a keystone authentication uri
+    set_fact:
+      auth_uri: "{{
+                stackenv.content
+                | b64decode
+                | regex_replace('\n', ' ')
+                | regex_replace('^.*KEYSTONE_SERVICE_URI=([^ ]+).*$', '\\1')
+                }}"
+    when:
+      - p.stat.exists
+
+  - name: Replace keystone auth uri in test-setup-k8s-vim.sh
+    replace:
+       path: "{{ item }}"
+       regexp: "http://127.0.0.1/identity"
+       replace: "{{ auth_uri }}"
+    with_items:
+       - "{{ zuul_work_dir }}/tools/test-setup-k8s-vim.sh"
+    when:
+      - p.stat.exists
+
+  - name: Replace k8s auth uri in local-k8s-vim.yaml
+    replace:
+       path: "{{ item }}"
+       regexp: "https://127.0.0.1:6443"
+       replace: "{{ kuryr_k8s_api_url }}"
+    with_items:
+       - "{{ zuul_work_dir }}/tacker/tests/etc/samples/local-k8s-vim.yaml"
+    when:
+      - p.stat.exists
+
+  - name: Replace the config file path in the test-setup-k8s-vim.sh
+    replace:
+       path: "{{ zuul_work_dir }}/tools/test-setup-k8s-vim.sh"
+       regexp: '(?<=config-file )([^ ]+)(?= )'
+       replace: "{{ ansible_env.HOME }}/{{ zuul_work_dir }}/tacker/tests/etc/samples/local-k8s-vim.yaml"
+
+  - name: Run tools/test-setup-k8s-vim.sh
+    command: tools/test-setup-k8s-vim.sh
+    args:
+      chdir: "{{ zuul_work_dir }}"
+    when:
+      - p.stat.exists
+      - p.stat.executable
+
+  when:
+    - inventory_hostname == 'controller-tacker'
+    - kuryr_k8s_api_url is defined

tacker/tests/etc/samples/etsi/nfv/common_artifact/Definitions/helloworld3_df_simple.yaml

@@ -220,7 +220,7 @@ topology_template:
         properties:
           aspect: worker_instance
           deltas:
-            delta_2:
+            delta_1:
               number_of_instances: 1
         targets: [ VDU2 ]
 

tacker/tests/etc/samples/local-k8s-vim.yaml

@@ -0,0 +1,6 @@
+auth_url: "https://127.0.0.1:6443"
+username: "admin"
+password: "admin"
+project_name: "default"
+ssl_ca_cert: None
+type: "kubernetes"

tools/test-setup-k8s-vim.sh

@@ -0,0 +1,27 @@
+#!/bin/bash -xe
+
+# This script is used to set up default vim
+# for functional testing, which cannot be put
+# in devstack/plugin.sh because new zuul3 CI
+# cannot keep the devstack plugins order
+#
+# Also, this script updates the following
+# parameter which has been modified
+# unintentionally by ansible playbook
+# `roles/setup-default-vim/tasks/main.yaml`
+# according to the execution environment of
+# Zuul.
+#
+#  --os-auth-url
+#  --config-file
+
+openstack vim register \
+    --os-username nfv_user \
+    --os-project-name nfv \
+    --os-password devstack \
+    --os-auth-url http://127.0.0.1/identity \
+    --os-project-domain-name Default \
+    --os-user-domain-name Default \
+    --description "Kubernetes VIM" \
+    --config-file /opt/stack/tacker/tacker/tests/etc/samples/local-k8s-vim.yaml \
+    vim-kubernetes

tox.ini

@@ -58,6 +58,12 @@ setenv = {[testenv]setenv}
 commands =
   stestr --test-path=./tacker/tests/functional/sol_separated_nfvo run --slowest --concurrency 1 {posargs}
 
+[testenv:dsvm-functional-sol-kubernetes]
+setenv = {[testenv]setenv}
+
+commands =
+  stestr --test-path=./tacker/tests/functional/sol_kubernetes run --slowest --concurrency 2 {posargs}
+
 [testenv:debug]
 commands = oslo_debug_helper {posargs}