From fe46bd2612171545ca8c451792bab00bdd43109f Mon Sep 17 00:00:00 2001 From: Naoaki Horie Date: Mon, 14 Dec 2020 05:45:04 +0000 Subject: [PATCH] Add a k8s cluster for the k8s related FT in zuul This patch add a kubernetes cluster for the kubernetes related functional tests of the VNF LCM in the zuul environment. There is no impact to the existing jobs because this patch only add a new job, however we may need to watch the load on the Zuul environment due to its parallel jobs. A new node-set consists of four nodes; * Controller: Keystone, Nova, Neutron, Glance, Cinder, Octavia, MySQL, MQ, ETCD * Controller-tacker: Tacker, Tacker-conductor * Controller-k8s: kuryr-k8s, kuryr-CNI, k8s-api, kubelet * Compute: Nova-compute All kubernetes resources are created on the controller-k8s node. This patch includes the following changes: * Added a execution command for the functional test of related k8s for the VNF LCM in tox.ini. * Registered a vim of the `kubernetes` type by ansible. Also added related materials. * Moved the functional test files for k8s to other new directory. * Fixed a minor invalid definition in the definition file used for functional testing. Change-Id: I1621b904450e94d6793b4c524de6785520f2e805 --- .zuul.yaml | 218 ++++++++++++++++++ roles/setup-default-vim/tasks/main.yaml | 78 +++++++ .../Definitions/helloworld3_df_simple.yaml | 2 +- tacker/tests/etc/samples/local-k8s-vim.yaml | 6 + .../functional/sol_kubernetes/__init__.py | 0 .../sol_kubernetes/vnflcm/__init__.py | 0 .../vnflcm/test_kubernetes.py | 0 tools/test-setup-k8s-vim.sh | 27 +++ tox.ini | 6 + 9 files changed, 336 insertions(+), 1 deletion(-) create mode 100644 tacker/tests/etc/samples/local-k8s-vim.yaml create mode 100644 tacker/tests/functional/sol_kubernetes/__init__.py create mode 100644 tacker/tests/functional/sol_kubernetes/vnflcm/__init__.py rename tacker/tests/functional/{sol => sol_kubernetes}/vnflcm/test_kubernetes.py (100%) create mode 100755 tools/test-setup-k8s-vim.sh diff --git a/.zuul.yaml b/.zuul.yaml index b73e31458..f8c490a91 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -32,6 +32,36 @@ - compute1 - compute2 +- nodeset: + name: openstack-k8s-4-nodes-focal + nodes: + - name: controller + label: ubuntu-focal + - name: controller-tacker + label: ubuntu-focal + - name: controller-k8s + label: ubuntu-focal + - name: compute1 + label: ubuntu-focal + groups: + - name: compute + nodes: + - controller-k8s + - compute1 + - name: subnode + nodes: + - controller-tacker + - controller-k8s + - compute1 + - name: switch + nodes: + - controller + - name: peers + nodes: + - controller-tacker + - controller-k8s + - compute1 + - job: name: tacker-functional-devstack-multinode-legacy parent: devstack @@ -232,6 +262,193 @@ base_url: http://127.0.0.1:9990/grant/v1/grants tox_envlist: dsvm-functional-sol-separated-nfvo +- job: + name: tacker-functional-devstack-multinode-sol-kubernetes + parent: devstack + description: | + Multinodes job for SOL devstack-based kubernetes functional tests + nodeset: openstack-k8s-4-nodes-focal + pre-run: playbooks/devstack/pre.yaml + run: playbooks/devstack/run.yaml + post-run: playbooks/devstack/post.yaml + roles: + - zuul: openstack-infra/devstack + timeout: 10800 + required-projects: + - openstack/barbican + - openstack/cinder + - openstack/devstack-gate + - openstack/devstack-plugin-container + - openstack/glance + - openstack/heat + - openstack/horizon + - openstack/keystone + - openstack/kuryr-kubernetes + - openstack/mistral + - openstack/neutron + - openstack/nova + - openstack/octavia + - openstack/placement + - openstack/python-barbicanclient + - openstack/python-blazarclient + - openstack/python-mistralclient + - openstack/python-octaviaclient + - openstack/python-tackerclient + - openstack/tacker + - openstack/tacker-horizon + host-vars: + controller: + devstack_plugins: + barbican: https://opendev.org/openstack/barbican + heat: https://opendev.org/openstack/heat + mistral: https://opendev.org/openstack/mistral + octavia: https://opendev.org/openstack/octavia + devstack_services: + base: false + c-api: true + c-bak: false + c-sch: true + c-vol: true + cinder: true + coredns: false + etcd3: true + g-api: true + g-reg: true + horizon: false + key: true + mysql: true + n-api-meta: true + n-api: true + n-cond: true + n-cpu: false + n-novnc: true + n-sch: true + neutron: true + o-api: true + o-cw: true + o-hk: true + o-hm: true + octavia: true + placement-api: true + placement-client: true + q-agt: true + q-dhcp: true + q-l3: true + q-meta: true + q-metering: true + q-qos: true + q-svc: true + rabbit: true + s-account: false + s-container: false + s-object: false + s-proxy: false + swift: false + tempest: false + tls-proxy: false + tox_install_siblings: false + controller-tacker: + devstack_local_conf: {} + devstack_services: + q-agt: true + tacker: true + tacker-conductor: true + devstack_plugins: + tacker: https://opendev.org/openstack/tacker + tox_envlist: dsvm-functional-sol-kubernetes + controller-k8s: + devstack_local_conf: {} + devstack_plugins: + devstack-plugin-container: https://opendev.org/openstack/devstack-plugin-container + kuryr-kubernetes: https://opendev.org/openstack/kuryr-kubernetes + devstack_services: + etcd3: false + kubelet: true + kubernetes-api: true + kubernetes-controller-manager: true + kubernetes-scheduler: true + kuryr-daemon: true + kuryr-kubernetes: true + octavia: false + q-agt: true + tox_install_siblings: false + group-vars: + subnode: + devstack_localrc: + CELLSV2_SETUP: singleconductor + DATABASE_TYPE: mysql + IS_ZUUL_FT: True + KEYSTONE_SERVICE_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}" + KURYR_FORCE_IMAGE_BUILD: true + KURYR_K8S_API_PORT: 8080 + KURYR_K8S_API_URL: "http://{{ hostvars['controller-k8s']['nodepool']['private_ipv4'] }}:${KURYR_K8S_API_PORT}" + KURYR_K8S_CLOUD_PROVIDER: false + KURYR_K8S_CONTAINERIZED_DEPLOYMENT: false + KURYR_K8S_MULTI_WORKER_TESTS: false + KURYR_NEUTRON_DEFAULT_SUBNETPOOL_ID: shared-default-subnetpool-v4 + MYSQL_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}" + OCTAVIA_AMP_IMAGE_FILE: "/tmp/test-only-amphora-x64-haproxy-ubuntu-bionic.qcow2" + OCTAVIA_AMP_IMAGE_NAME: "test-only-amphora-x64-haproxy-ubuntu-bionic" + OCTAVIA_AMP_IMAGE_SIZE: 3 + OVS_BRIDGE_MAPPINGS: public:br-ex,mgmtphysnet0:br-infra + PHYSICAL_NETWORK: mgmtphysnet0 + Q_DVR_MODE: dvr + TACKER_HOST: "{{ hostvars['controller-tacker']['nodepool']['private_ipv4'] }}" + TACKER_MODE: standalone + USE_PYTHON3: true + devstack_services: + dstat: false + horizon: false + n-api-meta: false + n-api: false + n-cauth: false + n-cond: false + n-cpu: true + n-novnc: false + n-obj: false + n-sch: false + q-agt: true + tls-proxy: false + vars: + devstack_localrc: + CELLSV2_SETUP: singleconductor + DATABASE_TYPE: mysql + ETCD_USE_RAMDISK: true + IS_ZUUL_FT: True + KEYSTONE_SERVICE_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}" + L2_AGENT_EXTENSIONS: qos + MYSQL_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}" + OCTAVIA_AMP_IMAGE_FILE: "/tmp/test-only-amphora-x64-haproxy-ubuntu-bionic.qcow2" + OCTAVIA_AMP_IMAGE_NAME: "test-only-amphora-x64-haproxy-ubuntu-bionic" + OCTAVIA_AMP_IMAGE_SIZE: 3 + OVS_BRIDGE_MAPPINGS: public:br-ex,mgmtphysnet0:br-infra + PHYSICAL_NETWORK: mgmtphysnet0 + Q_DVR_MODE: dvr + Q_ML2_PLUGIN_EXT_DRIVERS: port_security,qos + Q_SERVICE_PLUGIN_CLASSES: router,neutron.services.metering.metering_plugin.MeteringPlugin,neutron.services.qos.qos_plugin.QoSPlugin,qos + TACKER_HOST: "{{ hostvars['controller-tacker']['nodepool']['private_ipv4'] }}" + TACKER_MODE: standalone + USE_PYTHON3: true + devstack_local_conf: + post-config: + $NEUTRON_DHCP_CONF: + DEFAULT: + enable_isolated_metadata: True + $OCTAVIA_CONF: + controller_worker: + amp_active_retries: 9999 + kuryr_k8s_api_url: "http://{{ hostvars['controller-k8s']['nodepool']['private_ipv4'] }}:8080" + test_matrix_configs: [neutron] + zuul_work_dir: src/opendev.org/openstack/tacker + zuul_copy_output: + '{{ devstack_log_dir }}/kubernetes': 'logs' + irrelevant-files: + - ^.*\.rst$ + - ^doc/.*$ + - ^releasenotes/.*$ + - ^contrib/.*$ + voting: false + - project: templates: - check-requirements @@ -245,3 +462,4 @@ - tacker-functional-devstack-multinode-legacy - tacker-functional-devstack-multinode-sol - tacker-functional-devstack-multinode-sol-separated-nfvo + - tacker-functional-devstack-multinode-sol-kubernetes diff --git a/roles/setup-default-vim/tasks/main.yaml b/roles/setup-default-vim/tasks/main.yaml index c965a33ce..a96455465 100644 --- a/roles/setup-default-vim/tasks/main.yaml +++ b/roles/setup-default-vim/tasks/main.yaml @@ -65,3 +65,81 @@ when: - inventory_hostname == 'controller-tacker' + +- block: + - name: Copy tools/test-setup-k8s-vim.sh + copy: + remote_src=True + src={{ devstack_base_dir }}/tacker/tools/test-setup-k8s-vim.sh + dest={{ zuul_work_dir }}/tools/test-setup-k8s-vim.sh + mode=0755 + + - name: Copy test k8s vim file + copy: + remote_src=True + src={{ devstack_base_dir }}/tacker/tacker/tests/etc/samples/local-k8s-vim.yaml + dest={{ zuul_work_dir }}/tacker/tests/etc/samples/local-k8s-vim.yaml + + - name: Check if project's tools/test-setup-k8s-vim.sh exists + stat: + path: "{{ zuul_work_dir }}/tools/test-setup-k8s-vim.sh" + register: p + - fail: + msg: > + {{ zuul_work_dir }}/tools/test-setup-k8s-vim.sh doesn't exists + or it doesn't have execute permission. + when: p.stat.exists != True or p.stat.executable != True + + - name: Get stackenv from devstack environment + slurp: + src: "{{ devstack_base_dir }}/devstack/.stackenv" + register: stackenv + + - name: Set a keystone authentication uri + set_fact: + auth_uri: "{{ + stackenv.content + | b64decode + | regex_replace('\n', ' ') + | regex_replace('^.*KEYSTONE_SERVICE_URI=([^ ]+).*$', '\\1') + }}" + when: + - p.stat.exists + + - name: Replace keystone auth uri in test-setup-k8s-vim.sh + replace: + path: "{{ item }}" + regexp: "http://127.0.0.1/identity" + replace: "{{ auth_uri }}" + with_items: + - "{{ zuul_work_dir }}/tools/test-setup-k8s-vim.sh" + when: + - p.stat.exists + + - name: Replace k8s auth uri in local-k8s-vim.yaml + replace: + path: "{{ item }}" + regexp: "https://127.0.0.1:6443" + replace: "{{ kuryr_k8s_api_url }}" + with_items: + - "{{ zuul_work_dir }}/tacker/tests/etc/samples/local-k8s-vim.yaml" + when: + - p.stat.exists + + - name: Replace the config file path in the test-setup-k8s-vim.sh + replace: + path: "{{ zuul_work_dir }}/tools/test-setup-k8s-vim.sh" + regexp: '(?<=config-file )([^ ]+)(?= )' + replace: "{{ ansible_env.HOME }}/{{ zuul_work_dir }}/tacker/tests/etc/samples/local-k8s-vim.yaml" + + - name: Run tools/test-setup-k8s-vim.sh + command: tools/test-setup-k8s-vim.sh + args: + chdir: "{{ zuul_work_dir }}" + when: + - p.stat.exists + - p.stat.executable + + when: + - inventory_hostname == 'controller-tacker' + - kuryr_k8s_api_url is defined diff --git a/tacker/tests/etc/samples/etsi/nfv/common_artifact/Definitions/helloworld3_df_simple.yaml b/tacker/tests/etc/samples/etsi/nfv/common_artifact/Definitions/helloworld3_df_simple.yaml index 26280e99a..df269aa14 100644 --- a/tacker/tests/etc/samples/etsi/nfv/common_artifact/Definitions/helloworld3_df_simple.yaml +++ b/tacker/tests/etc/samples/etsi/nfv/common_artifact/Definitions/helloworld3_df_simple.yaml @@ -220,7 +220,7 @@ topology_template: properties: aspect: worker_instance deltas: - delta_2: + delta_1: number_of_instances: 1 targets: [ VDU2 ] diff --git a/tacker/tests/etc/samples/local-k8s-vim.yaml b/tacker/tests/etc/samples/local-k8s-vim.yaml new file mode 100644 index 000000000..5f830e8b0 --- /dev/null +++ b/tacker/tests/etc/samples/local-k8s-vim.yaml @@ -0,0 +1,6 @@ +auth_url: "https://127.0.0.1:6443" +username: "admin" +password: "admin" +project_name: "default" +ssl_ca_cert: None +type: "kubernetes" diff --git a/tacker/tests/functional/sol_kubernetes/__init__.py b/tacker/tests/functional/sol_kubernetes/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tacker/tests/functional/sol_kubernetes/vnflcm/__init__.py b/tacker/tests/functional/sol_kubernetes/vnflcm/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tacker/tests/functional/sol/vnflcm/test_kubernetes.py b/tacker/tests/functional/sol_kubernetes/vnflcm/test_kubernetes.py similarity index 100% rename from tacker/tests/functional/sol/vnflcm/test_kubernetes.py rename to tacker/tests/functional/sol_kubernetes/vnflcm/test_kubernetes.py diff --git a/tools/test-setup-k8s-vim.sh b/tools/test-setup-k8s-vim.sh new file mode 100755 index 000000000..b3d1fc5ff --- /dev/null +++ b/tools/test-setup-k8s-vim.sh @@ -0,0 +1,27 @@ +#!/bin/bash -xe + +# This script is used to set up default vim +# for functional testing, which cannot be put +# in devstack/plugin.sh because new zuul3 CI +# cannot keep the devstack plugins order +# +# Also, this script updates the following +# parameter which has been modified +# unintentionally by ansible playbook +# `roles/setup-default-vim/tasks/main.yaml` +# according to the execution environment of +# Zuul. +# +# --os-auth-url +# --config-file + +openstack vim register \ + --os-username nfv_user \ + --os-project-name nfv \ + --os-password devstack \ + --os-auth-url http://127.0.0.1/identity \ + --os-project-domain-name Default \ + --os-user-domain-name Default \ + --description "Kubernetes VIM" \ + --config-file /opt/stack/tacker/tacker/tests/etc/samples/local-k8s-vim.yaml \ + vim-kubernetes diff --git a/tox.ini b/tox.ini index 83986a939..eed919b76 100644 --- a/tox.ini +++ b/tox.ini @@ -58,6 +58,12 @@ setenv = {[testenv]setenv} commands = stestr --test-path=./tacker/tests/functional/sol_separated_nfvo run --slowest --concurrency 1 {posargs} +[testenv:dsvm-functional-sol-kubernetes] +setenv = {[testenv]setenv} + +commands = + stestr --test-path=./tacker/tests/functional/sol_kubernetes run --slowest --concurrency 2 {posargs} + [testenv:debug] commands = oslo_debug_helper {posargs}