Add a k8s cluster for the k8s related FT in zuul

This patch add a kubernetes cluster for the kubernetes related
functional tests of the VNF LCM in the zuul environment.

There is no impact to the existing jobs because this patch only
add a new job, however we may need to watch the load on the Zuul
environment due to its parallel jobs.

A new node-set consists of four nodes;

  * Controller: Keystone, Nova, Neutron, Glance, Cinder, Octavia,
                MySQL, MQ, ETCD
  * Controller-tacker: Tacker, Tacker-conductor
  * Controller-k8s: kuryr-k8s, kuryr-CNI, k8s-api, kubelet
  * Compute: Nova-compute

All kubernetes resources are created on the controller-k8s node.

This patch includes the following changes:

  * Added a execution command for the functional test of related
    k8s for the VNF LCM in tox.ini.
  * Registered a vim of the `kubernetes` type by ansible. Also
    added related materials.
  * Moved the functional test files for k8s to other new directory.
  * Fixed a minor invalid definition in the definition file used
    for functional testing.

Change-Id: I1621b904450e94d6793b4c524de6785520f2e805
This commit is contained in:
Naoaki Horie 2020-12-14 05:45:04 +00:00
parent 3bb268a2a7
commit fe46bd2612
9 changed files with 336 additions and 1 deletions

View File

@ -32,6 +32,36 @@
- compute1
- compute2
- nodeset:
name: openstack-k8s-4-nodes-focal
nodes:
- name: controller
label: ubuntu-focal
- name: controller-tacker
label: ubuntu-focal
- name: controller-k8s
label: ubuntu-focal
- name: compute1
label: ubuntu-focal
groups:
- name: compute
nodes:
- controller-k8s
- compute1
- name: subnode
nodes:
- controller-tacker
- controller-k8s
- compute1
- name: switch
nodes:
- controller
- name: peers
nodes:
- controller-tacker
- controller-k8s
- compute1
- job:
name: tacker-functional-devstack-multinode-legacy
parent: devstack
@ -232,6 +262,193 @@
base_url: http://127.0.0.1:9990/grant/v1/grants
tox_envlist: dsvm-functional-sol-separated-nfvo
- job:
name: tacker-functional-devstack-multinode-sol-kubernetes
parent: devstack
description: |
Multinodes job for SOL devstack-based kubernetes functional tests
nodeset: openstack-k8s-4-nodes-focal
pre-run: playbooks/devstack/pre.yaml
run: playbooks/devstack/run.yaml
post-run: playbooks/devstack/post.yaml
roles:
- zuul: openstack-infra/devstack
timeout: 10800
required-projects:
- openstack/barbican
- openstack/cinder
- openstack/devstack-gate
- openstack/devstack-plugin-container
- openstack/glance
- openstack/heat
- openstack/horizon
- openstack/keystone
- openstack/kuryr-kubernetes
- openstack/mistral
- openstack/neutron
- openstack/nova
- openstack/octavia
- openstack/placement
- openstack/python-barbicanclient
- openstack/python-blazarclient
- openstack/python-mistralclient
- openstack/python-octaviaclient
- openstack/python-tackerclient
- openstack/tacker
- openstack/tacker-horizon
host-vars:
controller:
devstack_plugins:
barbican: https://opendev.org/openstack/barbican
heat: https://opendev.org/openstack/heat
mistral: https://opendev.org/openstack/mistral
octavia: https://opendev.org/openstack/octavia
devstack_services:
base: false
c-api: true
c-bak: false
c-sch: true
c-vol: true
cinder: true
coredns: false
etcd3: true
g-api: true
g-reg: true
horizon: false
key: true
mysql: true
n-api-meta: true
n-api: true
n-cond: true
n-cpu: false
n-novnc: true
n-sch: true
neutron: true
o-api: true
o-cw: true
o-hk: true
o-hm: true
octavia: true
placement-api: true
placement-client: true
q-agt: true
q-dhcp: true
q-l3: true
q-meta: true
q-metering: true
q-qos: true
q-svc: true
rabbit: true
s-account: false
s-container: false
s-object: false
s-proxy: false
swift: false
tempest: false
tls-proxy: false
tox_install_siblings: false
controller-tacker:
devstack_local_conf: {}
devstack_services:
q-agt: true
tacker: true
tacker-conductor: true
devstack_plugins:
tacker: https://opendev.org/openstack/tacker
tox_envlist: dsvm-functional-sol-kubernetes
controller-k8s:
devstack_local_conf: {}
devstack_plugins:
devstack-plugin-container: https://opendev.org/openstack/devstack-plugin-container
kuryr-kubernetes: https://opendev.org/openstack/kuryr-kubernetes
devstack_services:
etcd3: false
kubelet: true
kubernetes-api: true
kubernetes-controller-manager: true
kubernetes-scheduler: true
kuryr-daemon: true
kuryr-kubernetes: true
octavia: false
q-agt: true
tox_install_siblings: false
group-vars:
subnode:
devstack_localrc:
CELLSV2_SETUP: singleconductor
DATABASE_TYPE: mysql
IS_ZUUL_FT: True
KEYSTONE_SERVICE_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}"
KURYR_FORCE_IMAGE_BUILD: true
KURYR_K8S_API_PORT: 8080
KURYR_K8S_API_URL: "http://{{ hostvars['controller-k8s']['nodepool']['private_ipv4'] }}:${KURYR_K8S_API_PORT}"
KURYR_K8S_CLOUD_PROVIDER: false
KURYR_K8S_CONTAINERIZED_DEPLOYMENT: false
KURYR_K8S_MULTI_WORKER_TESTS: false
KURYR_NEUTRON_DEFAULT_SUBNETPOOL_ID: shared-default-subnetpool-v4
MYSQL_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}"
OCTAVIA_AMP_IMAGE_FILE: "/tmp/test-only-amphora-x64-haproxy-ubuntu-bionic.qcow2"
OCTAVIA_AMP_IMAGE_NAME: "test-only-amphora-x64-haproxy-ubuntu-bionic"
OCTAVIA_AMP_IMAGE_SIZE: 3
OVS_BRIDGE_MAPPINGS: public:br-ex,mgmtphysnet0:br-infra
PHYSICAL_NETWORK: mgmtphysnet0
Q_DVR_MODE: dvr
TACKER_HOST: "{{ hostvars['controller-tacker']['nodepool']['private_ipv4'] }}"
TACKER_MODE: standalone
USE_PYTHON3: true
devstack_services:
dstat: false
horizon: false
n-api-meta: false
n-api: false
n-cauth: false
n-cond: false
n-cpu: true
n-novnc: false
n-obj: false
n-sch: false
q-agt: true
tls-proxy: false
vars:
devstack_localrc:
CELLSV2_SETUP: singleconductor
DATABASE_TYPE: mysql
ETCD_USE_RAMDISK: true
IS_ZUUL_FT: True
KEYSTONE_SERVICE_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}"
L2_AGENT_EXTENSIONS: qos
MYSQL_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}"
OCTAVIA_AMP_IMAGE_FILE: "/tmp/test-only-amphora-x64-haproxy-ubuntu-bionic.qcow2"
OCTAVIA_AMP_IMAGE_NAME: "test-only-amphora-x64-haproxy-ubuntu-bionic"
OCTAVIA_AMP_IMAGE_SIZE: 3
OVS_BRIDGE_MAPPINGS: public:br-ex,mgmtphysnet0:br-infra
PHYSICAL_NETWORK: mgmtphysnet0
Q_DVR_MODE: dvr
Q_ML2_PLUGIN_EXT_DRIVERS: port_security,qos
Q_SERVICE_PLUGIN_CLASSES: router,neutron.services.metering.metering_plugin.MeteringPlugin,neutron.services.qos.qos_plugin.QoSPlugin,qos
TACKER_HOST: "{{ hostvars['controller-tacker']['nodepool']['private_ipv4'] }}"
TACKER_MODE: standalone
USE_PYTHON3: true
devstack_local_conf:
post-config:
$NEUTRON_DHCP_CONF:
DEFAULT:
enable_isolated_metadata: True
$OCTAVIA_CONF:
controller_worker:
amp_active_retries: 9999
kuryr_k8s_api_url: "http://{{ hostvars['controller-k8s']['nodepool']['private_ipv4'] }}:8080"
test_matrix_configs: [neutron]
zuul_work_dir: src/opendev.org/openstack/tacker
zuul_copy_output:
'{{ devstack_log_dir }}/kubernetes': 'logs'
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- ^releasenotes/.*$
- ^contrib/.*$
voting: false
- project:
templates:
- check-requirements
@ -245,3 +462,4 @@
- tacker-functional-devstack-multinode-legacy
- tacker-functional-devstack-multinode-sol
- tacker-functional-devstack-multinode-sol-separated-nfvo
- tacker-functional-devstack-multinode-sol-kubernetes

View File

@ -65,3 +65,81 @@
when:
- inventory_hostname == 'controller-tacker'
- block:
- name: Copy tools/test-setup-k8s-vim.sh
copy:
remote_src=True
src={{ devstack_base_dir }}/tacker/tools/test-setup-k8s-vim.sh
dest={{ zuul_work_dir }}/tools/test-setup-k8s-vim.sh
mode=0755
- name: Copy test k8s vim file
copy:
remote_src=True
src={{ devstack_base_dir }}/tacker/tacker/tests/etc/samples/local-k8s-vim.yaml
dest={{ zuul_work_dir }}/tacker/tests/etc/samples/local-k8s-vim.yaml
- name: Check if project's tools/test-setup-k8s-vim.sh exists
stat:
path: "{{ zuul_work_dir }}/tools/test-setup-k8s-vim.sh"
register: p
- fail:
msg: >
{{ zuul_work_dir }}/tools/test-setup-k8s-vim.sh doesn't exists
or it doesn't have execute permission.
when: p.stat.exists != True or p.stat.executable != True
- name: Get stackenv from devstack environment
slurp:
src: "{{ devstack_base_dir }}/devstack/.stackenv"
register: stackenv
- name: Set a keystone authentication uri
set_fact:
auth_uri: "{{
stackenv.content
| b64decode
| regex_replace('\n', ' ')
| regex_replace('^.*KEYSTONE_SERVICE_URI=([^ ]+).*$', '\\1')
}}"
when:
- p.stat.exists
- name: Replace keystone auth uri in test-setup-k8s-vim.sh
replace:
path: "{{ item }}"
regexp: "http://127.0.0.1/identity"
replace: "{{ auth_uri }}"
with_items:
- "{{ zuul_work_dir }}/tools/test-setup-k8s-vim.sh"
when:
- p.stat.exists
- name: Replace k8s auth uri in local-k8s-vim.yaml
replace:
path: "{{ item }}"
regexp: "https://127.0.0.1:6443"
replace: "{{ kuryr_k8s_api_url }}"
with_items:
- "{{ zuul_work_dir }}/tacker/tests/etc/samples/local-k8s-vim.yaml"
when:
- p.stat.exists
- name: Replace the config file path in the test-setup-k8s-vim.sh
replace:
path: "{{ zuul_work_dir }}/tools/test-setup-k8s-vim.sh"
regexp: '(?<=config-file )([^ ]+)(?= )'
replace: "{{ ansible_env.HOME }}/{{ zuul_work_dir }}/tacker/tests/etc/samples/local-k8s-vim.yaml"
- name: Run tools/test-setup-k8s-vim.sh
command: tools/test-setup-k8s-vim.sh
args:
chdir: "{{ zuul_work_dir }}"
when:
- p.stat.exists
- p.stat.executable
when:
- inventory_hostname == 'controller-tacker'
- kuryr_k8s_api_url is defined

View File

@ -220,7 +220,7 @@ topology_template:
properties:
aspect: worker_instance
deltas:
delta_2:
delta_1:
number_of_instances: 1
targets: [ VDU2 ]

View File

@ -0,0 +1,6 @@
auth_url: "https://127.0.0.1:6443"
username: "admin"
password: "admin"
project_name: "default"
ssl_ca_cert: None
type: "kubernetes"

27
tools/test-setup-k8s-vim.sh Executable file
View File

@ -0,0 +1,27 @@
#!/bin/bash -xe
# This script is used to set up default vim
# for functional testing, which cannot be put
# in devstack/plugin.sh because new zuul3 CI
# cannot keep the devstack plugins order
#
# Also, this script updates the following
# parameter which has been modified
# unintentionally by ansible playbook
# `roles/setup-default-vim/tasks/main.yaml`
# according to the execution environment of
# Zuul.
#
# --os-auth-url
# --config-file
openstack vim register \
--os-username nfv_user \
--os-project-name nfv \
--os-password devstack \
--os-auth-url http://127.0.0.1/identity \
--os-project-domain-name Default \
--os-user-domain-name Default \
--description "Kubernetes VIM" \
--config-file /opt/stack/tacker/tacker/tests/etc/samples/local-k8s-vim.yaml \
vim-kubernetes

View File

@ -58,6 +58,12 @@ setenv = {[testenv]setenv}
commands =
stestr --test-path=./tacker/tests/functional/sol_separated_nfvo run --slowest --concurrency 1 {posargs}
[testenv:dsvm-functional-sol-kubernetes]
setenv = {[testenv]setenv}
commands =
stestr --test-path=./tacker/tests/functional/sol_kubernetes run --slowest --concurrency 2 {posargs}
[testenv:debug]
commands = oslo_debug_helper {posargs}