From a0b8ca43e498da4b514f46c165d8c395070a17e4 Mon Sep 17 00:00:00 2001 From: Ghanshyam Date: Wed, 17 Jul 2019 09:46:41 +0000 Subject: [PATCH] Setup framework for IPv6-only deployment verification Devstack IPv6 base job 'devstack-IPv6' setup the IPv6 setting to deploy the services to listen on IPv6 address. Tempest 'devstack-tempest-ipv6' job derived from 'devstack-IPv6' job adds the playbooks to run the tests. As part of Train community goal 'Support IPv6-Only Deployments', we will expand the 'devstack-tempest-ipv6' job to do IPv6-only deployments verification. This commit define the new roles of 'ipv6-only-deployments-verification' which will be invoked as part of run phase of 'devstack-tempest-ipv6' job. This role will do IPv6-only setting and deployments verification via tools/verify-ipv6-only-deployement.sh IPv6 verification script can be extended further to perform more checks and via project specific test case. Those tests will run as part of project specific child jobs. The 'devstack-tempest-ipv6' job will be used as parent for project specific IPv6-only job. Those child job can extend the project specific IPv6 verification by defining new playebook for post-run. That way the base verification done in 'devstack-tempest-ipv6' will still run in addition to project specific verificaiton and tests run. Verification structure will be: - 'devstack-IPv6' deploy the service on IPv6 - 'devstack-tempest-ipv6' run will verify the IPv6-only setting and listen address - Child jobs derived from 'devstack-tempest-ipv6' will run the IPv6 related test case or any further IPv6 deployment verification. This commit also adds the new job 'tempest-ipv6-only' which will run smoke and ipv6 related tests present in Tempest. This job will be used to run on 6 services (Nova, Neutron, Cinder, Keystone, Glance, Swift) deployed by devstack. Story: #2005477 Task: #35923 Change-Id: Ie1e5e5e02ee45ab7a4aae0f35bdb42afc2c3fe2e --- .zuul.yaml | 66 +++++++------- playbooks/devstack-tempest-ipv6.yaml | 24 ++++++ .../README.rst | 16 ++++ .../defaults/main.yaml | 1 + .../tasks/main.yaml | 4 + tools/verify-ipv6-only-deployments.sh | 86 +++++++++++++++++++ tox.ini | 12 +++ 7 files changed, 178 insertions(+), 31 deletions(-) create mode 100644 playbooks/devstack-tempest-ipv6.yaml create mode 100644 roles/ipv6-only-deployments-verification/README.rst create mode 100644 roles/ipv6-only-deployments-verification/defaults/main.yaml create mode 100644 roles/ipv6-only-deployments-verification/tasks/main.yaml create mode 100755 tools/verify-ipv6-only-deployments.sh diff --git a/.zuul.yaml b/.zuul.yaml index cdc63a7c3c..85bf1bb016 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -7,12 +7,12 @@ This Tempest job provides the base for both the single and multi-node test setup. To run a multi-node test inherit from devstack-tempest and set the nodeset to a multi-node one. - required-projects: + required-projects: &base_required-projects - opendev.org/openstack/tempest timeout: 7200 - roles: + roles: &base_roles - zuul: opendev.org/openstack/devstack - vars: + vars: &base_vars devstack_services: tempest: true devstack_local_conf: @@ -53,36 +53,36 @@ name: devstack-tempest-ipv6 parent: devstack-ipv6 description: | - Base Tempest IPv6 job. - required-projects: - - opendev.org/openstack/tempest + Base Tempest IPv6 job. This job is derived from 'devstack-ipv6' + which set the IPv6-only setting for OpenStack services. As part of + run phase, this job will verify the IPv6 setting and check the services + endpoints and listen addresses are IPv6. Basically it will run the script + ./tool/verify-ipv6-only-deployments.sh + + Child jobs of this job can run their own set of tests and can + add post-run playebooks to extend the IPv6 verification specific + to their deployed services. + Check the wiki page for more details about project jobs setup + - https://wiki.openstack.org/wiki/Goal-IPv6-only-deployments-and-testing + required-projects: *base_required-projects timeout: 7200 - roles: - - zuul: opendev.org/openstack/devstack - vars: - devstack_services: - tempest: true - devstack_local_conf: - test-config: - $TEMPEST_CONFIG: - compute: - min_compute_nodes: "{{ groups['compute'] | default(['controller']) | length }}" - test_results_stage_name: test_results - zuul_copy_output: - '{{ devstack_base_dir }}/tempest/etc/tempest.conf': logs - '{{ devstack_base_dir }}/tempest/etc/accounts.yaml': logs - '{{ devstack_base_dir }}/tempest/tempest.log': logs - '{{ stage_dir }}/{{ test_results_stage_name }}.subunit': logs - '{{ stage_dir }}/{{ test_results_stage_name }}.html': logs - '{{ stage_dir }}/stackviz': logs - extensions_to_txt: - conf: true - log: true - yaml: true - yml: true - run: playbooks/devstack-tempest.yaml + roles: *base_roles + vars: *base_vars + run: playbooks/devstack-tempest-ipv6.yaml post-run: playbooks/post-tempest.yaml +- job: + name: tempest-ipv6-only + parent: devstack-tempest-ipv6 + # This currently works from stable/pike on. + branches: ^(?!stable/ocata).*$ + description: | + Integration test of IPv6-only deployments. This job runs + smoke and IPv6 relates tests only. Basic idea is to test + whether OpenStack Services listen on IPv6 addrress or not. + vars: + tox_envlist: ipv6-only + - job: name: tempest-full parent: devstack-tempest @@ -729,7 +729,7 @@ - tempest-multinode-full-py3: irrelevant-files: *tempest-irrelevant-files - tempest-tox-plugin-sanity-check: - irrelevant-files: + irrelevant-files: &tempest-irrelevant-files-2 - ^.*\.rst$ - ^doc/.*$ - ^etc/.*$ @@ -738,6 +738,8 @@ - ^tempest/hacking/.*$ - ^tempest/tests/.*$ # tools/ is not here since this relies on a script in tools/. + - tempest-ipv6-only: + irrelevant-files: *tempest-irrelevant-files-2 - tempest-slow: irrelevant-files: *tempest-irrelevant-files - tempest-slow-py3: @@ -790,6 +792,8 @@ irrelevant-files: *tempest-irrelevant-files - grenade-py3: irrelevant-files: *tempest-irrelevant-files + - tempest-ipv6-only: + irrelevant-files: *tempest-irrelevant-files-2 experimental: jobs: - tempest-cinder-v2-api: diff --git a/playbooks/devstack-tempest-ipv6.yaml b/playbooks/devstack-tempest-ipv6.yaml new file mode 100644 index 0000000000..5f72345c38 --- /dev/null +++ b/playbooks/devstack-tempest-ipv6.yaml @@ -0,0 +1,24 @@ +# Changes that run through devstack-tempest-ipv6 are likely to have an impact on +# the devstack part of the job, so we keep devstack in the main play to +# avoid zuul retrying on legitimate failures. +- hosts: all + roles: + - orchestrate-devstack + +# We run tests only on one node, regardless how many nodes are in the system +- hosts: tempest + environment: + # This enviroment variable is used by the optional tempest-gabbi + # job provided by the gabbi-tempest plugin. It can be safely ignored + # if that plugin is not being used. + GABBI_TEMPEST_PATH: "{{ gabbi_tempest_path | default('') }}" + roles: + - setup-tempest-run-dir + - setup-tempest-data-dir + - acl-devstack-files + # Verify the IPv6-only deployments. This role will perform check for + # IPv6 only env for example Devstack IPv6 settings and services listen + # address is IPv6 etc. This is invoked before tests are run so that we can + # fail early if anything missing the IPv6 settings or deployments. + - ipv6-only-deployments-verification + - run-tempest diff --git a/roles/ipv6-only-deployments-verification/README.rst b/roles/ipv6-only-deployments-verification/README.rst new file mode 100644 index 0000000000..400a8da222 --- /dev/null +++ b/roles/ipv6-only-deployments-verification/README.rst @@ -0,0 +1,16 @@ +Verify the IPv6-only deployments + +This role needs to be invoked from a playbook that +run tests. This role verifies the IPv6 setting on +devstack side and devstack deploy services on IPv6. +This role is invoked before tests are run so that +if any missing IPv6 setting or deployments can fail +the job early. + + +**Role Variables** + +.. zuul:rolevar:: devstack_base_dir + :default: /opt/stack + + The devstack base directory. diff --git a/roles/ipv6-only-deployments-verification/defaults/main.yaml b/roles/ipv6-only-deployments-verification/defaults/main.yaml new file mode 100644 index 0000000000..fea05c8146 --- /dev/null +++ b/roles/ipv6-only-deployments-verification/defaults/main.yaml @@ -0,0 +1 @@ +devstack_base_dir: /opt/stack diff --git a/roles/ipv6-only-deployments-verification/tasks/main.yaml b/roles/ipv6-only-deployments-verification/tasks/main.yaml new file mode 100644 index 0000000000..d73c79c06f --- /dev/null +++ b/roles/ipv6-only-deployments-verification/tasks/main.yaml @@ -0,0 +1,4 @@ +- name: Verify the ipv6-only deployments + become: true + become_user: stack + shell: "{{ devstack_base_dir }}/tempest/tools/verify-ipv6-only-deployments.sh" diff --git a/tools/verify-ipv6-only-deployments.sh b/tools/verify-ipv6-only-deployments.sh new file mode 100755 index 0000000000..90807a3161 --- /dev/null +++ b/tools/verify-ipv6-only-deployments.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# +# +# NOTE(gmann): This script is used in 'devstack-tempest-ipv6' zuul job to verify that +# services are deployed on IPv6 properly or not. This will capture if any devstck or devstack +# plugins are missing the required setting to listen on IPv6 address. This is run as part of +# run phase of zuul job and before test run. Child job of 'devstack-tempest-ipv6' +# can expand the IPv6 verification specific to project by defining the new post-run script which +# will run along with this base script. +# If there are more common verification for IPv6 then we can always extent this script. + +# Keep track of the DevStack directory +TOP_DIR=$(cd $(dirname "$0")/../../devstack && pwd) +source $TOP_DIR/stackrc +source $TOP_DIR/openrc admin admin + +function verify_devstack_ipv6_setting { + local _service_host=$(echo $SERVICE_HOST | tr -d []) + local _host_ipv6=$(echo $HOST_IPV6 | tr -d []) + local _service_listen_address=$(echo $SERVICE_LISTEN_ADDRESS | tr -d []) + local _service_local_host=$(echo $SERVICE_LOCAL_HOST | tr -d []) + if [[ "$SERVICE_IP_VERSION" != 6 ]]; then + echo $SERVICE_IP_VERSION "SERVICE_IP_VERSION is not set to 6 which is must for devstack to deploy services with IPv6 address." + exit 1 + fi + is_service_host_ipv6=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_service_host'"))') + if [[ "$is_service_host_ipv6" != "True" ]]; then + echo $SERVICE_HOST "SERVICE_HOST is not ipv6 which means devstack cannot deploy services on IPv6 address." + exit 1 + fi + is_host_ipv6=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_host_ipv6'"))') + if [[ "$is_host_ipv6" != "True" ]]; then + echo $HOST_IPV6 "HOST_IPV6 is not ipv6 which means devstack cannot deploy services on IPv6 address." + exit 1 + fi + is_service_listen_address=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_service_listen_address'"))') + if [[ "$is_service_listen_address" != "True" ]]; then + echo $SERVICE_LISTEN_ADDRESS "SERVICE_LISTEN_ADDRESS is not ipv6 which means devstack cannot deploy services on IPv6 address." + exit 1 + fi + is_service_local_host=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_service_local_host'"))') + if [[ "$is_service_local_host" != "True" ]]; then + echo $SERVICE_LOCAL_HOST "SERVICE_LOCAL_HOST is not ipv6 which means devstack cannot deploy services on IPv6 address." + exit 1 + fi + echo "Devstack is properly configured with IPv6" + echo "SERVICE_IP_VERSION: " $SERVICE_IP_VERSION "HOST_IPV6: " $HOST_IPV6 "SERVICE_HOST: " $SERVICE_HOST "SERVICE_LISTEN_ADDRESS: " $SERVICE_LISTEN_ADDRESS "SERVICE_LOCAL_HOST: " $SERVICE_LOCAL_HOST +} + +function sanity_check_system_ipv6_enabled { + system_ipv6_enabled=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_ipv6_enabled())') + if [[ $system_ipv6_enabled != "True" ]]; then + echo "IPv6 is disabled in system" + exit 1 + fi + echo "IPv6 is enabled in system" +} + +function verify_service_listen_address_is_ipv6 { + local endpoints_verified=False + local all_ipv6=True + endpoints=$(openstack endpoint list -f value -c URL) + for endpoint in ${endpoints}; do + local endpoint_address=$(echo "$endpoint" | awk -F/ '{print $3}' | awk -F] '{print $1}') + endpoint_address=$(echo $endpoint_address | tr -d []) + local is_endpoint_ipv6=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$endpoint_address'"))') + if [[ "$is_endpoint_ipv6" != "True" ]]; then + all_ipv6=False + echo $endpoint ": This is not ipv6 endpoint which means corresponding service is not listening on IPv6 address." + continue + fi + endpoints_verified=True + done + if [[ "$all_ipv6" == "False" ]] || [[ "$endpoints_verified" == "False" ]]; then + exit 1 + fi + echo "All services deployed by devstack is on IPv6 endpoints" + echo $endpoints +} + +#First thing to verify if system has IPv6 enabled or not +sanity_check_system_ipv6_enabled +#Verify whether devstack is configured properly with IPv6 setting +verify_devstack_ipv6_setting +#Get all registrfed endpoints by devstack in keystone and verify that each endpoints address is IPv6. +verify_service_listen_address_is_ipv6 diff --git a/tox.ini b/tox.ini index 8a7a5090dd..53d7355c79 100644 --- a/tox.ini +++ b/tox.ini @@ -231,6 +231,18 @@ commands = find . -type f -name "*.pyc" -delete tempest run --serial --regex '\[.*\bslow\b.*\]' {posargs} +[testenv:ipv6-only] +envdir = .tox/tempest +sitepackages = {[tempestenv]sitepackages} +setenv = {[tempestenv]setenv} +deps = {[tempestenv]deps} +# Run only smoke and ipv6 tests. This env is used to tests +# the ipv6 deployments and basic tests run fine so that we can +# verify that services listen on IPv6 address. +commands = + find . -type f -name "*.pyc" -delete + tempest run --regex '\[.*\bsmoke|ipv6|test_network_v6\b.*\]' {posargs} + [testenv:venv] deps = -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}