Browse Source

Migrate functional job to native zuulv3

The manilaclient functional test job
"manilaclient-dsvm-neutron-functional"
is currently broken due to a change to its
parent job in the devstack repository.

Instead of investing to fix that legacy style
job, we'd be better off rewriting it to be
in native zuulv3 style, i.e., without the
use of the deprecated devstack-gate project.

- Introduce a new job "python-manilaclient-functional"
  which replaces "manilaclient-dsvm-neutron-functional"
  but preserves the testing done by the latter.
- The new job runs on a minimal devstack (keystone,
  neutron, manila, mysql and rabbit) and uses an ansible
  role introduced to setup test configuration.
- Add a "bindep.txt" file to call out test system
  dependencies, and install them via the new job.
- Cleanup the old job, the associated devstack gate
  hooks and playbooks.

Change-Id: I3b67f3b566bdadf1cc228f885b7253435a4507b5
Signed-off-by: Goutham Pacha Ravi <gouthampravi@gmail.com>
changes/19/737919/15
Goutham Pacha Ravi 1 year ago
parent
commit
d65d7f05a7
  1. 31
      .zuul.yaml
  2. 13
      bindep.txt
  3. 111
      contrib/ci/post_test_hook.sh
  4. 32
      contrib/ci/pre_test_hook.sh
  5. 28
      playbooks/legacy/manilaclient-dsvm-neutron-functional/post.yaml
  6. 70
      playbooks/legacy/manilaclient-dsvm-neutron-functional/run.yaml
  7. 6
      playbooks/python-manilaclient-functional/post.yaml
  8. 12
      playbooks/python-manilaclient-functional/run.yaml
  9. 46
      roles/populate-manilaclient-config/README.rst
  10. 6
      roles/populate-manilaclient-config/defaults/main.yaml
  11. 53
      roles/populate-manilaclient-config/tasks/main.yaml
  12. 18
      zuul.d/project.yaml
  13. 82
      zuul.d/python-manilaclient-jobs.yaml

31
.zuul.yaml

@ -1,31 +0,0 @@
- project:
templates:
- publish-openstack-docs-pti
- openstack-cover-jobs
- openstack-lower-constraints-jobs
- openstack-python3-victoria-jobs
- check-requirements
- release-notes-jobs-python3
check:
jobs:
- manilaclient-dsvm-neutron-functional
- rally-dsvm-manila-multibackend:
voting: false
- rally-dsvm-manila-multibackend-no-ss:
voting: false
gate:
jobs:
- manilaclient-dsvm-neutron-functional
- job:
name: manilaclient-dsvm-neutron-functional
parent: legacy-dsvm-base
run: playbooks/legacy/manilaclient-dsvm-neutron-functional/run.yaml
post-run: playbooks/legacy/manilaclient-dsvm-neutron-functional/post.yaml
timeout: 5400
required-projects:
- openstack/devstack-gate
- openstack/manila
- openstack/neutron
- openstack/python-manilaclient
- openstack/manila-tempest-plugin

13
bindep.txt

@ -0,0 +1,13 @@
# This is a cross-platform list tracking distribution packages needed by tests;
# see https://docs.openstack.org/infra/bindep/ for additional information.
gettext
libffi-dev [platform:dpkg]
libffi-devel [platform:rpm]
libssl-dev [platform:ubuntu-xenial]
locales [platform:debian]
python-dev [platform:dpkg]
python-devel [platform:rpm]
python3-all-dev [platform:ubuntu !platform:ubuntu-precise]
python3-dev [platform:dpkg]
python3-devel [platform:fedora]

111
contrib/ci/post_test_hook.sh

@ -1,111 +0,0 @@
#!/bin/bash -xe
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This script is executed inside post_test_hook function in devstack gate.
export MANILACLIENT_DIR="$BASE/new/python-manilaclient"
export MANILACLIENT_CONF="$MANILACLIENT_DIR/etc/manilaclient/manilaclient.conf"
# Go to the manilaclient dir
cd $MANILACLIENT_DIR
# Give permissions
sudo chown -R $USER:stack .
# Create manilaclient config file
touch $MANILACLIENT_CONF
# Import functions from devstack
source $BASE/new/devstack/functions
env | grep OS_
# Set options to config client.
source $BASE/new/devstack/openrc demo demo
env | grep OS_
export OS_TENANT_NAME=${OS_PROJECT_NAME:-$OS_TENANT_NAME}
iniset $MANILACLIENT_CONF DEFAULT username $OS_USERNAME
iniset $MANILACLIENT_CONF DEFAULT tenant_name $OS_TENANT_NAME
iniset $MANILACLIENT_CONF DEFAULT password $OS_PASSWORD
iniset $MANILACLIENT_CONF DEFAULT auth_url $OS_AUTH_URL
iniset $MANILACLIENT_CONF DEFAULT project_domain_name $OS_PROJECT_DOMAIN_NAME
iniset $MANILACLIENT_CONF DEFAULT user_domain_name $OS_USER_DOMAIN_NAME
iniset $MANILACLIENT_CONF DEFAULT project_domain_id $OS_PROJECT_DOMAIN_ID
iniset $MANILACLIENT_CONF DEFAULT user_domain_id $OS_USER_DOMAIN_ID
source $BASE/new/devstack/openrc admin demo
env | grep OS_
export OS_TENANT_NAME=${OS_PROJECT_NAME:-$OS_TENANT_NAME}
iniset $MANILACLIENT_CONF DEFAULT admin_username $OS_USERNAME
iniset $MANILACLIENT_CONF DEFAULT admin_tenant_name $OS_TENANT_NAME
iniset $MANILACLIENT_CONF DEFAULT admin_password $OS_PASSWORD
iniset $MANILACLIENT_CONF DEFAULT admin_auth_url $OS_AUTH_URL
iniset $MANILACLIENT_CONF DEFAULT admin_project_domain_name $OS_PROJECT_DOMAIN_NAME
iniset $MANILACLIENT_CONF DEFAULT admin_user_domain_name $OS_USER_DOMAIN_NAME
iniset $MANILACLIENT_CONF DEFAULT admin_project_domain_id $OS_PROJECT_DOMAIN_ID
iniset $MANILACLIENT_CONF DEFAULT admin_user_domain_id $OS_USER_DOMAIN_ID
# Suppress errors in cleanup of resources
SUPPRESS_ERRORS=${SUPPRESS_ERRORS_IN_CLEANUP:-True}
iniset $MANILACLIENT_CONF DEFAULT suppress_errors_in_cleanup $SUPPRESS_ERRORS
# Set access type usage specific to dummy driver that we are using in CI
iniset $MANILACLIENT_CONF DEFAULT access_types_mapping "nfs:ip,cifs:user"
# Dummy driver is capable of running share migration tests
iniset $MANILACLIENT_CONF DEFAULT run_migration_tests "True"
# Dummy driver is capable of running share manage tests
iniset $MANILACLIENT_CONF DEFAULT run_manage_tests "True"
# Running mountable snapshot tests in dummy driver
iniset $MANILACLIENT_CONF DEFAULT run_mount_snapshot_tests "True"
# Create share network and use it for functional tests if required
USE_SHARE_NETWORK=$(trueorfalse True USE_SHARE_NETWORK)
if [[ ${USE_SHARE_NETWORK} = True ]]; then
SHARE_NETWORK_NAME=${SHARE_NETWORK_NAME:-ci}
DEFAULT_NEUTRON_NET=$(openstack network show private -c id -f value)
DEFAULT_NEUTRON_SUBNET=$(openstack subnet show private-subnet -c id -f value)
NEUTRON_NET=${NEUTRON_NET:-$DEFAULT_NEUTRON_NET}
NEUTRON_SUBNET=${NEUTRON_SUBNET:-$DEFAULT_NEUTRON_SUBNET}
manila share-network-create \
--name $SHARE_NETWORK_NAME \
--neutron-net $NEUTRON_NET \
--neutron-subnet $NEUTRON_SUBNET
iniset $MANILACLIENT_CONF DEFAULT share_network $SHARE_NETWORK_NAME
iniset $MANILACLIENT_CONF DEFAULT admin_share_network $SHARE_NETWORK_NAME
fi
# Set share type if required
if [[ "$SHARE_TYPE" ]]; then
iniset $MANILACLIENT_CONF DEFAULT share_type $SHARE_TYPE
fi
# let us control if we die or not
set +o errexit
CONCURRENCY=${CONCURRENCY:-8}
# Run functional tests
sudo -H -u $USER tox -e functional -v -- --concurrency=$CONCURRENCY
EXIT_CODE=$?
# Copy artifacts into ZUUL's workspace
sudo -H -u $USER cp -r $MANILACLIENT_DIR $WORKSPACE
return $EXIT_CODE

32
contrib/ci/pre_test_hook.sh

@ -1,32 +0,0 @@
#!/bin/bash -xe
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This script is executed inside pre_test_hook function in devstack gate.
# Run manila's pre_test_hook.sh script first
source $BASE/new/manila/contrib/ci/pre_test_hook.sh True dummy multibackend
sudo -H pip install virtualenv
virtualenv /tmp/devstack-tools
/tmp/devstack-tools/bin/pip install -U devstack-tools==0.4.0
localconf=$BASE/new/devstack/local.conf
DSCONF=/tmp/devstack-tools/bin/dsconf
# Set big quota for share networks to avoid limit exceedances
$DSCONF setlc $localconf MANILA_OPTGROUP_DEFAULT_quota_share_networks 50
$DSCONF setlc $localconf MANILA_CONFIGURE_DEFAULT_TYPES True
# Enable and use only v3 of Identity API
$DSCONF setlc $localconf IDENTITY_API_VERSION 3
$DSCONF setlc $localconf ENABLE_IDENTITY_V2 False

28
playbooks/legacy/manilaclient-dsvm-neutron-functional/post.yaml

@ -1,28 +0,0 @@
- hosts: primary
roles:
- role: fetch-tox-output
zuul_work_dir: "{{ ansible_user_dir }}/workspace/python-manilaclient"
tox_envlist: functional
- role: fetch-subunit-output
zuul_work_dir: "{{ ansible_user_dir }}/workspace/python-manilaclient"
tox_envlist: functional
tasks:
- name: Copy job config files
synchronize:
src: "{{ ansible_user_dir }}/workspace/python-manilaclient/etc"
dest: "{{ zuul.executor.log_root }}"
mode: pull
verify_host: true
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: "{{ ansible_user_dir }}/workspace/"
dest: "{{ zuul.executor.log_root }}"
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=/logs/**
- --include=*/
- --exclude=*
- --prune-empty-dirs

70
playbooks/legacy/manilaclient-dsvm-neutron-functional/run.yaml

@ -1,70 +0,0 @@
- hosts: all
name: Autoconverted job legacy-manilaclient-dsvm-neutron-functional from old job
gate-manilaclient-dsvm-neutron-functional-ubuntu-xenial
tasks:
- name: Ensure legacy workspace directory
file:
path: '{{ ansible_user_dir }}/workspace'
state: directory
- shell:
cmd: |
set -e
set -x
cat > clonemap.yaml << EOF
clonemap:
- name: openstack/devstack-gate
dest: devstack-gate
EOF
/usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
https://opendev.org \
openstack/devstack-gate
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'
- shell:
cmd: |
set -e
set -x
cat << 'EOF' >>"/tmp/dg-local.conf"
[[local|localrc]]
enable_plugin manila https://opendev.org/openstack/manila
EOF
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'
- shell:
cmd: |
set -e
set -x
export PYTHONUNBUFFERED=true
export DEVSTACK_GATE_NEUTRON=1
export DEVSTACK_PROJECT_FROM_GIT="python-manilaclient"
export PROJECTS="openstack/manila-tempest-plugin $PROJECTS"
# Keep localrc to be able to set some vars in pre_test_hook
export KEEP_LOCALRC=1
export DEVSTACK_GATE_USE_PYTHON3=True
export PYTHON3_VERSION="3.6"
function pre_test_hook {
source $BASE/new/python-manilaclient/contrib/ci/pre_test_hook.sh
}
export -f pre_test_hook
function post_test_hook {
# Configure and run functional tests
source $BASE/new/python-manilaclient/contrib/ci/post_test_hook.sh
}
export -f post_test_hook
cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
./safe-devstack-vm-gate-wrap.sh
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'

6
playbooks/python-manilaclient-functional/post.yaml

@ -0,0 +1,6 @@
- hosts: all
vars:
tox_envlist: functional
roles:
- fetch-tox-output
- fetch-subunit-output

12
playbooks/python-manilaclient-functional/run.yaml

@ -0,0 +1,12 @@
- hosts: all
roles:
- ensure-python
- run-devstack
# Run bindep and test-setup after devstack so that they won't interfere
- role: bindep
bindep_profile: test
bindep_dir: "{{ zuul_work_dir }}"
- test-setup
- populate-manilaclient-config
- ensure-tox
- tox

46
roles/populate-manilaclient-config/README.rst

@ -0,0 +1,46 @@
populate-manilaclient-config
============================
An ansible role to use devstack's helper scripts and create a configuration
file for running python-manilaclient's functional tests.
Role Variables
--------------
.. zuul:rolevar:: base_dir
:type: string
:default: /opt/stack
The base directory for the installation. The devstack folder is expected
to be found in this location
.. zuul:rolevar:: manilaclient_config
:type: string
:default: "{{ zuul.project.src_dir }}/etc/manilaclient/manilaclient.conf"
The location of the manilaclient test configuration file.
.. zuul:rolevar:: neutron_network_name
:type: string
:default: private
A pre-created neutron network that the tests can use. This network must
be accessible to the "demo" and "admin" users within the "demo" project.
.. zuul:rolevar:: neutron_subnet_name
:type: string
:default: private-subnet
A pre-created neutron subnet that the tests can use. This network must
be accessible to the "demo" and "admin" users within the "demo" project.
.. zuul:rolevar:: share_network_name
:type: string
:default: ci
The name to give the share network created by this role, and configured
for use by the tests.

6
roles/populate-manilaclient-config/defaults/main.yaml

@ -0,0 +1,6 @@
---
base_dir: /opt/stack
manilaclient_config: "{{ zuul.project.src_dir }}/etc/manilaclient/manilaclient.conf"
neutron_network_name: private
neutron_subnet_name: private-subnet
share_network_name: ci

53
roles/populate-manilaclient-config/tasks/main.yaml

@ -0,0 +1,53 @@
- name: create the manilaclient config file
file:
path: "{{ manilaclient_config }}"
state: touch
- name: Set OS_ env variables to config
shell:
cmd: |
source {{ base_dir }}/devstack/functions
source {{ base_dir }}/devstack/openrc demo demo &>/dev/null
iniset {{ manilaclient_config }} DEFAULT username $OS_USERNAME
iniset {{ manilaclient_config }} DEFAULT tenant_name $OS_PROJECT_NAME
iniset {{ manilaclient_config }} DEFAULT password $OS_PASSWORD
iniset {{ manilaclient_config }} DEFAULT auth_url $OS_AUTH_URL
iniset {{ manilaclient_config }} DEFAULT project_domain_id $OS_PROJECT_DOMAIN_ID
iniset {{ manilaclient_config }} DEFAULT user_domain_id $OS_USER_DOMAIN_ID
source {{ base_dir }}/devstack/openrc admin demo &>/dev/null
iniset {{ manilaclient_config }} DEFAULT admin_username $OS_USERNAME
iniset {{ manilaclient_config }} DEFAULT admin_tenant_name $OS_TENANT_NAME
iniset {{ manilaclient_config }} DEFAULT admin_password $OS_PASSWORD
iniset {{ manilaclient_config }} DEFAULT admin_auth_url $OS_AUTH_URL
iniset {{ manilaclient_config }} DEFAULT admin_project_domain_id $OS_PROJECT_DOMAIN_ID
iniset {{ manilaclient_config }} DEFAULT admin_user_domain_id $OS_USER_DOMAIN_ID
iniset {{ manilaclient_config }} DEFAULT access_types_mapping "nfs:ip,cifs:user"
# Tests
iniset {{ manilaclient_config }} DEFAULT suppress_errors_in_cleanup false
iniset {{ manilaclient_config }} DEFAULT run_migration_tests true
iniset {{ manilaclient_config }} DEFAULT run_manage_tests true
iniset {{ manilaclient_config }} DEFAULT run_mount_snapshot_tests true
args:
executable: "/bin/bash"
- name: create a share network
shell:
cmd: |
source {{ base_dir }}/devstack/functions
source {{ base_dir }}/devstack/openrc demo demo &>/dev/null
NEUTRON_NET=$(openstack network show {{ neutron_network_name }} -c id -f value)
NEUTRON_SUBNET=$(openstack subnet show {{ neutron_subnet_name }} -c id -f value)
manila share-network-create \
--name {{ share_network_name }} \
--neutron-net $NEUTRON_NET \
--neutron-subnet $NEUTRON_SUBNET
iniset {{ manilaclient_config }} DEFAULT share_network {{ share_network_name }}
iniset {{ manilaclient_config }} DEFAULT admin_share_network {{ share_network_name }}
args:
executable: "/bin/bash"

18
zuul.d/project.yaml

@ -0,0 +1,18 @@
- project:
templates:
- publish-openstack-docs-pti
- openstack-cover-jobs
- openstack-lower-constraints-jobs
- openstack-python3-victoria-jobs
- check-requirements
- release-notes-jobs-python3
check:
jobs:
- python-manilaclient-functional
- rally-dsvm-manila-multibackend:
voting: false
- rally-dsvm-manila-multibackend-no-ss:
voting: false
gate:
jobs:
- python-manilaclient-functional

82
zuul.d/python-manilaclient-jobs.yaml

@ -0,0 +1,82 @@
- job:
name: python-manilaclient-functional
parent: devstack-minimal
irrelevant-files:
- ^manilaclient/tests/unit/.*$
- ^setup.cfg$
- ^tools/.*$
- ^tox.ini$
required-projects:
- openstack/keystone
- openstack/manila
- openstack/neutron
- openstack/python-manilaclient
timeout: 5400
run: playbooks/python-manilaclient-functional/run.yaml
post-run: playbooks/python-manilaclient-functional/post.yaml
vars:
zuul_copy_output:
"{{ zuul.project.src_dir }}/etc/manilaclient/manilaclient.conf": logs
tox_environment:
PYTHONUNBUFFERED: 'true'
tox_envlist: functional
devstack_plugins:
manila: https://opendev.org/openstack/manila
devstack_localrc:
USE_PYTHON3: true
INSTALL_TEMPEST: false
# Enable manila with a fake driver that supports all capabilities
MANILA_CONFIGURE_DEFAULT_TYPES: true
MANILA_OPTGROUP_DEFAULT_quota_share_networks: 50
MANILA_DEFAULT_SHARE_TYPE_EXTRA_SPECS: 'snapshot_support=True create_share_from_snapshot_support=True revert_to_snapshot_support=True mount_snapshot_support=True'
MANILA_INSTALL_TEMPEST_PLUGIN_SYSTEMWIDE: false
MANILA_SERVICE_IMAGE_ENABLED: false
MANILA_SHARE_MIGRATION_PERIOD_TASK_INTERVAL: 1
SHARE_DRIVER: manila.tests.share.drivers.dummy.DummyDriver
MANILA_REPLICA_STATE_UPDATE_INTERVAL: 10
MANILA_ENABLED_BACKENDS: alpha,beta,gamma,delta
MANILA_CONFIGURE_GROUPS: alpha,beta,gamma,delta,membernet,adminnet
MANILA_OPTGROUP_alpha_share_driver: manila.tests.share.drivers.dummy.DummyDriver
MANILA_OPTGROUP_alpha_driver_handles_share_servers: true
MANILA_OPTGROUP_alpha_share_backend_name: ALPHA
MANILA_OPTGROUP_alpha_network_config_group: membernet
MANILA_OPTGROUP_alpha_admin_network_config_group: adminnet
MANILA_OPTGROUP_alpha_replication_domain: DUMMY_DOMAIN_2
MANILA_OPTGROUP_beta_share_driver: manila.tests.share.drivers.dummy.DummyDriver
MANILA_OPTGROUP_beta_driver_handles_share_servers: true
MANILA_OPTGROUP_beta_share_backend_name: BETA
MANILA_OPTGROUP_beta_network_config_group: membernet
MANILA_OPTGROUP_beta_admin_network_config_group: adminnet
MANILA_OPTGROUP_beta_replication_domain: DUMMY_DOMAIN_2
MANILA_OPTGROUP_gamma_share_driver: manila.tests.share.drivers.dummy.DummyDriver
MANILA_OPTGROUP_gamma_driver_handles_share_servers: false
MANILA_OPTGROUP_gamma_share_backend_name: GAMMA
MANILA_OPTGROUP_gamma_replication_domain: DUMMY_DOMAIN
MANILA_OPTGROUP_delta_share_driver: manila.tests.share.drivers.dummy.DummyDriver
MANILA_OPTGROUP_delta_driver_handles_share_servers: false
MANILA_OPTGROUP_delta_share_backend_name: DELTA
MANILA_OPTGROUP_delta_replication_domain: DUMMY_DOMAIN
MANILA_OPTGROUP_membernet_network_api_class: manila.network.standalone_network_plugin.StandaloneNetworkPlugin
MANILA_OPTGROUP_membernet_standalone_network_plugin_gateway: 10.0.0.1
MANILA_OPTGROUP_membernet_standalone_network_plugin_mask: 24
MANILA_OPTGROUP_membernet_standalone_network_plugin_network_type: vlan
MANILA_OPTGROUP_membernet_standalone_network_plugin_segmentation_id: 1010
MANILA_OPTGROUP_membernet_standalone_network_plugin_allowed_ip_ranges: 10.0.0.10-10.0.0.209
MANILA_OPTGROUP_membernet_network_plugin_ipv4_enabled: true
MANILA_OPTGROUP_adminnet_network_api_class: manila.network.standalone_network_plugin.StandaloneNetworkPlugin
MANILA_OPTGROUP_adminnet_standalone_network_plugin_gateway: 11.0.0.1
MANILA_OPTGROUP_adminnet_standalone_network_plugin_mask: 24
MANILA_OPTGROUP_adminnet_standalone_network_plugin_network_type: vlan
MANILA_OPTGROUP_adminnet_standalone_network_plugin_segmentation_id: 1011
MANILA_OPTGROUP_adminnet_standalone_network_plugin_allowed_ip_ranges: 11.0.0.10-11.0.0.19,11.0.0.30-11.0.0.39,11.0.0.50-11.0.0.199
MANILA_OPTGROUP_adminnet_network_plugin_ipv4_enabled: true
devstack_services:
# Keystone
key: true
# Neutron
q-agt: true
q-dhcp: true
q-l3: true
q-meta: true
q-metering: true
q-svc: true
Loading…
Cancel
Save