Migrate neutron-functional job to zuul v3 syntax

This patch migrates also neutron-functional-python27 job to
zuul v3 syntax.

Neutron's functional tests have to be run on host with
deployed minimal devstack but without running any e.g. neutron
services. Because of that new job's template inherits from
devstack-minimal job instead of devstack-tox-functional and
we need to have own run_functional_job and post_functional_job
playbooks.

It also adds ansible roles:
- configure_functional_tests
- setup_logdir
- fetch_journal_log

which are used in new neutron-functional job definition.
Those roles can be also used later e.g. for fullstack job.

Change-Id: I80bc17c8c9f43050ac0c21176fbc4be46c11ce35
Related-bug: #1804844
This commit is contained in:
Slawek Kaplonski 2018-11-23 15:53:10 +01:00
parent 6920727fe1
commit 33110fb7e8
18 changed files with 150 additions and 293 deletions

View File

@ -102,24 +102,46 @@
- job: - job:
name: neutron-functional name: neutron-functional
parent: legacy-dsvm-base parent: devstack-minimal
run: playbooks/legacy/neutron-functional/run.yaml description: Run neutron functional tests
post-run: playbooks/legacy/neutron-functional/post.yaml
timeout: 7800 timeout: 7800
required-projects: required-projects:
- openstack-infra/devstack-gate - git.openstack.org/openstack-dev/devstack
- openstack/neutron - openstack/neutron
roles:
- zuul: openstack-dev/devstack
pre-run: playbooks/configure_functional_job.yaml
run: playbooks/run_functional_job.yaml
post-run: playbooks/post_functional_job.yaml
irrelevant-files: irrelevant-files:
- ^.*\.rst$ - ^.*\.rst$
- ^doc/.*$ - ^doc/.*$
- ^neutron/locale/.*$ - ^neutron/locale/.*$
- ^releasenotes/.*$ - ^releasenotes/.*$
vars:
devstack_services:
# Ignore any default set by devstack. Emit a "disable_all_services".
base: false
devstack_localrc:
USE_PYTHON3: true
INSTALL_TESTONLY_PACKAGES: true
DATABASE_PASSWORD: stackdb
tox_envlist: dsvm-functional
tox_install_siblings: false
zuul_copy_output:
# We need to copy directory with logs to have it in job artifacts also,
# /opt/stack/logs is default logs directory defined in neutron's
# tox.ini file
'{{ devstack_base_dir }}/logs/dsvm-functional-logs': logs
- job: - job:
name: neutron-functional-python27 name: neutron-functional-python27
parent: neutron-functional parent: neutron-functional
run: playbooks/legacy/neutron-functional-python27/run.yaml description: Run neutron functional tests with Python 2.7
post-run: playbooks/legacy/neutron-functional-python27/post.yaml vars:
devstack_localrc:
USE_PYTHON3: false
tox_envlist: dsvm-functional-python27
- job: - job:
name: neutron-fullstack name: neutron-fullstack

View File

@ -2,7 +2,7 @@
set -ex set -ex
VENV=${1:-"dsvm-functional"} VENV=${1:-"dsvm-fullstack"}
FLAVOR=${2:-"all"} FLAVOR=${2:-"all"}
GATE_DEST=$BASE/new GATE_DEST=$BASE/new
@ -55,7 +55,7 @@ function load_rc_for_rally {
case $VENV in case $VENV in
"dsvm-functional"|"dsvm-functional-python27"|"dsvm-fullstack") "dsvm-fullstack")
# The following need to be set before sourcing # The following need to be set before sourcing
# configure_for_func_testing. # configure_for_func_testing.
GATE_STACK_USER=stack GATE_STACK_USER=stack

View File

@ -0,0 +1,4 @@
- hosts: all
roles:
- setup_logdir
- configure_functional_tests

View File

@ -1,80 +0,0 @@
- hosts: primary
tasks:
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=**/*nose_results.html
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=**/*testr_results.html.gz
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=/.testrepository/tmp*
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=**/*testrepository.subunit.gz
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}/tox'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=/.tox/*/log/*
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=/logs/**
- --include=*/
- --exclude=*
- --prune-empty-dirs

View File

@ -1,54 +0,0 @@
- hosts: all
name: Autoconverted job legacy-neutron-dsvm-functional from old job gate-neutron-dsvm-functional-ubuntu-xenial-nv
tasks:
- name: Ensure legacy workspace directory
file:
path: '{{ ansible_user_dir }}/workspace'
state: directory
- shell:
cmd: |
set -e
set -x
cat > clonemap.yaml << EOF
clonemap:
- name: openstack-infra/devstack-gate
dest: devstack-gate
EOF
/usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
git://git.openstack.org \
openstack-infra/devstack-gate
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'
- shell:
cmd: |
set -e
set -x
export PYTHONUNBUFFERED=true
export DEVSTACK_GATE_TEMPEST=0
export DEVSTACK_GATE_EXERCISES=0
export DEVSTACK_GATE_NEUTRON=1
export DEVSTACK_GATE_INSTALL_TESTONLY=1
export BRANCH_OVERRIDE=default
if [ "$BRANCH_OVERRIDE" != "default" ] ; then
export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
fi
function gate_hook {
bash -xe $BASE/new/neutron/neutron/tests/contrib/gate_hook.sh dsvm-functional-python27
}
export -f gate_hook
function post_test_hook {
bash -xe $BASE/new/neutron/neutron/tests/contrib/post_test_hook.sh dsvm-functional-python27
}
export -f post_test_hook
cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
./safe-devstack-vm-gate-wrap.sh
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'

View File

@ -1,96 +0,0 @@
- hosts: primary
tasks:
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=**/*nose_results.html
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=**/*testr_results.html.gz
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=/.testrepository/tmp*
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=**/*testrepository.subunit.gz
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}/tox'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=/.tox/*/log/*
- --include=*/
- --exclude=*
- --prune-empty-dirs
# TODO(slaweq): this should be moved to separate role and used in zuulv3
# jobs definitions also
- name: Store journal logs in {{ ansible_user_dir }}/workspace/logs/journal.log
become: yes
shell:
cmd: |
/bin/journalctl -a > {{ ansible_user_dir }}/workspace/logs/journal.log
- name: Set journal.log file permissions
become: yes
file:
path: '{{ ansible_user_dir }}/workspace/logs/journal.log'
owner: '{{ ansible_user }}'
group: '{{ ansible_user }}'
mode: 0644
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=/logs/**
- --include=*/
- --exclude=*
- --prune-empty-dirs

View File

@ -1,55 +0,0 @@
- hosts: all
name: Autoconverted job legacy-neutron-dsvm-functional from old job gate-neutron-dsvm-functional-ubuntu-xenial-nv
tasks:
- name: Ensure legacy workspace directory
file:
path: '{{ ansible_user_dir }}/workspace'
state: directory
- shell:
cmd: |
set -e
set -x
cat > clonemap.yaml << EOF
clonemap:
- name: openstack-infra/devstack-gate
dest: devstack-gate
EOF
/usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
git://git.openstack.org \
openstack-infra/devstack-gate
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'
- shell:
cmd: |
set -e
set -x
export USE_PYTHON3=true
export PYTHONUNBUFFERED=true
export DEVSTACK_GATE_TEMPEST=0
export DEVSTACK_GATE_EXERCISES=0
export DEVSTACK_GATE_NEUTRON=1
export DEVSTACK_GATE_INSTALL_TESTONLY=1
export BRANCH_OVERRIDE=default
if [ "$BRANCH_OVERRIDE" != "default" ] ; then
export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
fi
function gate_hook {
bash -xe $BASE/new/neutron/neutron/tests/contrib/gate_hook.sh dsvm-functional
}
export -f gate_hook
function post_test_hook {
bash -xe $BASE/new/neutron/neutron/tests/contrib/post_test_hook.sh dsvm-functional
}
export -f post_test_hook
cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
./safe-devstack-vm-gate-wrap.sh
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'

View File

@ -0,0 +1,5 @@
- hosts: all
roles:
- fetch_journal_log
- fetch-tox-output
- fetch-subunit-output

View File

@ -0,0 +1,10 @@
- hosts: all
roles:
- run-devstack
# Run bindep and test-setup after devstack so that they won't interfere
- role: bindep
bindep_profile: test
bindep_dir: "{{ zuul_work_dir }}"
- test-setup
- ensure-tox
- tox

View File

@ -0,0 +1,18 @@
Configure host to run on it Neutron functional/fullstack tests
**Role Variables**
.. zuul:rolevar:: tests_venv
:default: {{ tox_envlist }}
.. zuul:rolevar:: base_dir
:default: {{ ansible_user_dir }}/src/git.openstack.org
.. zuul:rolevar:: gate_dest_dir
:default: {{ base_dir }}/openstack
.. zuul:rolevar:: devstack_dir
:default: {{ base_dir }}/openstack-dev/devstack
.. zuul:rolevar:: neutron_dir
:default: {{ gate_dest_dir }}/neutron

View File

@ -0,0 +1,5 @@
tests_venv: "{{ tox_envlist }}"
base_dir: "{{ ansible_user_dir }}/src/git.openstack.org"
gate_dest_dir: "{{ base_dir }}/openstack"
devstack_dir: "{{ base_dir }}/openstack-dev/devstack"
neutron_dir: "{{ gate_dest_dir }}/neutron"

View File

@ -0,0 +1,19 @@
- shell:
cmd: |
set -e
set -x
GATE_STACK_USER={{ ansible_user }}
IS_GATE=True
BASE_DIR={{ base_dir }}
GATE_DEST={{ gate_dest_dir }}
NEUTRON_PATH={{ neutron_dir }}
DEVSTACK_PATH={{ devstack_dir }}
VENV={{ tests_venv }}
source $DEVSTACK_PATH/functions
source $NEUTRON_PATH/devstack/lib/ovs
source $NEUTRON_PATH/tools/configure_for_func_testing.sh
configure_host_for_func_testing
executable: /bin/bash

View File

@ -0,0 +1,16 @@
Collect journal log from test run
By default, this stores journal log into log file and store it in
"journal_log_file_path"
**Role Variables**
.. zuul:rolevar:: journal_log_path
:default: {{ ansible_user_dir }}/workspace/logs
Path where journal log file will be stored on job's node.
.. zuul:rolevar:: journal_log_file_name
:default: {{ journal_log_path }}/journal.log
Name of journal log file.

View File

@ -0,0 +1,2 @@
journal_log_path: "{{ ansible_user_dir }}/logs"
journal_log_file_name: "{{ journal_log_path }}/journal.log"

View File

@ -0,0 +1,23 @@
- name: Ensure {{ journal_log_path }} exists
become: yes
msg: "Ensure logdir {{ journal_log_path }} exists and have correct permission"
file:
path: "{{ journal_log_path }}"
state: directory
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: 0775
- name: Store journal logs in {{ journal_log_file_name }}
become: yes
shell:
cmd: |
/bin/journalctl -a > {{ journal_log_file_name }}
- name: Set journal.log file permissions
become: yes
file:
path: '{{ journal_log_file_name }}'
owner: '{{ ansible_user }}'
group: '{{ ansible_user }}'
mode: 0644

View File

@ -0,0 +1,8 @@
Configure logs dir to be accessible for ``stack`` user.
**Role Variables**
.. zuul:rolevar:: logdir
:default: /opt/stack/logs
Name of the directory where logs will be stored.

View File

@ -0,0 +1 @@
logdir: /opt/stack/logs

View File

@ -0,0 +1,9 @@
- name: Ensure logdir exists
become: yes
msg: "Ensure logdir {{ logdir }} exists and have correct permission"
file:
path: "{{ logdir }}"
state: directory
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: 0775