Browse Source

Migrate neutron-functional job to zuul v3 syntax

This patch migrates also neutron-functional-python27 job to
zuul v3 syntax.

Neutron's functional tests have to be run on host with
deployed minimal devstack but without running any e.g. neutron
services. Because of that new job's template inherits from
devstack-minimal job instead of devstack-tox-functional and
we need to have own run_functional_job and post_functional_job
playbooks.

It also adds ansible roles:
- configure_functional_tests
- setup_logdir
- fetch_journal_log

which are used in new neutron-functional job definition.
Those roles can be also used later e.g. for fullstack job.

Change-Id: I80bc17c8c9f43050ac0c21176fbc4be46c11ce35
Related-bug: #1804844
changes/42/619742/65
Slawek Kaplonski 4 years ago
parent
commit
33110fb7e8
  1. 34
      .zuul.yaml
  2. 4
      neutron/tests/contrib/gate_hook.sh
  3. 4
      playbooks/configure_functional_job.yaml
  4. 80
      playbooks/legacy/neutron-functional-python27/post.yaml
  5. 54
      playbooks/legacy/neutron-functional-python27/run.yaml
  6. 96
      playbooks/legacy/neutron-functional/post.yaml
  7. 55
      playbooks/legacy/neutron-functional/run.yaml
  8. 5
      playbooks/post_functional_job.yaml
  9. 10
      playbooks/run_functional_job.yaml
  10. 18
      roles/configure_functional_tests/README.rst
  11. 5
      roles/configure_functional_tests/defaults/main.yaml
  12. 19
      roles/configure_functional_tests/tasks/main.yaml
  13. 16
      roles/fetch_journal_log/README.rst
  14. 2
      roles/fetch_journal_log/defaults/main.yaml
  15. 23
      roles/fetch_journal_log/tasks/main.yaml
  16. 8
      roles/setup_logdir/README.rst
  17. 1
      roles/setup_logdir/defaults/main.yaml
  18. 9
      roles/setup_logdir/tasks/main.yaml

34
.zuul.yaml

@ -102,24 +102,46 @@
- job:
name: neutron-functional
parent: legacy-dsvm-base
run: playbooks/legacy/neutron-functional/run.yaml
post-run: playbooks/legacy/neutron-functional/post.yaml
parent: devstack-minimal
description: Run neutron functional tests
timeout: 7800
required-projects:
- openstack-infra/devstack-gate
- git.openstack.org/openstack-dev/devstack
- openstack/neutron
roles:
- zuul: openstack-dev/devstack
pre-run: playbooks/configure_functional_job.yaml
run: playbooks/run_functional_job.yaml
post-run: playbooks/post_functional_job.yaml
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- ^neutron/locale/.*$
- ^releasenotes/.*$
vars:
devstack_services:
# Ignore any default set by devstack. Emit a "disable_all_services".
base: false
devstack_localrc:
USE_PYTHON3: true
INSTALL_TESTONLY_PACKAGES: true
DATABASE_PASSWORD: stackdb
tox_envlist: dsvm-functional
tox_install_siblings: false
zuul_copy_output:
# We need to copy directory with logs to have it in job artifacts also,
# /opt/stack/logs is default logs directory defined in neutron's
# tox.ini file
'{{ devstack_base_dir }}/logs/dsvm-functional-logs': logs
- job:
name: neutron-functional-python27
parent: neutron-functional
run: playbooks/legacy/neutron-functional-python27/run.yaml
post-run: playbooks/legacy/neutron-functional-python27/post.yaml
description: Run neutron functional tests with Python 2.7
vars:
devstack_localrc:
USE_PYTHON3: false
tox_envlist: dsvm-functional-python27
- job:
name: neutron-fullstack

4
neutron/tests/contrib/gate_hook.sh

@ -2,7 +2,7 @@
set -ex
VENV=${1:-"dsvm-functional"}
VENV=${1:-"dsvm-fullstack"}
FLAVOR=${2:-"all"}
GATE_DEST=$BASE/new
@ -55,7 +55,7 @@ function load_rc_for_rally {
case $VENV in
"dsvm-functional"|"dsvm-functional-python27"|"dsvm-fullstack")
"dsvm-fullstack")
# The following need to be set before sourcing
# configure_for_func_testing.
GATE_STACK_USER=stack

4
playbooks/configure_functional_job.yaml

@ -0,0 +1,4 @@
- hosts: all
roles:
- setup_logdir
- configure_functional_tests

80
playbooks/legacy/neutron-functional-python27/post.yaml

@ -1,80 +0,0 @@
- hosts: primary
tasks:
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=**/*nose_results.html
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=**/*testr_results.html.gz
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=/.testrepository/tmp*
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=**/*testrepository.subunit.gz
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}/tox'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=/.tox/*/log/*
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=/logs/**
- --include=*/
- --exclude=*
- --prune-empty-dirs

54
playbooks/legacy/neutron-functional-python27/run.yaml

@ -1,54 +0,0 @@
- hosts: all
name: Autoconverted job legacy-neutron-dsvm-functional from old job gate-neutron-dsvm-functional-ubuntu-xenial-nv
tasks:
- name: Ensure legacy workspace directory
file:
path: '{{ ansible_user_dir }}/workspace'
state: directory
- shell:
cmd: |
set -e
set -x
cat > clonemap.yaml << EOF
clonemap:
- name: openstack-infra/devstack-gate
dest: devstack-gate
EOF
/usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
git://git.openstack.org \
openstack-infra/devstack-gate
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'
- shell:
cmd: |
set -e
set -x
export PYTHONUNBUFFERED=true
export DEVSTACK_GATE_TEMPEST=0
export DEVSTACK_GATE_EXERCISES=0
export DEVSTACK_GATE_NEUTRON=1
export DEVSTACK_GATE_INSTALL_TESTONLY=1
export BRANCH_OVERRIDE=default
if [ "$BRANCH_OVERRIDE" != "default" ] ; then
export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
fi
function gate_hook {
bash -xe $BASE/new/neutron/neutron/tests/contrib/gate_hook.sh dsvm-functional-python27
}
export -f gate_hook
function post_test_hook {
bash -xe $BASE/new/neutron/neutron/tests/contrib/post_test_hook.sh dsvm-functional-python27
}
export -f post_test_hook
cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
./safe-devstack-vm-gate-wrap.sh
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'

96
playbooks/legacy/neutron-functional/post.yaml

@ -1,96 +0,0 @@
- hosts: primary
tasks:
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=**/*nose_results.html
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=**/*testr_results.html.gz
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=/.testrepository/tmp*
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=**/*testrepository.subunit.gz
- --include=*/
- --exclude=*
- --prune-empty-dirs
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}/tox'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=/.tox/*/log/*
- --include=*/
- --exclude=*
- --prune-empty-dirs
# TODO(slaweq): this should be moved to separate role and used in zuulv3
# jobs definitions also
- name: Store journal logs in {{ ansible_user_dir }}/workspace/logs/journal.log
become: yes
shell:
cmd: |
/bin/journalctl -a > {{ ansible_user_dir }}/workspace/logs/journal.log
- name: Set journal.log file permissions
become: yes
file:
path: '{{ ansible_user_dir }}/workspace/logs/journal.log'
owner: '{{ ansible_user }}'
group: '{{ ansible_user }}'
mode: 0644
- name: Copy files from {{ ansible_user_dir }}/workspace/ on node
synchronize:
src: '{{ ansible_user_dir }}/workspace/'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- --include=/logs/**
- --include=*/
- --exclude=*
- --prune-empty-dirs

55
playbooks/legacy/neutron-functional/run.yaml

@ -1,55 +0,0 @@
- hosts: all
name: Autoconverted job legacy-neutron-dsvm-functional from old job gate-neutron-dsvm-functional-ubuntu-xenial-nv
tasks:
- name: Ensure legacy workspace directory
file:
path: '{{ ansible_user_dir }}/workspace'
state: directory
- shell:
cmd: |
set -e
set -x
cat > clonemap.yaml << EOF
clonemap:
- name: openstack-infra/devstack-gate
dest: devstack-gate
EOF
/usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
git://git.openstack.org \
openstack-infra/devstack-gate
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'
- shell:
cmd: |
set -e
set -x
export USE_PYTHON3=true
export PYTHONUNBUFFERED=true
export DEVSTACK_GATE_TEMPEST=0
export DEVSTACK_GATE_EXERCISES=0
export DEVSTACK_GATE_NEUTRON=1
export DEVSTACK_GATE_INSTALL_TESTONLY=1
export BRANCH_OVERRIDE=default
if [ "$BRANCH_OVERRIDE" != "default" ] ; then
export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
fi
function gate_hook {
bash -xe $BASE/new/neutron/neutron/tests/contrib/gate_hook.sh dsvm-functional
}
export -f gate_hook
function post_test_hook {
bash -xe $BASE/new/neutron/neutron/tests/contrib/post_test_hook.sh dsvm-functional
}
export -f post_test_hook
cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
./safe-devstack-vm-gate-wrap.sh
executable: /bin/bash
chdir: '{{ ansible_user_dir }}/workspace'
environment: '{{ zuul | zuul_legacy_vars }}'

5
playbooks/post_functional_job.yaml

@ -0,0 +1,5 @@
- hosts: all
roles:
- fetch_journal_log
- fetch-tox-output
- fetch-subunit-output

10
playbooks/run_functional_job.yaml

@ -0,0 +1,10 @@
- hosts: all
roles:
- run-devstack
# Run bindep and test-setup after devstack so that they won't interfere
- role: bindep
bindep_profile: test
bindep_dir: "{{ zuul_work_dir }}"
- test-setup
- ensure-tox
- tox

18
roles/configure_functional_tests/README.rst

@ -0,0 +1,18 @@
Configure host to run on it Neutron functional/fullstack tests
**Role Variables**
.. zuul:rolevar:: tests_venv
:default: {{ tox_envlist }}
.. zuul:rolevar:: base_dir
:default: {{ ansible_user_dir }}/src/git.openstack.org
.. zuul:rolevar:: gate_dest_dir
:default: {{ base_dir }}/openstack
.. zuul:rolevar:: devstack_dir
:default: {{ base_dir }}/openstack-dev/devstack
.. zuul:rolevar:: neutron_dir
:default: {{ gate_dest_dir }}/neutron

5
roles/configure_functional_tests/defaults/main.yaml

@ -0,0 +1,5 @@
tests_venv: "{{ tox_envlist }}"
base_dir: "{{ ansible_user_dir }}/src/git.openstack.org"
gate_dest_dir: "{{ base_dir }}/openstack"
devstack_dir: "{{ base_dir }}/openstack-dev/devstack"
neutron_dir: "{{ gate_dest_dir }}/neutron"

19
roles/configure_functional_tests/tasks/main.yaml

@ -0,0 +1,19 @@
- shell:
cmd: |
set -e
set -x
GATE_STACK_USER={{ ansible_user }}
IS_GATE=True
BASE_DIR={{ base_dir }}
GATE_DEST={{ gate_dest_dir }}
NEUTRON_PATH={{ neutron_dir }}
DEVSTACK_PATH={{ devstack_dir }}
VENV={{ tests_venv }}
source $DEVSTACK_PATH/functions
source $NEUTRON_PATH/devstack/lib/ovs
source $NEUTRON_PATH/tools/configure_for_func_testing.sh
configure_host_for_func_testing
executable: /bin/bash

16
roles/fetch_journal_log/README.rst

@ -0,0 +1,16 @@
Collect journal log from test run
By default, this stores journal log into log file and store it in
"journal_log_file_path"
**Role Variables**
.. zuul:rolevar:: journal_log_path
:default: {{ ansible_user_dir }}/workspace/logs
Path where journal log file will be stored on job's node.
.. zuul:rolevar:: journal_log_file_name
:default: {{ journal_log_path }}/journal.log
Name of journal log file.

2
roles/fetch_journal_log/defaults/main.yaml

@ -0,0 +1,2 @@
journal_log_path: "{{ ansible_user_dir }}/logs"
journal_log_file_name: "{{ journal_log_path }}/journal.log"

23
roles/fetch_journal_log/tasks/main.yaml

@ -0,0 +1,23 @@
- name: Ensure {{ journal_log_path }} exists
become: yes
msg: "Ensure logdir {{ journal_log_path }} exists and have correct permission"
file:
path: "{{ journal_log_path }}"
state: directory
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: 0775
- name: Store journal logs in {{ journal_log_file_name }}
become: yes
shell:
cmd: |
/bin/journalctl -a > {{ journal_log_file_name }}
- name: Set journal.log file permissions
become: yes
file:
path: '{{ journal_log_file_name }}'
owner: '{{ ansible_user }}'
group: '{{ ansible_user }}'
mode: 0644

8
roles/setup_logdir/README.rst

@ -0,0 +1,8 @@
Configure logs dir to be accessible for ``stack`` user.
**Role Variables**
.. zuul:rolevar:: logdir
:default: /opt/stack/logs
Name of the directory where logs will be stored.

1
roles/setup_logdir/defaults/main.yaml

@ -0,0 +1 @@
logdir: /opt/stack/logs

9
roles/setup_logdir/tasks/main.yaml

@ -0,0 +1,9 @@
- name: Ensure logdir exists
become: yes
msg: "Ensure logdir {{ logdir }} exists and have correct permission"
file:
path: "{{ logdir }}"
state: directory
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: 0775
Loading…
Cancel
Save