Browse Source

Migrate neutron-functional job to zuul v3 syntax

This patch migrates also neutron-functional-python27 job to
zuul v3 syntax.

Neutron's functional tests have to be run on host with
deployed minimal devstack but without running any e.g. neutron
services. Because of that new job's template inherits from
devstack-minimal job instead of devstack-tox-functional and
we need to have own run_functional_job and post_functional_job
playbooks.

It also adds ansible roles:
- configure_functional_tests
- setup_logdir
- fetch_journal_log

which are used in new neutron-functional job definition.
Those roles can be also used later e.g. for fullstack job.

Change-Id: I80bc17c8c9f43050ac0c21176fbc4be46c11ce35
Related-bug: #1804844
Slawek Kaplonski 5 months ago
parent
commit
33110fb7e8

+ 28
- 6
.zuul.yaml View File

@@ -102,24 +102,46 @@
102 102
 
103 103
 - job:
104 104
     name: neutron-functional
105
-    parent: legacy-dsvm-base
106
-    run: playbooks/legacy/neutron-functional/run.yaml
107
-    post-run: playbooks/legacy/neutron-functional/post.yaml
105
+    parent: devstack-minimal
106
+    description: Run neutron functional tests
108 107
     timeout: 7800
109 108
     required-projects:
110
-      - openstack-infra/devstack-gate
109
+      - git.openstack.org/openstack-dev/devstack
111 110
       - openstack/neutron
111
+    roles:
112
+      - zuul: openstack-dev/devstack
113
+    pre-run: playbooks/configure_functional_job.yaml
114
+    run: playbooks/run_functional_job.yaml
115
+    post-run: playbooks/post_functional_job.yaml
112 116
     irrelevant-files:
113 117
       - ^.*\.rst$
114 118
       - ^doc/.*$
115 119
       - ^neutron/locale/.*$
116 120
       - ^releasenotes/.*$
121
+    vars:
122
+      devstack_services:
123
+        # Ignore any default set by devstack. Emit a "disable_all_services".
124
+        base: false
125
+      devstack_localrc:
126
+        USE_PYTHON3: true
127
+        INSTALL_TESTONLY_PACKAGES: true
128
+        DATABASE_PASSWORD: stackdb
129
+      tox_envlist: dsvm-functional
130
+      tox_install_siblings: false
131
+      zuul_copy_output:
132
+        # We need to copy directory with logs to have it in job artifacts also,
133
+        # /opt/stack/logs is default logs directory defined in neutron's
134
+        # tox.ini file
135
+        '{{ devstack_base_dir }}/logs/dsvm-functional-logs': logs
117 136
 
118 137
 - job:
119 138
     name: neutron-functional-python27
120 139
     parent: neutron-functional
121
-    run: playbooks/legacy/neutron-functional-python27/run.yaml
122
-    post-run: playbooks/legacy/neutron-functional-python27/post.yaml
140
+    description: Run neutron functional tests with Python 2.7
141
+    vars:
142
+      devstack_localrc:
143
+        USE_PYTHON3: false
144
+      tox_envlist: dsvm-functional-python27
123 145
 
124 146
 - job:
125 147
     name: neutron-fullstack

+ 2
- 2
neutron/tests/contrib/gate_hook.sh View File

@@ -2,7 +2,7 @@
2 2
 
3 3
 set -ex
4 4
 
5
-VENV=${1:-"dsvm-functional"}
5
+VENV=${1:-"dsvm-fullstack"}
6 6
 FLAVOR=${2:-"all"}
7 7
 
8 8
 GATE_DEST=$BASE/new
@@ -55,7 +55,7 @@ function load_rc_for_rally {
55 55
 
56 56
 
57 57
 case $VENV in
58
-"dsvm-functional"|"dsvm-functional-python27"|"dsvm-fullstack")
58
+"dsvm-fullstack")
59 59
     # The following need to be set before sourcing
60 60
     # configure_for_func_testing.
61 61
     GATE_STACK_USER=stack

+ 4
- 0
playbooks/configure_functional_job.yaml View File

@@ -0,0 +1,4 @@
1
+- hosts: all
2
+  roles:
3
+    - setup_logdir
4
+    - configure_functional_tests

+ 0
- 80
playbooks/legacy/neutron-functional-python27/post.yaml View File

@@ -1,80 +0,0 @@
1
-- hosts: primary
2
-  tasks:
3
-
4
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
5
-      synchronize:
6
-        src: '{{ ansible_user_dir }}/workspace/'
7
-        dest: '{{ zuul.executor.log_root }}'
8
-        mode: pull
9
-        copy_links: true
10
-        verify_host: true
11
-        rsync_opts:
12
-          - --include=**/*nose_results.html
13
-          - --include=*/
14
-          - --exclude=*
15
-          - --prune-empty-dirs
16
-
17
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
18
-      synchronize:
19
-        src: '{{ ansible_user_dir }}/workspace/'
20
-        dest: '{{ zuul.executor.log_root }}'
21
-        mode: pull
22
-        copy_links: true
23
-        verify_host: true
24
-        rsync_opts:
25
-          - --include=**/*testr_results.html.gz
26
-          - --include=*/
27
-          - --exclude=*
28
-          - --prune-empty-dirs
29
-
30
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
31
-      synchronize:
32
-        src: '{{ ansible_user_dir }}/workspace/'
33
-        dest: '{{ zuul.executor.log_root }}'
34
-        mode: pull
35
-        copy_links: true
36
-        verify_host: true
37
-        rsync_opts:
38
-          - --include=/.testrepository/tmp*
39
-          - --include=*/
40
-          - --exclude=*
41
-          - --prune-empty-dirs
42
-
43
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
44
-      synchronize:
45
-        src: '{{ ansible_user_dir }}/workspace/'
46
-        dest: '{{ zuul.executor.log_root }}'
47
-        mode: pull
48
-        copy_links: true
49
-        verify_host: true
50
-        rsync_opts:
51
-          - --include=**/*testrepository.subunit.gz
52
-          - --include=*/
53
-          - --exclude=*
54
-          - --prune-empty-dirs
55
-
56
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
57
-      synchronize:
58
-        src: '{{ ansible_user_dir }}/workspace/'
59
-        dest: '{{ zuul.executor.log_root }}/tox'
60
-        mode: pull
61
-        copy_links: true
62
-        verify_host: true
63
-        rsync_opts:
64
-          - --include=/.tox/*/log/*
65
-          - --include=*/
66
-          - --exclude=*
67
-          - --prune-empty-dirs
68
-
69
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
70
-      synchronize:
71
-        src: '{{ ansible_user_dir }}/workspace/'
72
-        dest: '{{ zuul.executor.log_root }}'
73
-        mode: pull
74
-        copy_links: true
75
-        verify_host: true
76
-        rsync_opts:
77
-          - --include=/logs/**
78
-          - --include=*/
79
-          - --exclude=*
80
-          - --prune-empty-dirs

+ 0
- 54
playbooks/legacy/neutron-functional-python27/run.yaml View File

@@ -1,54 +0,0 @@
1
-- hosts: all
2
-  name: Autoconverted job legacy-neutron-dsvm-functional from old job gate-neutron-dsvm-functional-ubuntu-xenial-nv
3
-  tasks:
4
-
5
-    - name: Ensure legacy workspace directory
6
-      file:
7
-        path: '{{ ansible_user_dir }}/workspace'
8
-        state: directory
9
-
10
-    - shell:
11
-        cmd: |
12
-          set -e
13
-          set -x
14
-          cat > clonemap.yaml << EOF
15
-          clonemap:
16
-            - name: openstack-infra/devstack-gate
17
-              dest: devstack-gate
18
-          EOF
19
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
20
-              git://git.openstack.org \
21
-              openstack-infra/devstack-gate
22
-        executable: /bin/bash
23
-        chdir: '{{ ansible_user_dir }}/workspace'
24
-      environment: '{{ zuul | zuul_legacy_vars }}'
25
-
26
-    - shell:
27
-        cmd: |
28
-          set -e
29
-          set -x
30
-          export PYTHONUNBUFFERED=true
31
-          export DEVSTACK_GATE_TEMPEST=0
32
-          export DEVSTACK_GATE_EXERCISES=0
33
-          export DEVSTACK_GATE_NEUTRON=1
34
-          export DEVSTACK_GATE_INSTALL_TESTONLY=1
35
-          export BRANCH_OVERRIDE=default
36
-          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
37
-              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
38
-          fi
39
-
40
-          function gate_hook {
41
-              bash -xe $BASE/new/neutron/neutron/tests/contrib/gate_hook.sh dsvm-functional-python27
42
-          }
43
-          export -f gate_hook
44
-
45
-          function post_test_hook {
46
-              bash -xe $BASE/new/neutron/neutron/tests/contrib/post_test_hook.sh dsvm-functional-python27
47
-          }
48
-          export -f post_test_hook
49
-
50
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
51
-          ./safe-devstack-vm-gate-wrap.sh
52
-        executable: /bin/bash
53
-        chdir: '{{ ansible_user_dir }}/workspace'
54
-      environment: '{{ zuul | zuul_legacy_vars }}'

+ 0
- 96
playbooks/legacy/neutron-functional/post.yaml View File

@@ -1,96 +0,0 @@
1
-- hosts: primary
2
-  tasks:
3
-
4
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
5
-      synchronize:
6
-        src: '{{ ansible_user_dir }}/workspace/'
7
-        dest: '{{ zuul.executor.log_root }}'
8
-        mode: pull
9
-        copy_links: true
10
-        verify_host: true
11
-        rsync_opts:
12
-          - --include=**/*nose_results.html
13
-          - --include=*/
14
-          - --exclude=*
15
-          - --prune-empty-dirs
16
-
17
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
18
-      synchronize:
19
-        src: '{{ ansible_user_dir }}/workspace/'
20
-        dest: '{{ zuul.executor.log_root }}'
21
-        mode: pull
22
-        copy_links: true
23
-        verify_host: true
24
-        rsync_opts:
25
-          - --include=**/*testr_results.html.gz
26
-          - --include=*/
27
-          - --exclude=*
28
-          - --prune-empty-dirs
29
-
30
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
31
-      synchronize:
32
-        src: '{{ ansible_user_dir }}/workspace/'
33
-        dest: '{{ zuul.executor.log_root }}'
34
-        mode: pull
35
-        copy_links: true
36
-        verify_host: true
37
-        rsync_opts:
38
-          - --include=/.testrepository/tmp*
39
-          - --include=*/
40
-          - --exclude=*
41
-          - --prune-empty-dirs
42
-
43
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
44
-      synchronize:
45
-        src: '{{ ansible_user_dir }}/workspace/'
46
-        dest: '{{ zuul.executor.log_root }}'
47
-        mode: pull
48
-        copy_links: true
49
-        verify_host: true
50
-        rsync_opts:
51
-          - --include=**/*testrepository.subunit.gz
52
-          - --include=*/
53
-          - --exclude=*
54
-          - --prune-empty-dirs
55
-
56
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
57
-      synchronize:
58
-        src: '{{ ansible_user_dir }}/workspace/'
59
-        dest: '{{ zuul.executor.log_root }}/tox'
60
-        mode: pull
61
-        copy_links: true
62
-        verify_host: true
63
-        rsync_opts:
64
-          - --include=/.tox/*/log/*
65
-          - --include=*/
66
-          - --exclude=*
67
-          - --prune-empty-dirs
68
-
69
-    # TODO(slaweq): this should be moved to separate role and used in zuulv3
70
-    # jobs definitions also
71
-    - name: Store journal logs in {{ ansible_user_dir }}/workspace/logs/journal.log
72
-      become: yes
73
-      shell:
74
-        cmd: |
75
-          /bin/journalctl -a > {{ ansible_user_dir }}/workspace/logs/journal.log
76
-
77
-    - name: Set journal.log file permissions
78
-      become: yes
79
-      file:
80
-        path: '{{ ansible_user_dir }}/workspace/logs/journal.log'
81
-        owner: '{{ ansible_user }}'
82
-        group: '{{ ansible_user }}'
83
-        mode: 0644
84
-
85
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
86
-      synchronize:
87
-        src: '{{ ansible_user_dir }}/workspace/'
88
-        dest: '{{ zuul.executor.log_root }}'
89
-        mode: pull
90
-        copy_links: true
91
-        verify_host: true
92
-        rsync_opts:
93
-          - --include=/logs/**
94
-          - --include=*/
95
-          - --exclude=*
96
-          - --prune-empty-dirs

+ 0
- 55
playbooks/legacy/neutron-functional/run.yaml View File

@@ -1,55 +0,0 @@
1
-- hosts: all
2
-  name: Autoconverted job legacy-neutron-dsvm-functional from old job gate-neutron-dsvm-functional-ubuntu-xenial-nv
3
-  tasks:
4
-
5
-    - name: Ensure legacy workspace directory
6
-      file:
7
-        path: '{{ ansible_user_dir }}/workspace'
8
-        state: directory
9
-
10
-    - shell:
11
-        cmd: |
12
-          set -e
13
-          set -x
14
-          cat > clonemap.yaml << EOF
15
-          clonemap:
16
-            - name: openstack-infra/devstack-gate
17
-              dest: devstack-gate
18
-          EOF
19
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
20
-              git://git.openstack.org \
21
-              openstack-infra/devstack-gate
22
-        executable: /bin/bash
23
-        chdir: '{{ ansible_user_dir }}/workspace'
24
-      environment: '{{ zuul | zuul_legacy_vars }}'
25
-
26
-    - shell:
27
-        cmd: |
28
-          set -e
29
-          set -x
30
-          export USE_PYTHON3=true
31
-          export PYTHONUNBUFFERED=true
32
-          export DEVSTACK_GATE_TEMPEST=0
33
-          export DEVSTACK_GATE_EXERCISES=0
34
-          export DEVSTACK_GATE_NEUTRON=1
35
-          export DEVSTACK_GATE_INSTALL_TESTONLY=1
36
-          export BRANCH_OVERRIDE=default
37
-          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
38
-              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
39
-          fi
40
-
41
-          function gate_hook {
42
-              bash -xe $BASE/new/neutron/neutron/tests/contrib/gate_hook.sh dsvm-functional
43
-          }
44
-          export -f gate_hook
45
-
46
-          function post_test_hook {
47
-              bash -xe $BASE/new/neutron/neutron/tests/contrib/post_test_hook.sh dsvm-functional
48
-          }
49
-          export -f post_test_hook
50
-
51
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
52
-          ./safe-devstack-vm-gate-wrap.sh
53
-        executable: /bin/bash
54
-        chdir: '{{ ansible_user_dir }}/workspace'
55
-      environment: '{{ zuul | zuul_legacy_vars }}'

+ 5
- 0
playbooks/post_functional_job.yaml View File

@@ -0,0 +1,5 @@
1
+- hosts: all
2
+  roles:
3
+    - fetch_journal_log
4
+    - fetch-tox-output
5
+    - fetch-subunit-output

+ 10
- 0
playbooks/run_functional_job.yaml View File

@@ -0,0 +1,10 @@
1
+- hosts: all
2
+  roles:
3
+    - run-devstack
4
+    # Run bindep and test-setup after devstack so that they won't interfere
5
+    - role: bindep
6
+      bindep_profile: test
7
+      bindep_dir: "{{ zuul_work_dir }}"
8
+    - test-setup
9
+    - ensure-tox
10
+    - tox

+ 18
- 0
roles/configure_functional_tests/README.rst View File

@@ -0,0 +1,18 @@
1
+Configure host to run on it Neutron functional/fullstack tests
2
+
3
+**Role Variables**
4
+
5
+.. zuul:rolevar:: tests_venv
6
+   :default: {{ tox_envlist }}
7
+
8
+.. zuul:rolevar:: base_dir
9
+   :default: {{ ansible_user_dir }}/src/git.openstack.org
10
+
11
+.. zuul:rolevar:: gate_dest_dir
12
+   :default: {{ base_dir }}/openstack
13
+
14
+.. zuul:rolevar:: devstack_dir
15
+   :default: {{ base_dir }}/openstack-dev/devstack
16
+
17
+.. zuul:rolevar:: neutron_dir
18
+   :default: {{ gate_dest_dir }}/neutron

+ 5
- 0
roles/configure_functional_tests/defaults/main.yaml View File

@@ -0,0 +1,5 @@
1
+tests_venv: "{{ tox_envlist }}"
2
+base_dir: "{{ ansible_user_dir }}/src/git.openstack.org"
3
+gate_dest_dir: "{{ base_dir }}/openstack"
4
+devstack_dir: "{{ base_dir }}/openstack-dev/devstack"
5
+neutron_dir: "{{ gate_dest_dir }}/neutron"

+ 19
- 0
roles/configure_functional_tests/tasks/main.yaml View File

@@ -0,0 +1,19 @@
1
+- shell:
2
+    cmd: |
3
+        set -e
4
+        set -x
5
+        GATE_STACK_USER={{ ansible_user }}
6
+        IS_GATE=True
7
+
8
+        BASE_DIR={{ base_dir }}
9
+        GATE_DEST={{ gate_dest_dir }}
10
+        NEUTRON_PATH={{ neutron_dir }}
11
+        DEVSTACK_PATH={{ devstack_dir }}
12
+        VENV={{ tests_venv }}
13
+
14
+        source $DEVSTACK_PATH/functions
15
+        source $NEUTRON_PATH/devstack/lib/ovs
16
+        source $NEUTRON_PATH/tools/configure_for_func_testing.sh
17
+
18
+        configure_host_for_func_testing
19
+    executable: /bin/bash

+ 16
- 0
roles/fetch_journal_log/README.rst View File

@@ -0,0 +1,16 @@
1
+Collect journal log from test run
2
+
3
+By default, this stores journal log into log file and store it in
4
+"journal_log_file_path"
5
+
6
+**Role Variables**
7
+
8
+.. zuul:rolevar:: journal_log_path
9
+   :default: {{ ansible_user_dir }}/workspace/logs
10
+
11
+   Path where journal log file will be stored on job's node.
12
+
13
+.. zuul:rolevar:: journal_log_file_name
14
+   :default: {{ journal_log_path }}/journal.log
15
+
16
+   Name of journal log file.

+ 2
- 0
roles/fetch_journal_log/defaults/main.yaml View File

@@ -0,0 +1,2 @@
1
+journal_log_path: "{{ ansible_user_dir }}/logs"
2
+journal_log_file_name: "{{ journal_log_path }}/journal.log"

+ 23
- 0
roles/fetch_journal_log/tasks/main.yaml View File

@@ -0,0 +1,23 @@
1
+- name: Ensure {{ journal_log_path }} exists
2
+  become: yes
3
+  msg: "Ensure logdir {{ journal_log_path }} exists and have correct permission"
4
+  file:
5
+    path: "{{ journal_log_path }}"
6
+    state: directory
7
+    owner: "{{ ansible_user }}"
8
+    group: "{{ ansible_user }}"
9
+    mode: 0775
10
+
11
+- name: Store journal logs in {{ journal_log_file_name }}
12
+  become: yes
13
+  shell:
14
+    cmd: |
15
+      /bin/journalctl -a > {{ journal_log_file_name }}
16
+
17
+- name: Set journal.log file permissions
18
+  become: yes
19
+  file:
20
+    path: '{{ journal_log_file_name }}'
21
+    owner: '{{ ansible_user }}'
22
+    group: '{{ ansible_user }}'
23
+    mode: 0644

+ 8
- 0
roles/setup_logdir/README.rst View File

@@ -0,0 +1,8 @@
1
+Configure logs dir to be accessible for ``stack`` user.
2
+
3
+**Role Variables**
4
+
5
+.. zuul:rolevar:: logdir
6
+   :default: /opt/stack/logs
7
+
8
+   Name of the directory where logs will be stored.

+ 1
- 0
roles/setup_logdir/defaults/main.yaml View File

@@ -0,0 +1 @@
1
+logdir: /opt/stack/logs

+ 9
- 0
roles/setup_logdir/tasks/main.yaml View File

@@ -0,0 +1,9 @@
1
+- name: Ensure logdir exists
2
+  become: yes
3
+  msg: "Ensure logdir {{ logdir }} exists and have correct permission"
4
+  file:
5
+    path: "{{ logdir }}"
6
+    state: directory
7
+    owner: "{{ ansible_user }}"
8
+    group: "{{ ansible_user }}"
9
+    mode: 0775

Loading…
Cancel
Save