Migrate neutron-functional job to zuul v3 syntax

This patch migrates also neutron-functional-python27 job to
zuul v3 syntax.

Neutron's functional tests have to be run on host with
deployed minimal devstack but without running any e.g. neutron
services. Because of that new job's template inherits from
devstack-minimal job instead of devstack-tox-functional and
we need to have own run_functional_job and post_functional_job
playbooks.

It also adds ansible roles:
- configure_functional_tests
- setup_logdir
- fetch_journal_log

which are used in new neutron-functional job definition.
Those roles can be also used later e.g. for fullstack job.

Change-Id: I80bc17c8c9f43050ac0c21176fbc4be46c11ce35
Related-bug: #1804844

.zuul.yaml

@@ -102,24 +102,46 @@
 
 - job:
     name: neutron-functional
-    parent: legacy-dsvm-base
+    parent: devstack-minimal
-    run: playbooks/legacy/neutron-functional/run.yaml
+    description: Run neutron functional tests
-    post-run: playbooks/legacy/neutron-functional/post.yaml
     timeout: 7800
     required-projects:
-      - openstack-infra/devstack-gate
+      - git.openstack.org/openstack-dev/devstack
       - openstack/neutron
+    roles:
+      - zuul: openstack-dev/devstack
+    pre-run: playbooks/configure_functional_job.yaml
+    run: playbooks/run_functional_job.yaml
+    post-run: playbooks/post_functional_job.yaml
     irrelevant-files:
       - ^.*\.rst$
       - ^doc/.*$
       - ^neutron/locale/.*$
       - ^releasenotes/.*$
+    vars:
+      devstack_services:
+        # Ignore any default set by devstack. Emit a "disable_all_services".
+        base: false
+      devstack_localrc:
+        USE_PYTHON3: true
+        INSTALL_TESTONLY_PACKAGES: true
+        DATABASE_PASSWORD: stackdb
+      tox_envlist: dsvm-functional
+      tox_install_siblings: false
+      zuul_copy_output:
+        # We need to copy directory with logs to have it in job artifacts also,
+        # /opt/stack/logs is default logs directory defined in neutron's
+        # tox.ini file
+        '{{ devstack_base_dir }}/logs/dsvm-functional-logs': logs
 
 - job:
     name: neutron-functional-python27
     parent: neutron-functional
-    run: playbooks/legacy/neutron-functional-python27/run.yaml
+    description: Run neutron functional tests with Python 2.7
-    post-run: playbooks/legacy/neutron-functional-python27/post.yaml
+    vars:
+      devstack_localrc:
+        USE_PYTHON3: false
+      tox_envlist: dsvm-functional-python27
 
 - job:
     name: neutron-fullstack

neutron/tests/contrib/gate_hook.sh

@@ -2,7 +2,7 @@
 
 set -ex
 
-VENV=${1:-"dsvm-functional"}
+VENV=${1:-"dsvm-fullstack"}
 FLAVOR=${2:-"all"}
 
 GATE_DEST=$BASE/new
@@ -55,7 +55,7 @@ function load_rc_for_rally {
 
 
 case $VENV in
-"dsvm-functional"|"dsvm-functional-python27"|"dsvm-fullstack")
+"dsvm-fullstack")
     # The following need to be set before sourcing
     # configure_for_func_testing.
     GATE_STACK_USER=stack

playbooks/configure_functional_job.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - setup_logdir
+    - configure_functional_tests

playbooks/legacy/neutron-functional-python27/post.yaml

@@ -1,80 +0,0 @@
-- hosts: primary
-  tasks:
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*nose_results.html
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*testr_results.html.gz
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/.testrepository/tmp*
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*testrepository.subunit.gz
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}/tox'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/.tox/*/log/*
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/logs/**
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs

playbooks/legacy/neutron-functional-python27/run.yaml

@@ -1,54 +0,0 @@
-- hosts: all
-  name: Autoconverted job legacy-neutron-dsvm-functional from old job gate-neutron-dsvm-functional-ubuntu-xenial-nv
-  tasks:
-
-    - name: Ensure legacy workspace directory
-      file:
-        path: '{{ ansible_user_dir }}/workspace'
-        state: directory
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat > clonemap.yaml << EOF
-          clonemap:
-            - name: openstack-infra/devstack-gate
-              dest: devstack-gate
-          EOF
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
-              git://git.openstack.org \
-              openstack-infra/devstack-gate
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          export PYTHONUNBUFFERED=true
-          export DEVSTACK_GATE_TEMPEST=0
-          export DEVSTACK_GATE_EXERCISES=0
-          export DEVSTACK_GATE_NEUTRON=1
-          export DEVSTACK_GATE_INSTALL_TESTONLY=1
-          export BRANCH_OVERRIDE=default
-          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
-              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
-          fi
-
-          function gate_hook {
-              bash -xe $BASE/new/neutron/neutron/tests/contrib/gate_hook.sh dsvm-functional-python27
-          }
-          export -f gate_hook
-
-          function post_test_hook {
-              bash -xe $BASE/new/neutron/neutron/tests/contrib/post_test_hook.sh dsvm-functional-python27
-          }
-          export -f post_test_hook
-
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
-          ./safe-devstack-vm-gate-wrap.sh
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'

playbooks/legacy/neutron-functional/post.yaml

@@ -1,96 +0,0 @@
-- hosts: primary
-  tasks:
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*nose_results.html
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*testr_results.html.gz
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/.testrepository/tmp*
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*testrepository.subunit.gz
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}/tox'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/.tox/*/log/*
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    # TODO(slaweq): this should be moved to separate role and used in zuulv3
-    # jobs definitions also
-    - name: Store journal logs in {{ ansible_user_dir }}/workspace/logs/journal.log
-      become: yes
-      shell:
-        cmd: |
-          /bin/journalctl -a > {{ ansible_user_dir }}/workspace/logs/journal.log
-
-    - name: Set journal.log file permissions
-      become: yes
-      file:
-        path: '{{ ansible_user_dir }}/workspace/logs/journal.log'
-        owner: '{{ ansible_user }}'
-        group: '{{ ansible_user }}'
-        mode: 0644
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/logs/**
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs

playbooks/legacy/neutron-functional/run.yaml

@@ -1,55 +0,0 @@
-- hosts: all
-  name: Autoconverted job legacy-neutron-dsvm-functional from old job gate-neutron-dsvm-functional-ubuntu-xenial-nv
-  tasks:
-
-    - name: Ensure legacy workspace directory
-      file:
-        path: '{{ ansible_user_dir }}/workspace'
-        state: directory
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat > clonemap.yaml << EOF
-          clonemap:
-            - name: openstack-infra/devstack-gate
-              dest: devstack-gate
-          EOF
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
-              git://git.openstack.org \
-              openstack-infra/devstack-gate
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          export USE_PYTHON3=true
-          export PYTHONUNBUFFERED=true
-          export DEVSTACK_GATE_TEMPEST=0
-          export DEVSTACK_GATE_EXERCISES=0
-          export DEVSTACK_GATE_NEUTRON=1
-          export DEVSTACK_GATE_INSTALL_TESTONLY=1
-          export BRANCH_OVERRIDE=default
-          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
-              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
-          fi
-
-          function gate_hook {
-              bash -xe $BASE/new/neutron/neutron/tests/contrib/gate_hook.sh dsvm-functional
-          }
-          export -f gate_hook
-
-          function post_test_hook {
-              bash -xe $BASE/new/neutron/neutron/tests/contrib/post_test_hook.sh dsvm-functional
-          }
-          export -f post_test_hook
-
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
-          ./safe-devstack-vm-gate-wrap.sh
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'

playbooks/post_functional_job.yaml

@@ -0,0 +1,5 @@
+- hosts: all
+  roles:
+    - fetch_journal_log
+    - fetch-tox-output
+    - fetch-subunit-output

playbooks/run_functional_job.yaml

@@ -0,0 +1,10 @@
+- hosts: all
+  roles:
+    - run-devstack
+    # Run bindep and test-setup after devstack so that they won't interfere
+    - role: bindep
+      bindep_profile: test
+      bindep_dir: "{{ zuul_work_dir }}"
+    - test-setup
+    - ensure-tox
+    - tox

roles/configure_functional_tests/README.rst

@@ -0,0 +1,18 @@
+Configure host to run on it Neutron functional/fullstack tests
+
+**Role Variables**
+
+.. zuul:rolevar:: tests_venv
+   :default: {{ tox_envlist }}
+
+.. zuul:rolevar:: base_dir
+   :default: {{ ansible_user_dir }}/src/git.openstack.org
+
+.. zuul:rolevar:: gate_dest_dir
+   :default: {{ base_dir }}/openstack
+
+.. zuul:rolevar:: devstack_dir
+   :default: {{ base_dir }}/openstack-dev/devstack
+
+.. zuul:rolevar:: neutron_dir
+   :default: {{ gate_dest_dir }}/neutron

roles/configure_functional_tests/defaults/main.yaml

@@ -0,0 +1,5 @@
+tests_venv: "{{ tox_envlist }}"
+base_dir: "{{ ansible_user_dir }}/src/git.openstack.org"
+gate_dest_dir: "{{ base_dir }}/openstack"
+devstack_dir: "{{ base_dir }}/openstack-dev/devstack"
+neutron_dir: "{{ gate_dest_dir }}/neutron"

roles/configure_functional_tests/tasks/main.yaml

@@ -0,0 +1,19 @@
+- shell:
+    cmd: |
+        set -e
+        set -x
+        GATE_STACK_USER={{ ansible_user }}
+        IS_GATE=True
+
+        BASE_DIR={{ base_dir }}
+        GATE_DEST={{ gate_dest_dir }}
+        NEUTRON_PATH={{ neutron_dir }}
+        DEVSTACK_PATH={{ devstack_dir }}
+        VENV={{ tests_venv }}
+
+        source $DEVSTACK_PATH/functions
+        source $NEUTRON_PATH/devstack/lib/ovs
+        source $NEUTRON_PATH/tools/configure_for_func_testing.sh
+
+        configure_host_for_func_testing
+    executable: /bin/bash

roles/fetch_journal_log/README.rst

@@ -0,0 +1,16 @@
+Collect journal log from test run
+
+By default, this stores journal log into log file and store it in
+"journal_log_file_path"
+
+**Role Variables**
+
+.. zuul:rolevar:: journal_log_path
+   :default: {{ ansible_user_dir }}/workspace/logs
+
+   Path where journal log file will be stored on job's node.
+
+.. zuul:rolevar:: journal_log_file_name
+   :default: {{ journal_log_path }}/journal.log
+
+   Name of journal log file.

roles/fetch_journal_log/defaults/main.yaml

@@ -0,0 +1,2 @@
+journal_log_path: "{{ ansible_user_dir }}/logs"
+journal_log_file_name: "{{ journal_log_path }}/journal.log"

roles/fetch_journal_log/tasks/main.yaml

@@ -0,0 +1,23 @@
+- name: Ensure {{ journal_log_path }} exists
+  become: yes
+  msg: "Ensure logdir {{ journal_log_path }} exists and have correct permission"
+  file:
+    path: "{{ journal_log_path }}"
+    state: directory
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    mode: 0775
+
+- name: Store journal logs in {{ journal_log_file_name }}
+  become: yes
+  shell:
+    cmd: |
+      /bin/journalctl -a > {{ journal_log_file_name }}
+
+- name: Set journal.log file permissions
+  become: yes
+  file:
+    path: '{{ journal_log_file_name }}'
+    owner: '{{ ansible_user }}'
+    group: '{{ ansible_user }}'
+    mode: 0644

roles/setup_logdir/README.rst

@@ -0,0 +1,8 @@
+Configure logs dir to be accessible for ``stack`` user.
+
+**Role Variables**
+
+.. zuul:rolevar:: logdir
+   :default: /opt/stack/logs
+
+   Name of the directory where logs will be stored.

roles/setup_logdir/defaults/main.yaml

@@ -0,0 +1 @@
+logdir: /opt/stack/logs

roles/setup_logdir/tasks/main.yaml

@@ -0,0 +1,9 @@
+- name: Ensure logdir exists
+  become: yes
+  msg: "Ensure logdir {{ logdir }} exists and have correct permission"
+  file:
+    path: "{{ logdir }}"
+    state: directory
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    mode: 0775