Convert nova-next to a zuul v3 job

For the most part this should be a pretty straight-forward
port of the run.yaml. The most complicated thing is executing
the post_test_hook.sh script. For that, a new post-run playbook
and role are added.

The relative path to devstack scripts in post_test_hook.sh itself
had to drop the 'new' directory since we are no longer executing
the script through devstack-gate anymore the 'new' path does not
exist.

Change-Id: Ie3dc90862c895a8bd9bff4511a16254945f45478

.zuul.yaml

@@ -181,7 +181,7 @@
 
 - job:
     name: nova-next
-    parent: nova-dsvm-base
+    parent: devstack-tempest
     description: |
       This job was added in Newton when placement and cellsv2
       were optional. Placement and cellsv2 are required starting in Ocata. In
@@ -194,8 +194,52 @@
       volume multi-attach testing.
       Starting in Train, the job enabled counting quota usage from placement.
       Runs all tempest compute API and most scenario tests concurrently.
-    run: playbooks/legacy/nova-next/run.yaml
-    post-run: playbooks/legacy/nova-next/post.yaml
+    irrelevant-files: *dsvm-irrelevant-files
+    # Run post-tempest tests like for nova-manage commands.
+    post-run: playbooks/nova-next/post.yaml
+    vars:
+      # We use the "all" environment for tempest_test_regex and
+      # tempest_black_regex.
+      tox_envlist: all
+      # Run all compute API tests and most scenario tests at the default
+      # concurrency (nproc/2 which is normally 4 in the gate).
+      tempest_test_regex: ^tempest\.(scenario|api\.compute)
+      # The tempest.scenario.test_network* tests are skipped because they
+      # (1) take a long time and (2) are already covered in the
+      # tempest-slow* job. If this regex gets more complicated use
+      # tempest_test_blacklist.
+      tempest_black_regex: ^tempest.scenario.test_network
+      devstack_local_conf:
+        post-config:
+          $NOVA_CONF:
+            compute:
+              # Switch off the provider association refresh, which should
+              # reduce the number of placement calls in steady state. Added in
+              # Stein.
+              # TODO(efried): Due to bug #1802143, nova-cpu.conf gets
+              # overwritten after post-config|$NOVA_CPU_CONF gets effected, so
+              # use $NOVA_CONF instead for now.
+              resource_provider_association_refresh: 0
+            quota:
+              # Added in Train.
+              count_usage_from_placement: True
+            scheduler:
+              # Added in Train.
+              query_placement_for_image_type_support: True
+      devstack_localrc:
+        # Added in Pike.
+        NOVA_USE_SERVICE_TOKEN: True
+        # Enable TLS between the noVNC proxy & compute nodes; this requires
+        # the tls-proxy service to be enabled. Added in Queens.
+        NOVA_CONSOLE_PROXY_COMPUTE_TLS: True
+        # Added in Stein.
+        USE_PYTHON3: True
+        # Added in Stein.
+        ENABLE_VOLUME_MULTIATTACH: True
+      devstack_services:
+        tls-proxy: true
+        # TODO(mriedem): Consider disabling other non-essential services like
+        # c-bak.
 
 - job:
     name: nova-tempest-v2-api

gate/post_test_hook.sh

@@ -34,8 +34,8 @@ function purge_db {
 }
 
 BASE=${BASE:-/opt/stack}
-source ${BASE}/new/devstack/functions-common
-source ${BASE}/new/devstack/lib/nova
+source ${BASE}/devstack/functions-common
+source ${BASE}/devstack/lib/nova
 cell_conf=$(conductor_conf 1)
 # NOTE(danms): We need to pass the main config to get the api db
 # bits, and then also the cell config for the cell1 db (instead of
@@ -49,7 +49,7 @@ purge_db
 set -e
 # We need to get the admin credentials to run the OSC CLIs for Placement.
 set +x
-source $BASE/new/devstack/openrc admin
+source $BASE/devstack/openrc admin
 set -x
 
 # TODO(mriedem): Consider checking for instances in ERROR state because

playbooks/legacy/nova-next/post.yaml

@@ -1,15 +0,0 @@
-- hosts: primary
-  tasks:
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/logs/**
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs

playbooks/legacy/nova-next/run.yaml

@@ -1,85 +0,0 @@
-- hosts: all
-  name: nova-next
-  tasks:
-
-    - name: Ensure workspace directory
-      file:
-        path: '{{ ansible_user_dir }}/workspace'
-        state: directory
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat > clonemap.yaml << EOF
-          clonemap:
-            - name: openstack/devstack-gate
-              dest: devstack-gate
-          EOF
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
-              https://opendev.org \
-              openstack/devstack-gate
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        # Enable TLS between the noVNC proxy & compute nodes; this requires
-        # the tls-proxy service to be enabled.
-
-        # Switch off the provider association refresh, which should reduce the
-        # number of placement calls in steady state.
-        # TODO(efried): Due to bug #1802143, nova-cpu.conf gets overwritten
-        # after post-config|$NOVA_CPU_CONF gets effected, so use $NOVA_CONF
-        # instead for now.
-        cmd: |
-          set -e
-          set -x
-          cat << 'EOF' >>"/tmp/dg-local.conf"
-          [[local|localrc]]
-          NOVA_USE_SERVICE_TOKEN=True
-          NOVA_CONSOLE_PROXY_COMPUTE_TLS=True
-          USE_PYTHON3=True
-          ENABLE_VOLUME_MULTIATTACH=True
-
-          [[post-config|$NOVA_CONF]]
-          [compute]
-          resource_provider_association_refresh = 0
-          [quota]
-          count_usage_from_placement = True
-
-          [scheduler]
-          query_placement_for_image_type_support = True
-
-          EOF
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          export ENABLED_SERVICES=tls-proxy
-          export PYTHONUNBUFFERED=true
-          # Yes we want to run Tempest.
-          export DEVSTACK_GATE_TEMPEST=1
-          # Run all compute API tests and most scenario tests at the default
-          # concurrency (nproc/2 which is normally 4 in the gate).
-          # The tempest.scenario.test_network* tests are skipped because they
-          # (1) take a long time and (2) are already covered in the
-          # tempest-slow* job.
-          export DEVSTACK_GATE_TEMPEST_REGEX="^tempest\.((scenario(?!\.test_network))|(api\.compute))"
-          # The post_test_hook runs some post-test CLIs for things that
-          # Tempest does not test, like archiving deleted records.
-          function post_test_hook {
-              if [ -f $BASE/new/nova/gate/post_test_hook.sh ]; then
-                  $BASE/new/nova/gate/post_test_hook.sh
-              fi
-          }
-          export -f post_test_hook
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
-          ./safe-devstack-vm-gate-wrap.sh
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'

playbooks/nova-next/post.yaml

@@ -0,0 +1,4 @@
+- hosts: controller
+  roles:
+    # Runs gate/post_test_hook.sh to test things like nova-manage commands.
+    - run-post-test-hook

roles/run-post-test-hook/README.rst

@@ -0,0 +1,8 @@
+Runs gate/post_test_hook.sh.
+
+**Role Variables**
+
+.. zuul:rolevar:: devstack_base_dir
+   :default: /opt/stack
+
+   The devstack base directory.

roles/run-post-test-hook/defaults/main.yaml

@@ -0,0 +1 @@
+devstack_base_dir: /opt/stack

roles/run-post-test-hook/tasks/main.yaml

@@ -0,0 +1,3 @@
+- name: Run post_test_hook.sh
+  command: "{{ devstack_base_dir }}/nova/gate/post_test_hook.sh"
+  become: yes