Functional Job fix

This patch includes various changes in order to get the functional
job gate for the neutron-classifier to work as expected.

Previous patch includes changes made to openstack tox py36 and py37
jobs. Link to merged patch: https://review.opendev.org/#/c/672725/
Due to the dependency on neutron-lib the patch had to remove
the failing functional job from the gate in order to get merged.

Changes Include:
 - removes files that were used by the Zuul v2 functional test
and are not needed anymore.
 - adds dsvm-functional tox.ini changes as carried out by
 https://review.opendev.org/#/c/674336/ patch.
- remove unneeded extra job variables, the defaults are just fine
  and include these settings
- make test_path overridable in .stestr.conf
- make job voting

Change-Id: I180ac6df807864a4fd2b129ab525c7e78bca67c0
Co-Authored-By: Andreas Jaeger <aj@suse.com>

.stestr.conf

@@ -1,3 +1,3 @@
 [DEFAULT]
-test_path=./neutron_classifier/tests/unit
+test_path=${OS_TEST_PATH:-./neutron_classifier/tests/unit}
 top_dir=./

.zuul.yaml

@@ -2,57 +2,20 @@
     templates:
       - build-openstack-docs-pti
       - check-requirements
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
       - openstack-python3-train-jobs-neutron
       - openstack-lower-constraints-jobs-neutron
     check:
       jobs:
-        - neutron-classifier-functional:
-            voting: false
+        - neutron-classifier-functional
     gate:
       queue: neutron-classifier
+      jobs:
+        - neutron-classifier-functional
 
 - job:
     name: neutron-classifier-functional
     parent: neutron-functional
-    timeout: 7800
-    required-projects:
-      - openstack/keystone
-      - openstack/neutron
-      - openstack/glance
-      - openstack/nova
-      - openstack/placement
-      - openstack/python-openstackclient
     vars:
+      project_name: neutron-classifier
       tox_install_siblings: true
-      devstack_localrc:
-        LIBS_FROM_GIT: python-openstackclient
-        GLANCE_V1_ENABLED: true
-      devstack_services:
-        dstat: true
-        etcd3: true
-        mysql: true
-        peakmem_tracker: true
-        rabbit: true
-        tls-proxy: true
-        key: true
-        n-api: true
-        n-api-meta: true
-        n-cauth: true
-        n-cond: true
-        n-cpu: true
-        n-novnc: true
-        n-obj: true
-        n-sch: true
-        placement-api: true
-        q-agt: true
-        q-dhcp: true
-        q-l3: true
-        q-meta: true
-        q-metering: true
-        q-svc: true
-        horizon: false
-        tempest: false
-      osc_environment:
-        PYTHONUNBUFFERED: 'true'
-        OS_CLOUD: devstack-admin

neutron_classifier/tests/contrib/gate_hook.sh

@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-
-set -ex
-
-VENV=${1:-"dsvm-functional"}
-
-GATE_DEST=$BASE/new
-NEUTRON_PATH=$GATE_DEST/neutron
-NETWORKING_CCF_PATH=$GATE_DEST/neutron-classifier
-GATE_HOOKS=$NETWORKING_CCF_PATH/neutron_classifier/tests/contrib/hooks
-DEVSTACK_PATH=$GATE_DEST/devstack
-LOCAL_CONF=$DEVSTACK_PATH/late-local.conf
-DSCONF=/tmp/devstack-tools/bin/dsconf
-
-# Install devstack-tools used to produce local.conf; we can't rely on
-# test-requirements.txt because the gate hook is triggered before neutron is
-# installed
-sudo -H pip install virtualenv
-virtualenv /tmp/devstack-tools
-/tmp/devstack-tools/bin/pip install -U devstack-tools==0.4.0
-
-case $VENV in
-"dsvm-functional"|"dsvm-fullstack")
-    # The following need to be set before sourcing
-    # configure_for_func_testing.
-    GATE_STACK_USER=stack
-    PROJECT_NAME=neutron-classifier
-    IS_GATE=True
-    LOCAL_CONF=$DEVSTACK_PATH/local.conf
-
-    source $DEVSTACK_PATH/functions
-
-    source $NEUTRON_PATH/devstack/lib/ovs
-    source $NEUTRON_PATH/tools/configure_for_func_testing.sh
-
-    configure_host_for_func_testing
-
-    # Make the workspace owned by the stack user
-    sudo chown -R $STACK_USER:$STACK_USER $BASE
-    ;;
-"dsvm-neutron-classifier")
-    export DEVSTACK_LOCALCONF=$(cat $LOCAL_CONF)
-    $BASE/new/devstack-gate/devstack-vm-gate.sh
-    ;;
-*)
-    echo "Unrecognized environment $VENV".
-    exit 1
-esac
-

neutron_classifier/tests/contrib/post_test_hook.sh

@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-
-set -xe
-
-CCF_DIR="$BASE/new/neutron-classifier"
-SCRIPTS_DIR="/usr/os-testr-env/bin/"
-
-venv=${1:-"dsvm-functional"}
-
-function generate_testr_results {
-    # Give job user rights to access tox logs
-    sudo -H -u $owner chmod o+rw .
-    sudo -H -u $owner chmod o+rw -R .stestr
-    if [ -f ".stestr/0" ] ; then
-        .tox/$venv/bin/subunit-1to2 < .stestr/0 > ./stestr.subunit
-        $SCRIPTS_DIR/subunit2html ./stestr.subunit testr_results.html
-        gzip -9 ./stestr.subunit
-        gzip -9 ./testr_results.html
-        sudo mv ./*.gz /opt/stack/logs/
-    fi
-}
-
-if [[ "$venv" == dsvm-functional* ]] || [[ "$venv" == dsvm-fullstack* ]]
-then
-    owner=stack
-    sudo_env=
-
-    # Set owner permissions according to job's requirements.
-    cd $CCF_DIR
-    sudo chown -R $owner:stack $CCF_DIR
-
-    # Run tests
-    echo "Running neutron-classifier $venv test suite"
-    set +e
-    sudo -H -u $owner $sudo_env tox -e $venv
-    testr_exit_code=$?
-    set -e
-
-    # Collect and parse results
-    generate_testr_results
-    exit $testr_exit_code
-fi

playbooks/legacy/neutron-classifier-functional-dsvm/post.yaml

@@ -1,80 +0,0 @@
-- hosts: primary
-  tasks:
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*nose_results.html
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*testr_results.html.gz
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/.testrepository/tmp*
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*testrepository.subunit.gz
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}/tox'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/.tox/*/log/*
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/logs/**
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs

playbooks/legacy/neutron-classifier-functional-dsvm/run.yaml

@@ -1,70 +0,0 @@
-- hosts: all
-  name: neutron-classifier-functional-dsvm
-  tasks:
-
-    - name: Ensure legacy workspace directory
-      file:
-        path: '{{ ansible_user_dir }}/workspace'
-        state: directory
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat > clonemap.yaml << EOF
-          clonemap:
-            - name: openstack/devstack-gate
-              dest: devstack-gate
-          EOF
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
-              https://opendev.org \
-              openstack/devstack-gate
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat << 'EOF' >>"/tmp/dg-local.conf"
-          [[local|localrc]]
-          enable_plugin neutron-classifier https://opendev.org/x/neutron-classifier
-          EOF
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          export PYTHONUNBUFFERED=true
-          export DEVSTACK_GATE_UNSTACK=1
-          export DEVSTACK_GATE_TEMPEST=0
-          export DEVSTACK_GATE_EXERCISES=0
-          export DEVSTACK_GATE_NEUTRON=1
-          export DEVSTACK_GATE_INSTALL_TESTONLY=1
-          export BRANCH_OVERRIDE=default
-          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
-              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
-          fi
-          # Because we are testing a non standard project, add
-          # our project repository. This makes zuul do the right
-          # reference magic for testing changes.
-          export PROJECTS="x/neutron-classifier $PROJECTS"
-          # Keep localrc to be able to set some vars in pre_test_hook
-          export KEEP_LOCALRC=1
-          function gate_hook {
-              bash -xe $BASE/new/neutron-classifier/neutron_classifier/tests/contrib/gate_hook.sh dsvm-functional
-          }
-          export -f gate_hook
-          function post_test_hook {
-             bash -xe $BASE/new/neutron-classifier/neutron_classifier/tests/contrib/post_test_hook.sh dsvm-functional
-          }
-          export -f post_test_hook
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
-          ./safe-devstack-vm-gate-wrap.sh
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'

tox.ini

@@ -53,37 +53,19 @@ commands = {posargs}
 commands =
     pip install -q -e "git+https://opendev.org/openstack/neutron#egg=neutron"
 
-[testenv:functional]
-basepython = python3
-setenv = {[testenv]setenv}
-         OS_TEST_TIMEOUT=180
-         OS_TEST_PATH=./neutron_classifier/tests/functional
-         OS_LOG_PATH={env:OS_LOG_PATH:/opt/stack/logs}
-deps =
-  {[testenv]deps}
-  -r{toxinidir}/neutron_classifier/tests/functional/requirements.txt
-commands =
-  stestr run {posargs}
-
-[testenv:functional-py35]
-basepython = python3.5
-setenv = {[testenv]setenv}
-         OS_TEST_TIMEOUT=180
-         OS_TEST_PATH=./neutron_classifier/tests/functional
-         OS_LOG_PATH={env:OS_LOG_PATH:/opt/stack/logs}
-deps =
-  {[testenv]deps}
-  -r{toxinidir}/neutron_classifier/tests/functional/requirements.txt
-commands = stestr run {posargs}
-
 [testenv:dsvm-functional]
-basepython = python2.7
-setenv = {[testenv:functional]setenv}
-         {[testenv:dsvm]setenv}
-         OS_TEST_PATH=./neutron_classifier/tests/functional
-sitepackages=True
+basepython = python3
+setenv =
+    {[testenv]setenv}
+    OS_TEST_PATH=./neutron_classifier/tests/functional
+    OS_SUDO_TESTING=1
+    OS_FAIL_ON_MISSING_DEPS=1
+    OS_TEST_TIMEOUT=180
+    OS_TESTR_CONCURRENCY=1
+    OS_LOG_PATH={env:OS_LOG_PATH:/opt/stack/logs}
 deps =
-  {[testenv:functional]deps}
+  {[testenv]deps}
+  -r{toxinidir}/neutron_classifier/tests/functional/requirements.txt
 commands =
   stestr run {posargs}