Migrate jobs to zuulV3

This migrates the converted legacy jobs to the keystone
repo. This follows Step 1 in the legacy migration docs[0].

[0] https://docs.openstack.org/infra/manual/zuulv3.html#legacy-job-migration-details

Needed-By: I33e09b0f7835eb038c76bb5c7343570762e52ab0
Needed-By: I3ca75444be1961c4870d3afd07febaa70ead05fb
Change-Id: If0d0db326f25a95f6a0c16f49c841a4db7cb6022

.zuul.yaml

@@ -11,6 +11,62 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- job:
+    name: keystone-dsvm-functional
+    parent: legacy-dsvm-base
+    run: playbooks/legacy/keystone-dsvm-functional/run.yaml
+    post-run: playbooks/legacy/keystone-dsvm-functional/post.yaml
+    timeout: 4200
+    required-projects:
+      - openstack-infra/devstack-gate
+      - openstack/keystone
+      - openstack/keystone-tempest-plugin
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^api-ref/.*$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^keystone/tests/unit/.*$
+      - ^releasenotes/.*$
+
+- job:
+    name: keystone-dsvm-functional-v3-only
+    parent: legacy-dsvm-base
+    run: playbooks/legacy/keystone-dsvm-functional-v3-only/run.yaml
+    post-run: playbooks/legacy/keystone-dsvm-functional-v3-only/post.yaml
+    timeout: 4200
+    voting: false
+    required-projects:
+      - openstack-infra/devstack-gate
+      - openstack/keystone
+      - openstack/keystone-tempest-plugin
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^api-ref/.*$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^keystone/tests/unit/.*$
+      - ^releasenotes/.*$
+
+- job:
+    name: keystone-dsvm-py35-functional-v3-only
+    parent: legacy-dsvm-base
+    run: playbooks/legacy/keystone-dsvm-py35-functional-v3-only/run.yaml
+    post-run: playbooks/legacy/keystone-dsvm-py35-functional-v3-only/post.yaml
+    timeout: 4200
+    voting: false
+    required-projects:
+      - openstack-infra/devstack-gate
+      - openstack/keystone
+      - openstack/keystone-tempest-plugin
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^api-ref/.*$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^keystone/tests/unit/.*$
+      - ^releasenotes/.*$
+
 # This job will execute 'tox -e upgrade' from the OSA
 # repo specified in 'osa_test_repo'.
 - job:
@@ -30,8 +86,55 @@
       tox_env: upgrade
       osa_test_repo: "openstack/openstack-ansible-os_keystone"
 
+# Experimental
+- job:
+    name: keystone-tox-patch_cover
+    parent: legacy-base
+    run: playbooks/legacy/keystone-tox-patch_cover/run.yaml
+    post-run: playbooks/legacy/keystone-tox-patch_cover/post.yaml
+    timeout: 2400
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^api-ref/.*$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^keystone/tests/unit/.*$
+      - ^releasenotes/.*$
+    required-projects:
+      - openstack/requirements
+
+# Experimental
+- job:
+    name: keystone-dsvm-grenade-multinode
+    parent: legacy-dsvm-base-multinode
+    run: playbooks/legacy/keystone-dsvm-grenade-multinode/run.yaml
+    post-run: playbooks/legacy/keystone-dsvm-grenade-multinode/post.yaml
+    timeout: 10800
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^api-ref/.*$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^keystone/tests/unit/.*$
+      - ^releasenotes/.*$
+    required-projects:
+      - openstack-dev/grenade
+      - openstack-infra/devstack-gate
+      - openstack/keystone
+    nodeset: legacy-ubuntu-xenial-2-node
+
 - project:
     name: openstack/keystone
     check:
       jobs:
+        - keystone-dsvm-functional
+        - keystone-dsvm-functional-v3-only
+        - keystone-dsvm-py35-functional-v3-only
         - openstack-ansible-keystone-rolling-upgrade
+    gate:
+      jobs:
+        - keystone-dsvm-functional
+    experimental:
+      jobs:
+        - keystone-tox-patch_cover
+        - keystone-dsvm-grenade-multinode

playbooks/legacy/keystone-dsvm-functional-v3-only/post.yaml

@@ -0,0 +1,15 @@
+- hosts: primary
+  tasks:
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/logs/**
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs

playbooks/legacy/keystone-dsvm-functional-v3-only/run.yaml

@@ -0,0 +1,60 @@
+- hosts: all
+  name: Autoconverted job legacy-keystone-dsvm-functional-v3-only from old job gate-keystone-dsvm-functional-v3-only-ubuntu-xenial-nv
+  tasks:
+
+    - name: Ensure legacy workspace directory
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat > clonemap.yaml << EOF
+          clonemap:
+            - name: openstack-infra/devstack-gate
+              dest: devstack-gate
+          EOF
+          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
+              git://git.openstack.org \
+              openstack-infra/devstack-gate
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat << 'EOF' >>"/tmp/dg-local.conf"
+          [[local|localrc]]
+          TEMPEST_PLUGINS='/opt/stack/new/keystone-tempest-plugin'
+          ENABLE_IDENTITY_V2=False
+          enable_plugin keystone git://git.openstack.org/openstack/keystone
+
+          EOF
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          export PYTHONUNBUFFERED=true
+          export PROJECTS="openstack/keystone-tempest-plugin $PROJECTS"
+          export DEVSTACK_GATE_TEMPEST=1
+          export BRANCH_OVERRIDE=default
+          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
+              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
+          fi
+
+          export DEVSTACK_GATE_TEMPEST_REGEX='keystone_tempest_plugin'
+          export ENABLED_SERVICES=keystone-saml2-federation
+
+          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
+          ./safe-devstack-vm-gate-wrap.sh
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'

playbooks/legacy/keystone-dsvm-functional/post.yaml

@@ -0,0 +1,15 @@
+- hosts: primary
+  tasks:
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/logs/**
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs

playbooks/legacy/keystone-dsvm-functional/run.yaml

@@ -0,0 +1,57 @@
+- hosts: all
+  name: Autoconverted job legacy-keystone-dsvm-functional from old job gate-keystone-dsvm-functional-ubuntu-xenial
+  tasks:
+
+    - name: Ensure legacy workspace directory
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat > clonemap.yaml << EOF
+          clonemap:
+            - name: openstack-infra/devstack-gate
+              dest: devstack-gate
+          EOF
+          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
+              git://git.openstack.org \
+              openstack-infra/devstack-gate
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat << 'EOF' >>"/tmp/dg-local.conf"
+          [[local|localrc]]
+          TEMPEST_PLUGINS='/opt/stack/new/keystone-tempest-plugin'
+
+          EOF
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          export PYTHONUNBUFFERED=true
+          export PROJECTS="openstack/keystone-tempest-plugin $PROJECTS"
+          export DEVSTACK_GATE_TEMPEST=1
+          export BRANCH_OVERRIDE=default
+          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
+              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
+          fi
+
+          export DEVSTACK_GATE_TEMPEST_REGEX='keystone_tempest_plugin'
+
+          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
+          ./safe-devstack-vm-gate-wrap.sh
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'

playbooks/legacy/keystone-dsvm-grenade-multinode/post.yaml

@@ -0,0 +1,15 @@
+- hosts: primary
+  tasks:
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/logs/**
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs

playbooks/legacy/keystone-dsvm-grenade-multinode/run.yaml

@@ -0,0 +1,47 @@
+- hosts: primary
+  name: Autoconverted job legacy-keystone-dsvm-grenade-multinode from old job gate-keystone-dsvm-grenade-multinode-ubuntu-xenial-nv
+  tasks:
+
+    - name: Ensure legacy workspace directory
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat > clonemap.yaml << EOF
+          clonemap:
+            - name: openstack-infra/devstack-gate
+              dest: devstack-gate
+          EOF
+          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
+              git://git.openstack.org \
+              openstack-infra/devstack-gate
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          export PYTHONUNBUFFERED=true
+          export DEVSTACK_GATE_CONFIGDRIVE=0
+          export DEVSTACK_GATE_NEUTRON=1
+          export DEVSTACK_GATE_GRENADE=pullup
+          export PROJECTS="openstack-dev/grenade $PROJECTS"
+          export BRANCH_OVERRIDE=default
+          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
+              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
+          fi
+          export DEVSTACK_GATE_TOPOLOGY="multinode"
+          export MULTI_KEYSTONE=1
+          export DEVSTACK_LOCAL_CONFIG="enable_plugin keystone git://git.openstack.org/openstack/keystone"
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"MULTI_KEYSTONE=1"
+          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
+          ./safe-devstack-vm-gate-wrap.sh
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'

playbooks/legacy/keystone-dsvm-py35-functional-v3-only/post.yaml

@@ -0,0 +1,15 @@
+- hosts: primary
+  tasks:
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/logs/**
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs

playbooks/legacy/keystone-dsvm-py35-functional-v3-only/run.yaml

@@ -0,0 +1,68 @@
+- hosts: all
+  name: Autoconverted job legacy-keystone-dsvm-py35-functional-v3-only from old job
+    gate-keystone-dsvm-py35-functional-v3-only-ubuntu-xenial-nv
+  tasks:
+
+    - name: Ensure legacy workspace directory
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat > clonemap.yaml << EOF
+          clonemap:
+            - name: openstack-infra/devstack-gate
+              dest: devstack-gate
+          EOF
+          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
+              git://git.openstack.org \
+              openstack-infra/devstack-gate
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat << 'EOF' >>"/tmp/dg-local.conf"
+          [[local|localrc]]
+          TEMPEST_PLUGINS='/opt/stack/new/keystone-tempest-plugin'
+          # swift is not ready for python3 yet
+          disable_service s-account
+          disable_service s-container
+          disable_service s-object
+          disable_service s-proxy
+          ENABLE_IDENTITY_V2=False
+          enable_plugin keystone git://git.openstack.org/openstack/keystone
+
+          EOF
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          export DEVSTACK_GATE_USE_PYTHON3=True
+          export PYTHONUNBUFFERED=true
+          export PROJECTS="openstack/keystone-tempest-plugin $PROJECTS"
+          export DEVSTACK_GATE_TEMPEST=1
+
+          export BRANCH_OVERRIDE=default
+          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
+              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
+          fi
+
+          export DEVSTACK_GATE_TEMPEST_REGEX='keystone_tempest_plugin'
+          export ENABLED_SERVICES=keystone-saml2-federation
+
+          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
+          ./safe-devstack-vm-gate-wrap.sh
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'

playbooks/legacy/keystone-tox-patch_cover/post.yaml

@@ -0,0 +1,67 @@
+- hosts: primary
+  tasks:
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=**/*nose_results.html
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=**/*testr_results.html.gz
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/.testrepository/tmp*
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=**/*testrepository.subunit.gz
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}/tox'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/.tox/*/log/*
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs

playbooks/legacy/keystone-tox-patch_cover/run.yaml

@@ -0,0 +1,86 @@
+- hosts: all
+  name: Autoconverted job legacy-keystone-tox-patch_cover from old job gate-keystone-tox-patch_cover-ubuntu-xenial
+  tasks:
+
+    - name: Ensure legacy workspace directory
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          CLONEMAP=`mktemp`
+          REQS_DIR=`mktemp -d`
+          function cleanup {
+              mkdir -p $WORKSPACE
+              rm -rf $CLONEMAP $REQS_DIR
+          }
+          trap cleanup EXIT
+          cat > $CLONEMAP << EOF
+          clonemap:
+            - name: $ZUUL_PROJECT
+              dest: .
+          EOF
+          # zuul cloner works poorly if there are 2 names that are the
+          # same in here.
+          if [[ "$ZUUL_PROJECT" != "openstack/requirements" ]]; then
+          cat >> $CLONEMAP << EOF
+            - name: openstack/requirements
+              dest: $REQS_DIR
+          EOF
+          fi
+          /usr/zuul-env/bin/zuul-cloner -m $CLONEMAP --cache-dir /opt/git \
+              git://git.openstack.org $ZUUL_PROJECT openstack/requirements
+          # REQS_DIR is not set for openstack/requirements and there is also
+          # no need to copy in this case.
+          if [[ "$ZUUL_PROJECT" != "openstack/requirements" ]]; then
+              cp $REQS_DIR/upper-constraints.txt ./
+          fi
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: /usr/local/jenkins/slave_scripts/install-distro-packages.sh
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          if [ -x tools/test-setup.sh ] ; then
+            tools/test-setup.sh
+          fi
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -x
+          sudo rm -f /etc/sudoers.d/zuul
+          # Prove that general sudo access is actually revoked
+          ! sudo -n true
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: /usr/local/jenkins/slave_scripts/run-tox.sh patch_cover
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          OUT=`git ls-files --other --exclude-standard --directory`
+          if [ -z "$OUT" ]; then
+              echo "No extra files created during test."
+              exit 0
+          else
+              echo "The following un-ignored files were created during the test:"
+              echo "$OUT"
+              exit 0  # TODO: change to 1 to fail tests.
+          fi
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'