Begin work to modernize pbr's integration testing

PBR does integration testing by installing all of openstack's python
projects to ensure we can install all of them in a variety of different
ways with different pip versions. Unfortunately, these jobs are still
using devstack-gate. We can simplify them significantly because zuul is
now doing what devstack-gate does and provides repos configured the way
we want them to be.

This change updates the jobs to drop devstack-gate and simply consume
zuul's repo prep.

A follow up will remove the old legacy jobs as they need to be removed
from project-config first.

Needed-By: https://review.opendev.org/745189
Change-Id: I77a38c4611dc28db79d6f25d96ad0de36b224c98

.zuul.yaml

@@ -101,6 +101,113 @@
     run: playbooks/legacy/pbr-installation-upstream-devstack/run.yaml
     post-run: playbooks/legacy/pbr-installation-upstream-devstack/post.yaml
 
+- job:
+    name: pbr-installation-openstack-base
+    timeout: 5400
+    description: |
+      Base job for pbr jobs that install openstack packages with current
+      pbr. This ensures we don't break our ability to install openstack.
+    required-projects:
+      # TODO update this list with current active python projects
+      - openstack/pbr
+      - openstack/tripleo-ci
+      - openstack/aodh
+      - openstack/automaton
+      - openstack/ceilometer
+      - openstack/ceilometermiddleware
+      - openstack/cinder
+      - openstack/cliff
+      - openstack/debtcollector
+      - openstack/dib-utils
+      - openstack/diskimage-builder
+      - openstack/django_openstack_auth
+      - openstack/futurist
+      - openstack/glance
+      - openstack/glance_store
+      - openstack/heat
+      - openstack/heat-cfntools
+      - openstack/heat-templates
+      - openstack/horizon
+      - openstack/ironic
+      - openstack/ironic-lib
+      - openstack/ironic-python-agent
+      - openstack/keystone
+      - openstack/keystoneauth
+      - openstack/keystonemiddleware
+      - openstack/manila
+      - openstack/manila-ui
+      - openstack/neutron
+      - openstack/neutron-vpnaas
+      - openstack/nova
+      - openstack/octavia
+      - openstack/os-apply-config
+      - openstack/os-brick
+      - openstack/os-client-config
+      - openstack/os-collect-config
+      - openstack/os-net-config
+      - openstack/os-refresh-config
+      - openstack/osc-lib
+      - openstack/oslo.cache
+      - openstack/oslo.concurrency
+      - openstack/oslo.config
+      - openstack/oslo.context
+      - openstack/oslo.db
+      - openstack/oslo.i18n
+      - openstack/oslo.log
+      - openstack/oslo.messaging
+      - openstack/oslo.middleware
+      - openstack/oslo.policy
+      - openstack/oslo.reports
+      - openstack/oslo.rootwrap
+      - openstack/oslo.serialization
+      - openstack/oslo.service
+      - openstack/oslo.utils
+      - openstack/oslo.versionedobjects
+      - openstack/oslo.vmware
+      - openstack/pycadf
+      - openstack/python-cinderclient
+      - openstack/python-glanceclient
+      - openstack/python-heatclient
+      - openstack/python-ironicclient
+      - openstack/python-keystoneclient
+      - openstack/python-manilaclient
+      - openstack/python-neutronclient
+      - openstack/python-novaclient
+      - openstack/python-openstackclient
+      - openstack/python-saharaclient
+      - openstack/python-swiftclient
+      - openstack/python-troveclient
+      - openstack/python-zaqarclient
+      - openstack/requirements
+      - openstack/sahara
+      - openstack/sahara-dashboard
+      - openstack/stevedore
+      - openstack/swift
+      - openstack/taskflow
+      - openstack/tempest
+      - openstack/tooz
+      - openstack/tripleo-heat-templates
+      - openstack/tripleo-image-elements
+      - openstack/trove
+      - openstack/trove-dashboard
+      - openstack/zaqar
+
+- job:
+    name: pbr-installation-openstack
+    parent: pbr-installation-openstack-base
+    pre-run: playbooks/pbr-installation-openstack/pre.yaml
+    run: playbooks/pbr-installation-openstack/run.yaml
+    vars:
+      pbr_pip_version: ''
+
+- job:
+    name: pbr-installation-openstack-pip-dev
+    description: |
+      This job runs the pbr installations with pip trunk.
+    parent: pbr-installation-openstack
+    vars:
+      pbr_pip_version: 'git+https://github.com/pypa/pip.git#egg=pip'
+
 - project:
     templates:
       - lib-forward-testing
@@ -113,9 +220,13 @@
       - publish-openstack-docs-pti
     check:
       jobs:
-        - pbr-installation-devstack
-        - pbr-installation-upstream-devstack
+        - pbr-installation-openstack
+        - pbr-installation-openstack-pip-dev
     gate:
       jobs:
-        - pbr-installation-devstack
-        - pbr-installation-upstream-devstack
+        - pbr-installation-openstack
+        - pbr-installation-openstack-pip-dev
+    periodic:
+      jobs:
+        - pbr-installation-openstack
+        - pbr-installation-openstack-pip-dev

playbooks/pbr-installation-openstack/pre.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - ensure-pip
+    - ensure-virtualenv

playbooks/pbr-installation-openstack/run.yaml

@@ -0,0 +1,6 @@
+- hosts: all
+  tasks:
+    - shell:
+        cmd: |
+          export PBR_PIP_VERSION="{{ pbr_pip_version }}"
+          bash -xe /home/zuul/src/opendev.org/openstack/pbr/tools/integration.sh $(cat /home/zuul/src/opendev.org/openstack/requirements/projects.txt)

tools/integration.sh

@@ -13,7 +13,7 @@ function mkvenv {
     venv=$1
 
     rm -rf $venv
-    virtualenv $venv
+    virtualenv -p python3 $venv
     $venv/bin/pip install $PIPFLAGS -U $PIPVERSION wheel requests
 
     # If a change to PBR is being tested, preinstall the wheel for it
@@ -22,11 +22,11 @@ function mkvenv {
     fi
 }
 
-# BASE should be a directory with a subdir called "new" and in that
+# BASE should be a directory with a subdir called "openstack" and in that
 #      dir, there should be a git repository for every entry in PROJECTS
-BASE=${BASE:-/opt/stack}
+BASE=${BASE:-/home/zuul/src/opendev.org/}
 
-REPODIR=${REPODIR:-$BASE/new}
+REPODIR=${REPODIR:-$BASE/openstack}
 
 # TODO: Figure out how to get this on to the box properly
 sudo apt-get update