Update the tempest install environment

While this should be backported this purpose of this commit
is to unblock gating and provide for the always changing
requirements within tempest while also allowing us to ensure
that we are defcore complaint.

Tempest is not an integrated service within OpenStack and does
not adhere to any given package or requirement set that would
in a real work function with the rest of OpenStack. Because
tempest is intended to be a standalone system that is not
installed along side of the rest of OpenStack it general will
break and or introduce new requirements that break the services
that depend on various versions of packages as found within
global requirements. To fix this issue tempest is now being
installed within a VENV. The virtual environment will ensure
that tempest is installed in a location where it can resolve
its own dependencies without general impact to the rest of the
system.

Additionally, we removed the heat_contrib_extraroute heat
plugin from the build process because its presently
incompatible with PBR >= 0.11.0 which is related to issue
https://bugs.launchpad.net/openstack-ansible/+bug/1450733 .
However we have already built wheels in our repo which will
still allow this contributing plugin to be installed as an
integrated part of the system. Currently, we git clone heat
source onto heat_engine containers and install selected plugins
via a 'python setup.py install'.  This change removes the tasks
that do that and simply adds heat-contrib-extraroute
to heat_pip_packages so it gets installed on all heat-related
containers.  This is actually only required on the heat_engine
containers but the package is tiny and should not cause any issues
being present on the heat_api containers.

Change-Id: Ib972704084ead5748b19362b142fb161fea4a734

defaults/main.yml

@@ -62,12 +62,8 @@ tempest_volume_backup_enabled: False
 
 tempest_main_group: tempest_all
 
-# fixtures, testscenarios, pbr locked due to issue https://bugs.launchpad.net/openstack-ansible/+bug/1450733
-tempest_pip_packages:
-  - "fixtures<1.1.0"
-  - nose
+tempest_requires_pip_packages:
   - oslo.serialization
-  - "pbr>=0.6,!=0.7,<0.11.0"
   - python-barbicanclient
   - python-ceilometerclient
   - python-cinderclient
@@ -87,8 +83,13 @@ tempest_pip_packages:
   - python-troveclient
   - python-tuskarclient
   - python-memcached
+  - virtualenv
+
+tempest_pip_packages:
+  - fixtures
+  - nose
   - testrepository
-  - "testscenarios<=0.4"
+  - testscenarios
   - testtools
 
 tempest_image_dir: "/opt/images"

tasks/tempest_install.yml

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Install pip packages
+- name: Install required pip packages
   pip:
     name: "{{ item }}"
     state: present
@@ -22,9 +22,10 @@
   retries: 5
   delay: 2
   with_items:
-    - "{{ tempest_pip_packages }}"
+    - "{{ tempest_requires_pip_packages }}"
   tags:
-    - tempest-pip-packages
+    - tempest-pip-requires-packages
+    - tempest-pip-install
 
 - name: Get tempest from git
   git:
@@ -38,13 +39,20 @@
   tags:
     - tempest-git-clone
 
-- name: Install local tempest
+- name: Install pip packages for tempest
   pip:
-    name: "{{ tempest_git_dest }}"
+    name: "{{ item }}"
     state: present
+    virtualenv: "{{ tempest_git_dest }}"
+    virtualenv_site_packages: "yes"
+    extra_args: "--isolated"
   register: install_packages
   until: install_packages|success
   retries: 5
   delay: 2
+  with_items:
+    - "{{ tempest_pip_packages }}"
+    - "{{ tempest_git_dest }}"
   tags:
+    - tempest-pip-packages
     - tempest-pip-install

templates/openstack_tempest_gate.sh.j2

@@ -141,6 +141,9 @@ done
 # work in tempest directory
 pushd {{ tempest_git_dest }}
 
+# Load the tempest venv for a tempest run
+source {{ tempest_git_dest }}/bin/activate
+
 # read creds into environment
 source /root/openrc
 
@@ -190,4 +193,7 @@ else
   echo "TEMPEST FAIL $test_list_summary"
 fi
 
+# Deactivate the venv after run
+deactivate || true
+
 exit $result