Update the tempest install environment

While this should be backported this purpose of this commit
is to unblock gating and provide for the always changing
requirements within tempest while also allowing us to ensure
that we are defcore complaint.

Tempest is not an integrated service within OpenStack and does
not adhere to any given package or requirement set that would
in a real work function with the rest of OpenStack. Because
tempest is intended to be a standalone system that is not
installed along side of the rest of OpenStack it general will
break and or introduce new requirements that break the services
that depend on various versions of packages as found within
global requirements. To fix this issue tempest is now being
installed within a VENV. The virtual environment will ensure
that tempest is installed in a location where it can resolve
its own dependencies without general impact to the rest of the
system.

Additionally, we removed the heat_contrib_extraroute heat
plugin from the build process because its presently
incompatible with PBR >= 0.11.0 which is related to issue
https://bugs.launchpad.net/openstack-ansible/+bug/1450733 .
However we have already built wheels in our repo which will
still allow this contributing plugin to be installed as an
integrated part of the system. Currently, we git clone heat
source onto heat_engine containers and install selected plugins
via a 'python setup.py install'.  This change removes the tasks
that do that and simply adds heat-contrib-extraroute
to heat_pip_packages so it gets installed on all heat-related
containers.  This is actually only required on the heat_engine
containers but the package is tiny and should not cause any issues
being present on the heat_api containers.

Change-Id: Ib972704084ead5748b19362b142fb161fea4a734

playbooks/roles/os_heat/defaults/main.yml

@@ -149,6 +149,7 @@ heat_pip_packages:
   - python-swiftclient
   - python-troveclient
   - heat
+  - heat-contrib-extraroute
 
 ## Service Names
 heat_service_names:

playbooks/roles/os_heat/tasks/heat_install_plugins.yml

@@ -1,41 +0,0 @@
----
-# Copyright 2014, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Clone heat repo
-  git:
-    repo: "{{ heat_git_repo }}"
-    dest: "{{ heat_git_dest }}"
-    version: "{{ heat_git_install_branch }}"
-  register: git_clone
-  until: git_clone|success
-  retries: 5
-  delay: 2
-  tags:
-    - heat-git-clone
-
-- name: Install git repo plugins
-  shell: |
-    python setup.py install
-  args:
-    chdir: "{{ heat_git_dest }}/{{ item.path }}/{{ item.package }}"
-  with_items: heat_repo_plugins
-  register: pip_install
-  until: pip_install|success
-  retries: 5
-  delay: 2
-  notify:
-    - Restart heat services
-  tags:
-    - heat-plugin-install

playbooks/roles/os_heat/tasks/main.yml

@@ -16,10 +16,6 @@
 - include: heat_pre_install.yml
 - include: heat_install.yml
 
-- include: heat_install_plugins.yml
-  when: >
-    inventory_hostname in groups['heat_engine']
-
 - include: heat_domain_setup.yml
   when: >
     inventory_hostname == groups['heat_all'][0]

playbooks/roles/os_tempest/defaults/main.yml

@@ -62,12 +62,8 @@ tempest_volume_backup_enabled: False
 
 tempest_main_group: tempest_all
 
-# fixtures, testscenarios, pbr locked due to issue https://bugs.launchpad.net/openstack-ansible/+bug/1450733
-tempest_pip_packages:
-  - "fixtures<1.1.0"
-  - nose
+tempest_requires_pip_packages:
   - oslo.serialization
-  - "pbr>=0.6,!=0.7,<0.11.0"
   - python-barbicanclient
   - python-ceilometerclient
   - python-cinderclient
@@ -87,8 +83,13 @@ tempest_pip_packages:
   - python-troveclient
   - python-tuskarclient
   - python-memcached
+  - virtualenv
+
+tempest_pip_packages:
+  - fixtures
+  - nose
   - testrepository
-  - "testscenarios<=0.4"
+  - testscenarios
   - testtools
 
 tempest_image_dir: "/opt/images"

playbooks/roles/os_tempest/tasks/tempest_install.yml

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Install pip packages
+- name: Install required pip packages
   pip:
     name: "{{ item }}"
     state: present
@@ -22,9 +22,10 @@
   retries: 5
   delay: 2
   with_items:
-    - "{{ tempest_pip_packages }}"
+    - "{{ tempest_requires_pip_packages }}"
   tags:
-    - tempest-pip-packages
+    - tempest-pip-requires-packages
+    - tempest-pip-install
 
 - name: Get tempest from git
   git:
@@ -38,13 +39,20 @@
   tags:
     - tempest-git-clone
 
-- name: Install local tempest
+- name: Install pip packages for tempest
   pip:
-    name: "{{ tempest_git_dest }}"
+    name: "{{ item }}"
     state: present
+    virtualenv: "{{ tempest_git_dest }}"
+    virtualenv_site_packages: "yes"
+    extra_args: "--isolated"
   register: install_packages
   until: install_packages|success
   retries: 5
   delay: 2
+  with_items:
+    - "{{ tempest_pip_packages }}"
+    - "{{ tempest_git_dest }}"
   tags:
+    - tempest-pip-packages
     - tempest-pip-install

playbooks/roles/os_tempest/templates/openstack_tempest_gate.sh.j2

@@ -141,6 +141,9 @@ done
 # work in tempest directory
 pushd {{ tempest_git_dest }}
 
+# Load the tempest venv for a tempest run
+source {{ tempest_git_dest }}/bin/activate
+
 # read creds into environment
 source /root/openrc
 
@@ -190,4 +193,7 @@ else
   echo "TEMPEST FAIL $test_list_summary"
 fi
 
+# Deactivate the venv after run
+deactivate || true
+
 exit $result

playbooks/roles/repo_server/defaults/main.yml

@@ -69,12 +69,8 @@ repo_apt_packages:
   - sqlite3
   - vlan
 
-# The version of PBR is being locked down to resolve an upstream issue with
-# all of the services transitioning to Liberty
-# https://bugs.launchpad.net/openstack-ansible/+bug/1450733
 repo_pip_packages:
   - cloudlib
-  - "pbr>=0.6,!=0.7,<0.11.0"
   - PyCrypto
   - python-memcached
   - PyYAML

playbooks/vars/repo_packages/openstack_services.yml

@@ -44,8 +44,14 @@ glance_git_dest: "/opt/glance_{{ glance_git_install_branch | replace('/', '_') }
 heat_git_repo: https://github.com/openstack/heat
 heat_git_install_branch: 5854f1d121d11faebd6e962e251d2e84bda801de # HEAD of "master" as of 20.04.2015
 heat_git_dest: "/opt/heat_{{ heat_git_install_branch | replace('/', '_') }}"
-heat_repo_plugins:
-  - { path: "contrib", package: "extraroute" }
+# The version of PBR >= 0.11.0 breaks this plugin (contrib package) so the package has been removed
+# from the build process until the PBR issues are resolve upstream which is related to all of the
+# services transitioning to Liberty. Note that the package will still be available as its presently
+# part of the upstream index as found here: http://rpc-repo.rackspace.com/pools/heat_contrib_extraroute/
+# BUG: https://bugs.launchpad.net/openstack-ansible/+bug/1450733
+# TODO(cloudnull) Re-enable this plugin once the build issues are resolved upstream.
+# heat_repo_plugins:
+#   - { path: "contrib", package: "extraroute" }
 
 
 ## Horizon service

requirements.txt

@@ -7,5 +7,3 @@ PrettyTable>=0.7,<0.8   # scripts/inventory-manage.py
 pycrypto>=2.6           # ansible
 PyYAML>=3.1.0           # ansible
 requests>=2.2.0,!=2.4.0 # scripts/os-ansible-role-requirements.py
-pbr>=0.6,!=0.7,<0.11.0  # locked to lower than 0.11.0 due to issue https://bugs.launchpad.net/openstack-ansible/+bug/1450733
-