Implement shippable venvs

The change builds venvs in a single repo container and then
ships them to to all targets. The built venvs will be within
the repo servers and will allow for faster deployments,
upgrades, and more consistent deployments for the life cycle
of the deployment.

This will create a versioned tarball that will allow for
greater visablility into the build process as well as giving
deployers/developers the ability to compair a release in
place.

Change-Id: Ieef0b89ebc009d1453c99e19e53a36eb2d70edae
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

playbooks/os-aodh-install.yml

@@ -63,6 +63,7 @@
   roles:
     - role: "os_aodh"
       aodh_venv_tag: "{{ openstack_release }}"
+      aodh_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/aodh-{{ openstack_release }}.tgz"
       tags:
         - "os-aodh"
     - { role: "openstack_openrc", tags: [ "openstack-openrc" ] }

playbooks/os-ceilometer-install.yml

@@ -74,6 +74,7 @@
   roles:
     - role: "os_ceilometer"
       ceilometer_venv_tag: "{{ openstack_release }}"
+      ceilometer_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/ceilometer-{{ openstack_release }}.tgz"
       tags:
         - "os-ceilometer"
     - { role: "openstack_openrc", tags: [ "openstack-openrc" ] }

playbooks/os-cinder-install.yml

@@ -157,6 +157,7 @@
   roles:
     - role: "os_cinder"
       cinder_venv_tag: "{{ openstack_release }}"
+      cinder_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/cinder-{{ openstack_release }}.tgz"
       cinder_galera_address: "{{ internal_lb_vip_address }}"
       cinder_storage_address: "{{ container_address }}"
       cinder_glance_host: "{{ internal_lb_vip_address }}"

playbooks/os-glance-install.yml

@@ -92,6 +92,7 @@
     - role: "os_glance"
       glance_galera_address: "{{ galera_address }}"
       glance_venv_tag: "{{ openstack_release }}"
+      glance_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/glance-{{ openstack_release }}.tgz"
       tags:
         - "os-glance"
     - { role: "openstack_openrc", tags: [ "openstack-openrc" ] }

playbooks/os-heat-install.yml

@@ -74,6 +74,7 @@
   roles:
     - role: "os_heat"
       heat_venv_tag: "{{ openstack_release }}"
+      heat_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/heat-{{ openstack_release }}.tgz"
       heat_galera_address: "{{ galera_address }}"
       tags:
         - "os-heat"

playbooks/os-horizon-install.yml

@@ -76,6 +76,7 @@
       horizon_galera_address: "{{ galera_address }}"
       horizon_server_name: "{{ container_name }}"
       horizon_venv_tag: "{{ openstack_release }}"
+      horizon_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/horizon-{{ openstack_release }}.tgz"
       tags:
         - "os-horizon"
     - role: "rsyslog_client"

playbooks/os-keystone-install.yml

@@ -74,6 +74,7 @@
   roles:
     - role: "os_keystone"
       keystone_venv_tag: "{{ openstack_release }}"
+      keystone_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/keystone-{{ openstack_release }}.tgz"
       keystone_galera_address: "{{ galera_address }}"
       tags:
         - "os-keystone"

playbooks/os-neutron-install.yml

@@ -125,6 +125,7 @@
   roles:
     - role: "os_neutron"
       neutron_venv_tag: "{{ openstack_release }}"
+      neutron_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/neutron-{{ openstack_release }}.tgz"
       neutron_galera_address: "{{ galera_address }}"
       neutron_local_ip: "{{ _local_ip }}"
       neutron_overlay_network: "{{ _overlay_network }}"

playbooks/os-nova-install.yml

@@ -160,6 +160,7 @@
       nova_galera_address: "{{ internal_lb_vip_address }}"
       nova_management_address: "{{ management_address }}"
       nova_venv_tag: "{{ openstack_release }}"
+      nova_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/nova-{{ openstack_release }}.tgz"
       tags:
         - "os-nova"
     - role: "ceph_client"

playbooks/os-swift-setup.yml

@@ -133,6 +133,7 @@
   roles:
     - role: "os_swift"
       swift_venv_tag: "{{ openstack_release }}"
+      swift_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/swift-{{ openstack_release }}.tgz"
       swift_storage_address: "{{ storage_address }}"
       swift_replication_address: "{{ replication_address }}"
       tags:

playbooks/os-tempest-install.yml

@@ -18,7 +18,11 @@
   max_fail_percentage: 20
   user: root
   roles:
-    - { role: "os_tempest", tags: [ "os-tempest" ] }
+    - role: "os_tempest"
+      tempest_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/tempest-{{ openstack_release }}.tgz"
+      tempest_git_repo: "{{ openstack_repo_git_url }}/tempest"
+      tags:
+        - "os-tempest"
     - role: "rsyslog_client"
       rsyslog_client_log_rotate_file: utility_log_rotate
       rsyslog_client_log_dir: "/var/log/utility"

playbooks/plugins/lookups/py_pkgs.py

@@ -107,6 +107,7 @@ class DependencyFileProcessor(object):
         self.pip = dict()
         self.pip['git_package'] = list()
         self.pip['py_package'] = list()
+        self.pip['role_packages'] = dict()
         self.git_pip_install = 'git+%s@%s'
         self.file_names = self._get_files(path=local_path)
 
@@ -231,6 +232,7 @@ class DependencyFileProcessor(object):
             ext=ext
         )
 
+        role_name = None
         for file_name in file_names:
             with open(file_name, 'rb') as f:
                 # If there is an exception loading the file continue
@@ -244,6 +246,11 @@ class DependencyFileProcessor(object):
                     if not loaded_config:
                         continue
 
+                    if 'roles' in file_name:
+                        _role_name = file_name.split('roles%s' % os.sep)[-1]
+                        role_name = _role_name.split(os.sep)[0]
+
+
             for key, values in loaded_config.items():
                 # This conditional is set to ensure we're not processes git repos
                 #  from the defaults file which may conflict with what is being set
@@ -258,6 +265,17 @@ class DependencyFileProcessor(object):
                 if [i for i in BUILT_IN_PIP_PACKAGE_VARS if i in key]:
                     self.pip['py_package'].extend(values)
 
+                    if role_name:
+                        if not role_name in self.pip['role_packages']:
+                            self.pip['role_packages'][role_name] = values
+                        else:
+                            self.pip['role_packages'][role_name].extend(values)
+                            self.pip['role_packages'][role_name] = list(
+                                set(
+                                    self.pip['role_packages'][role_name]
+                                )
+                            )
+
 
 def _abs_path(path):
     return os.path.abspath(
@@ -334,5 +352,6 @@ class LookupModule(object):
                     ) for i in return_data['remote_packages']
                 ]
                 return_data['remote_package_parts'] = remote_package_parts
+                return_data['role_packages'] = dfp.pip['role_packages']
 
                 return [return_data]

playbooks/repo-build.yml

@@ -31,6 +31,9 @@
         - repo-set-requirement-names-filtered
         - repo-set-constraints
         - repo-build-constraints-file
+        - repo-create-venv-archive
+        - repo-venv-compress-archive
+        - repo-build-venvs
   roles:
     - role: "repo_build"
       repo_build_release_tag: "{{ openstack_release }}"

playbooks/roles/os_aodh/defaults/main.yml

@@ -29,6 +29,8 @@ aodh_venv_enabled: true
 #  system path used when the installing.
 aodh_bin: "{{ aodh_venv_bin }}"
 
+aodh_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/aodh.tgz
+
 ## System info
 aodh_system_user_name: aodh
 aodh_system_group_name: aodh
@@ -85,6 +87,7 @@ aodh_apt_packages:
 # aodh packages that must be installed before anything else
 aodh_requires_pip_packages:
   - virtualenv
+  - virtualenv-tools
   - python-keystoneclient # Keystoneclient needed to OSA keystone lib
 
 # Common pip packages

playbooks/roles/os_aodh/tasks/aodh_install.yml

@@ -43,6 +43,58 @@
     - aodh-install
     - aodh-pip-packages
 
+- name: Attempt venv download
+  get_url:
+    url: "{{ aodh_venv_download_url }}"
+    dest: "/var/cache/{{ aodh_venv_download_url | basename }}"
+  ignore_errors: true
+  register: get_venv
+  when: aodh_venv_enabled | bool
+  tags:
+    - aodh-install
+    - aodh-pip-packages
+
+- name: Set aodh get_venv fact
+  set_fact:
+    aodh_get_venv: "{{ get_venv }}"
+  when: aodh_venv_enabled | bool
+  tags:
+    - aodh-install
+    - aodh-pip-packages
+
+- name: Create aodh venv dir
+  file:
+    path: "{{ aodh_venv_bin | dirname }}"
+    state: directory
+  when:
+    - aodh_venv_enabled | bool
+    - aodh_get_venv | success
+  tags:
+    - aodh-install
+    - aodh-pip-packages
+
+- name: Unarchive pre-built venv
+  unarchive:
+    src: "/var/cache/{{ aodh_venv_download_url | basename }}"
+    dest: "{{ aodh_venv_bin | dirname }}"
+    copy: "no"
+  when:
+    - aodh_venv_enabled | bool
+    - aodh_get_venv | success
+  tags:
+    - aodh-install
+    - aodh-pip-packages
+
+- name: Update virtualenv path
+  command: >
+    virtualenv-tools --update-path=auto {{ aodh_venv_bin | dirname }}
+  when:
+    - aodh_venv_enabled | bool
+    - aodh_get_venv | success
+  tags:
+    - aodh-install
+    - aodh-pip-packages
+
 - name: Install pip packages (venv)
   pip:
     name: "{{ item }}"
@@ -56,7 +108,9 @@
   delay: 2
   with_items:
     - "{{ aodh_pip_packages }}"
-  when: aodh_venv_enabled | bool
+  when:
+    - aodh_venv_enabled | bool
+    - aodh_get_venv | failed
   tags:
     - aodh-install
     - aodh-pip-packages

playbooks/roles/os_ceilometer/defaults/main.yml

@@ -29,6 +29,8 @@ ceilometer_venv_enabled: true
 #  system path used when the installing.
 ceilometer_bin: "{{ ceilometer_venv_bin }}"
 
+ceilometer_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/ceilometer.tgz
+
 ## System info
 ceilometer_system_user_name: ceilometer
 ceilometer_system_group_name: ceilometer
@@ -87,6 +89,7 @@ ceilometer_apt_packages:
 # ceilometer packages that must be installed before anything else
 ceilometer_requires_pip_packages:
   - virtualenv
+  - virtualenv-tools
   - python-keystoneclient # Keystoneclient needed to OSA keystone lib
 
 # Common pip packages

playbooks/roles/os_ceilometer/tasks/ceilometer_install.yml

@@ -43,6 +43,58 @@
     - ceilometer-install
     - ceilometer-pip-packages
 
+- name: Attempt venv download
+  get_url:
+    url: "{{ ceilometer_venv_download_url }}"
+    dest: "/var/cache/{{ ceilometer_venv_download_url | basename }}"
+  ignore_errors: true
+  register: get_venv
+  when: ceilometer_venv_enabled | bool
+  tags:
+    - ceilometer-install
+    - ceilometer-pip-packages
+
+- name: Set ceilometer get_venv fact
+  set_fact:
+    ceilometer_get_venv: "{{ get_venv }}"
+  when: ceilometer_venv_enabled | bool
+  tags:
+    - ceilometer-install
+    - ceilometer-pip-packages
+
+- name: Create ceilometer venv dir
+  file:
+    path: "{{ ceilometer_venv_bin | dirname }}"
+    state: directory
+  when:
+    - ceilometer_venv_enabled | bool
+    - ceilometer_get_venv | success
+  tags:
+    - ceilometer-install
+    - ceilometer-pip-packages
+
+- name: Unarchive pre-built venv
+  unarchive:
+    src: "/var/cache/{{ ceilometer_venv_download_url | basename }}"
+    dest: "{{ ceilometer_venv_bin | dirname }}"
+    copy: "no"
+  when:
+    - ceilometer_venv_enabled | bool
+    - ceilometer_get_venv | success
+  tags:
+    - ceilometer-install
+    - ceilometer-pip-packages
+
+- name: Update virtualenv path
+  command: >
+    virtualenv-tools --update-path=auto {{ ceilometer_venv_bin | dirname }}
+  when:
+    - ceilometer_venv_enabled | bool
+    - ceilometer_get_venv | success
+  tags:
+    - ceilometer-install
+    - ceilometer-pip-packages
+
 - name: Install pip packages (venv)
   pip:
     name: "{{ item }}"
@@ -56,7 +108,9 @@
   delay: 2
   with_items:
     - "{{ ceilometer_pip_packages }}"
-  when: ceilometer_venv_enabled | bool
+  when:
+    - ceilometer_venv_enabled | bool
+    - ceilometer_get_venv | failed
   tags:
     - ceilometer-install
     - ceilometer-pip-packages

playbooks/roles/os_cinder/defaults/main.yml

@@ -28,6 +28,8 @@ cinder_venv_enabled: true
 #  system path used when the installing.
 cinder_bin: "{{ cinder_venv_bin }}"
 
+cinder_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/cinder.tgz
+
 # Enable/Disable Ceilometer
 cinder_ceilometer_enabled: False
 
@@ -226,6 +228,7 @@ cinder_apt_packages:
 # Cinder packages that must be installed before anything else
 cinder_requires_pip_packages:
   - virtualenv
+  - virtualenv-tools
   - python-keystoneclient # Keystoneclient needed to OSA keystone lib
 
 # Common pip packages

playbooks/roles/os_cinder/tasks/cinder_install.yml

@@ -52,6 +52,58 @@
     - cinder-install
     - cinder-pip-packages
 
+- name: Attempt venv download
+  get_url:
+    url: "{{ cinder_venv_download_url }}"
+    dest: "/var/cache/{{ cinder_venv_download_url | basename }}"
+  ignore_errors: true
+  register: get_venv
+  when: cinder_venv_enabled | bool
+  tags:
+    - cinder-install
+    - cinder-pip-packages
+
+- name: Set cinder get_venv fact
+  set_fact:
+    cinder_get_venv: "{{ get_venv }}"
+  when: cinder_venv_enabled | bool
+  tags:
+    - cinder-install
+    - cinder-pip-packages
+
+- name: Create cinder venv dir
+  file:
+    path: "{{ cinder_venv_bin | dirname }}"
+    state: directory
+  when:
+    - cinder_venv_enabled | bool
+    - cinder_get_venv | success
+  tags:
+    - cinder-install
+    - cinder-pip-packages
+
+- name: Unarchive pre-built venv
+  unarchive:
+    src: "/var/cache/{{ cinder_venv_download_url | basename }}"
+    dest: "{{ cinder_venv_bin | dirname }}"
+    copy: "no"
+  when:
+    - cinder_venv_enabled | bool
+    - cinder_get_venv | success
+  tags:
+    - cinder-install
+    - cinder-pip-packages
+
+- name: Update virtualenv path
+  command: >
+    virtualenv-tools --update-path=auto {{ cinder_venv_bin | dirname }}
+  when:
+    - cinder_venv_enabled | bool
+    - cinder_get_venv | success
+  tags:
+    - cinder-install
+    - cinder-pip-packages
+
 - name: Install pip packages (venv)
   pip:
     name: "{{ item }}"
@@ -65,7 +117,9 @@
   delay: 2
   with_items:
     - "{{ cinder_pip_packages }}"
-  when: cinder_venv_enabled | bool
+  when:
+    - cinder_venv_enabled | bool
+    - cinder_get_venv | failed
   tags:
     - cinder-install
     - cinder-pip-packages

playbooks/roles/os_glance/defaults/main.yml

@@ -37,6 +37,8 @@ glance_venv_etc_dir: "{{ glance_bin | dirname }}/etc/glance"
 glance_non_venv_etc_dir: "/usr/local/etc/glance"
 glance_etc_dir: "{{ (glance_venv_enabled | bool) | ternary(glance_venv_etc_dir, glance_non_venv_etc_dir) }}"
 
+glance_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/glance.tgz
+
 # Enable/Disable Ceilometer
 glance_ceilometer_enabled: False
 
@@ -175,6 +177,7 @@ glance_apt_packages:
 # Cinder packages that must be installed before anything else
 glance_requires_pip_packages:
   - virtualenv
+  - virtualenv-tools
   - python-keystoneclient # Keystoneclient needed to OSA keystone lib
 
 # Common pip packages

playbooks/roles/os_glance/tasks/glance_install.yml

@@ -52,6 +52,58 @@
     - glance-install
     - glance-pip-packages
 
+- name: Attempt venv download
+  get_url:
+    url: "{{ glance_venv_download_url }}"
+    dest: "/var/cache/{{ glance_venv_download_url | basename }}"
+  ignore_errors: true
+  register: get_venv
+  when: glance_venv_enabled | bool
+  tags:
+    - glance-install
+    - glance-pip-packages
+
+- name: Set glance get_venv fact
+  set_fact:
+    glance_get_venv: "{{ get_venv }}"
+  when: glance_venv_enabled | bool
+  tags:
+    - glance-install
+    - glance-pip-packages
+
+- name: Create glance venv dir
+  file:
+    path: "{{ glance_venv_bin | dirname }}"
+    state: directory
+  when:
+    - glance_venv_enabled | bool
+    - glance_get_venv | success
+  tags:
+    - glance-install
+    - glance-pip-packages
+
+- name: Unarchive pre-built venv
+  unarchive:
+    src: "/var/cache/{{ glance_venv_download_url | basename }}"
+    dest: "{{ glance_venv_bin | dirname }}"
+    copy: "no"
+  when:
+    - glance_venv_enabled | bool
+    - glance_get_venv | success
+  tags:
+    - glance-install
+    - glance-pip-packages
+
+- name: Update virtualenv path
+  command: >
+    virtualenv-tools --update-path=auto {{ glance_venv_bin | dirname }}
+  when:
+    - glance_venv_enabled | bool
+    - glance_get_venv | success
+  tags:
+    - glance-install
+    - glance-pip-packages
+
 - name: Install pip packages (venv)
   pip:
     name: "{{ item }}"
@@ -65,7 +117,9 @@
   delay: 2
   with_items:
     - "{{ glance_pip_packages }}"
-  when: glance_venv_enabled | bool
+  when:
+    - glance_venv_enabled | bool
+    - glance_get_venv | failed
   tags:
     - glance-install
     - glance-pip-packages

playbooks/roles/os_heat/defaults/main.yml

@@ -35,6 +35,8 @@ heat_venv_enabled: true
 #  system path used when the installing.
 heat_bin: "{{ heat_venv_bin }}"
 
+heat_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/heat.tgz
+
 heat_fatal_deprecations: False
 
 heat_clients_endpoint: internalURL
@@ -157,6 +159,7 @@ heat_apt_packages:
 # Heat packages that must be installed before anything else
 heat_requires_pip_packages:
   - virtualenv
+  - virtualenv-tools
   - python-keystoneclient # Keystoneclient needed to OSA keystone lib
 
 # Common pip packages

playbooks/roles/os_heat/tasks/heat_install.yml

@@ -52,6 +52,58 @@
     - heat-install
     - heat-pip-packages
 
+- name: Attempt venv download
+  get_url:
+    url: "{{ heat_venv_download_url }}"
+    dest: "/var/cache/{{ heat_venv_download_url | basename }}"
+  ignore_errors: true
+  register: get_venv
+  when: heat_venv_enabled | bool
+  tags:
+    - heat-install
+    - heat-pip-packages
+
+- name: Set heat get_venv fact
+  set_fact:
+    heat_get_venv: "{{ get_venv }}"
+  when: heat_venv_enabled | bool
+  tags:
+    - heat-install
+    - heat-pip-packages
+
+- name: Create heat venv dir
+  file:
+    path: "{{ heat_venv_bin | dirname }}"
+    state: directory
+  when:
+    - heat_venv_enabled | bool
+    - heat_get_venv | success
+  tags:
+    - heat-install
+    - heat-pip-packages
+
+- name: Unarchive pre-built venv
+  unarchive:
+    src: "/var/cache/{{ heat_venv_download_url | basename }}"
+    dest: "{{ heat_venv_bin | dirname }}"
+    copy: "no"
+  when:
+    - heat_venv_enabled | bool
+    - heat_get_venv | success
+  tags:
+    - heat-install
+    - heat-pip-packages
+
+- name: Update virtualenv path
+  command: >
+    virtualenv-tools --update-path=auto {{ heat_venv_bin | dirname }}
+  when:
+    - heat_venv_enabled | bool
+    - heat_get_venv | success
+  tags:
+    - heat-install
+    - heat-pip-packages
+
 - name: Install pip packages (venv)
   pip:
     name: "{{ item }}"
@@ -65,7 +117,9 @@
   delay: 2
   with_items:
     - "{{ heat_pip_packages }}"
-  when: heat_venv_enabled | bool
+  when:
+    - heat_venv_enabled | bool
+    - heat_get_venv | failed
   tags:
     - heat-install
     - heat-pip-packages

playbooks/roles/os_horizon/defaults/main.yml

@@ -29,6 +29,8 @@ horizon_venv_enabled: true
 #  system path used when the installing.
 horizon_bin: "{{ horizon_venv_bin }}"
 
+horizon_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/horizon.tgz
+
 ## System info
 horizon_system_user_name: horizon
 horizon_system_group_name: www-data
@@ -137,6 +139,7 @@ horizon_apt_packages:
 # horizon packages that must be installed before anything else
 horizon_requires_pip_packages:
   - virtualenv
+  - virtualenv-tools
   - python-keystoneclient # Keystoneclient needed to OSA keystone lib
 
 horizon_pip_packages:

playbooks/roles/os_horizon/tasks/horizon_install.yml

@@ -50,7 +50,9 @@
   delay: 2
   with_items:
     - "{{ horizon_pip_packages }}"
-  when: horizon_venv_enabled | bool
+  when:
+    - horizon_venv_enabled | bool
+    - horizon_get_venv | failed
   tags:
     - horizon-pip-packages
 

playbooks/roles/os_horizon/tasks/horizon_pre_install.yml

@@ -71,6 +71,58 @@
   tags:
     - horizon-pip-packages
 
+- name: Attempt venv download
+  get_url:
+    url: "{{ horizon_venv_download_url }}"
+    dest: "/var/cache/{{ horizon_venv_download_url | basename }}"
+  ignore_errors: true
+  register: get_venv
+  when: horizon_venv_enabled | bool
+  tags:
+    - horizon-install
+    - horizon-pip-packages
+
+- name: Set horizon get_venv fact
+  set_fact:
+    horizon_get_venv: "{{ get_venv }}"
+  when: horizon_venv_enabled | bool
+  tags:
+    - horizon-install
+    - horizon-pip-packages
+
+- name: Create horizon venv dir
+  file:
+    path: "{{ horizon_venv_bin | dirname }}"
+    state: directory
+  when:
+    - horizon_venv_enabled | bool
+    - horizon_get_venv | success
+  tags:
+    - horizon-install
+    - horizon-pip-packages
+
+- name: Unarchive pre-built venv
+  unarchive:
+    src: "/var/cache/{{ horizon_venv_download_url | basename }}"
+    dest: "{{ horizon_venv_bin | dirname }}"
+    copy: "no"
+  when:
+    - horizon_venv_enabled | bool
+    - horizon_get_venv | success
+  tags:
+    - horizon-install
+    - horizon-pip-packages
+
+- name: Update virtualenv path
+  command: >
+    virtualenv-tools --update-path=auto {{ horizon_venv_bin | dirname }}
+  when:
+    - horizon_venv_enabled | bool
+    - horizon_get_venv | success
+  tags:
+    - horizon-install
+    - horizon-pip-packages
+
 - name: Create horizon venv
   pip:
     name: "{{ item }}"
@@ -80,8 +132,11 @@
     extra_args: "{{ pip_install_options|default('') }}"
   with_items:
     - "{{ horizon_requires_pip_packages }}"
-  when: horizon_venv_enabled | bool
+  when:
+    - horizon_venv_enabled | bool
+    - horizon_get_venv | failed
   tags:
+    - horizon-install
     - horizon-pip-packages
 
 - name: Create horizon links for venv
@@ -96,6 +151,7 @@
       dest: "{{ horizon_lib_dir }}"
   when: horizon_venv_enabled | bool
   tags:
+    - horizon-install
     - horizon-configs
 
 - name: Create static horizon dir

playbooks/roles/os_keystone/defaults/main.yml

@@ -29,6 +29,8 @@ keystone_venv_enabled: true
 #  system path used when the installing.
 keystone_bin: "{{ keystone_venv_bin }}"
 
+keystone_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/keystone.tgz
+
 keystone_fatal_deprecations: False
 
 ## System info
@@ -347,6 +349,7 @@ keystone_idp_apt_packages:
 # Keystone packages that must be installed before anything else
 keystone_requires_pip_packages:
   - virtualenv
+  - virtualenv-tools
   - python-keystoneclient # Keystoneclient needed to OSA keystone lib
 
 # Common pip packages

playbooks/roles/os_keystone/tasks/keystone_install.yml

@@ -80,6 +80,58 @@
     - keystone-install
     - keystone-pip-packages
 
+- name: Attempt venv download
+  get_url:
+    url: "{{ keystone_venv_download_url }}"
+    dest: "/var/cache/{{ keystone_venv_download_url | basename }}"
+  ignore_errors: true
+  register: get_venv
+  when: keystone_venv_enabled | bool
+  tags:
+    - keystone-install
+    - keystone-pip-packages
+
+- name: Set keystone get_venv fact
+  set_fact:
+    keystone_get_venv: "{{ get_venv }}"
+  when: keystone_venv_enabled | bool
+  tags:
+    - keystone-install
+    - keystone-pip-packages
+
+- name: Create keystone venv dir
+  file:
+    path: "{{ keystone_venv_bin | dirname }}"
+    state: directory
+  when:
+    - keystone_venv_enabled | bool
+    - keystone_get_venv | success
+  tags:
+    - keystone-install
+    - keystone-pip-packages
+
+- name: Unarchive pre-built venv
+  unarchive:
+    src: "/var/cache/{{ keystone_venv_download_url | basename }}"
+    dest: "{{ keystone_venv_bin | dirname }}"
+    copy: "no"
+  when:
+    - keystone_venv_enabled | bool
+    - keystone_get_venv | success
+  tags:
+    - keystone-install
+    - keystone-pip-packages
+
+- name: Update virtualenv path
+  command: >
+    virtualenv-tools --update-path=auto {{ keystone_venv_bin | dirname }}
+  when:
+    - keystone_venv_enabled | bool
+    - keystone_get_venv | success
+  tags:
+    - keystone-install
+    - keystone-pip-packages
+
 - name: Install pip packages (venv)
   pip:
     name: "{{ item }}"
@@ -93,7 +145,9 @@
   delay: 2
   with_items:
     - "{{ keystone_pip_packages }}"
-  when: keystone_venv_enabled | bool
+  when:
+    - keystone_venv_enabled | bool
+    - keystone_get_venv | failed
   tags:
     - keystone-install
     - keystone-pip-packages

playbooks/roles/os_neutron/defaults/main.yml

@@ -32,6 +32,8 @@ neutron_venv_enabled: true
 #  system path used when the installing.
 neutron_bin: "{{ neutron_venv_bin }}"
 
+neutron_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/neutron.tgz
+
 # Set the lib dir path to that of the local python path where neutron is installed.
 #  This is used for role access to the db migrations.
 #  Example:
@@ -311,6 +313,7 @@ neutron_apt_remove_packages:
 # neutron packages that must be installed before anything else
 neutron_requires_pip_packages:
   - virtualenv
+  - virtualenv-tools
   - python-keystoneclient # Keystoneclient needed to OSA keystone lib
 
 neutron_pip_packages:

playbooks/roles/os_neutron/tasks/neutron_install.yml

@@ -65,6 +65,58 @@
     - neutron-install
     - neutron-pip-packages
 
+- name: Attempt venv download
+  get_url:
+    url: "{{ neutron_venv_download_url }}"
+    dest: "/var/cache/{{ neutron_venv_download_url | basename }}"
+  ignore_errors: true
+  register: get_venv
+  when: neutron_venv_enabled | bool
+  tags:
+    - neutron-install
+    - neutron-pip-packages
+
+- name: Set neutron get_venv fact
+  set_fact:
+    neutron_get_venv: "{{ get_venv }}"
+  when: neutron_venv_enabled | bool
+  tags:
+    - neutron-install
+    - neutron-pip-packages
+
+- name: Create neutron venv dir
+  file:
+    path: "{{ neutron_venv_bin | dirname }}"
+    state: directory
+  when:
+    - neutron_venv_enabled | bool
+    - neutron_get_venv | success
+  tags:
+    - neutron-install
+    - neutron-pip-packages
+
+- name: Unarchive pre-built venv
+  unarchive:
+    src: "/var/cache/{{ neutron_venv_download_url | basename }}"
+    dest: "{{ neutron_venv_bin | dirname }}"
+    copy: "no"
+  when:
+    - neutron_venv_enabled | bool
+    - neutron_get_venv | success
+  tags:
+    - neutron-install
+    - neutron-pip-packages
+
+- name: Update virtualenv path
+  command: >
+    virtualenv-tools --update-path=auto {{ neutron_venv_bin | dirname }}
+  when:
+    - neutron_venv_enabled | bool
+    - neutron_get_venv | success
+  tags:
+    - neutron-install
+    - neutron-pip-packages
+
 - name: Install pip packages (venv)
   pip:
     name: "{{ item }}"
@@ -78,7 +130,9 @@
   delay: 2
   with_items:
     - "{{ neutron_pip_packages }}"
-  when: neutron_venv_enabled | bool
+  when:
+    - neutron_venv_enabled | bool
+    - neutron_get_venv | failed
   tags:
     - neutron-install
     - neutron-pip-packages

playbooks/roles/os_neutron/tasks/plumgrid_config.yml

@@ -35,6 +35,7 @@
   delay: 2
   when:
     - inventory_hostname in groups['neutron_server']
+    - neutron_get_venv | failed
   tags:
     - neutron-install
     - neutron-pip-packages

playbooks/roles/os_nova/defaults/main.yml

@@ -32,6 +32,8 @@ nova_venv_enabled: true
 #  system path used when the installing.
 nova_bin: "{{ nova_venv_bin }}"
 
+nova_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/nova.tgz
+
 nova_fatal_deprecations: False
 
 ## System info
@@ -289,6 +291,7 @@ nova_compute_kvm_apt_packages:
 # nova packages that must be installed before anything else
 nova_requires_pip_packages:
   - virtualenv
+  - virtualenv-tools
   - python-keystoneclient # Keystoneclient needed to OSA keystone lib
 
 nova_compute_pip_packages:

playbooks/roles/os_nova/tasks/nova_compute_kvm_install.yml

@@ -51,7 +51,9 @@
   delay: 2
   with_items:
     - "{{ nova_compute_pip_packages }}"
-  when: nova_venv_enabled | bool
+  when:
+    - nova_venv_enabled | bool
+    - nova_get_venv | failed
   tags:
     - nova-install
     - nova-pip-packages

playbooks/roles/os_nova/tasks/nova_console_novnc_install.yml

@@ -68,7 +68,9 @@
   delay: 2
   with_items:
     - "{{ nova_novnc_pip_packages }}"
-  when: nova_venv_enabled | bool
+  when:
+    - nova_venv_enabled | bool
+    - nova_get_venv | failed
   tags:
     - nova-install
     - nova-pip-packages

playbooks/roles/os_nova/tasks/nova_install.yml

@@ -52,6 +52,58 @@
     - nova-install
     - nova-pip-packages
 
+- name: Attempt venv download
+  get_url:
+    url: "{{ nova_venv_download_url }}"
+    dest: "/var/cache/{{ nova_venv_download_url | basename }}"
+  ignore_errors: true
+  register: get_venv
+  when: nova_venv_enabled | bool
+  tags:
+    - nova-install
+    - nova-pip-packages
+
+- name: Set nova get_venv fact
+  set_fact:
+    nova_get_venv: "{{ get_venv }}"
+  when: nova_venv_enabled | bool
+  tags:
+    - nova-install
+    - nova-pip-packages
+
+- name: Create nova venv dir
+  file:
+    path: "{{ nova_venv_bin | dirname }}"
+    state: directory
+  when:
+    - nova_venv_enabled | bool
+    - nova_get_venv | success
+  tags:
+    - nova-install
+    - nova-pip-packages
+
+- name: Unarchive pre-built venv
+  unarchive:
+    src: "/var/cache/{{ nova_venv_download_url | basename }}"
+    dest: "{{ nova_venv_bin | dirname }}"
+    copy: "no"
+  when:
+    - nova_venv_enabled | bool
+    - nova_get_venv | success
+  tags:
+    - nova-install
+    - nova-pip-packages
+
+- name: Update virtualenv path
+  command: >
+    virtualenv-tools --update-path=auto {{ nova_venv_bin | dirname }}
+  when:
+    - nova_venv_enabled | bool
+    - nova_get_venv | success
+  tags:
+    - nova-install
+    - nova-pip-packages
+
 - name: Install pip packages (venv)
   pip:
     name: "{{ item }}"
@@ -65,7 +117,9 @@
   delay: 2
   with_items:
     - "{{ nova_pip_packages }}"
-  when: nova_venv_enabled | bool
+  when:
+    - nova_venv_enabled | bool
+    - nova_get_venv | failed
   tags:
     - nova-install
     - nova-pip-packages

playbooks/roles/os_swift/defaults/main.yml

@@ -32,6 +32,8 @@ swift_venv_enabled: true
 #  system path used when the installing.
 swift_bin: "{{ swift_venv_bin }}"
 
+swift_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/swift.tgz
+
 # Set the full path to the swift recon cron
 recon_cron_path: "{{ swift_bin }}/swift-recon-cron"
 
@@ -177,6 +179,7 @@ swift: {}
 # swift packages that must be installed before anything else
 swift_requires_pip_packages:
   - virtualenv
+  - virtualenv-tools
   - python-keystoneclient # Keystoneclient needed to OSA keystone lib
 
 swift_pip_packages:

playbooks/roles/os_swift/tasks/swift_install.yml

@@ -52,6 +52,58 @@
     - swift-install
     - swift-pip-packages
 
+- name: Attempt venv download
+  get_url:
+    url: "{{ swift_venv_download_url }}"
+    dest: "/var/cache/{{ swift_venv_download_url | basename }}"
+  ignore_errors: true
+  register: get_venv
+  when: swift_venv_enabled | bool
+  tags:
+    - swift-install
+    - swift-pip-packages
+
+- name: Set swift get_venv fact
+  set_fact:
+    swift_get_venv: "{{ get_venv }}"
+  when: swift_venv_enabled | bool
+  tags:
+    - swift-install
+    - swift-pip-packages
+
+- name: Create swift venv dir
+  file:
+    path: "{{ swift_venv_bin | dirname }}"
+    state: directory
+  when:
+    - swift_venv_enabled | bool
+    - swift_get_venv | success
+  tags:
+    - swift-install
+    - swift-pip-packages
+
+- name: Unarchive pre-built venv
+  unarchive:
+    src: "/var/cache/{{ swift_venv_download_url | basename }}"
+    dest: "{{ swift_venv_bin | dirname }}"
+    copy: "no"
+  when:
+    - swift_venv_enabled | bool
+    - swift_get_venv | success
+  tags:
+    - swift-install
+    - swift-pip-packages
+
+- name: Update virtualenv path
+  command: >
+    virtualenv-tools --update-path=auto {{ swift_venv_bin | dirname }}
+  when:
+    - swift_venv_enabled | bool
+    - swift_get_venv | success
+  tags:
+    - swift-install
+    - swift-pip-packages
+
 - name: Install pip packages (venv)
   pip:
     name: "{{ item }}"
@@ -65,7 +117,9 @@
   delay: 2
   with_items:
     - "{{ swift_pip_packages }}"
-  when: swift_venv_enabled | bool
+  when:
+    - swift_venv_enabled | bool
+    - swift_get_venv | failed
   tags:
     - swift-install
     - swift-pip-packages

playbooks/roles/os_tempest/defaults/main.yml

@@ -19,6 +19,7 @@ is_metal: true
 ## Verbosity Options
 debug: False
 verbose: True
+tempest_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/tempest.tgz
 tempest_fatal_deprecations: False
 
 tempest_private_subnet_cidr: "192.168.74.0/24"
@@ -71,6 +72,17 @@ tempest_volume_multi_backend_enabled: False
 tempest_main_group: tempest_all
 
 tempest_requires_pip_packages:
+  - python-glanceclient
+  - python-keystoneclient
+  - python-neutronclient
+  - python-novaclient
+  - virtualenv
+  - virtualenv-tools
+
+tempest_pip_packages:
+  - fixtures
+  - junitxml
+  - nose
   - oslo.serialization
   - python-ceilometerclient
   - python-cinderclient
@@ -81,17 +93,12 @@ tempest_requires_pip_packages:
   - python-neutronclient
   - python-novaclient
   - python-openstackclient
+  - python-subunit
   - python-swiftclient
-  - virtualenv
+  - tempest-lib
-
-tempest_pip_packages:
-  - fixtures
-  - nose
   - testrepository
   - testscenarios
   - testtools
-  - python-subunit
-  - junitxml
 
 # Please update SHA in tempest_images below when changing the cirros version.
 cirros_version: 0.3.4

playbooks/roles/os_tempest/tasks/main.yml

@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- include: tempest_install.yml
+
 - include: tempest_resources.yml
   when: >
     inventory_hostname == groups[tempest_main_group][0]
@@ -22,7 +24,6 @@
   when: >
     inventory_hostname != groups[tempest_main_group][0]
 
-- include: tempest_install.yml
 - include: tempest_post_install.yml
 
 - name: Flush handlers

playbooks/roles/os_tempest/tasks/tempest_install.yml

@@ -26,7 +26,8 @@
     - "{{ tempest_requires_pip_packages }}"
   tags:
     - tempest-pip-requires-packages
-    - tempest-pip-install
+    - tempest-install
+    - tempest-pip-packages
 
 - name: Get tempest from git
   git:
@@ -42,17 +43,83 @@
     - tempest-git-clone
     - tempest-pip-install
 
-- name: Install pip packages for tempest
+- name: Attempt venv download
+  get_url:
+    url: "{{ tempest_venv_download_url }}"
+    dest: "/var/cache/{{ tempest_venv_download_url | basename }}"
+  ignore_errors: true
+  register: get_venv
+  tags:
+    - tempest-install
+    - tempest-pip-packages
+
+- name: Set tempest get_venv fact
+  set_fact:
+    tempest_get_venv: "{{ get_venv }}"
+  tags:
+    - tempest-install
+    - tempest-pip-packages
+
+- name: Create tempest venv dir
+  file:
+    path: "{{ tempest_git_dest }}"
+    state: directory
+  when:
+    - tempest_get_venv | success
+  tags:
+    - tempest-install
+    - tempest-pip-packages
+
+- name: Unarchive pre-built venv
+  unarchive:
+    src: "/var/cache/{{ tempest_venv_download_url | basename }}"
+    dest: "{{ tempest_git_dest }}"
+    copy: "no"
+  when:
+    - tempest_get_venv | success
+  tags:
+    - tempest-install
+    - tempest-pip-packages
+
+- name: Update virtualenv path
+  command: >
+    virtualenv-tools --update-path=auto {{ tempest_git_dest }}
+  when:
+    - tempest_get_venv | success
+  tags:
+    - tempest-install
+    - tempest-pip-packages
+
+- name: Install pip packages for tempest (prebuilt venv)
   pip:
-    name: "{{ item }}"
+    name: "{{ tempest_git_dest }}"
     state: present
     virtualenv: "{{ tempest_git_dest }}"
-    virtualenv_site_packages: "yes"
+    virtualenv_site_packages: "no"
     extra_args: "{{ tempest_pip_instructions }}"
   register: install_packages
   until: install_packages|success
   retries: 5
   delay: 2
+  when:
+    - tempest_get_venv | success
+  tags:
+    - tempest-pip-packages
+    - tempest-pip-install
+
+- name: Install pip packages for tempest (no prebuilt venv)
+  pip:
+    name: "{{ item }}"
+    state: present
+    virtualenv: "{{ tempest_git_dest }}"
+    virtualenv_site_packages: "no"
+    extra_args: "{{ tempest_pip_instructions }}"
+  register: install_packages
+  until: install_packages|success
+  retries: 5
+  delay: 2
+  when:
+    - tempest_get_venv | failed
   with_items:
     - "{{ tempest_pip_packages }}"
     - "{{ tempest_git_dest }}"

playbooks/roles/repo_build/defaults/main.yml

@@ -30,3 +30,13 @@ repo_build_pip_default_index: "https://pypi.python.org/simple"
 repo_build_pip_extra_index: "https://pypi.python.org/simple"
 
 repo_build_timeout: 120
+
+repo_build_venv_force_rebuild: false
+repo_build_venv_build_dir: "/tmp/openstack-venv-builder"
+repo_build_venv_dir: "/var/www/repo/venvs"
+repo_build_venv_pip_install_options: >
+  --timeout 120
+  --find-links {{ repo_build_release_path }}/{{ repo_build_release_tag }}
+  --no-index
+  --verbose
+  --log /var/log/repo/repo_venv_builder.log

playbooks/roles/repo_build/tasks/main.yml

@@ -13,8 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Wheel building
 - include: repo_clone_git.yml
 - include: repo_set_facts.yml
 - include: repo_pre_build.yml
 - include: repo_build.yml
 - include: repo_post_build.yml
+
+# Venv building
+- include: repo_venv.yml
+  tags:
+    - repo-build-venvs

playbooks/roles/repo_build/tasks/repo_venv.yml

@@ -0,0 +1,18 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- include: repo_venv_pre_build.yml
+- include: repo_venv_build.yml
+- include: repo_venv_post_build.yml

playbooks/roles/repo_build/tasks/repo_venv_build.yml

@@ -0,0 +1,81 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Get venv command path
+  command: which virtualenv
+  register: virtualenv_path
+  tags:
+    - repo-command-bin
+    - repo-create-venv
+
+- name: Set virtualenv command path
+  set_fact:
+    virtualenv_bin: "{{ virtualenv_path.stdout }}"
+  tags:
+    - repo-command-bin
+    - repo-create-venv
+
+- name: Check for created venvs
+  command: >
+    ls -1 "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}/"
+  register: created_venvs
+  tags:
+    - repo-create-venv
+
+- name: Set existing venv fact
+  set_fact:
+    existing_venvs: "{{ created_venvs.stdout_lines }}"
+  tags:
+    - repo-command-bin
+    - repo-create-venv
+
+- name: Create role based venv
+  pip:
+    name: "{{ item.value | join(' ') }}"
+    state: present
+    virtualenv: "{{ repo_build_venv_build_dir }}/venvs/{{ item.key | replace('os_', '') }}"
+    virtualenv_site_packages: "no"
+    virtualenv_command: "{{ virtualenv_bin }} --always-copy"
+    extra_args: "{{ repo_build_venv_pip_install_options }}"
+  register: install_packages
+  until: install_packages|success
+  retries: 5
+  delay: 2
+  with_dict: local_packages.results.0.item.role_packages
+  when:
+    - '"os_" in item.key'
+    - "'{{ item.key | replace('os_', '') }}.tgz' not in existing_venvs"
+  tags:
+    - repo-create-venv
+
+- name: Create venv archive
+  shell: |
+    # This is to clean up pyc files which makes the archived venv smaller
+    #  TODO(cloudnull) This should use the find module when we move to Ansible 2.0
+    find "{{ repo_build_venv_build_dir }}/venvs/{{ item.key | replace('os_', '') }}" -name '*.pyc' -delete
+    # Create archive
+    tar -czf "{{ item.key | replace('os_', '') }}-{{ repo_build_release_tag }}.tgz" \
+        -C "{{ repo_build_venv_build_dir }}/venvs/{{ item.key | replace('os_', '') }}" .
+  args:
+    chdir: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}"
+    creates: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}/{{ item.key | replace('os_', '') }}.tgz"
+  with_dict: local_packages.results.0.item.role_packages
+  when:
+    - '"os_" in item.key'
+  tags:
+    - skip_ansible_lint
+    - repo-venv-compress-archive
+    - repo-create-venv-archive
+    - repo-create-venv

playbooks/roles/repo_build/tasks/repo_venv_post_build.yml

@@ -0,0 +1,23 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Cleanup venv directory
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - "{{ repo_build_venv_build_dir }}"
+  tags:
+    - repo-cleanup-venv-location

playbooks/roles/repo_build/tasks/repo_venv_pre_build.yml

@@ -0,0 +1,49 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Make sure old venv build directories are clean
+  file:
+    path: "{{ item }}"
+    state: "absent"
+  with_items:
+    - "{{ repo_build_venv_build_dir }}"
+  tags:
+    - repo-create-venv-location
+    - repo-venv-compress-archive
+    - repo-create-venv-archive
+
+- name: Destroy base venvs to rebuild them
+  file:
+    path: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}"
+    state: "absent"
+  when: repo_build_venv_force_rebuild | bool
+  tags:
+    - repo-create-venv-location
+    - repo-venv-compress-archive
+    - repo-create-venv-archive
+
+- name: Create venv directory
+  file:
+    path: "{{ item }}"
+    state: "directory"
+    owner: "{{ repo_build_service_user_name }}"
+    mode: "2755"
+  with_items:
+    - "{{ repo_build_venv_build_dir }}/venvs"
+    - "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}"
+  tags:
+    - repo-create-venv-location
+    - repo-venv-compress-archive
+    - repo-create-venv-archive

playbooks/roles/repo_server/defaults/main.yml

@@ -85,6 +85,8 @@ repo_pip_packages:
   - turbolift
   - wheel
   - yaprt
+  - virtualenv
+  - virtualenv-tools
 
 # Main web server port
 repo_server_port: 8181