From 8d2e5f62a79c4690cf2f7c9aa8e569a90ac9a9eb Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Fri, 16 Oct 2015 22:22:14 -0500 Subject: [PATCH] Implement shippable venvs The change builds venvs in a single repo container and then ships them to to all targets. The built venvs will be within the repo servers and will allow for faster deployments, upgrades, and more consistent deployments for the life cycle of the deployment. This will create a versioned tarball that will allow for greater visablility into the build process as well as giving deployers/developers the ability to compair a release in place. Change-Id: Ieef0b89ebc009d1453c99e19e53a36eb2d70edae Signed-off-by: Kevin Carter --- playbooks/os-aodh-install.yml | 1 + playbooks/os-ceilometer-install.yml | 1 + playbooks/os-cinder-install.yml | 1 + playbooks/os-glance-install.yml | 1 + playbooks/os-heat-install.yml | 1 + playbooks/os-horizon-install.yml | 1 + playbooks/os-keystone-install.yml | 1 + playbooks/os-neutron-install.yml | 1 + playbooks/os-nova-install.yml | 1 + playbooks/os-swift-setup.yml | 1 + playbooks/os-tempest-install.yml | 6 +- playbooks/plugins/lookups/py_pkgs.py | 19 +++++ playbooks/repo-build.yml | 3 + playbooks/roles/os_aodh/defaults/main.yml | 3 + .../roles/os_aodh/tasks/aodh_install.yml | 56 ++++++++++++- .../roles/os_ceilometer/defaults/main.yml | 3 + .../tasks/ceilometer_install.yml | 56 ++++++++++++- playbooks/roles/os_cinder/defaults/main.yml | 3 + .../roles/os_cinder/tasks/cinder_install.yml | 56 ++++++++++++- playbooks/roles/os_glance/defaults/main.yml | 3 + .../roles/os_glance/tasks/glance_install.yml | 56 ++++++++++++- playbooks/roles/os_heat/defaults/main.yml | 3 + .../roles/os_heat/tasks/heat_install.yml | 56 ++++++++++++- playbooks/roles/os_horizon/defaults/main.yml | 3 + .../os_horizon/tasks/horizon_install.yml | 4 +- .../os_horizon/tasks/horizon_pre_install.yml | 58 ++++++++++++- playbooks/roles/os_keystone/defaults/main.yml | 3 + .../os_keystone/tasks/keystone_install.yml | 56 ++++++++++++- playbooks/roles/os_neutron/defaults/main.yml | 3 + .../os_neutron/tasks/neutron_install.yml | 56 ++++++++++++- .../os_neutron/tasks/plumgrid_config.yml | 1 + playbooks/roles/os_nova/defaults/main.yml | 3 + .../tasks/nova_compute_kvm_install.yml | 4 +- .../tasks/nova_console_novnc_install.yml | 4 +- .../roles/os_nova/tasks/nova_install.yml | 56 ++++++++++++- playbooks/roles/os_swift/defaults/main.yml | 3 + .../roles/os_swift/tasks/swift_install.yml | 56 ++++++++++++- playbooks/roles/os_tempest/defaults/main.yml | 21 +++-- playbooks/roles/os_tempest/tasks/main.yml | 3 +- .../os_tempest/tasks/tempest_install.yml | 75 ++++++++++++++++- playbooks/roles/repo_build/defaults/main.yml | 10 +++ playbooks/roles/repo_build/tasks/main.yml | 6 ++ .../roles/repo_build/tasks/repo_venv.yml | 18 +++++ .../repo_build/tasks/repo_venv_build.yml | 81 +++++++++++++++++++ .../repo_build/tasks/repo_venv_post_build.yml | 23 ++++++ .../repo_build/tasks/repo_venv_pre_build.yml | 49 +++++++++++ playbooks/roles/repo_server/defaults/main.yml | 2 + 47 files changed, 905 insertions(+), 26 deletions(-) create mode 100644 playbooks/roles/repo_build/tasks/repo_venv.yml create mode 100644 playbooks/roles/repo_build/tasks/repo_venv_build.yml create mode 100644 playbooks/roles/repo_build/tasks/repo_venv_post_build.yml create mode 100644 playbooks/roles/repo_build/tasks/repo_venv_pre_build.yml diff --git a/playbooks/os-aodh-install.yml b/playbooks/os-aodh-install.yml index 26def617f1..7618252e76 100644 --- a/playbooks/os-aodh-install.yml +++ b/playbooks/os-aodh-install.yml @@ -63,6 +63,7 @@ roles: - role: "os_aodh" aodh_venv_tag: "{{ openstack_release }}" + aodh_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/aodh-{{ openstack_release }}.tgz" tags: - "os-aodh" - { role: "openstack_openrc", tags: [ "openstack-openrc" ] } diff --git a/playbooks/os-ceilometer-install.yml b/playbooks/os-ceilometer-install.yml index 93e137a58e..815b428123 100644 --- a/playbooks/os-ceilometer-install.yml +++ b/playbooks/os-ceilometer-install.yml @@ -74,6 +74,7 @@ roles: - role: "os_ceilometer" ceilometer_venv_tag: "{{ openstack_release }}" + ceilometer_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/ceilometer-{{ openstack_release }}.tgz" tags: - "os-ceilometer" - { role: "openstack_openrc", tags: [ "openstack-openrc" ] } diff --git a/playbooks/os-cinder-install.yml b/playbooks/os-cinder-install.yml index f211abffac..be37b9dfdc 100644 --- a/playbooks/os-cinder-install.yml +++ b/playbooks/os-cinder-install.yml @@ -157,6 +157,7 @@ roles: - role: "os_cinder" cinder_venv_tag: "{{ openstack_release }}" + cinder_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/cinder-{{ openstack_release }}.tgz" cinder_galera_address: "{{ internal_lb_vip_address }}" cinder_storage_address: "{{ container_address }}" cinder_glance_host: "{{ internal_lb_vip_address }}" diff --git a/playbooks/os-glance-install.yml b/playbooks/os-glance-install.yml index e29137de97..cf5698c7b8 100644 --- a/playbooks/os-glance-install.yml +++ b/playbooks/os-glance-install.yml @@ -92,6 +92,7 @@ - role: "os_glance" glance_galera_address: "{{ galera_address }}" glance_venv_tag: "{{ openstack_release }}" + glance_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/glance-{{ openstack_release }}.tgz" tags: - "os-glance" - { role: "openstack_openrc", tags: [ "openstack-openrc" ] } diff --git a/playbooks/os-heat-install.yml b/playbooks/os-heat-install.yml index 0e94bec9c4..dd7fdb40f5 100644 --- a/playbooks/os-heat-install.yml +++ b/playbooks/os-heat-install.yml @@ -74,6 +74,7 @@ roles: - role: "os_heat" heat_venv_tag: "{{ openstack_release }}" + heat_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/heat-{{ openstack_release }}.tgz" heat_galera_address: "{{ galera_address }}" tags: - "os-heat" diff --git a/playbooks/os-horizon-install.yml b/playbooks/os-horizon-install.yml index 3c314d0b74..c1108297c2 100644 --- a/playbooks/os-horizon-install.yml +++ b/playbooks/os-horizon-install.yml @@ -76,6 +76,7 @@ horizon_galera_address: "{{ galera_address }}" horizon_server_name: "{{ container_name }}" horizon_venv_tag: "{{ openstack_release }}" + horizon_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/horizon-{{ openstack_release }}.tgz" tags: - "os-horizon" - role: "rsyslog_client" diff --git a/playbooks/os-keystone-install.yml b/playbooks/os-keystone-install.yml index 1391152ea7..b6836ff4f2 100644 --- a/playbooks/os-keystone-install.yml +++ b/playbooks/os-keystone-install.yml @@ -74,6 +74,7 @@ roles: - role: "os_keystone" keystone_venv_tag: "{{ openstack_release }}" + keystone_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/keystone-{{ openstack_release }}.tgz" keystone_galera_address: "{{ galera_address }}" tags: - "os-keystone" diff --git a/playbooks/os-neutron-install.yml b/playbooks/os-neutron-install.yml index 66f1d16b88..228ec74a63 100644 --- a/playbooks/os-neutron-install.yml +++ b/playbooks/os-neutron-install.yml @@ -125,6 +125,7 @@ roles: - role: "os_neutron" neutron_venv_tag: "{{ openstack_release }}" + neutron_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/neutron-{{ openstack_release }}.tgz" neutron_galera_address: "{{ galera_address }}" neutron_local_ip: "{{ _local_ip }}" neutron_overlay_network: "{{ _overlay_network }}" diff --git a/playbooks/os-nova-install.yml b/playbooks/os-nova-install.yml index a341f4bb8c..62b5130445 100644 --- a/playbooks/os-nova-install.yml +++ b/playbooks/os-nova-install.yml @@ -160,6 +160,7 @@ nova_galera_address: "{{ internal_lb_vip_address }}" nova_management_address: "{{ management_address }}" nova_venv_tag: "{{ openstack_release }}" + nova_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/nova-{{ openstack_release }}.tgz" tags: - "os-nova" - role: "ceph_client" diff --git a/playbooks/os-swift-setup.yml b/playbooks/os-swift-setup.yml index a2b28c14c3..5a1442d452 100644 --- a/playbooks/os-swift-setup.yml +++ b/playbooks/os-swift-setup.yml @@ -133,6 +133,7 @@ roles: - role: "os_swift" swift_venv_tag: "{{ openstack_release }}" + swift_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/swift-{{ openstack_release }}.tgz" swift_storage_address: "{{ storage_address }}" swift_replication_address: "{{ replication_address }}" tags: diff --git a/playbooks/os-tempest-install.yml b/playbooks/os-tempest-install.yml index 01663cfb9c..a4a1a83db3 100644 --- a/playbooks/os-tempest-install.yml +++ b/playbooks/os-tempest-install.yml @@ -18,7 +18,11 @@ max_fail_percentage: 20 user: root roles: - - { role: "os_tempest", tags: [ "os-tempest" ] } + - role: "os_tempest" + tempest_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/tempest-{{ openstack_release }}.tgz" + tempest_git_repo: "{{ openstack_repo_git_url }}/tempest" + tags: + - "os-tempest" - role: "rsyslog_client" rsyslog_client_log_rotate_file: utility_log_rotate rsyslog_client_log_dir: "/var/log/utility" diff --git a/playbooks/plugins/lookups/py_pkgs.py b/playbooks/plugins/lookups/py_pkgs.py index f625578ba2..a06c7b9727 100644 --- a/playbooks/plugins/lookups/py_pkgs.py +++ b/playbooks/plugins/lookups/py_pkgs.py @@ -107,6 +107,7 @@ class DependencyFileProcessor(object): self.pip = dict() self.pip['git_package'] = list() self.pip['py_package'] = list() + self.pip['role_packages'] = dict() self.git_pip_install = 'git+%s@%s' self.file_names = self._get_files(path=local_path) @@ -231,6 +232,7 @@ class DependencyFileProcessor(object): ext=ext ) + role_name = None for file_name in file_names: with open(file_name, 'rb') as f: # If there is an exception loading the file continue @@ -244,6 +246,11 @@ class DependencyFileProcessor(object): if not loaded_config: continue + if 'roles' in file_name: + _role_name = file_name.split('roles%s' % os.sep)[-1] + role_name = _role_name.split(os.sep)[0] + + for key, values in loaded_config.items(): # This conditional is set to ensure we're not processes git repos # from the defaults file which may conflict with what is being set @@ -258,6 +265,17 @@ class DependencyFileProcessor(object): if [i for i in BUILT_IN_PIP_PACKAGE_VARS if i in key]: self.pip['py_package'].extend(values) + if role_name: + if not role_name in self.pip['role_packages']: + self.pip['role_packages'][role_name] = values + else: + self.pip['role_packages'][role_name].extend(values) + self.pip['role_packages'][role_name] = list( + set( + self.pip['role_packages'][role_name] + ) + ) + def _abs_path(path): return os.path.abspath( @@ -334,5 +352,6 @@ class LookupModule(object): ) for i in return_data['remote_packages'] ] return_data['remote_package_parts'] = remote_package_parts + return_data['role_packages'] = dfp.pip['role_packages'] return [return_data] diff --git a/playbooks/repo-build.yml b/playbooks/repo-build.yml index f7e59cc925..e11019e78c 100644 --- a/playbooks/repo-build.yml +++ b/playbooks/repo-build.yml @@ -31,6 +31,9 @@ - repo-set-requirement-names-filtered - repo-set-constraints - repo-build-constraints-file + - repo-create-venv-archive + - repo-venv-compress-archive + - repo-build-venvs roles: - role: "repo_build" repo_build_release_tag: "{{ openstack_release }}" diff --git a/playbooks/roles/os_aodh/defaults/main.yml b/playbooks/roles/os_aodh/defaults/main.yml index d737566f4f..c2f6284ca1 100644 --- a/playbooks/roles/os_aodh/defaults/main.yml +++ b/playbooks/roles/os_aodh/defaults/main.yml @@ -29,6 +29,8 @@ aodh_venv_enabled: true # system path used when the installing. aodh_bin: "{{ aodh_venv_bin }}" +aodh_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/aodh.tgz + ## System info aodh_system_user_name: aodh aodh_system_group_name: aodh @@ -85,6 +87,7 @@ aodh_apt_packages: # aodh packages that must be installed before anything else aodh_requires_pip_packages: - virtualenv + - virtualenv-tools - python-keystoneclient # Keystoneclient needed to OSA keystone lib # Common pip packages diff --git a/playbooks/roles/os_aodh/tasks/aodh_install.yml b/playbooks/roles/os_aodh/tasks/aodh_install.yml index d71fd3069c..9d15b3cc8b 100644 --- a/playbooks/roles/os_aodh/tasks/aodh_install.yml +++ b/playbooks/roles/os_aodh/tasks/aodh_install.yml @@ -43,6 +43,58 @@ - aodh-install - aodh-pip-packages +- name: Attempt venv download + get_url: + url: "{{ aodh_venv_download_url }}" + dest: "/var/cache/{{ aodh_venv_download_url | basename }}" + ignore_errors: true + register: get_venv + when: aodh_venv_enabled | bool + tags: + - aodh-install + - aodh-pip-packages + +- name: Set aodh get_venv fact + set_fact: + aodh_get_venv: "{{ get_venv }}" + when: aodh_venv_enabled | bool + tags: + - aodh-install + - aodh-pip-packages + +- name: Create aodh venv dir + file: + path: "{{ aodh_venv_bin | dirname }}" + state: directory + when: + - aodh_venv_enabled | bool + - aodh_get_venv | success + tags: + - aodh-install + - aodh-pip-packages + +- name: Unarchive pre-built venv + unarchive: + src: "/var/cache/{{ aodh_venv_download_url | basename }}" + dest: "{{ aodh_venv_bin | dirname }}" + copy: "no" + when: + - aodh_venv_enabled | bool + - aodh_get_venv | success + tags: + - aodh-install + - aodh-pip-packages + +- name: Update virtualenv path + command: > + virtualenv-tools --update-path=auto {{ aodh_venv_bin | dirname }} + when: + - aodh_venv_enabled | bool + - aodh_get_venv | success + tags: + - aodh-install + - aodh-pip-packages + - name: Install pip packages (venv) pip: name: "{{ item }}" @@ -56,7 +108,9 @@ delay: 2 with_items: - "{{ aodh_pip_packages }}" - when: aodh_venv_enabled | bool + when: + - aodh_venv_enabled | bool + - aodh_get_venv | failed tags: - aodh-install - aodh-pip-packages diff --git a/playbooks/roles/os_ceilometer/defaults/main.yml b/playbooks/roles/os_ceilometer/defaults/main.yml index 6bc23b9696..97f8e7b5ca 100644 --- a/playbooks/roles/os_ceilometer/defaults/main.yml +++ b/playbooks/roles/os_ceilometer/defaults/main.yml @@ -29,6 +29,8 @@ ceilometer_venv_enabled: true # system path used when the installing. ceilometer_bin: "{{ ceilometer_venv_bin }}" +ceilometer_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/ceilometer.tgz + ## System info ceilometer_system_user_name: ceilometer ceilometer_system_group_name: ceilometer @@ -87,6 +89,7 @@ ceilometer_apt_packages: # ceilometer packages that must be installed before anything else ceilometer_requires_pip_packages: - virtualenv + - virtualenv-tools - python-keystoneclient # Keystoneclient needed to OSA keystone lib # Common pip packages diff --git a/playbooks/roles/os_ceilometer/tasks/ceilometer_install.yml b/playbooks/roles/os_ceilometer/tasks/ceilometer_install.yml index 504673c9b6..3bd033885e 100644 --- a/playbooks/roles/os_ceilometer/tasks/ceilometer_install.yml +++ b/playbooks/roles/os_ceilometer/tasks/ceilometer_install.yml @@ -43,6 +43,58 @@ - ceilometer-install - ceilometer-pip-packages +- name: Attempt venv download + get_url: + url: "{{ ceilometer_venv_download_url }}" + dest: "/var/cache/{{ ceilometer_venv_download_url | basename }}" + ignore_errors: true + register: get_venv + when: ceilometer_venv_enabled | bool + tags: + - ceilometer-install + - ceilometer-pip-packages + +- name: Set ceilometer get_venv fact + set_fact: + ceilometer_get_venv: "{{ get_venv }}" + when: ceilometer_venv_enabled | bool + tags: + - ceilometer-install + - ceilometer-pip-packages + +- name: Create ceilometer venv dir + file: + path: "{{ ceilometer_venv_bin | dirname }}" + state: directory + when: + - ceilometer_venv_enabled | bool + - ceilometer_get_venv | success + tags: + - ceilometer-install + - ceilometer-pip-packages + +- name: Unarchive pre-built venv + unarchive: + src: "/var/cache/{{ ceilometer_venv_download_url | basename }}" + dest: "{{ ceilometer_venv_bin | dirname }}" + copy: "no" + when: + - ceilometer_venv_enabled | bool + - ceilometer_get_venv | success + tags: + - ceilometer-install + - ceilometer-pip-packages + +- name: Update virtualenv path + command: > + virtualenv-tools --update-path=auto {{ ceilometer_venv_bin | dirname }} + when: + - ceilometer_venv_enabled | bool + - ceilometer_get_venv | success + tags: + - ceilometer-install + - ceilometer-pip-packages + - name: Install pip packages (venv) pip: name: "{{ item }}" @@ -56,7 +108,9 @@ delay: 2 with_items: - "{{ ceilometer_pip_packages }}" - when: ceilometer_venv_enabled | bool + when: + - ceilometer_venv_enabled | bool + - ceilometer_get_venv | failed tags: - ceilometer-install - ceilometer-pip-packages diff --git a/playbooks/roles/os_cinder/defaults/main.yml b/playbooks/roles/os_cinder/defaults/main.yml index 0fd14f00ea..7fad37b472 100644 --- a/playbooks/roles/os_cinder/defaults/main.yml +++ b/playbooks/roles/os_cinder/defaults/main.yml @@ -28,6 +28,8 @@ cinder_venv_enabled: true # system path used when the installing. cinder_bin: "{{ cinder_venv_bin }}" +cinder_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/cinder.tgz + # Enable/Disable Ceilometer cinder_ceilometer_enabled: False @@ -226,6 +228,7 @@ cinder_apt_packages: # Cinder packages that must be installed before anything else cinder_requires_pip_packages: - virtualenv + - virtualenv-tools - python-keystoneclient # Keystoneclient needed to OSA keystone lib # Common pip packages diff --git a/playbooks/roles/os_cinder/tasks/cinder_install.yml b/playbooks/roles/os_cinder/tasks/cinder_install.yml index 9eb0f27b72..fb6c49ddd6 100644 --- a/playbooks/roles/os_cinder/tasks/cinder_install.yml +++ b/playbooks/roles/os_cinder/tasks/cinder_install.yml @@ -52,6 +52,58 @@ - cinder-install - cinder-pip-packages +- name: Attempt venv download + get_url: + url: "{{ cinder_venv_download_url }}" + dest: "/var/cache/{{ cinder_venv_download_url | basename }}" + ignore_errors: true + register: get_venv + when: cinder_venv_enabled | bool + tags: + - cinder-install + - cinder-pip-packages + +- name: Set cinder get_venv fact + set_fact: + cinder_get_venv: "{{ get_venv }}" + when: cinder_venv_enabled | bool + tags: + - cinder-install + - cinder-pip-packages + +- name: Create cinder venv dir + file: + path: "{{ cinder_venv_bin | dirname }}" + state: directory + when: + - cinder_venv_enabled | bool + - cinder_get_venv | success + tags: + - cinder-install + - cinder-pip-packages + +- name: Unarchive pre-built venv + unarchive: + src: "/var/cache/{{ cinder_venv_download_url | basename }}" + dest: "{{ cinder_venv_bin | dirname }}" + copy: "no" + when: + - cinder_venv_enabled | bool + - cinder_get_venv | success + tags: + - cinder-install + - cinder-pip-packages + +- name: Update virtualenv path + command: > + virtualenv-tools --update-path=auto {{ cinder_venv_bin | dirname }} + when: + - cinder_venv_enabled | bool + - cinder_get_venv | success + tags: + - cinder-install + - cinder-pip-packages + - name: Install pip packages (venv) pip: name: "{{ item }}" @@ -65,7 +117,9 @@ delay: 2 with_items: - "{{ cinder_pip_packages }}" - when: cinder_venv_enabled | bool + when: + - cinder_venv_enabled | bool + - cinder_get_venv | failed tags: - cinder-install - cinder-pip-packages diff --git a/playbooks/roles/os_glance/defaults/main.yml b/playbooks/roles/os_glance/defaults/main.yml index cf7d9f79b6..b9c0cfd2fb 100644 --- a/playbooks/roles/os_glance/defaults/main.yml +++ b/playbooks/roles/os_glance/defaults/main.yml @@ -37,6 +37,8 @@ glance_venv_etc_dir: "{{ glance_bin | dirname }}/etc/glance" glance_non_venv_etc_dir: "/usr/local/etc/glance" glance_etc_dir: "{{ (glance_venv_enabled | bool) | ternary(glance_venv_etc_dir, glance_non_venv_etc_dir) }}" +glance_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/glance.tgz + # Enable/Disable Ceilometer glance_ceilometer_enabled: False @@ -175,6 +177,7 @@ glance_apt_packages: # Cinder packages that must be installed before anything else glance_requires_pip_packages: - virtualenv + - virtualenv-tools - python-keystoneclient # Keystoneclient needed to OSA keystone lib # Common pip packages diff --git a/playbooks/roles/os_glance/tasks/glance_install.yml b/playbooks/roles/os_glance/tasks/glance_install.yml index 010bc42f8d..0df0eaa1f0 100644 --- a/playbooks/roles/os_glance/tasks/glance_install.yml +++ b/playbooks/roles/os_glance/tasks/glance_install.yml @@ -52,6 +52,58 @@ - glance-install - glance-pip-packages +- name: Attempt venv download + get_url: + url: "{{ glance_venv_download_url }}" + dest: "/var/cache/{{ glance_venv_download_url | basename }}" + ignore_errors: true + register: get_venv + when: glance_venv_enabled | bool + tags: + - glance-install + - glance-pip-packages + +- name: Set glance get_venv fact + set_fact: + glance_get_venv: "{{ get_venv }}" + when: glance_venv_enabled | bool + tags: + - glance-install + - glance-pip-packages + +- name: Create glance venv dir + file: + path: "{{ glance_venv_bin | dirname }}" + state: directory + when: + - glance_venv_enabled | bool + - glance_get_venv | success + tags: + - glance-install + - glance-pip-packages + +- name: Unarchive pre-built venv + unarchive: + src: "/var/cache/{{ glance_venv_download_url | basename }}" + dest: "{{ glance_venv_bin | dirname }}" + copy: "no" + when: + - glance_venv_enabled | bool + - glance_get_venv | success + tags: + - glance-install + - glance-pip-packages + +- name: Update virtualenv path + command: > + virtualenv-tools --update-path=auto {{ glance_venv_bin | dirname }} + when: + - glance_venv_enabled | bool + - glance_get_venv | success + tags: + - glance-install + - glance-pip-packages + - name: Install pip packages (venv) pip: name: "{{ item }}" @@ -65,7 +117,9 @@ delay: 2 with_items: - "{{ glance_pip_packages }}" - when: glance_venv_enabled | bool + when: + - glance_venv_enabled | bool + - glance_get_venv | failed tags: - glance-install - glance-pip-packages diff --git a/playbooks/roles/os_heat/defaults/main.yml b/playbooks/roles/os_heat/defaults/main.yml index 1c543ca29b..12b33a23ff 100644 --- a/playbooks/roles/os_heat/defaults/main.yml +++ b/playbooks/roles/os_heat/defaults/main.yml @@ -35,6 +35,8 @@ heat_venv_enabled: true # system path used when the installing. heat_bin: "{{ heat_venv_bin }}" +heat_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/heat.tgz + heat_fatal_deprecations: False heat_clients_endpoint: internalURL @@ -157,6 +159,7 @@ heat_apt_packages: # Heat packages that must be installed before anything else heat_requires_pip_packages: - virtualenv + - virtualenv-tools - python-keystoneclient # Keystoneclient needed to OSA keystone lib # Common pip packages diff --git a/playbooks/roles/os_heat/tasks/heat_install.yml b/playbooks/roles/os_heat/tasks/heat_install.yml index 4b0157ec7b..85b5f5a96f 100644 --- a/playbooks/roles/os_heat/tasks/heat_install.yml +++ b/playbooks/roles/os_heat/tasks/heat_install.yml @@ -52,6 +52,58 @@ - heat-install - heat-pip-packages +- name: Attempt venv download + get_url: + url: "{{ heat_venv_download_url }}" + dest: "/var/cache/{{ heat_venv_download_url | basename }}" + ignore_errors: true + register: get_venv + when: heat_venv_enabled | bool + tags: + - heat-install + - heat-pip-packages + +- name: Set heat get_venv fact + set_fact: + heat_get_venv: "{{ get_venv }}" + when: heat_venv_enabled | bool + tags: + - heat-install + - heat-pip-packages + +- name: Create heat venv dir + file: + path: "{{ heat_venv_bin | dirname }}" + state: directory + when: + - heat_venv_enabled | bool + - heat_get_venv | success + tags: + - heat-install + - heat-pip-packages + +- name: Unarchive pre-built venv + unarchive: + src: "/var/cache/{{ heat_venv_download_url | basename }}" + dest: "{{ heat_venv_bin | dirname }}" + copy: "no" + when: + - heat_venv_enabled | bool + - heat_get_venv | success + tags: + - heat-install + - heat-pip-packages + +- name: Update virtualenv path + command: > + virtualenv-tools --update-path=auto {{ heat_venv_bin | dirname }} + when: + - heat_venv_enabled | bool + - heat_get_venv | success + tags: + - heat-install + - heat-pip-packages + - name: Install pip packages (venv) pip: name: "{{ item }}" @@ -65,7 +117,9 @@ delay: 2 with_items: - "{{ heat_pip_packages }}" - when: heat_venv_enabled | bool + when: + - heat_venv_enabled | bool + - heat_get_venv | failed tags: - heat-install - heat-pip-packages diff --git a/playbooks/roles/os_horizon/defaults/main.yml b/playbooks/roles/os_horizon/defaults/main.yml index 32e4e20be1..a2eb8581aa 100644 --- a/playbooks/roles/os_horizon/defaults/main.yml +++ b/playbooks/roles/os_horizon/defaults/main.yml @@ -29,6 +29,8 @@ horizon_venv_enabled: true # system path used when the installing. horizon_bin: "{{ horizon_venv_bin }}" +horizon_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/horizon.tgz + ## System info horizon_system_user_name: horizon horizon_system_group_name: www-data @@ -137,6 +139,7 @@ horizon_apt_packages: # horizon packages that must be installed before anything else horizon_requires_pip_packages: - virtualenv + - virtualenv-tools - python-keystoneclient # Keystoneclient needed to OSA keystone lib horizon_pip_packages: diff --git a/playbooks/roles/os_horizon/tasks/horizon_install.yml b/playbooks/roles/os_horizon/tasks/horizon_install.yml index beb6a70b0c..16e8ad403d 100644 --- a/playbooks/roles/os_horizon/tasks/horizon_install.yml +++ b/playbooks/roles/os_horizon/tasks/horizon_install.yml @@ -50,7 +50,9 @@ delay: 2 with_items: - "{{ horizon_pip_packages }}" - when: horizon_venv_enabled | bool + when: + - horizon_venv_enabled | bool + - horizon_get_venv | failed tags: - horizon-pip-packages diff --git a/playbooks/roles/os_horizon/tasks/horizon_pre_install.yml b/playbooks/roles/os_horizon/tasks/horizon_pre_install.yml index fe0ca767c9..96f925f216 100644 --- a/playbooks/roles/os_horizon/tasks/horizon_pre_install.yml +++ b/playbooks/roles/os_horizon/tasks/horizon_pre_install.yml @@ -71,6 +71,58 @@ tags: - horizon-pip-packages +- name: Attempt venv download + get_url: + url: "{{ horizon_venv_download_url }}" + dest: "/var/cache/{{ horizon_venv_download_url | basename }}" + ignore_errors: true + register: get_venv + when: horizon_venv_enabled | bool + tags: + - horizon-install + - horizon-pip-packages + +- name: Set horizon get_venv fact + set_fact: + horizon_get_venv: "{{ get_venv }}" + when: horizon_venv_enabled | bool + tags: + - horizon-install + - horizon-pip-packages + +- name: Create horizon venv dir + file: + path: "{{ horizon_venv_bin | dirname }}" + state: directory + when: + - horizon_venv_enabled | bool + - horizon_get_venv | success + tags: + - horizon-install + - horizon-pip-packages + +- name: Unarchive pre-built venv + unarchive: + src: "/var/cache/{{ horizon_venv_download_url | basename }}" + dest: "{{ horizon_venv_bin | dirname }}" + copy: "no" + when: + - horizon_venv_enabled | bool + - horizon_get_venv | success + tags: + - horizon-install + - horizon-pip-packages + +- name: Update virtualenv path + command: > + virtualenv-tools --update-path=auto {{ horizon_venv_bin | dirname }} + when: + - horizon_venv_enabled | bool + - horizon_get_venv | success + tags: + - horizon-install + - horizon-pip-packages + - name: Create horizon venv pip: name: "{{ item }}" @@ -80,8 +132,11 @@ extra_args: "{{ pip_install_options|default('') }}" with_items: - "{{ horizon_requires_pip_packages }}" - when: horizon_venv_enabled | bool + when: + - horizon_venv_enabled | bool + - horizon_get_venv | failed tags: + - horizon-install - horizon-pip-packages - name: Create horizon links for venv @@ -96,6 +151,7 @@ dest: "{{ horizon_lib_dir }}" when: horizon_venv_enabled | bool tags: + - horizon-install - horizon-configs - name: Create static horizon dir diff --git a/playbooks/roles/os_keystone/defaults/main.yml b/playbooks/roles/os_keystone/defaults/main.yml index 418ce5b51e..ef1f8904df 100644 --- a/playbooks/roles/os_keystone/defaults/main.yml +++ b/playbooks/roles/os_keystone/defaults/main.yml @@ -29,6 +29,8 @@ keystone_venv_enabled: true # system path used when the installing. keystone_bin: "{{ keystone_venv_bin }}" +keystone_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/keystone.tgz + keystone_fatal_deprecations: False ## System info @@ -347,6 +349,7 @@ keystone_idp_apt_packages: # Keystone packages that must be installed before anything else keystone_requires_pip_packages: - virtualenv + - virtualenv-tools - python-keystoneclient # Keystoneclient needed to OSA keystone lib # Common pip packages diff --git a/playbooks/roles/os_keystone/tasks/keystone_install.yml b/playbooks/roles/os_keystone/tasks/keystone_install.yml index c5a21337ca..3d17530eff 100644 --- a/playbooks/roles/os_keystone/tasks/keystone_install.yml +++ b/playbooks/roles/os_keystone/tasks/keystone_install.yml @@ -80,6 +80,58 @@ - keystone-install - keystone-pip-packages +- name: Attempt venv download + get_url: + url: "{{ keystone_venv_download_url }}" + dest: "/var/cache/{{ keystone_venv_download_url | basename }}" + ignore_errors: true + register: get_venv + when: keystone_venv_enabled | bool + tags: + - keystone-install + - keystone-pip-packages + +- name: Set keystone get_venv fact + set_fact: + keystone_get_venv: "{{ get_venv }}" + when: keystone_venv_enabled | bool + tags: + - keystone-install + - keystone-pip-packages + +- name: Create keystone venv dir + file: + path: "{{ keystone_venv_bin | dirname }}" + state: directory + when: + - keystone_venv_enabled | bool + - keystone_get_venv | success + tags: + - keystone-install + - keystone-pip-packages + +- name: Unarchive pre-built venv + unarchive: + src: "/var/cache/{{ keystone_venv_download_url | basename }}" + dest: "{{ keystone_venv_bin | dirname }}" + copy: "no" + when: + - keystone_venv_enabled | bool + - keystone_get_venv | success + tags: + - keystone-install + - keystone-pip-packages + +- name: Update virtualenv path + command: > + virtualenv-tools --update-path=auto {{ keystone_venv_bin | dirname }} + when: + - keystone_venv_enabled | bool + - keystone_get_venv | success + tags: + - keystone-install + - keystone-pip-packages + - name: Install pip packages (venv) pip: name: "{{ item }}" @@ -93,7 +145,9 @@ delay: 2 with_items: - "{{ keystone_pip_packages }}" - when: keystone_venv_enabled | bool + when: + - keystone_venv_enabled | bool + - keystone_get_venv | failed tags: - keystone-install - keystone-pip-packages diff --git a/playbooks/roles/os_neutron/defaults/main.yml b/playbooks/roles/os_neutron/defaults/main.yml index 6e85a1d23c..d03fef109c 100644 --- a/playbooks/roles/os_neutron/defaults/main.yml +++ b/playbooks/roles/os_neutron/defaults/main.yml @@ -32,6 +32,8 @@ neutron_venv_enabled: true # system path used when the installing. neutron_bin: "{{ neutron_venv_bin }}" +neutron_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/neutron.tgz + # Set the lib dir path to that of the local python path where neutron is installed. # This is used for role access to the db migrations. # Example: @@ -311,6 +313,7 @@ neutron_apt_remove_packages: # neutron packages that must be installed before anything else neutron_requires_pip_packages: - virtualenv + - virtualenv-tools - python-keystoneclient # Keystoneclient needed to OSA keystone lib neutron_pip_packages: diff --git a/playbooks/roles/os_neutron/tasks/neutron_install.yml b/playbooks/roles/os_neutron/tasks/neutron_install.yml index bd41126eee..ae7689a1d3 100644 --- a/playbooks/roles/os_neutron/tasks/neutron_install.yml +++ b/playbooks/roles/os_neutron/tasks/neutron_install.yml @@ -65,6 +65,58 @@ - neutron-install - neutron-pip-packages +- name: Attempt venv download + get_url: + url: "{{ neutron_venv_download_url }}" + dest: "/var/cache/{{ neutron_venv_download_url | basename }}" + ignore_errors: true + register: get_venv + when: neutron_venv_enabled | bool + tags: + - neutron-install + - neutron-pip-packages + +- name: Set neutron get_venv fact + set_fact: + neutron_get_venv: "{{ get_venv }}" + when: neutron_venv_enabled | bool + tags: + - neutron-install + - neutron-pip-packages + +- name: Create neutron venv dir + file: + path: "{{ neutron_venv_bin | dirname }}" + state: directory + when: + - neutron_venv_enabled | bool + - neutron_get_venv | success + tags: + - neutron-install + - neutron-pip-packages + +- name: Unarchive pre-built venv + unarchive: + src: "/var/cache/{{ neutron_venv_download_url | basename }}" + dest: "{{ neutron_venv_bin | dirname }}" + copy: "no" + when: + - neutron_venv_enabled | bool + - neutron_get_venv | success + tags: + - neutron-install + - neutron-pip-packages + +- name: Update virtualenv path + command: > + virtualenv-tools --update-path=auto {{ neutron_venv_bin | dirname }} + when: + - neutron_venv_enabled | bool + - neutron_get_venv | success + tags: + - neutron-install + - neutron-pip-packages + - name: Install pip packages (venv) pip: name: "{{ item }}" @@ -78,7 +130,9 @@ delay: 2 with_items: - "{{ neutron_pip_packages }}" - when: neutron_venv_enabled | bool + when: + - neutron_venv_enabled | bool + - neutron_get_venv | failed tags: - neutron-install - neutron-pip-packages diff --git a/playbooks/roles/os_neutron/tasks/plumgrid_config.yml b/playbooks/roles/os_neutron/tasks/plumgrid_config.yml index c3a36c145b..752c9a9c72 100644 --- a/playbooks/roles/os_neutron/tasks/plumgrid_config.yml +++ b/playbooks/roles/os_neutron/tasks/plumgrid_config.yml @@ -35,6 +35,7 @@ delay: 2 when: - inventory_hostname in groups['neutron_server'] + - neutron_get_venv | failed tags: - neutron-install - neutron-pip-packages diff --git a/playbooks/roles/os_nova/defaults/main.yml b/playbooks/roles/os_nova/defaults/main.yml index 6080122c39..10a1a3f078 100644 --- a/playbooks/roles/os_nova/defaults/main.yml +++ b/playbooks/roles/os_nova/defaults/main.yml @@ -32,6 +32,8 @@ nova_venv_enabled: true # system path used when the installing. nova_bin: "{{ nova_venv_bin }}" +nova_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/nova.tgz + nova_fatal_deprecations: False ## System info @@ -289,6 +291,7 @@ nova_compute_kvm_apt_packages: # nova packages that must be installed before anything else nova_requires_pip_packages: - virtualenv + - virtualenv-tools - python-keystoneclient # Keystoneclient needed to OSA keystone lib nova_compute_pip_packages: diff --git a/playbooks/roles/os_nova/tasks/nova_compute_kvm_install.yml b/playbooks/roles/os_nova/tasks/nova_compute_kvm_install.yml index d99564155e..ba2564ac89 100644 --- a/playbooks/roles/os_nova/tasks/nova_compute_kvm_install.yml +++ b/playbooks/roles/os_nova/tasks/nova_compute_kvm_install.yml @@ -51,7 +51,9 @@ delay: 2 with_items: - "{{ nova_compute_pip_packages }}" - when: nova_venv_enabled | bool + when: + - nova_venv_enabled | bool + - nova_get_venv | failed tags: - nova-install - nova-pip-packages diff --git a/playbooks/roles/os_nova/tasks/nova_console_novnc_install.yml b/playbooks/roles/os_nova/tasks/nova_console_novnc_install.yml index bd7b77b9b3..9bcab62b0a 100644 --- a/playbooks/roles/os_nova/tasks/nova_console_novnc_install.yml +++ b/playbooks/roles/os_nova/tasks/nova_console_novnc_install.yml @@ -68,7 +68,9 @@ delay: 2 with_items: - "{{ nova_novnc_pip_packages }}" - when: nova_venv_enabled | bool + when: + - nova_venv_enabled | bool + - nova_get_venv | failed tags: - nova-install - nova-pip-packages diff --git a/playbooks/roles/os_nova/tasks/nova_install.yml b/playbooks/roles/os_nova/tasks/nova_install.yml index 5f75af1ba3..e5c217e573 100644 --- a/playbooks/roles/os_nova/tasks/nova_install.yml +++ b/playbooks/roles/os_nova/tasks/nova_install.yml @@ -52,6 +52,58 @@ - nova-install - nova-pip-packages +- name: Attempt venv download + get_url: + url: "{{ nova_venv_download_url }}" + dest: "/var/cache/{{ nova_venv_download_url | basename }}" + ignore_errors: true + register: get_venv + when: nova_venv_enabled | bool + tags: + - nova-install + - nova-pip-packages + +- name: Set nova get_venv fact + set_fact: + nova_get_venv: "{{ get_venv }}" + when: nova_venv_enabled | bool + tags: + - nova-install + - nova-pip-packages + +- name: Create nova venv dir + file: + path: "{{ nova_venv_bin | dirname }}" + state: directory + when: + - nova_venv_enabled | bool + - nova_get_venv | success + tags: + - nova-install + - nova-pip-packages + +- name: Unarchive pre-built venv + unarchive: + src: "/var/cache/{{ nova_venv_download_url | basename }}" + dest: "{{ nova_venv_bin | dirname }}" + copy: "no" + when: + - nova_venv_enabled | bool + - nova_get_venv | success + tags: + - nova-install + - nova-pip-packages + +- name: Update virtualenv path + command: > + virtualenv-tools --update-path=auto {{ nova_venv_bin | dirname }} + when: + - nova_venv_enabled | bool + - nova_get_venv | success + tags: + - nova-install + - nova-pip-packages + - name: Install pip packages (venv) pip: name: "{{ item }}" @@ -65,7 +117,9 @@ delay: 2 with_items: - "{{ nova_pip_packages }}" - when: nova_venv_enabled | bool + when: + - nova_venv_enabled | bool + - nova_get_venv | failed tags: - nova-install - nova-pip-packages diff --git a/playbooks/roles/os_swift/defaults/main.yml b/playbooks/roles/os_swift/defaults/main.yml index 4f8d1e60ac..897dc75b11 100644 --- a/playbooks/roles/os_swift/defaults/main.yml +++ b/playbooks/roles/os_swift/defaults/main.yml @@ -32,6 +32,8 @@ swift_venv_enabled: true # system path used when the installing. swift_bin: "{{ swift_venv_bin }}" +swift_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/swift.tgz + # Set the full path to the swift recon cron recon_cron_path: "{{ swift_bin }}/swift-recon-cron" @@ -177,6 +179,7 @@ swift: {} # swift packages that must be installed before anything else swift_requires_pip_packages: - virtualenv + - virtualenv-tools - python-keystoneclient # Keystoneclient needed to OSA keystone lib swift_pip_packages: diff --git a/playbooks/roles/os_swift/tasks/swift_install.yml b/playbooks/roles/os_swift/tasks/swift_install.yml index b5be1c18d2..41d3c073bb 100644 --- a/playbooks/roles/os_swift/tasks/swift_install.yml +++ b/playbooks/roles/os_swift/tasks/swift_install.yml @@ -52,6 +52,58 @@ - swift-install - swift-pip-packages +- name: Attempt venv download + get_url: + url: "{{ swift_venv_download_url }}" + dest: "/var/cache/{{ swift_venv_download_url | basename }}" + ignore_errors: true + register: get_venv + when: swift_venv_enabled | bool + tags: + - swift-install + - swift-pip-packages + +- name: Set swift get_venv fact + set_fact: + swift_get_venv: "{{ get_venv }}" + when: swift_venv_enabled | bool + tags: + - swift-install + - swift-pip-packages + +- name: Create swift venv dir + file: + path: "{{ swift_venv_bin | dirname }}" + state: directory + when: + - swift_venv_enabled | bool + - swift_get_venv | success + tags: + - swift-install + - swift-pip-packages + +- name: Unarchive pre-built venv + unarchive: + src: "/var/cache/{{ swift_venv_download_url | basename }}" + dest: "{{ swift_venv_bin | dirname }}" + copy: "no" + when: + - swift_venv_enabled | bool + - swift_get_venv | success + tags: + - swift-install + - swift-pip-packages + +- name: Update virtualenv path + command: > + virtualenv-tools --update-path=auto {{ swift_venv_bin | dirname }} + when: + - swift_venv_enabled | bool + - swift_get_venv | success + tags: + - swift-install + - swift-pip-packages + - name: Install pip packages (venv) pip: name: "{{ item }}" @@ -65,7 +117,9 @@ delay: 2 with_items: - "{{ swift_pip_packages }}" - when: swift_venv_enabled | bool + when: + - swift_venv_enabled | bool + - swift_get_venv | failed tags: - swift-install - swift-pip-packages diff --git a/playbooks/roles/os_tempest/defaults/main.yml b/playbooks/roles/os_tempest/defaults/main.yml index 9ddc703d3d..e9400e523a 100644 --- a/playbooks/roles/os_tempest/defaults/main.yml +++ b/playbooks/roles/os_tempest/defaults/main.yml @@ -19,6 +19,7 @@ is_metal: true ## Verbosity Options debug: False verbose: True +tempest_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/tempest.tgz tempest_fatal_deprecations: False tempest_private_subnet_cidr: "192.168.74.0/24" @@ -71,6 +72,17 @@ tempest_volume_multi_backend_enabled: False tempest_main_group: tempest_all tempest_requires_pip_packages: + - python-glanceclient + - python-keystoneclient + - python-neutronclient + - python-novaclient + - virtualenv + - virtualenv-tools + +tempest_pip_packages: + - fixtures + - junitxml + - nose - oslo.serialization - python-ceilometerclient - python-cinderclient @@ -81,17 +93,12 @@ tempest_requires_pip_packages: - python-neutronclient - python-novaclient - python-openstackclient + - python-subunit - python-swiftclient - - virtualenv - -tempest_pip_packages: - - fixtures - - nose + - tempest-lib - testrepository - testscenarios - testtools - - python-subunit - - junitxml # Please update SHA in tempest_images below when changing the cirros version. cirros_version: 0.3.4 diff --git a/playbooks/roles/os_tempest/tasks/main.yml b/playbooks/roles/os_tempest/tasks/main.yml index 4966e1d7bb..c5f89e2609 100644 --- a/playbooks/roles/os_tempest/tasks/main.yml +++ b/playbooks/roles/os_tempest/tasks/main.yml @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +- include: tempest_install.yml + - include: tempest_resources.yml when: > inventory_hostname == groups[tempest_main_group][0] @@ -22,7 +24,6 @@ when: > inventory_hostname != groups[tempest_main_group][0] -- include: tempest_install.yml - include: tempest_post_install.yml - name: Flush handlers diff --git a/playbooks/roles/os_tempest/tasks/tempest_install.yml b/playbooks/roles/os_tempest/tasks/tempest_install.yml index 377d1863d5..c739040b8a 100644 --- a/playbooks/roles/os_tempest/tasks/tempest_install.yml +++ b/playbooks/roles/os_tempest/tasks/tempest_install.yml @@ -26,7 +26,8 @@ - "{{ tempest_requires_pip_packages }}" tags: - tempest-pip-requires-packages - - tempest-pip-install + - tempest-install + - tempest-pip-packages - name: Get tempest from git git: @@ -42,17 +43,83 @@ - tempest-git-clone - tempest-pip-install -- name: Install pip packages for tempest +- name: Attempt venv download + get_url: + url: "{{ tempest_venv_download_url }}" + dest: "/var/cache/{{ tempest_venv_download_url | basename }}" + ignore_errors: true + register: get_venv + tags: + - tempest-install + - tempest-pip-packages + +- name: Set tempest get_venv fact + set_fact: + tempest_get_venv: "{{ get_venv }}" + tags: + - tempest-install + - tempest-pip-packages + +- name: Create tempest venv dir + file: + path: "{{ tempest_git_dest }}" + state: directory + when: + - tempest_get_venv | success + tags: + - tempest-install + - tempest-pip-packages + +- name: Unarchive pre-built venv + unarchive: + src: "/var/cache/{{ tempest_venv_download_url | basename }}" + dest: "{{ tempest_git_dest }}" + copy: "no" + when: + - tempest_get_venv | success + tags: + - tempest-install + - tempest-pip-packages + +- name: Update virtualenv path + command: > + virtualenv-tools --update-path=auto {{ tempest_git_dest }} + when: + - tempest_get_venv | success + tags: + - tempest-install + - tempest-pip-packages + +- name: Install pip packages for tempest (prebuilt venv) pip: - name: "{{ item }}" + name: "{{ tempest_git_dest }}" state: present virtualenv: "{{ tempest_git_dest }}" - virtualenv_site_packages: "yes" + virtualenv_site_packages: "no" extra_args: "{{ tempest_pip_instructions }}" register: install_packages until: install_packages|success retries: 5 delay: 2 + when: + - tempest_get_venv | success + tags: + - tempest-pip-packages + - tempest-pip-install + +- name: Install pip packages for tempest (no prebuilt venv) + pip: + name: "{{ item }}" + state: present + virtualenv: "{{ tempest_git_dest }}" + virtualenv_site_packages: "no" + extra_args: "{{ tempest_pip_instructions }}" + register: install_packages + until: install_packages|success + retries: 5 + delay: 2 + when: + - tempest_get_venv | failed with_items: - "{{ tempest_pip_packages }}" - "{{ tempest_git_dest }}" diff --git a/playbooks/roles/repo_build/defaults/main.yml b/playbooks/roles/repo_build/defaults/main.yml index 8b86d1ddd5..98a956f6a5 100644 --- a/playbooks/roles/repo_build/defaults/main.yml +++ b/playbooks/roles/repo_build/defaults/main.yml @@ -30,3 +30,13 @@ repo_build_pip_default_index: "https://pypi.python.org/simple" repo_build_pip_extra_index: "https://pypi.python.org/simple" repo_build_timeout: 120 + +repo_build_venv_force_rebuild: false +repo_build_venv_build_dir: "/tmp/openstack-venv-builder" +repo_build_venv_dir: "/var/www/repo/venvs" +repo_build_venv_pip_install_options: > + --timeout 120 + --find-links {{ repo_build_release_path }}/{{ repo_build_release_tag }} + --no-index + --verbose + --log /var/log/repo/repo_venv_builder.log diff --git a/playbooks/roles/repo_build/tasks/main.yml b/playbooks/roles/repo_build/tasks/main.yml index f713c52c8a..b1f0e975a7 100644 --- a/playbooks/roles/repo_build/tasks/main.yml +++ b/playbooks/roles/repo_build/tasks/main.yml @@ -13,8 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Wheel building - include: repo_clone_git.yml - include: repo_set_facts.yml - include: repo_pre_build.yml - include: repo_build.yml - include: repo_post_build.yml + +# Venv building +- include: repo_venv.yml + tags: + - repo-build-venvs diff --git a/playbooks/roles/repo_build/tasks/repo_venv.yml b/playbooks/roles/repo_build/tasks/repo_venv.yml new file mode 100644 index 0000000000..12cd64bfa6 --- /dev/null +++ b/playbooks/roles/repo_build/tasks/repo_venv.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2015, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- include: repo_venv_pre_build.yml +- include: repo_venv_build.yml +- include: repo_venv_post_build.yml diff --git a/playbooks/roles/repo_build/tasks/repo_venv_build.yml b/playbooks/roles/repo_build/tasks/repo_venv_build.yml new file mode 100644 index 0000000000..f4094a9de3 --- /dev/null +++ b/playbooks/roles/repo_build/tasks/repo_venv_build.yml @@ -0,0 +1,81 @@ +--- +# Copyright 2015, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Get venv command path + command: which virtualenv + register: virtualenv_path + tags: + - repo-command-bin + - repo-create-venv + +- name: Set virtualenv command path + set_fact: + virtualenv_bin: "{{ virtualenv_path.stdout }}" + tags: + - repo-command-bin + - repo-create-venv + +- name: Check for created venvs + command: > + ls -1 "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}/" + register: created_venvs + tags: + - repo-create-venv + +- name: Set existing venv fact + set_fact: + existing_venvs: "{{ created_venvs.stdout_lines }}" + tags: + - repo-command-bin + - repo-create-venv + +- name: Create role based venv + pip: + name: "{{ item.value | join(' ') }}" + state: present + virtualenv: "{{ repo_build_venv_build_dir }}/venvs/{{ item.key | replace('os_', '') }}" + virtualenv_site_packages: "no" + virtualenv_command: "{{ virtualenv_bin }} --always-copy" + extra_args: "{{ repo_build_venv_pip_install_options }}" + register: install_packages + until: install_packages|success + retries: 5 + delay: 2 + with_dict: local_packages.results.0.item.role_packages + when: + - '"os_" in item.key' + - "'{{ item.key | replace('os_', '') }}.tgz' not in existing_venvs" + tags: + - repo-create-venv + +- name: Create venv archive + shell: | + # This is to clean up pyc files which makes the archived venv smaller + # TODO(cloudnull) This should use the find module when we move to Ansible 2.0 + find "{{ repo_build_venv_build_dir }}/venvs/{{ item.key | replace('os_', '') }}" -name '*.pyc' -delete + # Create archive + tar -czf "{{ item.key | replace('os_', '') }}-{{ repo_build_release_tag }}.tgz" \ + -C "{{ repo_build_venv_build_dir }}/venvs/{{ item.key | replace('os_', '') }}" . + args: + chdir: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}" + creates: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}/{{ item.key | replace('os_', '') }}.tgz" + with_dict: local_packages.results.0.item.role_packages + when: + - '"os_" in item.key' + tags: + - skip_ansible_lint + - repo-venv-compress-archive + - repo-create-venv-archive + - repo-create-venv diff --git a/playbooks/roles/repo_build/tasks/repo_venv_post_build.yml b/playbooks/roles/repo_build/tasks/repo_venv_post_build.yml new file mode 100644 index 0000000000..a28ca5b9c5 --- /dev/null +++ b/playbooks/roles/repo_build/tasks/repo_venv_post_build.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2015, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Cleanup venv directory + file: + path: "{{ item }}" + state: absent + with_items: + - "{{ repo_build_venv_build_dir }}" + tags: + - repo-cleanup-venv-location diff --git a/playbooks/roles/repo_build/tasks/repo_venv_pre_build.yml b/playbooks/roles/repo_build/tasks/repo_venv_pre_build.yml new file mode 100644 index 0000000000..da8b21fc88 --- /dev/null +++ b/playbooks/roles/repo_build/tasks/repo_venv_pre_build.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2015, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Make sure old venv build directories are clean + file: + path: "{{ item }}" + state: "absent" + with_items: + - "{{ repo_build_venv_build_dir }}" + tags: + - repo-create-venv-location + - repo-venv-compress-archive + - repo-create-venv-archive + +- name: Destroy base venvs to rebuild them + file: + path: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}" + state: "absent" + when: repo_build_venv_force_rebuild | bool + tags: + - repo-create-venv-location + - repo-venv-compress-archive + - repo-create-venv-archive + +- name: Create venv directory + file: + path: "{{ item }}" + state: "directory" + owner: "{{ repo_build_service_user_name }}" + mode: "2755" + with_items: + - "{{ repo_build_venv_build_dir }}/venvs" + - "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}" + tags: + - repo-create-venv-location + - repo-venv-compress-archive + - repo-create-venv-archive diff --git a/playbooks/roles/repo_server/defaults/main.yml b/playbooks/roles/repo_server/defaults/main.yml index 963d61bd6c..d2cb481d27 100644 --- a/playbooks/roles/repo_server/defaults/main.yml +++ b/playbooks/roles/repo_server/defaults/main.yml @@ -85,6 +85,8 @@ repo_pip_packages: - turbolift - wheel - yaprt + - virtualenv + - virtualenv-tools # Main web server port repo_server_port: 8181