Set the default pip index dynamically

Before the repo server is up, only bare metal hosts
need pip installed, openstack-hosts-setup take care
of that part by default, with the openstack_host role
calling pip_install.

Once the repo server is up, the pypi cache will be available
to be used. This implements the changes to detect whether it
is available - this is important for the plays doing pip
installs before the repo server is available. Once it is
available the same plays will use it.

To be available, we need to wait for haproxy playbook to
load balance towards the repo container, or the baremetal
node if need be.

Once the repo server and the load balancer are up, we
can configure everything to use the repo server, and this
will be done with the repo-use.yml playbook, which is
calling the pip_install role.

We also remove the use of the pip_lock_to_internal_repo
from group variables, as we want to change the pip.conf
dynamically depending on the availability of the default
index.

Finally, we set the AIO in OpenStack-Infra to make use of
the appropriate infra mirror for the nginx reverse proxy.

Because the gates are blocked without this patch, this
patch should land in priority. The current upstream state
prevents that, so we temporarily disable the ceph testing,
which fails due to [1]. On top of it, we need to bump
shas to a known working state, else tempest would fail.
To avoid circular dependencies, we include the content
of patch [2], which is the sha bump pending this work.

[1]: https://bugs.launchpad.net/cinder/+bug/1737015
[2]: https://review.openstack.org/#/c/522850/

Change-Id: If19442918baa9ddacf7c19940c9b5007694bee61
Depends-On: I56f22f46ff849a7049ac7ae873af7a4bc526fa63
Depends-On: Ic4fd64f4dc82121a65088f3d7f4ae53f373df608
Depends-On: I7bd31f2d89d3fe9d48e32c79ddef7a8ef1392eb7
Depends-On: Icd93c0c801bfee1b4fdc8154d078067722c0640a
Depends-On: I1e9782eb0fd72690a9644c7a01e8c83a4486872b
Co-Authored-by: Jesse Pretorius <jesse.pretorius@rackspace.co.uk>

group_vars/all/pip.yml

@@ -17,7 +17,6 @@ pip_install_package_state: "{{ package_state }}"
 
 pip_links:
   - { name: "openstack_release", link: "{{ repo_release_path }}/" }
-pip_lock_to_internal_repo: "{{ (pip_links | length) >= 1 }}"
 
 # The upper constraints to apply to all pip installations
 pip_install_upper_constraints: "{{ repo_release_path }}/requirements_absolute_requirements.txt"
@@ -29,3 +28,6 @@ pip_offline_install: false
 
 # The URL to retrieve the get-pip.py installation script
 pip_upstream_url: "{{ (pip_offline_install | bool) | ternary('https://bootstrap.pypa.io/get-pip.py', repo_release_path ~ '/get-pip.py') }}"
+
+# The URL to the repo server's pypi reverse proxy simple index
+pip_default_index: "{{ openstack_repo_url }}/simple"

group_vars/repo_all.yml

@@ -43,9 +43,6 @@ repo_service_group_name: www-data
 repo_build_service_user_name: "{{ repo_service_user_name }}"
 repo_build_service_group_name: "{{ repo_service_group_name }}"
 
-# Disable the pip lock down for the repo servers
-pip_lock_to_internal_repo: False
-
 # The following package must always build from source.
 #
 # libvirt-python:
@@ -75,3 +72,14 @@ pkg_locations:
   - "{{ playbook_dir }}/../"
   - /etc/ansible/roles
   - /etc/openstack_deploy
+
+# On repo server, don't go through load balancer, directly hit the local
+# instances. It's all tested anyway before using.
+pip_links:
+  - { name: "openstack_release", link: "{{ repo_release_path | replace(internal_lb_vip_address, '127.0.0.1') }}/" }
+
+# The URL to retrieve the get-pip.py installation script locally
+pip_upstream_url: "{{ repo_release_path | replace(internal_lb_vip_address, '127.0.0.1') }}/get-pip.py"
+
+# The URL to the local PyPI reverse proxy
+pip_default_index: "{{ openstack_repo_url | replace(internal_lb_vip_address, '127.0.0.1') }}/simple"

playbooks/common-tasks/set-upper-constraints.yml

@@ -27,6 +27,15 @@
   tags:
     - common-constraints
 
+- name: Test internal repo URL for the availability of the pypi index
+  uri:
+    url: "{{ pip_default_index }}"
+    method: "HEAD"
+  register: pip_default_index_check
+  failed_when: false
+  tags:
+    - common-constraints
+
 - name: Remove global requirement pins file from host
   file:
     path: "/opt/global-requirement-pins.txt"
@@ -59,3 +68,11 @@
   when: (upper_constraints_check.status | default(503)) != 200
   tags:
     - common-constraints
+
+- name: Fallback to PyPI
+  set_fact:
+    pip_default_index: "https://pypi.python.org/simple"
+    pip_lock_to_internal_repo: False
+  when: (pip_default_index_check.status | default(503)) != 200
+  tags:
+    - common-constraints

playbooks/defaults/repo_packages/gnocchi.yml

@@ -33,5 +33,5 @@
 ## This service has a different stable branch strategy to the rest of OpenStack.
 ## The SHA is recorded here to make the SHA updating easier.
 gnocchi_git_repo: https://github.com/gnocchixyz/gnocchi
-gnocchi_git_install_branch: 81e2445ca7ef8c389c6c3f620a3f5b3ca144bd28 # HEAD of "master" as of 14.10.2017
+gnocchi_git_install_branch: fa065ac313ea9573782e89b4a1983afeeec4f059 # HEAD of "master" as of 24.11.2017
 gnocchi_git_project_group: gnocchi_all

playbooks/defaults/repo_packages/nova_consoles.yml

@@ -27,11 +27,11 @@
 
 ## NOVNC from source
 novncproxy_git_repo: https://github.com/novnc/noVNC
-novncproxy_git_install_branch: 3c5dd884d2e658bb23fd324570b48b86db153d14 # HEAD of "master" as of 14.10.2017
+novncproxy_git_install_branch: 7f3986815879421432ba6f4ea4b16c15925daccd # HEAD of "master" as of 24.11.2017
 novncproxy_git_project_group: nova_console
 
 ## spice-html5 from source
 spicehtml5_git_repo: https://github.com/SPICE/spice-html5
-spicehtml5_git_install_branch: 54cc41299bea8cd681ed0262735e0fd821cd774a # HEAD of "master" as of 14.10.2017
+spicehtml5_git_install_branch: 54cc41299bea8cd681ed0262735e0fd821cd774a # HEAD of "master" as of 24.11.2017
 spicehtml5_git_project_group: nova_console
 

playbooks/defaults/repo_packages/opendaylight.yml

@@ -14,5 +14,5 @@
 # limitations under the License.
 
 networking_odl_git_repo: https://git.openstack.org/openstack/networking-odl
-networking_odl_git_install_branch: b304fc9154d81e021d26327b39528ceaa3f0ac72 # HEAD of "master" as of 14.10.2017
+networking_odl_git_install_branch: 7a3c5fee7deb01d9237f5d1cc43a17931999af02 # HEAD of "master" as of 24.11.2017
 networking_odl_project_group: neutron_all

playbooks/defaults/repo_packages/openstack_services.yml

@@ -31,192 +31,192 @@
 
 ## Global Requirements
 requirements_git_repo: https://git.openstack.org/openstack/requirements
-requirements_git_install_branch: 6d9fe55d3697d46c8e10de829ebbdc8bcddf0111 # HEAD of "master" as of 14.10.2017
+requirements_git_install_branch: 691711c0effddd9cbaaadba3d494c15bc422fdd5 # HEAD of "master" as of 24.11.2017
 
 
 ## Aodh service
 aodh_git_repo: https://git.openstack.org/openstack/aodh
-aodh_git_install_branch: 348ec72208d9fc883eb93ae1b36bd501fe401cbc # HEAD of "master" as of 14.10.2017
+aodh_git_install_branch: 359043dc774be847cb539d18d13e336d40453e72 # HEAD of "master" as of 24.11.2017
 aodh_git_project_group: aodh_all
 
 
 ## Barbican service
 barbican_git_repo: https://git.openstack.org/openstack/barbican
-barbican_git_install_branch: d2ab56c61c0dc215dfbd078d00f4cabceaa2cdd7 # HEAD of "master" as of 14.10.2017
+barbican_git_install_branch: 5617d605f2e12840933e4a9d6417912cdbb811d5 # HEAD of "master" as of 24.11.2017
 barbican_git_project_group: barbican_all
 
 
 ## Ceilometer service
 ceilometer_git_repo: https://git.openstack.org/openstack/ceilometer
-ceilometer_git_install_branch: ecfdcc42a98efddc62838d3f6823a1cd5ff94cac # HEAD of "master" as of 14.10.2017
+ceilometer_git_install_branch: bd464f1f572ba150f52e284de430d13045dc6c18 # HEAD of "master" as of 24.11.2017
 ceilometer_git_project_group: ceilometer_all
 
 
 ## Cinder service
 cinder_git_repo: https://git.openstack.org/openstack/cinder
-cinder_git_install_branch: 625cf70f9690afa34ea717a7b3417890971e4c4a # HEAD of "master" as of 17.10.2017
+cinder_git_install_branch: 80558687d0fa55f2adf699e7369ebe3dbc3591bf # HEAD of "master" as of 24.11.2017
 cinder_git_project_group: cinder_all
 
 
 ## Designate service
 designate_git_repo: https://git.openstack.org/openstack/designate
-designate_git_install_branch: 284cdabe1e82934b05e20fda26423702c2628918 # HEAD of "master" as of 14.10.2017
+designate_git_install_branch: 2f75586379e8d611f37e06d385e79d0bc2c84ca1 # HEAD of "master" as of 24.11.2017
 designate_git_project_group: designate_all
 
 
 ## Horizon Designate dashboard plugin
 designate_dashboard_git_repo: https://git.openstack.org/openstack/designate-dashboard
-designate_dashboard_git_install_branch: 6213e7534f2c197bd2c577bd183d0bd53bf89857 # HEAD of "master" as of 14.10.2017
+designate_dashboard_git_install_branch: 571e127e5f853aa4dbdd377d831e32f8ff81eafe # HEAD of "master" as of 24.11.2017
 designate_dashboard_git_project_group: horizon_all
 
 
 ## Dragonflow service
 dragonflow_git_repo: https://git.openstack.org/openstack/dragonflow
-dragonflow_git_install_branch: 84f1a26ff8e976b753593dc445e09a4c1a675a21 # HEAD of "master" as of 14.10.2017
+dragonflow_git_install_branch: 7bf00cf315659252f03f6c65f6159a924da6f978 # HEAD of "master" as of 24.11.2017
 dragonflow_git_project_group: neutron_all
 
 
 ## Glance service
 glance_git_repo: https://git.openstack.org/openstack/glance
-glance_git_install_branch: 7f9853122252c38ecc536ea523bc8e3094c3adda # HEAD of "master" as of 14.10.2017
+glance_git_install_branch: d88bd2ca8ef95810441dae640d3c6b9e79eca353 # HEAD of "master" as of 24.11.2017
 glance_git_project_group: glance_all
 
 
 ## Heat service
 heat_git_repo: https://git.openstack.org/openstack/heat
-heat_git_install_branch: c5593161d9bcbddde69e38d9136ccb93367b51f8 # HEAD of "master" as of 14.10.2017
+heat_git_install_branch: f4a06c2a92a361dbb401107b4ea1ab60972f473e # HEAD of "master" as of 24.11.2017
 heat_git_project_group: heat_all
 
 
 ## Horizon service
 horizon_git_repo: https://git.openstack.org/openstack/horizon
-horizon_git_install_branch: 80643ca53f7db43eac496829290983126616ea4b # HEAD of "master" as of 14.10.2017
+horizon_git_install_branch: 846d269d90e01e463b510474040e0ad984a5679f # HEAD of "master" as of 24.11.2017
 horizon_git_project_group: horizon_all
 
 ## Horizon Ironic dashboard plugin
 ironic_dashboard_git_repo: https://git.openstack.org/openstack/ironic-ui
-ironic_dashboard_git_install_branch: 70933196786a780d690b11af9995b37942f28e3b # HEAD of "master" as of 14.10.2017
+ironic_dashboard_git_install_branch: d6199d51171e6c8700663b0b0618ee0adf033b4d # HEAD of "master" as of 24.11.2017
 ironic_dashboard_git_project_group: horizon_all
 
 ## Horizon Magnum dashboard plugin
 magnum_dashboard_git_repo: https://git.openstack.org/openstack/magnum-ui
-magnum_dashboard_git_install_branch: 02c2d06bdaebc892d37d5b51fc0555a5469bf017 # HEAD of "master" as of 14.10.2017
+magnum_dashboard_git_install_branch: 6160d903fae9c652b459c93c218e0ea75924a85d # HEAD of "master" as of 24.11.2017
 magnum_dashboard_git_project_group: horizon_all
 
 ## Horizon LBaaS dashboard plugin
 neutron_lbaas_dashboard_git_repo: https://git.openstack.org/openstack/neutron-lbaas-dashboard
-neutron_lbaas_dashboard_git_install_branch: 80a2f0f5c82acb58833c43e158afd974e555f92b # HEAD of "master" as of 14.10.2017
+neutron_lbaas_dashboard_git_install_branch: ef650294bcc7447d441e6a710c39d64e384e1b27 # HEAD of "master" as of 24.11.2017
 neutron_lbaas_dashboard_git_project_group: horizon_all
 
 ## Horizon FWaaS dashboard plugin
 neutron_fwaas_dashboard_git_repo: https://git.openstack.org//openstack/neutron-fwaas-dashboard
-neutron_fwaas_dashboard_git_install_branch: 58cdd72b1398ad7a4d4dcdba2ca9fe37fe9f63f8 # HEAD of "master" as of 14.10.2017
+neutron_fwaas_dashboard_git_install_branch: 6de122d4753a6db24d2dc4c22a71e702ed980e82 # HEAD of "master" as of 24.11.2017
 neutron_fwaas_dashboard_git_project_group: horizon_all
 
 ## Horizon Sahara dashboard plugin
 sahara_dashboard_git_repo: https://git.openstack.org/openstack/sahara-dashboard
-sahara_dashboard_git_install_branch: b8ccdf99971293a388b99d781b70fa0e0fb69d40 # HEAD of "master" as of 14.10.2017
+sahara_dashboard_git_install_branch: 3e5c59e6229dac8b303029058fcee9d61200ebc8 # HEAD of "master" as of 24.11.2017
 sahara_dashboard_git_project_group: horizon_all
 
 
 ## Keystone service
 keystone_git_repo: https://git.openstack.org/openstack/keystone
-keystone_git_install_branch: 47dbd256258d747d95cb5320bd02ae207ecf60d6 # HEAD of "master" as of 14.10.2017
+keystone_git_install_branch: 70fe4ec09b55def21361a32c8fa7f12e7c891ab1 # HEAD of "master" as of 24.11.2017
 keystone_git_project_group: keystone_all
 
 
 ## Neutron service
 neutron_git_repo: https://git.openstack.org/openstack/neutron
-neutron_git_install_branch: 62b788192014705ae1c31bed05fd5f6dbba9a2d9 # HEAD of "master" as of 14.10.2017
+neutron_git_install_branch: d1277c1630570ca45b490c48371e3f7e97be78c3 # HEAD of "master" as of 24.11.2017
 neutron_git_project_group: neutron_all
 
 neutron_lbaas_git_repo: https://git.openstack.org/openstack/neutron-lbaas
-neutron_lbaas_git_install_branch: a5294c1d1149bf3bcb3321937c9223f18b49ed7a # HEAD of "master" as of 14.10.2017
+neutron_lbaas_git_install_branch: b1123e7a759248dfa63afdf8b86aafd692572ebd # HEAD of "master" as of 24.11.2017
 neutron_lbaas_git_project_group: neutron_all
 
 neutron_vpnaas_git_repo: https://git.openstack.org/openstack/neutron-vpnaas
-neutron_vpnaas_git_install_branch: e3da5c10c472ece03828b076a9240b735295a42e # HEAD of "master" as of 14.10.2017
+neutron_vpnaas_git_install_branch: 79e4eb81dd05588bcf68b92d46c62f0d26153542 # HEAD of "master" as of 24.11.2017
 neutron_vpnaas_git_project_group: neutron_all
 
 neutron_fwaas_git_repo: https://git.openstack.org/openstack/neutron-fwaas
-neutron_fwaas_git_install_branch: 43f7ae3493a9e6333f3b9d533271a757de036736 # HEAD of "master" as of 14.10.2017
+neutron_fwaas_git_install_branch: 74eac2ca2980e6162d9c88ee6bd48830386c392a # HEAD of "master" as of 24.11.2017
 neutron_fwaas_git_project_group: neutron_all
 
 neutron_dynamic_routing_git_repo: https://git.openstack.org/openstack/neutron-dynamic-routing
-neutron_dynamic_routing_git_install_branch: 95d2bdfde10ba9622014620738d15e8c5bff0bb4 # HEAD of "master" as of 14.10.2017
+neutron_dynamic_routing_git_install_branch: 183c3fa4840d22be1974534eb9e1b28b552f4a42 # HEAD of "master" as of 24.11.2017
 neutron_dynamic_routing_git_project_group: neutron_all
 
 networking_calico_git_repo: https://git.openstack.org/openstack/networking-calico
-networking_calico_git_install_branch: 9688df1a3d1d8b3fd9ba367e82fe6b0559416728 # HEAD of "master" as of 14.10.2017
+networking_calico_git_install_branch: 9688df1a3d1d8b3fd9ba367e82fe6b0559416728 # HEAD of "master" as of 24.11.2017
 networking_calico_git_project_group: neutron_all
 
 ## Nova service
 nova_git_repo: https://git.openstack.org/openstack/nova
-nova_git_install_branch: c7832f7d9e60a18e95f3cfeef76d55989fbad803 # HEAD of "master" as of 14.10.2017
+nova_git_install_branch: 22a790ef45b0523e8cf2ed97d14e050431c90fd9 # HEAD of "master" as of 24.11.2017
 nova_git_project_group: nova_all
 
 
 ## PowerVM Virt Driver
 nova_powervm_git_repo: https://git.openstack.org/openstack/nova-powervm
-nova_powervm_git_install_branch: a58458507433d970b53e338df199dc8251812c23 # HEAD of "master" as of 14.10.2017
+nova_powervm_git_install_branch: f2de4441e39b0f66cf31f854b228e9e7037f04de # HEAD of "master" as of 24.11.2017
 nova_powervm_git_project_group: nova_all
 
 
 ## LXD Virt Driver
 nova_lxd_git_repo: https://git.openstack.org/openstack/nova-lxd
-nova_lxd_git_install_branch: aaa8b602150623a81432eb8fbad26c2d4d9b42b6 # HEAD of "master" as of 14.10.2017
+nova_lxd_git_install_branch: e498de603b31c189fd32a6067d45a36575b96b0a # HEAD of "master" as of 24.11.2017
 nova_lxd_git_project_group: nova_all
 
 
 ## Sahara service
 sahara_git_repo: https://git.openstack.org/openstack/sahara
-sahara_git_install_branch: cf4cf40eab5a485abb6052b4a9d263be4625dc4e # HEAD of "master" as of 14.10.2017
+sahara_git_install_branch: 395856c513b1efad82db8fa78fb1cbfe0f3a6749 # HEAD of "master" as of 24.11.2017
 sahara_git_project_group: sahara_all
 
 
 ## Swift service
 swift_git_repo: https://git.openstack.org/openstack/swift
-swift_git_install_branch: 98e7014aafbd52bd5e388314f3c2c1e1d2375806 # HEAD of "master" as of 14.10.2017
+swift_git_install_branch: 3135878d2fe9909f49fcadeeb9cc6c6933d06127 # HEAD of "master" as of 24.11.2017
 swift_git_project_group: swift_all
 
 
 ## Swift3 middleware
 swift_swift3_git_repo: https://git.openstack.org/openstack/swift3
-swift_swift3_git_install_branch: 1fb6a30ee59a16cd4b6c49bab963ff9e3f974580 # HEAD of "master" as of 14.10.2017
+swift_swift3_git_install_branch: 1fb6a30ee59a16cd4b6c49bab963ff9e3f974580 # HEAD of "master" as of 24.11.2017
 swift_swift3_git_project_group: swift_all
 
 
 ## Ironic service
 ironic_git_repo: https://git.openstack.org/openstack/ironic
-ironic_git_install_branch: 2d486ca8c0b1ed9251a76945457e0fc28239e7ac # HEAD of "master" as of 14.10.2017
+ironic_git_install_branch: 27ce77142bfb9ac56e85db37e0923a0eb47f2f7a # HEAD of "master" as of 24.11.2017
 ironic_git_project_group: ironic_all
 
 ## Magnum service
 magnum_git_repo: https://git.openstack.org/openstack/magnum
-magnum_git_install_branch: 4b88f7b78010299148fa8878e963c15fe5d5f525 # HEAD of "master" as of 14.10.2017
+magnum_git_install_branch: 4bf3b3263870a4ec81cf372713cacec446b3ee84 # HEAD of "master" as of 24.11.2017
 magnum_git_project_group: magnum_all
 
 ## Trove service
 trove_git_repo: https://git.openstack.org/openstack/trove
-trove_git_install_branch: f6c3c27a49d5cf1c84eec441923f9f4d8c12f3d2 # HEAD of "master" as of 14.10.2017
+trove_git_install_branch: b09d0eb3135047891a369d3c0eb2c6e9ae649f5b # HEAD of "master" as of 24.11.2017
 trove_git_project_group: trove_all
 
 ## Horizon Trove dashboard plugin
 trove_dashboard_git_repo: https://git.openstack.org/openstack/trove-dashboard
-trove_dashboard_git_install_branch: b4ebbbab63501793839d519f35d6d381f0334ca8 # HEAD of "master" as of 14.10.2017
+trove_dashboard_git_install_branch: 14a4609606d42cae827b8fc6b44453caea258976 # HEAD of "master" as of 24.11.2017
 trove_dashboard_git_project_group: horizon_all
 
 ## Octavia service
 octavia_git_repo: https://git.openstack.org/openstack/octavia
-octavia_git_install_branch: 859bb3bae35fb26e3305d4ee691b4d5eba99d1ff # HEAD of "master" as of 14.10.2017
+octavia_git_install_branch: bb9bb2d05b268cff9846e0a09ad3940be5fe5a80 # HEAD of "master" as of 24.11.2017
 octavia_git_project_group: octavia_all
 
 ## Molteniron service
 molteniron_git_repo: https://git.openstack.org/openstack/molteniron
-molteniron_git_install_branch: 094276cda77d814d07ad885e7d63de8d1243750a # HEAD of "master" as of 14.10.2017
+molteniron_git_install_branch: 094276cda77d814d07ad885e7d63de8d1243750a # HEAD of "master" as of 24.11.2017
 molteniron_git_project_group: molteniron_all
 
 ## Tacker service
 tacker_git_repo: https://git.openstack.org/openstack/tacker
-tacker_git_install_branch: 9ca9cf28cec8ae70a7795f7267ddd9909f5e307a # HEAD of "master" as of 14.10.2017
+tacker_git_install_branch: cc03b5d952527b8cad2e2e309a97d55afb1ca559 # HEAD of "master" as of 24.11.2017
 tacker_git_project_group: tacker_all

playbooks/defaults/repo_packages/openstack_testing.yml

@@ -27,13 +27,13 @@
 
 ## Tempest service
 tempest_git_repo: https://git.openstack.org/openstack/tempest
-tempest_git_install_branch: f48348533ebb2367463ada7402aad6e7cfee1ddd # HEAD of "master" as of 14.10.2017
+tempest_git_install_branch: 6e3b0b0d956e0cf4c83cf331b4e0eb8f5a6221a6 # HEAD of "master" as of 24.11.2017
 tempest_git_project_group: utility_all
 tempest_git_install_fragments: "venvwithindex=True&ignorerequirements=True"
 
 
 ## Rally service
 rally_git_repo: https://git.openstack.org/openstack/rally
-rally_git_install_branch: 425a2a7630df5e99ab287787081d88c5c140dedc # HEAD of "master" as of 14.10.2017
+rally_git_install_branch: 8e610c4f7f757a13fe96d09cc919c49d03af3746 # HEAD of "master" as of 24.11.2017
 rally_git_project_group: utility_all
 rally_git_install_fragments: "venvwithindex=True&ignorerequirements=True"

playbooks/lxc-containers-create.yml

@@ -41,7 +41,9 @@
   tags:
     - lxc-containers-create
 
-- name: Configure containers default software
+# TODO(evrardjp): Remove host_need_pip in the future
+# when the process building the repo is done before this step.
+- name: Configure containers default software, but don't run pip yet
   hosts: "{{ container_group|default('all_containers') }}"
   gather_facts: true
   user: root
@@ -49,6 +51,7 @@
     - role: "openstack_hosts"
   vars:
     is_metal: "{{ properties.is_metal|default(false) }}"
+    host_need_pip: False
   environment: "{{ deployment_environment_variables | default({}) }}"
   tags:
     - lxc-containers-create

playbooks/lxc-hosts-setup.yml

@@ -19,8 +19,6 @@
   max_fail_percentage: 20
   user: root
   pre_tasks:
-    - include: common-tasks/set-upper-constraints.yml
-    - include: common-tasks/set-pip-upstream-url.yml
     - name: Check the state of the default LXC service log directory
       stat:
         path: "/var/log/lxc"
@@ -51,10 +49,6 @@
       rsyslog_client_config_name: "99-lxc-rsyslog-client.conf"
       tags:
         - rsyslog
-  vars_files:
-    - defaults/repo_packages/openstack_services.yml
-  vars:
-    pip_lock_to_internal_repo: False
   environment: "{{ deployment_environment_variables | default({}) }}"
   tags:
     - lxc-hosts

playbooks/openstack-hosts-setup.yml

@@ -49,8 +49,12 @@
             (ansible_os_family == 'RedHat' and ansible_distribution_major_version == '7') or
             (ansible_os_family == 'Suse' and ansible_distribution_major_version == '42')
         msg: "The only supported platforms for this release are Ubuntu 16.04 LTS (Xenial), CentOS 7 (WIP) and openSUSE Leap 42.X (WIP)"
+    - include: common-tasks/set-upper-constraints.yml
+    - include: common-tasks/set-pip-upstream-url.yml
   roles:
     - role: "openstack_hosts"
+  vars_files:
+    - defaults/repo_packages/openstack_services.yml
   environment: "{{ deployment_environment_variables | default({}) }}"
   tags:
     - openstack-hosts

playbooks/repo-use.yml

@@ -0,0 +1,29 @@
+---
+# Copyright 2017, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# TODO(evrardjp): Remove this playbook when the repo_build process
+# is done upfront. This would mean that the openstack_hosts role
+# can run once and for all directly after the
+# lxc-container-create playbook.
+
+- name: Configure all nodes to use repo
+  hosts: all:!repo_all
+  roles:
+    - pip_install
+
+  environment: "{{ deployment_environment_variables | default({}) }}"
+  tags:
+    - always
+    - repo-config

playbooks/setup-infrastructure.yml

@@ -16,6 +16,10 @@
 - include: unbound-install.yml
 - include: repo-install.yml
 - include: haproxy-install.yml
+# TODO(evrardjp): Remove the following when repo_build is done
+# before lxc_container_create, and haproxy is moved with it as
+# second step.
+- include: repo-use.yml
 - include: memcached-install.yml
 - include: galera-install.yml
 - include: rabbitmq-install.yml

tests/roles/bootstrap-host/tasks/prepare_aio_config.yml

@@ -209,3 +209,15 @@
       line: 'security_package_state: "present"'
     - path: /etc/openstack_deploy/host_vars/localhost.yml
       line: 'tintin: "milou"'
+
+- name: Discover the pypi mirror URL when in nodepool
+  shell: |
+    source /etc/ci/mirror_info.sh
+    echo "${NODEPOOL_PYPI_MIRROR}"
+  args:
+    executable: /bin/bash
+  register: _pypi_mirror
+  when:
+    - nodepool_dir.stat.exists | bool
+  tags:
+    - skip_ansible_lint

tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2

@@ -205,3 +205,7 @@ security_rhel7_enable_chrony: no
 nova_service_negate:
   - "nova-agent.service"
   - "nova-resetnetwork.service"
+
+{% if _pypi_mirror is defined and _pypi_mirror.stdout is defined %}
+repo_nginx_pypi_upstream: "{{ _pypi_mirror.stdout | netloc }}"
+{% endif %}

zuul.d/jobs.yaml

@@ -50,6 +50,10 @@
 - job:
     name: openstack-ansible-deploy-ceph-ubuntu-xenial
     parent: openstack-ansible-deploy-aio-ubuntu-xenial
+    # temporarily moved to non voting until
+    # https://bugs.launchpad.net/cinder/+bug/1737015
+    # has been fixed.
+    voting: false
     vars:
       action: deploy
       scenario: ceph

zuul.d/project.yaml

@@ -33,7 +33,9 @@
         - openstack-ansible-linters
         - openstack-ansible-varstest-aio-ubuntu-xenial
         - openstack-ansible-deploy-aio-ubuntu-xenial
-        - openstack-ansible-deploy-ceph-ubuntu-xenial
+        # Removed temporarily until
+        # https://bugs.launchpad.net/cinder/+bug/1737015 is fixed
+        # - openstack-ansible-deploy-ceph-ubuntu-xenial
     periodic:
       jobs:
         - openstack-ansible-deploy-aio-centos-7