Updated repo-build to store package sources

This change enables our repo-build process to store the package sources
for all of our requirements within a given release. This change enables
the ability to have a multi-archetecture / multi-distro / multi-interpreter
deployment.

Because a python wheel can be tied to a particular archetecture, distro, or
interpreter wheels alone can not be relied on to power a mixed environment. The
source is needed to ensure pip is able to resolve a package depenedency even if
a pre-built wheel does not meet the python or system requirements. To enable
this a task has been added to the repo_build.yml file to first download all of
the sources in non-binary format, once downloaded the sources are used to build
all of the wheels locally.

Change-Id: Ib081c24f67e92c165cba14848ff86e20fe6c1530
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

defaults/main.yml

@@ -56,3 +56,4 @@ repo_pip_packages:
   - virtualenv
   - virtualenv-tools
 
+repo_build_store_pip_sources: false

releasenotes/notes/pip-source-store-d94ff2b68a99481a.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - The repo build process now has the ability to store the pip
+    sources within the build archive. This ability is useful when
+    deploying environments that are "multi-architecture",
+    "multi-distro", or "multi-interpreter" where specific pre-build
+    wheels may not be enough to support all of the deployment.
+    To enable the ability to store the python source code within a
+    given release, set the new option ``repo_build_store_pip_sources``
+    to ``true``.

tasks/repo_build.yml

@@ -13,6 +13,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- name: Create requirement download process script
+  template:
+    src: "op-pip-download-script.sh.j2"
+    dest: "/opt/op-pip-download-script.sh"
+  when: repo_build_store_pip_sources | bool
+  tags:
+    - repo-pip-download
+
+- name: Run requirement process script
+  shell: "bash /opt/op-pip-download-script.sh"
+  when: repo_build_store_pip_sources | bool
+  tags:
+    - repo-build-local-requirement-wheels
+    - repo-build-openstack-ansible-requirement-wheels
+
 - name: Create OpenStack-Ansible requirement wheels
   shell: >
     pip wheel --timeout {{ repo_build_timeout }}

tasks/repo_pre_build.yml

@@ -46,6 +46,7 @@
   with_items:
     - "{{ repo_build_release_path }}/{{ repo_build_release_tag }}"
     - "{{ repo_build_global_links_path }}"
+    - "{{ repo_build_output }}"
   tags:
     - repo-create-release-links-location
 

templates/op-pip-download-script.sh.j2

@@ -0,0 +1,33 @@
+#!/usr/local/env bash
+
+set -ev
+
+PID=()
+{% for item in local_requirement_normalized %}
+    yes i | pip install --timeout {{ repo_build_timeout }} \
+                        --download {{ repo_build_output }} \
+                        --no-binary :all: \
+                        --constraint {{ repo_build_release_path }}/{{ repo_build_release_tag }}/requirements_constraints.txt \
+                        {% if repo_build_pip_default_index is defined %}
+                        --index-url {{ repo_build_pip_default_index }} \
+                        --trusted-host {{ repo_build_pip_default_index | netloc_no_port }} \
+                        {% endif -%}
+                        {% if repo_build_pip_extra_index is defined %}
+                        --extra-index-url {{ repo_build_pip_extra_index }} \
+                        --trusted-host {{ repo_build_pip_extra_index | netloc_no_port }} \
+                        {% endif -%}
+                        {% if repo_build_pip_extra_indexes is defined %}
+                        --extra-index-url {{ repo_build_pip_extra_indexes | join(' --extra-index-url ') }} \
+                        --trusted-host {{ repo_build_pip_extra_indexes | map('netloc_no_port') | join(' --trusted-host ') }} \
+                        {% endif -%}
+                        --log /var/log/repo/repo_builder.log \
+                        "{{ item }}"
+    pid[{{ loop.index }}]=$!
+
+{% if loop.index is divisibleby(repo_build_concurrency | int) or loop.last %}
+    for job_pid in ${!pid[@]}; do
+        wait ${pid[$job_pid]} || exit 99
+    done
+{% endif %}
+
+{% endfor %}