Modified repo build for better multi-OS support

The repo build role will now store all wheels and venvs in directories
containg information about the OS in which the bits were build on. The
intention here is to ensure that all installation are able to support
multi-OS without running into issues caused by system packages and
different core libs. This change builds upon the multi-archetecture
support we already have.

Closes-Bug: 1641131
Change-Id: I3f36afa307e02a38d73b860fc23ed94a09882c34
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

defaults/main.yml

@@ -35,14 +35,16 @@ repo_build_service_user_name: "nginx"
 repo_build_service_group_name: "www-data"
 repo_build_base_path: "/var/www/repo"
 
-repo_build_global_links_path: "/var/www/repo/links"
-repo_build_release_path: "/var/www/repo/os-releases"
-repo_build_dir: "/tmp/openstack-builder"
-repo_build_output: "/tmp/openstack-wheel-output"
-repo_build_git_dir: "/var/www/repo/openstackgit"
-repo_build_pool_dir: "/var/www/repo/pools"
+repo_build_os_distro_version: "{{ ansible_distribution | lower }}-{{ ansible_distribution_version.split('.')[:2] | join('.') }}-{{ ansible_architecture | lower }}"
 
 repo_build_release_tag: "untagged"
+repo_build_global_links_path: "{{ repo_build_base_path }}/links"
+repo_build_release_version_path: "os-releases/{{ repo_build_release_tag }}"
+repo_build_release_path: "{{ repo_build_base_path }}/{{ repo_build_release_version_path }}/{{ repo_build_os_distro_version }}"
+repo_build_dir: "/tmp/openstack-builder"
+repo_build_output: "/tmp/openstack-wheel-output"
+repo_build_git_dir: "{{ repo_build_base_path }}/openstackgit"
+repo_build_pool_dir: "{{ repo_build_base_path }}/pools/{{ repo_build_os_distro_version }}"
 
 # Toggle whether git repositories should be cloned selectively or not
 repo_build_git_selective: "{{ true if (repo_build_wheel_selective | bool and repo_build_venv_selective | bool) else false }}"
@@ -73,15 +75,15 @@ repo_build_venv_selective: yes
 repo_build_timeout: 120
 repo_build_concurrency: "{{ (((ansible_processor_vcpus|default(1)) | int) > 4) | ternary(8, 4) }}"
 repo_build_venv_build_dir: "/tmp/openstack-venv-builder"
-repo_build_venv_dir: "/var/www/repo/venvs"
+repo_build_venv_dir: "{{ repo_build_base_path }}/venvs/{{ repo_build_release_tag }}/{{ repo_build_os_distro_version }}"
 repo_build_venv_pip_install_options: >
   --timeout 120
-  --find-links {{ repo_build_release_path }}/{{ repo_build_release_tag }}
+  --find-links {{ repo_build_release_path }}
   --log /var/log/repo/repo_venv_builder.log
 repo_build_venv_command_options: >
   {{ virtualenv_bin }}
   --always-copy
-  --extra-search-dir {{ repo_build_release_path }}/{{ repo_build_release_tag }}
+  --extra-search-dir {{ repo_build_release_path }}
   --never-download
 repo_pip_packages:
   - requests

releasenotes/notes/multi-os-wheel-repos-7910b13547ea59ad.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - The repo server file system structure has been updated to
+    allow for multiple Operating systems running multiple
+    architectures to be run at the same time and served from
+    a single server without impacting pools, venvs, wheel
+    archives, and manifests. The new structure follows the
+    following pattern `$RELEASE/$OS_TYPE-$ARCH` and has been
+    applied to *os-releases*, *venvs*, and *pools*.

tasks/repo_build.yml

@@ -18,7 +18,7 @@
     yes i | pip install --timeout {{ repo_build_timeout }}
                         --download {{ repo_build_output }}
                         --no-binary :all:
-                        --constraint {{ repo_build_release_path }}/{{ repo_build_release_tag }}/requirements_constraints.txt
+                        --constraint {{ repo_build_release_path }}/requirements_constraints.txt
                         {% if repo_build_pip_default_index is defined %}
                         --index-url {{ repo_build_pip_default_index }}
                         --trusted-host {{ repo_build_pip_default_index | netloc_no_port }}
@@ -32,7 +32,7 @@
                         --trusted-host {{ repo_build_pip_extra_indexes | map('netloc_no_port') | join(' --trusted-host ') }}
                         {% endif -%}
                         --log /var/log/repo/repo_builder.log
-                        --requirement {{ repo_build_release_path }}/{{ repo_build_release_tag }}/requirements.txt
+                        --requirement {{ repo_build_release_path }}/requirements.txt
   when: repo_build_store_pip_sources | bool
   tags:
     - repo-pip-download
@@ -43,7 +43,7 @@
     --wheel-dir {{ repo_build_output }}
     --find-links {{ repo_build_global_links_path }}
     --find-links {{ repo_build_output }}
-    --constraint {{ repo_build_release_path }}/{{ repo_build_release_tag }}/requirements_constraints.txt
+    --constraint {{ repo_build_release_path }}/requirements_constraints.txt
     {% if repo_build_pip_no_binary is defined %}--no-binary {{ repo_build_pip_no_binary | join(' --no-binary ') }}{% endif %}
     {% if repo_build_pip_default_index is defined %}--index-url {{ repo_build_pip_default_index }}
     --trusted-host {{ repo_build_pip_default_index | netloc_no_port }}{% endif %}
@@ -53,7 +53,7 @@
     --trusted-host {{ repo_build_pip_extra_indexes | map('netloc_no_port') | join(' --trusted-host ') }}{% endif %}
     --build {{ repo_build_dir }}
     --log /var/log/repo/repo_builder.log
-    --requirement {{ repo_build_release_path }}/{{ repo_build_release_tag }}/requirements.txt
+    --requirement {{ repo_build_release_path }}/requirements.txt
     {{ pip_install_options }}
   changed_when: false
   tags:

tasks/repo_clone_git.yml

@@ -40,7 +40,7 @@
 
 - name: Retrieve requirements content
   slurp:
-    src: "{{ repo_build_release_path }}/{{ repo_build_release_tag }}/requirements.txt"
+    src: "{{ repo_build_release_path }}/requirements.txt"
   register: slurp_requirements
   tags:
     - repo-clone-repos

tasks/repo_index.yml

@@ -16,8 +16,8 @@
 - name: Index all manifest items
   find:
     paths:
-      - "{{ repo_build_release_path }}/{{ repo_build_release_tag }}"
-      - "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}"
+      - "{{ repo_build_release_path }}"
+      - "{{ repo_build_venv_dir }}"
   register: manifest_items
   tags:
     - repo-create-release-manifest
@@ -25,7 +25,7 @@
 - name: Create release manifest
   template:
     src: "manifest.in.j2"
-    dest: "{{ repo_build_release_path }}/{{ repo_build_release_tag }}/MANIFEST.in"
+    dest: "{{ repo_build_release_path }}/MANIFEST.in"
   tags:
     - repo-create-release-manifest
 

tasks/repo_post_build.yml

@@ -43,14 +43,14 @@
 - name: Create absolute requirements
   template:
     src: "requirements_absolute_requirements.txt.j2"
-    dest: "{{ repo_build_release_path }}/{{ repo_build_release_tag }}/requirements_absolute_requirements.txt"
+    dest: "{{ repo_build_release_path }}/requirements_absolute_requirements.txt"
   tags:
     - repo-create-absolute-requirements
 
 - name: Copy get-pip script into release folder
   copy:
     src: "/opt/get-pip.py"
-    dest: "{{ repo_build_release_path }}/{{ repo_build_release_tag }}/"
+    dest: "{{ repo_build_release_path }}/"
     remote_src: yes
   tags:
     - repo-copy-pip

tasks/repo_pre_build.yml

@@ -26,7 +26,7 @@
 
 - name: Register os-release files
   find:
-    paths: "{{ repo_build_release_path }}/{{ repo_build_release_tag }}"
+    paths: "{{ repo_build_release_path }}"
     patterns:
       - "*{{ ansible_architecture | lower }}.whl"
       - "*none-any.whl"
@@ -51,7 +51,7 @@
     state: directory
     owner: "{{ repo_build_service_user_name }}"
   with_items:
-    - "{{ repo_build_release_path }}/{{ repo_build_release_tag }}"
+    - "{{ repo_build_release_path }}"
     - "{{ repo_build_global_links_path }}"
     - "{{ repo_build_output }}"
   tags:
@@ -60,7 +60,7 @@
 - name: Build package requirements file
   template:
     src: "requirements.txt.j2"
-    dest: "{{ repo_build_release_path }}/{{ repo_build_release_tag }}/requirements.txt"
+    dest: "{{ repo_build_release_path }}/requirements.txt"
   tags:
     - repo-build-filtered-package-files
 
@@ -70,7 +70,7 @@
 - name: Build package constraints file
   template:
     src: "requirements_constraints.txt.j2"
-    dest: "{{ repo_build_release_path }}/{{ repo_build_release_tag }}/requirements_constraints.txt"
+    dest: "{{ repo_build_release_path }}/requirements_constraints.txt"
   tags:
     - repo-build-constraints-file
 
@@ -78,7 +78,7 @@
   pip:
     name: "{{ repo_pip_packages }}"
     state: "{{ repo_build_pip_package_state }}"
-    extra_args: "--constraint {{ repo_build_release_path }}/{{ repo_build_release_tag }}/requirements_constraints.txt {{ pip_install_options }}"
+    extra_args: "--constraint {{ repo_build_release_path }}/requirements_constraints.txt {{ pip_install_options }}"
   register: install_packages
   until: install_packages|success
   retries: 5

tasks/repo_venv_build.yml

@@ -28,7 +28,7 @@
 
 - name: Check for created venvs
   command: >
-    ls -1 "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}/"
+    ls -1 "{{ repo_build_venv_dir }}/"
   changed_when: false
   register: created_venvs
   tags:

tasks/repo_venv_pre_build.yml

@@ -25,9 +25,9 @@
     - repo-create-venv-archive
 
 - name: Register arch base venvs
-  command: "find {{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}/ -name '*{{ ansible_architecture | lower }}*'"
+  command: "find {{ repo_build_venv_dir }}/ -name '*{{ ansible_architecture | lower }}*'"
   args:
-    removes: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}"
+    removes: "{{ repo_build_venv_dir }}"
   register: base_venvs
   tags:
     - repo-create-venv-location
@@ -52,7 +52,7 @@
     mode: 02755
   with_items:
     - "{{ repo_build_venv_build_dir }}/venvs"
-    - "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}"
+    - "{{ repo_build_venv_dir }}"
   tags:
     - repo-create-venv-location
     - repo-venv-compress-archive

templates/manifest.in.j2

@@ -1,8 +1,8 @@
 {% for item in manifest_items['files'] %}
 {%   set item_cleaned = item['path'] | regex_replace(repo_build_base_path ~ "/", '') %}
 {%   set item_basename = item_cleaned | basename %}
-{%   if item_cleaned | match("os-releases/.*\.whl") %}
-{{ repo_build_pool_dir | basename }}/{{ item_basename.split('-')[0] | lower }}/{{ item_basename | lower}}
+{%   if item_cleaned | match(repo_build_release_version_path + "/.*/.*\.whl") %}
+{{ repo_build_pool_dir | regex_replace(repo_build_base_path ~ "/", '') }}/{{ item_basename.split('-')[0] }}/{{ item_basename | lower }}
 {{ item_cleaned | lower }}
 {%   else %}
 {{ item_cleaned }}

templates/op-release-script.sh.j2

@@ -30,10 +30,10 @@ function build_repo {
     chown "{{ repo_build_service_user_name }}" "${PKG_DIRECTORY}/${PKG_LOWER}"
 
     # link the built package to the release reference
-    ln -sf "${PKG_DIRECTORY}/${PKG_LOWER}" "{{ repo_build_release_path }}/{{ repo_build_release_tag }}/${PKG_LOWER}"
+    ln -sf "${PKG_DIRECTORY}/${PKG_LOWER}" "{{ repo_build_release_path }}/${PKG_LOWER}"
 
     # Make sure the repo service user owns the link
-    chown -h "{{ repo_build_service_user_name }}" "{{ repo_build_release_path }}/{{ repo_build_release_tag }}/${PKG_LOWER}"
+    chown -h "{{ repo_build_service_user_name }}" "{{ repo_build_release_path }}/${PKG_LOWER}"
 
     # link the built package to the global links reference
     ln -sf "${PKG_DIRECTORY}/${PKG_LOWER}" "{{ repo_build_global_links_path }}/${PKG_LOWER}"

templates/op-venv-script.sh.j2

@@ -94,7 +94,7 @@ PID=()
 #  Venv creation is done parallel at a count of the known "ansible_processor_count" or using a default of 5.
 #  This loop will enter the venv build directory and create tagged venvs in a distribution directory
 #  If the venv archive already exists the creation process will be skipped
-pushd "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}"
+pushd "{{ repo_build_venv_dir }}"
 {% for key, value in os_group.items() %}
 
 {% set venvwithindex = [] %}

tests/test-repo-build.yml

@@ -40,7 +40,7 @@
     - "repo_build"
   post_tasks:
     - name: List the files in the venv folder
-      command: ls -1 /var/www/repo/venvs/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}/
+      command: ls -1 {{ repo_build_venv_dir }}/
       register: venv_folder_content
     - name: Ensure that the keystone venv is present
       assert:
@@ -72,7 +72,7 @@
       register: slurp_upper_constraints
     - name: Slurp requirements constraints
       slurp:
-        src: "{{ repo_build_release_path }}/{{ repo_build_release_tag }}/requirements_constraints.txt"
+        src: "{{ repo_build_release_path }}/requirements_constraints.txt"
       register: slurp_requirements_constraints
     - name: Set fact for requirements constraints content
       set_fact:

vars/redhat.yml

@@ -37,5 +37,5 @@ repo_build_distro_packages:
 # workaround for https://github.com/pypa/virtualenv/issues/565
 repo_build_venv_command_options: >
   {{ virtualenv_bin }}
-  --extra-search-dir {{ repo_build_release_path }}/{{ repo_build_release_tag }}
+  --extra-search-dir {{ repo_build_release_path }}
   --never-download