Updated the repo build templates

The change makes it so the upper constraints file is created
specifically using the repos indexed and cloned. Previously the
constraint file was created using the git repos found within the
local openstackgit directory and while that works for greenfield
deployments it does have the potential to create additional entries
that may or may not work with an environment.

The manifest file now uses normalized file names and will track the
exact git repos needed to power a build. This will allow for more
targetted syncs from an upstream mirror to happen using the
`repo-clone-mirror.yml` play.

The venv build process will now check for a prebuilt venv using the
versioned file name.

Change-Id: If4416f660133a6c65badba0c7c819a6409bfe11e
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

playbooks/repo-clone-mirror.yml

@@ -22,6 +22,11 @@
   hosts: repo_all[0]
   gather_facts: false
   user: root
+  pre_tasks:
+    - name: Delete old MANIFEST file if found
+      file:
+        state: absent
+        path: /tmp/MANIFEST.in
   tasks:
     - name: download MANIFEST.in
       get_url:
@@ -35,18 +40,27 @@
               {{ repo_upstream_url | netloc }}::{{ mirror_name }} {{ mirror_path }}
       sudo: yes
       sudo_user: "{{ repo_service_user_name }}"
+    - name: Create sync directories
+      file:
+        state: directory
+        path: "{{ mirror_path }}/{{ item }}"
+        owner: "{{ repo_service_user_name }}"
+      with_items: rsync_dirs
     - name: Sync supporting directories
       shell: |
         {{ rsync_commands }} \
               {{ rsync_flags }} \
-              {{ repo_upstream_url | netloc }}::{{ mirror_name }}/{{ item }} {{ mirror_path }}
-      with_items:
-        - container_images
-        - downloads
-        - openstackgit
+              {{ repo_upstream_url | netloc }}::{{ mirror_name }}/{{ item }}/ {{ mirror_path }}/{{ item }}
+      with_items: rsync_dirs
+      sudo: yes
+      sudo_user: "{{ repo_service_user_name }}"
   vars:
     rsync_commands: rsync
     rsync_flags: "-avzlHAX"
+    rsync_dirs:
+      - container_images
+      - downloads
+      - "venvs/{{ openstack_release }}"
     mirror_path: "{{ repo_service_home_folder }}/repo"
     mirror_name: "openstack_mirror"
     repo_upstream_url: "https://rpc-repo.rackspace.com"

playbooks/roles/repo_build/tasks/repo_pre_build.yml

@@ -50,14 +50,6 @@
   tags:
     - repo-build-filtered-package-files
 
-- name: Index cloned repos
-  shell: "find  {{ repo_build_git_dir }}/*  -maxdepth 0 -type d"
-  register: cloned_repos
-  tags:
-    - repo-index-cloned-repos
-    - repo-build-constraints-file
-    - repo-create-release-manifest
-
 - name: Build package constraints file
   template:
     src: "requirements_constraints.txt.j2"

playbooks/roles/repo_build/tasks/repo_venv_build.yml

@@ -56,7 +56,7 @@
   with_dict: local_packages.results.0.item.role_packages
   when:
     - '"os_" in item.key'
-    - "'{{ item.key | replace('os_', '') }}.tgz' not in existing_venvs"
+    - "'{{ item.key | replace('os_', '') }}-{{ repo_build_release_tag }}.tgz' not in existing_venvs"
   tags:
     - repo-create-venv
 
@@ -70,7 +70,7 @@
         -C "{{ repo_build_venv_build_dir }}/venvs/{{ item.key | replace('os_', '') }}" .
   args:
     chdir: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}"
-    creates: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}/{{ item.key | replace('os_', '') }}.tgz"
+    creates: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}/{{ item.key | replace('os_', '') }}-{{ repo_build_release_tag }}.tgz"
   with_dict: local_packages.results.0.item.role_packages
   when:
     - '"os_" in item.key'

playbooks/roles/repo_build/templates/manifest.in.j2

@@ -1,4 +1,7 @@
 {% for item in built_wheels.stdout_lines %}
-{{ repo_build_pool_dir | basename }}/{{ item.split('-')[0] | lower }}/{{ item }}
-{{ repo_build_release_path | basename }}/{{ repo_build_release_tag }}/{{ item }}
+{{ repo_build_pool_dir | basename }}/{{ item.split('-')[0] | lower }}/{{ item | lower }}
+{{ repo_build_release_path | basename }}/{{ repo_build_release_tag }}/{{ item | lower }}
+{% endfor %}
+{% for clone_item in local_packages.results.0.item.remote_package_parts -%}
+{{ repo_build_git_dir | basename }}/{{ clone_item['name'] }}
 {% endfor %}

playbooks/roles/repo_build/templates/requirements_constraints.txt.j2

@@ -1,8 +1,8 @@
 # Computed constraints
 {% set constraint_pkgs = [] -%}
-{% for clone_item in cloned_repos.stdout_lines -%}
-git+file://{{ clone_item }}#egg={{ clone_item | git_link_parse_name | replace('-', '_') | lower }}
-{% set _ = constraint_pkgs.append(clone_item | git_link_parse_name | replace('-', '_') | lower) %}
+{% for clone_item in local_packages.results.0.item.remote_package_parts -%}
+git+file://{{ repo_build_git_dir }}/{{ clone_item['name'] }}@{{ clone_item['version'] }}#egg={{ clone_item['name'] | replace('-', '_') | lower }}
+{% set _ = constraint_pkgs.append(clone_item['name'] | replace('-', '_') | lower) %}
 {% endfor %}
 # upper boundry constraints from requirements repo.
 {% for constraint_item in upper_constraints %}