From d7f496601f3ba99d1c02a940153e5e94915b0c31 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Fri, 23 Oct 2015 20:46:05 -0500 Subject: [PATCH] Updated the repo build templates The change makes it so the upper constraints file is created specifically using the repos indexed and cloned. Previously the constraint file was created using the git repos found within the local openstackgit directory and while that works for greenfield deployments it does have the potential to create additional entries that may or may not work with an environment. The manifest file now uses normalized file names and will track the exact git repos needed to power a build. This will allow for more targetted syncs from an upstream mirror to happen using the `repo-clone-mirror.yml` play. The venv build process will now check for a prebuilt venv using the versioned file name. Change-Id: If4416f660133a6c65badba0c7c819a6409bfe11e Signed-off-by: Kevin Carter --- playbooks/repo-clone-mirror.yml | 24 +++++++++++++++---- .../roles/repo_build/tasks/repo_pre_build.yml | 8 ------- .../repo_build/tasks/repo_venv_build.yml | 4 ++-- .../roles/repo_build/templates/manifest.in.j2 | 7 ++++-- .../templates/requirements_constraints.txt.j2 | 6 ++--- 5 files changed, 29 insertions(+), 20 deletions(-) diff --git a/playbooks/repo-clone-mirror.yml b/playbooks/repo-clone-mirror.yml index cc39e12621..b08deb7b06 100644 --- a/playbooks/repo-clone-mirror.yml +++ b/playbooks/repo-clone-mirror.yml @@ -22,6 +22,11 @@ hosts: repo_all[0] gather_facts: false user: root + pre_tasks: + - name: Delete old MANIFEST file if found + file: + state: absent + path: /tmp/MANIFEST.in tasks: - name: download MANIFEST.in get_url: @@ -35,18 +40,27 @@ {{ repo_upstream_url | netloc }}::{{ mirror_name }} {{ mirror_path }} sudo: yes sudo_user: "{{ repo_service_user_name }}" + - name: Create sync directories + file: + state: directory + path: "{{ mirror_path }}/{{ item }}" + owner: "{{ repo_service_user_name }}" + with_items: rsync_dirs - name: Sync supporting directories shell: | {{ rsync_commands }} \ {{ rsync_flags }} \ - {{ repo_upstream_url | netloc }}::{{ mirror_name }}/{{ item }} {{ mirror_path }} - with_items: - - container_images - - downloads - - openstackgit + {{ repo_upstream_url | netloc }}::{{ mirror_name }}/{{ item }}/ {{ mirror_path }}/{{ item }} + with_items: rsync_dirs + sudo: yes + sudo_user: "{{ repo_service_user_name }}" vars: rsync_commands: rsync rsync_flags: "-avzlHAX" + rsync_dirs: + - container_images + - downloads + - "venvs/{{ openstack_release }}" mirror_path: "{{ repo_service_home_folder }}/repo" mirror_name: "openstack_mirror" repo_upstream_url: "https://rpc-repo.rackspace.com" diff --git a/playbooks/roles/repo_build/tasks/repo_pre_build.yml b/playbooks/roles/repo_build/tasks/repo_pre_build.yml index 2ae1ad4e5c..4f3ccfe317 100644 --- a/playbooks/roles/repo_build/tasks/repo_pre_build.yml +++ b/playbooks/roles/repo_build/tasks/repo_pre_build.yml @@ -50,14 +50,6 @@ tags: - repo-build-filtered-package-files -- name: Index cloned repos - shell: "find {{ repo_build_git_dir }}/* -maxdepth 0 -type d" - register: cloned_repos - tags: - - repo-index-cloned-repos - - repo-build-constraints-file - - repo-create-release-manifest - - name: Build package constraints file template: src: "requirements_constraints.txt.j2" diff --git a/playbooks/roles/repo_build/tasks/repo_venv_build.yml b/playbooks/roles/repo_build/tasks/repo_venv_build.yml index f4094a9de3..338bc25a6f 100644 --- a/playbooks/roles/repo_build/tasks/repo_venv_build.yml +++ b/playbooks/roles/repo_build/tasks/repo_venv_build.yml @@ -56,7 +56,7 @@ with_dict: local_packages.results.0.item.role_packages when: - '"os_" in item.key' - - "'{{ item.key | replace('os_', '') }}.tgz' not in existing_venvs" + - "'{{ item.key | replace('os_', '') }}-{{ repo_build_release_tag }}.tgz' not in existing_venvs" tags: - repo-create-venv @@ -70,7 +70,7 @@ -C "{{ repo_build_venv_build_dir }}/venvs/{{ item.key | replace('os_', '') }}" . args: chdir: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}" - creates: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}/{{ item.key | replace('os_', '') }}.tgz" + creates: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}/{{ item.key | replace('os_', '') }}-{{ repo_build_release_tag }}.tgz" with_dict: local_packages.results.0.item.role_packages when: - '"os_" in item.key' diff --git a/playbooks/roles/repo_build/templates/manifest.in.j2 b/playbooks/roles/repo_build/templates/manifest.in.j2 index 29b1fc191d..7c6b5ae236 100644 --- a/playbooks/roles/repo_build/templates/manifest.in.j2 +++ b/playbooks/roles/repo_build/templates/manifest.in.j2 @@ -1,4 +1,7 @@ {% for item in built_wheels.stdout_lines %} -{{ repo_build_pool_dir | basename }}/{{ item.split('-')[0] | lower }}/{{ item }} -{{ repo_build_release_path | basename }}/{{ repo_build_release_tag }}/{{ item }} +{{ repo_build_pool_dir | basename }}/{{ item.split('-')[0] | lower }}/{{ item | lower }} +{{ repo_build_release_path | basename }}/{{ repo_build_release_tag }}/{{ item | lower }} +{% endfor %} +{% for clone_item in local_packages.results.0.item.remote_package_parts -%} +{{ repo_build_git_dir | basename }}/{{ clone_item['name'] }} {% endfor %} \ No newline at end of file diff --git a/playbooks/roles/repo_build/templates/requirements_constraints.txt.j2 b/playbooks/roles/repo_build/templates/requirements_constraints.txt.j2 index adc96c143e..36f5d344c6 100644 --- a/playbooks/roles/repo_build/templates/requirements_constraints.txt.j2 +++ b/playbooks/roles/repo_build/templates/requirements_constraints.txt.j2 @@ -1,8 +1,8 @@ # Computed constraints {% set constraint_pkgs = [] -%} -{% for clone_item in cloned_repos.stdout_lines -%} -git+file://{{ clone_item }}#egg={{ clone_item | git_link_parse_name | replace('-', '_') | lower }} -{% set _ = constraint_pkgs.append(clone_item | git_link_parse_name | replace('-', '_') | lower) %} +{% for clone_item in local_packages.results.0.item.remote_package_parts -%} +git+file://{{ repo_build_git_dir }}/{{ clone_item['name'] }}@{{ clone_item['version'] }}#egg={{ clone_item['name'] | replace('-', '_') | lower }} +{% set _ = constraint_pkgs.append(clone_item['name'] | replace('-', '_') | lower) %} {% endfor %} # upper boundry constraints from requirements repo. {% for constraint_item in upper_constraints %}