Make wheel build process idempotent
This patch implements the following: 1. The ability to use a boolean variable to always force a git clone or the wheel build to happen. 2. The use of local facts to mark whether a wheel build or venv build is required. This ensures that if there is a failure, the tasks will still be done when the repo build is executed again. 3. The git clone or wheel build tasks are only actioned if there is a change to the requirements or constraints. This ensures that when the repo build is executed without any changes, those processes will be skipped. 4. Re-arranges the tasks to ensure idempotency and to make the process easier to follow. A smaller set of tags are implemented which are intended to provide a clear code path for each of them. 5. Log output is added to the venv build process to make troubleshooting easier. 6. The stdout output for the wheel and venv build processes is made minimal to reduce confusion and make it easier to spot which item failed to build. The log output in /var/log/repo contains the verbose output from pip. Change-Id: I2008926b43653edf50c284f5068160e27915c90a
This commit is contained in:
parent
9137f824b0
commit
8c3933c069
@ -52,9 +52,15 @@ repo_build_pool_dir: "{{ repo_build_base_path }}/pools/{{ repo_build_os_distro_v
|
|||||||
# Toggle whether git repositories should be cloned selectively or not
|
# Toggle whether git repositories should be cloned selectively or not
|
||||||
repo_build_git_selective: "{{ true if (repo_build_wheel_selective | bool and repo_build_venv_selective | bool) else false }}"
|
repo_build_git_selective: "{{ true if (repo_build_wheel_selective | bool and repo_build_venv_selective | bool) else false }}"
|
||||||
|
|
||||||
|
# Toggle whether a git clone should be forced
|
||||||
|
repo_build_git_reclone: no
|
||||||
|
|
||||||
# Toggle whether wheels should be built selectively or not
|
# Toggle whether wheels should be built selectively or not
|
||||||
repo_build_wheel_selective: "{{ true if repo_build_venv_selective | bool else false }}"
|
repo_build_wheel_selective: "{{ true if repo_build_venv_selective | bool else false }}"
|
||||||
|
|
||||||
|
# Toggle whether a wheel rebuild should be forced
|
||||||
|
repo_build_wheel_rebuild: no
|
||||||
|
|
||||||
# Toggle whether venvs should be built selectively or not
|
# Toggle whether venvs should be built selectively or not
|
||||||
repo_build_venv_selective: yes
|
repo_build_venv_selective: yes
|
||||||
|
|
||||||
|
@ -24,20 +24,35 @@
|
|||||||
tags:
|
tags:
|
||||||
- always
|
- always
|
||||||
|
|
||||||
# Wheel building
|
|
||||||
- include: repo_build_install.yml
|
- include: repo_build_install.yml
|
||||||
- include: repo_pre_build.yml
|
tags:
|
||||||
- include: repo_build.yml
|
- repo-build-install
|
||||||
- include: repo_post_build.yml
|
|
||||||
|
|
||||||
# Venv building
|
- include: repo_build_prepare.yml
|
||||||
- include: repo_venv_build.yml
|
tags:
|
||||||
|
- always
|
||||||
|
|
||||||
|
- name: refresh local facts
|
||||||
|
setup:
|
||||||
|
filter: ansible_local
|
||||||
|
gather_subset: "!all"
|
||||||
|
tags:
|
||||||
|
- always
|
||||||
|
|
||||||
|
- include: repo_build_wheels.yml
|
||||||
|
when:
|
||||||
|
- ansible_local['openstack_ansible']['repo_build']['need_wheel_build'] | bool
|
||||||
|
tags:
|
||||||
|
- repo-build-wheels
|
||||||
|
|
||||||
|
- include: repo_build_venvs.yml
|
||||||
tags:
|
tags:
|
||||||
- repo-build-venvs
|
- repo-build-venvs
|
||||||
|
|
||||||
- include: repo_index.yml
|
- include: repo_build_index.yml
|
||||||
tags:
|
tags:
|
||||||
- repo-build-index
|
- repo-build-index
|
||||||
|
|
||||||
|
# Synchronize all built packages back to the repo master
|
||||||
- include: repo_package_sync.yml
|
- include: repo_package_sync.yml
|
||||||
when: inventory_hostname != groups['repo_all'][0]
|
when: inventory_hostname != groups['repo_all'][0]
|
||||||
|
@ -26,7 +26,3 @@
|
|||||||
until: install_packages | success
|
until: install_packages | success
|
||||||
retries: 5
|
retries: 5
|
||||||
delay: 2
|
delay: 2
|
||||||
tags:
|
|
||||||
- repo-build-apt-packages
|
|
||||||
- repo-build-yum-packages
|
|
||||||
- repo-build-zypper-packages
|
|
||||||
|
114
tasks/repo_build_prepare.yml
Normal file
114
tasks/repo_build_prepare.yml
Normal file
@ -0,0 +1,114 @@
|
|||||||
|
---
|
||||||
|
# Copyright 2015, Rackspace US, Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
- name: Initialize the wheel build local fact
|
||||||
|
ini_file:
|
||||||
|
dest: "/etc/ansible/facts.d/openstack_ansible.fact"
|
||||||
|
section: repo_build
|
||||||
|
option: need_wheel_build
|
||||||
|
value: False
|
||||||
|
when:
|
||||||
|
- "('openstack_ansible' not in ansible_local) or
|
||||||
|
('repo_build' not in ansible_local['openstack_ansible']) or
|
||||||
|
('need_wheel_build' not in ansible_local['openstack_ansible']['repo_build'])"
|
||||||
|
|
||||||
|
- name: Initialize the venv build local fact
|
||||||
|
ini_file:
|
||||||
|
dest: "/etc/ansible/facts.d/openstack_ansible.fact"
|
||||||
|
section: repo_build
|
||||||
|
option: need_venv_build
|
||||||
|
value: False
|
||||||
|
when:
|
||||||
|
- "('openstack_ansible' not in ansible_local) or
|
||||||
|
('repo_build' not in ansible_local['openstack_ansible']) or
|
||||||
|
('need_venv_build' not in ansible_local['openstack_ansible']['repo_build'])"
|
||||||
|
|
||||||
|
# TODO(odyssey4me):
|
||||||
|
# This is to clean up files created in early Ocata. It may be removed in Pike.
|
||||||
|
- name: Ensure old pool index file is cleaned up
|
||||||
|
file:
|
||||||
|
path: "{{ repo_build_pool_dir }}/index.html"
|
||||||
|
state: "absent"
|
||||||
|
|
||||||
|
- name: Create package directories
|
||||||
|
file:
|
||||||
|
path: "{{ item }}"
|
||||||
|
state: directory
|
||||||
|
owner: "{{ repo_build_service_user_name }}"
|
||||||
|
with_items:
|
||||||
|
- "{{ repo_build_release_path }}"
|
||||||
|
- "{{ repo_build_global_links_path }}"
|
||||||
|
|
||||||
|
- name: Build package requirements file
|
||||||
|
template:
|
||||||
|
src: "requirements.txt.j2"
|
||||||
|
dest: "{{ repo_build_release_path }}/requirements.txt"
|
||||||
|
register: _wheel_build_requirements
|
||||||
|
|
||||||
|
- include: repo_clone_git.yml
|
||||||
|
when:
|
||||||
|
- (repo_build_git_reclone | bool) or
|
||||||
|
(_wheel_build_requirements | changed)
|
||||||
|
tags:
|
||||||
|
- repo-clone-repos
|
||||||
|
|
||||||
|
- name: Retrieve upper constraints content
|
||||||
|
slurp:
|
||||||
|
src: "{{ repo_build_git_dir }}/requirements/upper-constraints.txt"
|
||||||
|
register: slurp_upper_constraints
|
||||||
|
|
||||||
|
- name: Decode the upper constraints content
|
||||||
|
set_fact:
|
||||||
|
upper_constraints: "{{ slurp_upper_constraints.content | b64decode | splitlines }}"
|
||||||
|
|
||||||
|
- name: Build package constraints file
|
||||||
|
template:
|
||||||
|
src: "requirements_constraints.txt.j2"
|
||||||
|
dest: "{{ repo_build_release_path }}/requirements_constraints.txt"
|
||||||
|
register: _wheel_build_constraints
|
||||||
|
|
||||||
|
- name: Record whether a wheel build is required
|
||||||
|
ini_file:
|
||||||
|
dest: "/etc/ansible/facts.d/openstack_ansible.fact"
|
||||||
|
section: repo_build
|
||||||
|
option: need_wheel_build
|
||||||
|
value: True
|
||||||
|
when:
|
||||||
|
- (_wheel_build_requirements | changed) or
|
||||||
|
(_wheel_build_constraints | changed) or
|
||||||
|
(repo_build_wheel_rebuild | bool)
|
||||||
|
|
||||||
|
- name: Record whether a venv build is required
|
||||||
|
ini_file:
|
||||||
|
dest: "/etc/ansible/facts.d/openstack_ansible.fact"
|
||||||
|
section: repo_build
|
||||||
|
option: need_venv_build
|
||||||
|
value: True
|
||||||
|
when:
|
||||||
|
- (_wheel_build_requirements | changed) or
|
||||||
|
(_wheel_build_constraints | changed) or
|
||||||
|
(repo_build_venv_rebuild | bool)
|
||||||
|
|
||||||
|
- name: Install pip packages
|
||||||
|
pip:
|
||||||
|
name: "{{ repo_pip_packages }}"
|
||||||
|
state: "{{ repo_build_pip_package_state }}"
|
||||||
|
extra_args: "--constraint {{ repo_build_release_path }}/requirements_constraints.txt {{ pip_install_options }}"
|
||||||
|
register: install_packages
|
||||||
|
until: install_packages|success
|
||||||
|
retries: 5
|
||||||
|
delay: 5
|
||||||
|
tags:
|
||||||
|
- repo-build-install
|
@ -27,14 +27,10 @@
|
|||||||
command: which virtualenv
|
command: which virtualenv
|
||||||
changed_when: false
|
changed_when: false
|
||||||
register: virtualenv_path
|
register: virtualenv_path
|
||||||
tags:
|
|
||||||
- always
|
|
||||||
|
|
||||||
- name: Set virtualenv command path
|
- name: Set virtualenv command path
|
||||||
set_fact:
|
set_fact:
|
||||||
virtualenv_bin: "{{ virtualenv_path.stdout }}"
|
virtualenv_bin: "{{ virtualenv_path.stdout }}"
|
||||||
tags:
|
|
||||||
- always
|
|
||||||
|
|
||||||
# This is removed so that virtual env is not installing unknown
|
# This is removed so that virtual env is not installing unknown
|
||||||
# packages as assumed wheels, IE pip8
|
# packages as assumed wheels, IE pip8
|
||||||
@ -63,7 +59,9 @@
|
|||||||
shell: "/opt/venv-build-script.sh {{ repo_build_release_path }}/venv-build-options-{{ item['item']['role_name'] }}.txt"
|
shell: "/opt/venv-build-script.sh {{ repo_build_release_path }}/venv-build-options-{{ item['item']['role_name'] }}.txt"
|
||||||
args:
|
args:
|
||||||
executable: "/bin/bash"
|
executable: "/bin/bash"
|
||||||
when: (item | changed) or (repo_build_venv_rebuild | bool)
|
when:
|
||||||
|
- (item | changed) or
|
||||||
|
(ansible_local['openstack_ansible']['repo_build']['need_venv_build'] | bool)
|
||||||
with_items:
|
with_items:
|
||||||
- "{{ _create_venv_options_files.results }}"
|
- "{{ _create_venv_options_files.results }}"
|
||||||
register: _build_venv
|
register: _build_venv
|
||||||
@ -85,3 +83,10 @@
|
|||||||
with_items: "{{ _build_venv['results'] }}"
|
with_items: "{{ _build_venv['results'] }}"
|
||||||
when:
|
when:
|
||||||
- item['ansible_job_id'] is defined
|
- item['ansible_job_id'] is defined
|
||||||
|
|
||||||
|
- name: Disable the venv build requirement now that it is complete
|
||||||
|
ini_file:
|
||||||
|
dest: "/etc/ansible/facts.d/openstack_ansible.fact"
|
||||||
|
section: repo_build
|
||||||
|
option: need_venv_build
|
||||||
|
value: False
|
@ -13,6 +13,23 @@
|
|||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
- name: Ensure that temporary folders from previous build failures are absent
|
||||||
|
file:
|
||||||
|
path: "{{ item }}"
|
||||||
|
state: "absent"
|
||||||
|
with_items:
|
||||||
|
- "{{ repo_build_dir }}"
|
||||||
|
- "{{ repo_build_output }}"
|
||||||
|
|
||||||
|
- name: Create temporary folders
|
||||||
|
file:
|
||||||
|
path: "{{ item }}"
|
||||||
|
state: directory
|
||||||
|
owner: "{{ repo_build_service_user_name }}"
|
||||||
|
with_items:
|
||||||
|
- "{{ repo_build_dir }}"
|
||||||
|
- "{{ repo_build_output }}"
|
||||||
|
|
||||||
- name: Download requirement pip sources
|
- name: Download requirement pip sources
|
||||||
shell: >
|
shell: >
|
||||||
yes i | pip install --timeout {{ repo_build_timeout }}
|
yes i | pip install --timeout {{ repo_build_timeout }}
|
||||||
@ -34,8 +51,6 @@
|
|||||||
--log /var/log/repo/repo_builder.log
|
--log /var/log/repo/repo_builder.log
|
||||||
--requirement {{ repo_build_release_path }}/requirements.txt
|
--requirement {{ repo_build_release_path }}/requirements.txt
|
||||||
when: repo_build_store_pip_sources | bool
|
when: repo_build_store_pip_sources | bool
|
||||||
tags:
|
|
||||||
- repo-pip-download
|
|
||||||
|
|
||||||
- name: Create OpenStack-Ansible requirement wheels
|
- name: Create OpenStack-Ansible requirement wheels
|
||||||
command: >
|
command: >
|
||||||
@ -56,6 +71,65 @@
|
|||||||
--requirement {{ repo_build_release_path }}/requirements.txt
|
--requirement {{ repo_build_release_path }}/requirements.txt
|
||||||
{{ pip_install_options }}
|
{{ pip_install_options }}
|
||||||
changed_when: false
|
changed_when: false
|
||||||
|
|
||||||
|
- name: Register os-release files
|
||||||
|
find:
|
||||||
|
paths: "{{ repo_build_release_path }}"
|
||||||
|
patterns:
|
||||||
|
- "*{{ ansible_architecture | lower }}.whl"
|
||||||
|
- "*none-any.whl"
|
||||||
|
register: os_release_files
|
||||||
|
|
||||||
|
- name: Ensure os-release files are cleaned up
|
||||||
|
file:
|
||||||
|
path: "{{ item['path'] }}"
|
||||||
|
state: "absent"
|
||||||
|
with_items: "{{ os_release_files.files }}"
|
||||||
|
|
||||||
|
- name: Index built wheels
|
||||||
|
find:
|
||||||
|
paths: "{{ repo_build_output }}"
|
||||||
|
register: built_wheels
|
||||||
|
|
||||||
|
- name: Create release process script
|
||||||
|
template:
|
||||||
|
src: "op-release-script.sh.j2"
|
||||||
|
dest: "/opt/op-release-script.sh"
|
||||||
|
mode: "0755"
|
||||||
|
|
||||||
|
- name: Run release process script
|
||||||
|
shell: "/opt/op-release-script.sh"
|
||||||
|
args:
|
||||||
|
executable: "/bin/bash"
|
||||||
|
changed_when: false
|
||||||
|
# This task requires the use of the shell module, so we skip lint
|
||||||
|
# to avoid:
|
||||||
|
# ANSIBLE0013 Use shell only when shell functionality is required
|
||||||
tags:
|
tags:
|
||||||
- repo-build-local-requirement-wheels
|
- skip_ansible_lint
|
||||||
- repo-build-openstack-ansible-requirement-wheels
|
|
||||||
|
- name: Create absolute requirements
|
||||||
|
template:
|
||||||
|
src: "requirements_absolute_requirements.txt.j2"
|
||||||
|
dest: "{{ repo_build_release_path }}/requirements_absolute_requirements.txt"
|
||||||
|
|
||||||
|
- name: Copy get-pip script into release folder
|
||||||
|
copy:
|
||||||
|
src: "/opt/get-pip.py"
|
||||||
|
dest: "{{ repo_build_release_path }}/"
|
||||||
|
remote_src: yes
|
||||||
|
|
||||||
|
- name: Clean up temporary build folders to save space
|
||||||
|
file:
|
||||||
|
path: "{{ item }}"
|
||||||
|
state: "absent"
|
||||||
|
with_items:
|
||||||
|
- "{{ repo_build_dir }}"
|
||||||
|
- "{{ repo_build_output }}"
|
||||||
|
|
||||||
|
- name: Disable the wheel build requirement now that it is complete
|
||||||
|
ini_file:
|
||||||
|
dest: "/etc/ansible/facts.d/openstack_ansible.fact"
|
||||||
|
section: repo_build
|
||||||
|
option: need_wheel_build
|
||||||
|
value: False
|
@ -18,15 +18,11 @@
|
|||||||
file:
|
file:
|
||||||
path: "/opt/op-clone-script.sh"
|
path: "/opt/op-clone-script.sh"
|
||||||
state: absent
|
state: absent
|
||||||
tags:
|
|
||||||
- repo-clone-repos
|
|
||||||
|
|
||||||
- name: Check if the git folder exists already
|
- name: Check if the git folder exists already
|
||||||
stat:
|
stat:
|
||||||
path: "{{ repo_build_git_dir }}"
|
path: "{{ repo_build_git_dir }}"
|
||||||
register: _git_folder
|
register: _git_folder
|
||||||
tags:
|
|
||||||
- repo-clone-repos
|
|
||||||
|
|
||||||
- name: Git service data folder setup
|
- name: Git service data folder setup
|
||||||
file:
|
file:
|
||||||
@ -35,15 +31,11 @@
|
|||||||
owner: "{{ repo_build_service_user_name }}"
|
owner: "{{ repo_build_service_user_name }}"
|
||||||
group: "{{ repo_build_service_group_name }}"
|
group: "{{ repo_build_service_group_name }}"
|
||||||
recurse: true
|
recurse: true
|
||||||
tags:
|
|
||||||
- repo-clone-repos
|
|
||||||
|
|
||||||
- name: Retrieve requirements content
|
- name: Retrieve requirements content
|
||||||
slurp:
|
slurp:
|
||||||
src: "{{ repo_build_release_path }}/requirements.txt"
|
src: "{{ repo_build_release_path }}/requirements.txt"
|
||||||
register: slurp_requirements
|
register: slurp_requirements
|
||||||
tags:
|
|
||||||
- repo-clone-repos
|
|
||||||
|
|
||||||
- name: Clone git repositories asynchronously
|
- name: Clone git repositories asynchronously
|
||||||
become: yes
|
become: yes
|
||||||
@ -61,8 +53,6 @@
|
|||||||
register: _git_clone
|
register: _git_clone
|
||||||
async: 1800
|
async: 1800
|
||||||
poll: 0
|
poll: 0
|
||||||
tags:
|
|
||||||
- repo-clone-repos
|
|
||||||
|
|
||||||
- name: Wait for git clones to complete
|
- name: Wait for git clones to complete
|
||||||
become: yes
|
become: yes
|
||||||
@ -70,7 +60,7 @@
|
|||||||
async_status:
|
async_status:
|
||||||
jid: "{{ item['ansible_job_id'] }}"
|
jid: "{{ item['ansible_job_id'] }}"
|
||||||
register: _git_jobs
|
register: _git_jobs
|
||||||
until: "{{ _git_jobs['finished'] | bool }}"
|
until: _git_jobs['finished'] | bool
|
||||||
delay: 5
|
delay: 5
|
||||||
retries: 360
|
retries: 360
|
||||||
with_items: "{{ _git_clone['results'] }}"
|
with_items: "{{ _git_clone['results'] }}"
|
||||||
|
@ -1,56 +0,0 @@
|
|||||||
---
|
|
||||||
# Copyright 2015, Rackspace US, Inc.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
- name: Index built wheels
|
|
||||||
command: "ls -1 {{ repo_build_output }}"
|
|
||||||
changed_when: false
|
|
||||||
register: built_wheels
|
|
||||||
tags:
|
|
||||||
- always
|
|
||||||
|
|
||||||
- name: Create release process script
|
|
||||||
template:
|
|
||||||
src: "op-release-script.sh.j2"
|
|
||||||
dest: "/opt/op-release-script.sh"
|
|
||||||
tags:
|
|
||||||
- repo-create-pool
|
|
||||||
|
|
||||||
- name: Run release process script
|
|
||||||
command: "bash /opt/op-release-script.sh"
|
|
||||||
changed_when: false
|
|
||||||
tags:
|
|
||||||
- repo-create-pool
|
|
||||||
|
|
||||||
- name: Remove release process script
|
|
||||||
file:
|
|
||||||
path: "/opt/op-release-script.sh"
|
|
||||||
state: absent
|
|
||||||
tags:
|
|
||||||
- repo-create-pool
|
|
||||||
|
|
||||||
- name: Create absolute requirements
|
|
||||||
template:
|
|
||||||
src: "requirements_absolute_requirements.txt.j2"
|
|
||||||
dest: "{{ repo_build_release_path }}/requirements_absolute_requirements.txt"
|
|
||||||
tags:
|
|
||||||
- repo-create-absolute-requirements
|
|
||||||
|
|
||||||
- name: Copy get-pip script into release folder
|
|
||||||
copy:
|
|
||||||
src: "/opt/get-pip.py"
|
|
||||||
dest: "{{ repo_build_release_path }}/"
|
|
||||||
remote_src: yes
|
|
||||||
tags:
|
|
||||||
- repo-copy-pip
|
|
@ -1,84 +0,0 @@
|
|||||||
---
|
|
||||||
# Copyright 2015, Rackspace US, Inc.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
- name: Ensure workspace files are cleaned up
|
|
||||||
file:
|
|
||||||
path: "{{ item }}"
|
|
||||||
state: "absent"
|
|
||||||
with_items:
|
|
||||||
- "{{ repo_build_dir }}"
|
|
||||||
- "{{ repo_build_output }}"
|
|
||||||
- "{{ repo_build_pool_dir }}/index.html" # pool directory index is removed because its no longer used
|
|
||||||
tags:
|
|
||||||
- repo-clean-workspace
|
|
||||||
|
|
||||||
- name: Register os-release files
|
|
||||||
find:
|
|
||||||
paths: "{{ repo_build_release_path }}"
|
|
||||||
patterns:
|
|
||||||
- "*{{ ansible_architecture | lower }}.whl"
|
|
||||||
- "*none-any.whl"
|
|
||||||
register: os_release_files
|
|
||||||
tags:
|
|
||||||
- repo-clean-workspace
|
|
||||||
|
|
||||||
- name: Ensure os-release files are cleaned up
|
|
||||||
file:
|
|
||||||
path: "{{ item.path }}"
|
|
||||||
state: "absent"
|
|
||||||
with_items: "{{ os_release_files.files }}"
|
|
||||||
tags:
|
|
||||||
- repo-clean-workspace
|
|
||||||
|
|
||||||
- name: Create release directory
|
|
||||||
file:
|
|
||||||
path: "{{ item }}"
|
|
||||||
state: directory
|
|
||||||
owner: "{{ repo_build_service_user_name }}"
|
|
||||||
with_items:
|
|
||||||
- "{{ repo_build_release_path }}"
|
|
||||||
- "{{ repo_build_global_links_path }}"
|
|
||||||
- "{{ repo_build_output }}"
|
|
||||||
tags:
|
|
||||||
- repo-create-release-links-location
|
|
||||||
|
|
||||||
- name: Build package requirements file
|
|
||||||
template:
|
|
||||||
src: "requirements.txt.j2"
|
|
||||||
dest: "{{ repo_build_release_path }}/requirements.txt"
|
|
||||||
tags:
|
|
||||||
- repo-build-filtered-package-files
|
|
||||||
|
|
||||||
- include: repo_clone_git.yml
|
|
||||||
- include: repo_set_facts.yml
|
|
||||||
|
|
||||||
- name: Build package constraints file
|
|
||||||
template:
|
|
||||||
src: "requirements_constraints.txt.j2"
|
|
||||||
dest: "{{ repo_build_release_path }}/requirements_constraints.txt"
|
|
||||||
tags:
|
|
||||||
- repo-build-constraints-file
|
|
||||||
|
|
||||||
- name: Install pip packages
|
|
||||||
pip:
|
|
||||||
name: "{{ repo_pip_packages }}"
|
|
||||||
state: "{{ repo_build_pip_package_state }}"
|
|
||||||
extra_args: "--constraint {{ repo_build_release_path }}/requirements_constraints.txt {{ pip_install_options }}"
|
|
||||||
register: install_packages
|
|
||||||
until: install_packages|success
|
|
||||||
retries: 5
|
|
||||||
delay: 5
|
|
||||||
tags:
|
|
||||||
- repo-pip-packages
|
|
@ -1,32 +0,0 @@
|
|||||||
---
|
|
||||||
# Copyright 2015, Rackspace US, Inc.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
|
|
||||||
- name: Retrieve upper constraints content
|
|
||||||
slurp:
|
|
||||||
src: "{{ repo_build_git_dir }}/requirements/upper-constraints.txt"
|
|
||||||
register: slurp_upper_constraints
|
|
||||||
tags:
|
|
||||||
- repo-set-constraints
|
|
||||||
- repo-get-upper-constraints
|
|
||||||
- repo-build-constraints-file
|
|
||||||
|
|
||||||
- name: Decode the upper constraints content
|
|
||||||
set_fact:
|
|
||||||
upper_constraints: "{{ slurp_upper_constraints.content | b64decode | splitlines }}"
|
|
||||||
when: slurp_upper_constraints | success
|
|
||||||
tags:
|
|
||||||
- repo-set-constraints
|
|
||||||
- repo-build-constraints-file
|
|
@ -1,5 +1,5 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
set -ve
|
set -e
|
||||||
|
|
||||||
|
|
||||||
function build_repo {
|
function build_repo {
|
||||||
@ -43,16 +43,19 @@ function build_repo {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
# Loop through all built wheel.
|
# Loop through all built wheels.
|
||||||
{% for item in built_wheels.stdout_lines %}
|
{% for file_data in built_wheels['files'] %}
|
||||||
|
{% set file_name = file_data['path'] | basename %}
|
||||||
|
|
||||||
# Set the PKG_DIRECTORY variable normalizing the name
|
# Set the PKG_DIRECTORY variable normalizing the name
|
||||||
# Set the package name variable normalizing the name
|
# Set the package name variable normalizing the name
|
||||||
DIRECTORY="{{ repo_build_pool_dir }}/{{ item.split('-')[0] | lower }}"
|
DIRECTORY="{{ repo_build_pool_dir }}/{{ file_name.split('-')[0] | lower }}"
|
||||||
NAME_LOWER="{{ item | lower }}"
|
NAME_LOWER="{{ file_name | lower }}"
|
||||||
RAW_NAME="{{ item }}"
|
RAW_NAME="{{ file_name }}"
|
||||||
|
|
||||||
|
echo -n "Moving ${RAW_NAME} to repo..."
|
||||||
build_repo "${DIRECTORY}" "${NAME_LOWER}" "${RAW_NAME}"
|
build_repo "${DIRECTORY}" "${NAME_LOWER}" "${RAW_NAME}"
|
||||||
|
echo "done"
|
||||||
|
|
||||||
unset PKG_DIRECTORY
|
unset PKG_DIRECTORY
|
||||||
unset PKG_LOWER
|
unset PKG_LOWER
|
||||||
|
@ -1,27 +1,56 @@
|
|||||||
{% if not repo_build_wheel_selective | bool %}
|
# This is a unique, sorted list of requirements compiled by the repo
|
||||||
{% for requirement in local_packages.results.0.item.packages %}
|
# build process. The requirements are compiled from all the roles
|
||||||
{{ requirement.split('#')[0].strip().replace('-', '_') }}
|
# using their *_pip_packages lists, the global-requirements-pins,
|
||||||
{% endfor %}
|
# and the git sources provided.
|
||||||
{% else %}
|
# Where a package is found to be provided from a git source, the
|
||||||
|
# designated git repository SHA is added as a comment.
|
||||||
|
|
||||||
{# #}
|
{# #}
|
||||||
{# We can't have duplicated requirements (pip wheel will fail). #}
|
{# To make it easier to add the comment for each git sourced package #}
|
||||||
{# To combat this we will use the overall package list requirements #}
|
{# when compiling the all_requirements list, we need to put together #}
|
||||||
{# as a reference set, and pull from there for whatever each role #}
|
{# a map of the package names to the version based on the git data #}
|
||||||
{# needs. #}
|
{# provided by the py_pkgs lookup. #}
|
||||||
|
{# #}
|
||||||
|
{% set git_packages = {} %}
|
||||||
|
{% for clone_item in local_packages.results.0.item.remote_package_parts %}
|
||||||
|
{% if 'ignorerequirements=true' not in clone_item['original'] %}
|
||||||
|
{% set name_normalized = clone_item['name'] | replace('-', '_') | lower %}
|
||||||
|
{% set _ = git_packages.update({name_normalized: clone_item['version']}) %}
|
||||||
|
{% endif %}
|
||||||
|
{% endfor %}
|
||||||
|
{# #}
|
||||||
|
{# The list provided by the py_pkgs lookup is a raw set which needs #}
|
||||||
|
{# some normalization. We also want to add the SHA/version as a #}
|
||||||
|
{# comment to anything provided from a git source to make it simple #}
|
||||||
|
{# for the git cloning and wheel building process to be idempotent. #}
|
||||||
{# #}
|
{# #}
|
||||||
{% set all_requirements={} %}
|
{% set all_requirements={} %}
|
||||||
{% for requirement_raw in local_packages.results.0.item.packages %}
|
{% for requirement_raw in local_packages.results.0.item.packages %}
|
||||||
{% set name = requirement_raw | regex_replace('(\[|>=|<=|>|<|==|~=|!=).*$','') %}
|
{% set name = requirement_raw | regex_replace('(\[|>=|<=|>|<|==|~=|!=).*$','') %}
|
||||||
{% set data = requirement_raw | regex_replace(name,'') %}
|
{% set data = requirement_raw | regex_replace(name,'') %}
|
||||||
{% set name_normalized = name | replace('-', '_') | lower %}
|
{% set name_normalized = name | replace('-', '_') | lower %}
|
||||||
{% set requirement_normalized = name_normalized + data %}
|
{% if name_normalized in git_packages %}
|
||||||
|
{% set requirement_normalized = name_normalized ~ data ~ ' # ' ~ git_packages[name_normalized] %}
|
||||||
|
{% else %}
|
||||||
|
{% set requirement_normalized = name_normalized ~ data %}
|
||||||
|
{% endif %}
|
||||||
{% set _ = all_requirements.update({name_normalized: requirement_normalized}) %}
|
{% set _ = all_requirements.update({name_normalized: requirement_normalized}) %}
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
{# #}
|
{# #}
|
||||||
{# Now we have a complete reference set to work with. We now need #}
|
{# Now we have a complete, normalised and commented reference set #}
|
||||||
{# to build a set of the packages we actually want to build which #}
|
{# to work with. Now, for a non-selective wheel build, we simply #}
|
||||||
{# we need to ensure is a unique set. We will use a key:value #}
|
{# output the resulting list. #}
|
||||||
{# mechanism to do this. #}
|
{# #}
|
||||||
|
{% if not repo_build_wheel_selective | bool %}
|
||||||
|
{% for requirement_name, requirement in all_requirements.iteritems() | sort %}
|
||||||
|
{{ requirement }}
|
||||||
|
{% endfor %}
|
||||||
|
{% else %}
|
||||||
|
{# #}
|
||||||
|
{# For a selective wheel build, we now need to build a set of the #}
|
||||||
|
{# packages we actually want to build. This list of packages must #}
|
||||||
|
{# be a unique set (or pip wheel will fail). We will use a #}
|
||||||
|
{# key:value mechanism to compile the set. #}
|
||||||
{# #}
|
{# #}
|
||||||
{% set selected_requirements={} %}
|
{% set selected_requirements={} %}
|
||||||
{# #}
|
{# #}
|
||||||
|
@ -1,3 +1,4 @@
|
|||||||
{% for item in built_wheels.stdout_lines %}
|
{% for file_data in built_wheels['files'] %}
|
||||||
{{ item.split('-')[0] | lower }}=={{ (item.split('-')[1].split('_')) | join('.post') | lower }}
|
{% set file_name = file_data['path'] | basename %}
|
||||||
|
{{ file_name.split('-')[0] | lower }}=={{ (file_name.split('-')[1].split('_')) | join('.post') | lower }}
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
|
@ -43,8 +43,14 @@ fi
|
|||||||
# Source the options file
|
# Source the options file
|
||||||
source "${ROLE_VENV_REQUIREMENTS_FILE}"
|
source "${ROLE_VENV_REQUIREMENTS_FILE}"
|
||||||
|
|
||||||
|
# Output the beginning of the build
|
||||||
|
echo -n "Building venv ${ROLE_VENV_FILE}..."
|
||||||
|
|
||||||
|
# Set the log file path
|
||||||
|
ROLE_VENV_LOG="/var/log/repo/venv_build_${ROLE_VENV_FILE}.log"
|
||||||
|
|
||||||
# Begin the venv build
|
# Begin the venv build
|
||||||
pushd "{{ repo_build_venv_dir }}"
|
pushd "{{ repo_build_venv_dir }}" &>/dev/null
|
||||||
|
|
||||||
# If the venv achive already exists, remove it
|
# If the venv achive already exists, remove it
|
||||||
[[ -e "${ROLE_VENV_FILE}.tgz" ]] && rm -f "${ROLE_VENV_FILE}.tgz"
|
[[ -e "${ROLE_VENV_FILE}.tgz" ]] && rm -f "${ROLE_VENV_FILE}.tgz"
|
||||||
@ -59,7 +65,7 @@ pushd "{{ repo_build_venv_dir }}"
|
|||||||
[[ -d "/tmp/${ROLE_VENV_FILE}" ]] && rm -rf "/tmp/${ROLE_VENV_FILE}"
|
[[ -d "/tmp/${ROLE_VENV_FILE}" ]] && rm -rf "/tmp/${ROLE_VENV_FILE}"
|
||||||
|
|
||||||
# Create the virtualenv shell
|
# Create the virtualenv shell
|
||||||
${VENV_CREATE_COMMAND} "${ROLE_VENV_PATH}"
|
${VENV_CREATE_COMMAND} "${ROLE_VENV_PATH}" &>${ROLE_VENV_LOG}
|
||||||
|
|
||||||
# Create the pip build directory
|
# Create the pip build directory
|
||||||
mkdir -p "/tmp/${ROLE_VENV_FILE}"
|
mkdir -p "/tmp/${ROLE_VENV_FILE}"
|
||||||
@ -69,25 +75,32 @@ pushd "{{ repo_build_venv_dir }}"
|
|||||||
|
|
||||||
# Install the packages into the venv
|
# Install the packages into the venv
|
||||||
${ROLE_VENV_PATH}/bin/pip install \
|
${ROLE_VENV_PATH}/bin/pip install \
|
||||||
|
--disable-pip-version-check \
|
||||||
|
--quiet --quiet \
|
||||||
--build "/tmp/${ROLE_VENV_FILE}" \
|
--build "/tmp/${ROLE_VENV_FILE}" \
|
||||||
${PIP_INSTALL_OPTIONS} \
|
${PIP_INSTALL_OPTIONS} \
|
||||||
${PIP_INDEX_OPTIONS} \
|
${PIP_INDEX_OPTIONS} \
|
||||||
${ROLE_VENV_REQUIREMENTS}
|
${ROLE_VENV_REQUIREMENTS} \
|
||||||
|
--log "${ROLE_VENV_LOG}"
|
||||||
|
|
||||||
# Deactivate the venv for good measure
|
# Deactivate the venv for good measure
|
||||||
deactivate
|
deactivate
|
||||||
|
|
||||||
# Find and remove all of the python pyc files
|
# Find and remove all of the python pyc files
|
||||||
find "${ROLE_VENV_PATH}" -type f -name '*.pyc' -delete
|
find "${ROLE_VENV_PATH}" -type f -name '*.pyc' -delete 2>>${ROLE_VENV_LOG}
|
||||||
|
|
||||||
# Create the archive
|
# Create the archive
|
||||||
tar czf "${ROLE_VENV_FILE}.tgz" -C "${ROLE_VENV_PATH}" .
|
tar czf "${ROLE_VENV_FILE}.tgz" -C "${ROLE_VENV_PATH}" . 2>>${ROLE_VENV_LOG}
|
||||||
|
|
||||||
# Create a checksum file for the archive
|
# Create a checksum file for the archive
|
||||||
sha1sum "${ROLE_VENV_FILE}.tgz" | awk '{print $1}' > "${ROLE_VENV_FILE}.checksum"
|
sha1sum "${ROLE_VENV_FILE}.tgz" | awk '{print $1}' > "${ROLE_VENV_FILE}.checksum" 2>>${ROLE_VENV_LOG}
|
||||||
|
|
||||||
# Delete working directories
|
# Delete working directories
|
||||||
rm -rf "${ROLE_VENV_PATH}"
|
rm -rf "${ROLE_VENV_PATH}"
|
||||||
rm -rf "/tmp/${ROLE_VENV_FILE}"
|
rm -rf "/tmp/${ROLE_VENV_FILE}"
|
||||||
|
|
||||||
popd
|
popd &>/dev/null
|
||||||
|
|
||||||
|
# Output the end of the build
|
||||||
|
echo "done"
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user