Updates to support ignored packages and external indexes
This change ensures that a venv create process can work when a dependency is crafted outside of the OpenStack global requirements system. The change keys off of the original package url setting the isolated perameter should "withindex=True" be found. The requirements.txt.j2 template file has been updated to allow packages to be ignored from wheel building should the `ignorerequirement=True` property be set within a given remote package. This ability is important as some packages, generally external testing repos, do not use requirement boundries and are known to break given enough time. Tagged tasks that MUST always be run have had tags set to "always". Change-Id: I5d11aca84dcb74c77f3d0e3f31ce6546458a3e84 Closes-Bug: #1631992 Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
This commit is contained in:
parent
05026c23bd
commit
baab4e2e7c
@ -81,7 +81,6 @@ repo_build_venv_dir: "/var/www/repo/venvs"
|
||||
repo_build_venv_pip_install_options: >
|
||||
--timeout 120
|
||||
--find-links {{ repo_build_release_path }}/{{ repo_build_release_tag }}
|
||||
--no-index
|
||||
--log /var/log/repo/repo_venv_builder.log
|
||||
repo_build_venv_command_options: >
|
||||
{{ virtualenv_bin }}
|
||||
|
@ -17,12 +17,7 @@
|
||||
command: "ls -1 {{ repo_build_output }}"
|
||||
register: built_wheels
|
||||
tags:
|
||||
- repo-build-index-wheels
|
||||
- repo-create-pool
|
||||
- repo-copy-wheels-to-pool
|
||||
- repo-create-release-links
|
||||
- repo-create-links-index
|
||||
- repo-create-release-manifest
|
||||
- always
|
||||
|
||||
- name: Create release process script
|
||||
template:
|
||||
|
@ -17,15 +17,13 @@
|
||||
command: which virtualenv
|
||||
register: virtualenv_path
|
||||
tags:
|
||||
- repo-command-bin
|
||||
- repo-create-venv
|
||||
- always
|
||||
|
||||
- name: Set virtualenv command path
|
||||
set_fact:
|
||||
virtualenv_bin: "{{ virtualenv_path.stdout }}"
|
||||
tags:
|
||||
- repo-command-bin
|
||||
- repo-create-venv
|
||||
- always
|
||||
|
||||
- name: Check for created venvs
|
||||
command: >
|
||||
|
@ -5,6 +5,8 @@ set -ev
|
||||
function venv_create {
|
||||
VENV_PATH="$1"
|
||||
VENV_FILE="$2"
|
||||
ROLE_VENV_WITH_INDEX="$3"
|
||||
VENV_VALUES="$4"
|
||||
|
||||
# If the venv working directory already exists remove it
|
||||
[[ -d "/tmp/${VENV_PATH}" ]] && rm -rf "/tmp/${VENV_PATH}"
|
||||
@ -23,7 +25,31 @@ function venv_create {
|
||||
|
||||
# Run the pip install within the venv and specify a specific build directory which
|
||||
# resolves pip locking issues when run in parallel.
|
||||
${VENV_PATH}/bin/pip install --build "/tmp/${VENV_FILE}" {{ repo_build_venv_pip_install_options.split() | join(' ') }} $3
|
||||
{% set pip_command = [] %}
|
||||
{% set _ = pip_command.append('${VENV_PATH}/bin/pip install') %}
|
||||
{% set _ = pip_command.append('--build "/tmp/${VENV_FILE}"') %}
|
||||
{% set _ = pip_command.append(repo_build_venv_pip_install_options.strip()) %}
|
||||
|
||||
if [ "${ROLE_VENV_WITH_INDEX}" = false ]; then
|
||||
{{ pip_command | join(' ') }} --no-index ${VENV_VALUES}
|
||||
|
||||
{% if repo_build_pip_default_index is defined %}
|
||||
{% set _ = pip_command.append('--index-url ' + repo_build_pip_default_index) %}
|
||||
{% set _ = pip_command.append('--trusted-host ' + repo_build_pip_default_index | netloc_no_port) %}
|
||||
{% endif %}
|
||||
{% if repo_build_pip_extra_index is defined %}
|
||||
{% set _ = pip_command.append('--extra-index-url ' + repo_build_pip_extra_index) %}
|
||||
{% set _ = pip_command.append('--trusted-host ' + repo_build_pip_extra_index | netloc_no_port) %}
|
||||
{% endif %}
|
||||
{% if repo_build_pip_extra_indexes is defined %}
|
||||
{% set _ = pip_command.append('--extra-index-url ' + repo_build_pip_extra_indexes | join(' --extra-index-url ')) %}
|
||||
{% set _ = pip_command.append('--trusted-host ' + repo_build_pip_extra_indexes | map('netloc_no_port') | join(' --trusted-host ')) %}
|
||||
{% endif %}
|
||||
|
||||
else
|
||||
{{ pip_command | join(' ') }} ${VENV_VALUES}
|
||||
|
||||
fi
|
||||
|
||||
# Deactivate the venv for good measure
|
||||
deactivate
|
||||
@ -70,14 +96,27 @@ PID=()
|
||||
# If the venv archive already exists the creation process will be skipped
|
||||
pushd "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}"
|
||||
{% for key, value in os_group.items() %}
|
||||
|
||||
{% set venvwithindex = [] %}
|
||||
{% for clone_item in local_packages.results.0.item.remote_package_parts %}
|
||||
{% if clone_item['name'] == (key | replace('os_', '')) and 'venvwithindex=True' in clone_item['original'] %}
|
||||
{% set _ = venvwithindex.append(true) %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% if venvwithindex %}
|
||||
ROLE_VENV_WITH_INDEX=true
|
||||
{% else %}
|
||||
ROLE_VENV_WITH_INDEX=false
|
||||
{% endif %}
|
||||
ROLE_VENV_PATH="{{ repo_build_venv_build_dir }}/venvs/{{ key | replace('os_', '') }}"
|
||||
ROLE_VENV_FILE="{{ key | replace('os_', '') }}-{{ repo_build_release_tag }}-{{ ansible_architecture | lower }}"
|
||||
if [ ! -f "${ROLE_VENV_FILE}.tgz" ];then
|
||||
venv_create "${ROLE_VENV_PATH}" "${ROLE_VENV_FILE}" "{{ value | join(' ') }}" &
|
||||
venv_create "${ROLE_VENV_PATH}" "${ROLE_VENV_FILE}" "${ROLE_VENV_WITH_INDEX}" "{{ value | join(' ') }}" &
|
||||
pid[{{ loop.index }}]=$!
|
||||
fi
|
||||
unset ROLE_VENV_PATH
|
||||
unset ROLE_VENV_FILE
|
||||
unset ROLE_VENV_WITH_INDEX
|
||||
{% if loop.index is divisibleby(repo_build_concurrency | int) or loop.last %}
|
||||
for job_pid in ${!pid[@]}; do
|
||||
wait ${pid[$job_pid]} || exit 99
|
||||
@ -86,3 +125,4 @@ done
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
popd
|
||||
|
||||
|
@ -58,8 +58,10 @@
|
||||
{% for requirement_raw in requirement_data %}
|
||||
{% set name = requirement_raw | regex_replace('(\[|>=|<=|>|<|==|~=|!=).*$','') %}
|
||||
{% set name_normalized = name | replace('-', '_') | lower %}
|
||||
{% set requirement_normalized = all_requirements[name_normalized] %}
|
||||
{% set _ = selected_requirements.update({name_normalized: requirement_normalized}) %}
|
||||
{% if name_normalized in all_requirements %}
|
||||
{% set requirement_normalized = all_requirements[name_normalized] %}
|
||||
{% set _ = selected_requirements.update({name_normalized: requirement_normalized}) %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
@ -295,7 +295,7 @@ local_packages:
|
||||
- egg_name: rally
|
||||
fragment: null
|
||||
name: rally
|
||||
original: git+https://git.openstack.org/openstack/rally@e6f4dafc9a3a788c3388abad32269a38f4809123#egg=rally&gitname=rally&projectgroup=utility_all
|
||||
original: git+https://git.openstack.org/openstack/rally@e6f4dafc9a3a788c3388abad32269a38f4809123#egg=rally&gitname=rally&venvwithindex=True&projectgroup=utility_all
|
||||
project_group: utility_all
|
||||
url: https://git.openstack.org/openstack/rally
|
||||
version: e6f4dafc9a3a788c3388abad32269a38f4809123
|
||||
@ -337,7 +337,7 @@ local_packages:
|
||||
- egg_name: tempest
|
||||
fragment: null
|
||||
name: tempest
|
||||
original: git+https://git.openstack.org/openstack/tempest@479bfaccf5b1db53afb9f0663de7544b25c4bca7#egg=tempest&gitname=tempest&projectgroup=utility_all
|
||||
original: git+https://git.openstack.org/openstack/tempest@479bfaccf5b1db53afb9f0663de7544b25c4bca7#egg=tempest&gitname=tempest&venvwithindex=True&projectgroup=utility_all
|
||||
project_group: utility_all
|
||||
url: https://git.openstack.org/openstack/tempest
|
||||
version: 479bfaccf5b1db53afb9f0663de7544b25c4bca7
|
||||
@ -364,12 +364,12 @@ local_packages:
|
||||
- git+https://git.openstack.org/openstack/nova-lxd@f81277ab66bed71a9c37dc8df99eface79e6156f#egg=nova_lxd&gitname=nova-lxd&projectgroup=nova_compute
|
||||
- git+https://git.openstack.org/openstack/nova-powervm@80e413814424c679215585c7a8e5d33281a1dea0#egg=nova_powervm&gitname=nova-powervm&projectgroup=nova_compute
|
||||
- git+https://git.openstack.org/openstack/nova@110c12b4145a5a978e3dd9e99e65be1d4a86ee06#egg=nova&gitname=nova&projectgroup=nova_all
|
||||
- git+https://git.openstack.org/openstack/rally@e6f4dafc9a3a788c3388abad32269a38f4809123#egg=rally&gitname=rally&projectgroup=utility_all
|
||||
- git+https://git.openstack.org/openstack/rally@e6f4dafc9a3a788c3388abad32269a38f4809123#egg=rally&gitname=rally&venvwithindex=True&projectgroup=utility_all
|
||||
- git+https://git.openstack.org/openstack/requirements@4f211a5002af4b19a2f050e543b34e2212250fbf#egg=requirements&gitname=requirements&projectgroup=all
|
||||
- git+https://git.openstack.org/openstack/sahara-dashboard@2bb9843690e583bc7944a86fdb27058d3cb2e887#egg=sahara_dashboard&gitname=sahara-dashboard&projectgroup=horizon_all
|
||||
- git+https://git.openstack.org/openstack/sahara@5eeeab432b95f59b2a1c2db85f1f1a297bd22668#egg=sahara&gitname=sahara&projectgroup=sahara_all
|
||||
- git+https://git.openstack.org/openstack/swift@d5e484e692c05c5f04069b181a5e4037239bc57c#egg=swift&gitname=swift&projectgroup=swift_all
|
||||
- git+https://git.openstack.org/openstack/tempest@479bfaccf5b1db53afb9f0663de7544b25c4bca7#egg=tempest&gitname=tempest&projectgroup=utility_all
|
||||
- git+https://git.openstack.org/openstack/tempest@479bfaccf5b1db53afb9f0663de7544b25c4bca7#egg=tempest&gitname=tempest&venvwithindex=True&projectgroup=utility_all
|
||||
- git+https://github.com/SPICE/spice-html5@54cc41299bea8cd681ed0262735e0fd821cd774a#egg=spice_html5&gitname=spice-html5&projectgroup=nova_console
|
||||
- git+https://github.com/kanaka/novnc@7a16304e52b3fbd466337601f05a7def2493c563#egg=novnc&gitname=novnc&projectgroup=nova_console
|
||||
- git+https://github.com/projectcalico/felix@a2daf3725e33a76bdb01c0311d5827f159969a29#egg=calico&gitname=calico&projectgroup=neutron_all
|
||||
@ -1567,4 +1567,4 @@ local_packages:
|
||||
- urllib3
|
||||
- virtualenv
|
||||
- virtualenv-tools
|
||||
- wheel
|
||||
- wheel
|
||||
|
Loading…
Reference in New Issue
Block a user