openstack-ansible-repo_build/templates/op-venv-script.sh.j2

#!/usr/local/env bash

set -ev

function venv_create {
    VENV_PATH="$1"
    VENV_FILE="$2"
    ROLE_VENV_WITH_INDEX="$3"
    VENV_VALUES="$4"

    # If the venv working directory already exists remove it
    [[ -d "/tmp/${VENV_PATH}" ]] && rm -rf "/tmp/${VENV_PATH}"

    # If the pip build directory already exists remove it
    [[ -d "/tmp/${VENV_FILE}" ]] && rm -rf "/tmp/${VENV_FILE}"

    # Create the virtualenv shell
    {{ repo_build_venv_command_options.split() | join(' ') }} "${VENV_PATH}"

    # Create the pip build directory
    mkdir -p "/tmp/${VENV_FILE}"

    # Activate the python virtual environment for good measure
    source "${VENV_PATH}/bin/activate"

    # Run the pip install within the venv and specify a specific build directory which
    #  resolves pip locking issues when run in parallel.
    {% set pip_command = [] %}
    {% set _ = pip_command.append('${VENV_PATH}/bin/pip install') %}
    {% set _ = pip_command.append('--build "/tmp/${VENV_FILE}"') %}
    {% set _ = pip_command.append(repo_build_venv_pip_install_options.strip()) %}

    if [ "${ROLE_VENV_WITH_INDEX}" = false ]; then
    {{ pip_command | join(' ') }} --no-index ${VENV_VALUES}

    {% if repo_build_pip_default_index is defined %}
    {% set _ = pip_command.append('--index-url ' + repo_build_pip_default_index) %}
    {% set _ = pip_command.append('--trusted-host ' + repo_build_pip_default_index | netloc_no_port) %}
    {% endif %}
    {% if repo_build_pip_extra_index is defined %}
    {% set _ = pip_command.append('--extra-index-url ' + repo_build_pip_extra_index)  %}
    {% set _ = pip_command.append('--trusted-host ' + repo_build_pip_extra_index | netloc_no_port) %}
    {% endif %}
    {% if repo_build_pip_extra_indexes is defined %}
    {% set _ = pip_command.append('--extra-index-url ' + repo_build_pip_extra_indexes | join(' --extra-index-url ')) %}
    {% set _ = pip_command.append('--trusted-host ' + repo_build_pip_extra_indexes | map('netloc_no_port') | join(' --trusted-host ')) %}
    {% endif %}

    else
    {{ pip_command | join(' ') }} ${VENV_VALUES}

    fi

    # Deactivate the venv for good measure
    deactivate

    # Find and remove all of the python pyc files
    find "${VENV_PATH}" -type f -name '*.pyc' -delete

    # Create the archive
    tar czf "${VENV_FILE}.tgz" -C "${VENV_PATH}" .

    # Create a checksum file for the archive
    sha1sum "${VENV_FILE}.tgz" | awk '{print $1}' > "${VENV_FILE}.checksum"

    # Remove the working directories
    rm -rf "${VENV_PATH}"
    rm -rf "/tmp/${VENV_FILE}"
}

# First operation is to sort and set the known os_* roles and create a unique dict.
#  NOTE: this is a Jinja loop and will not be rendered within the script. For debugging
#        purposes the group data will be rendered as a comment.
{% set os_group = {} %}
{% for role_name, role_data in local_packages.results.0.item.role_requirements.items() %}
{%   set _host_group = role_data['project_group'] %}
{%   if ("os_" in role_name) and ((not repo_build_venv_selective | bool) or ((groups[_host_group] is defined) and (groups[_host_group] | length > 0))) %}
{%     set requirement_list = [] %}
{%     for requirement_key, requirement_data in role_data.items() %}
{%       if '_pip_packages' in requirement_key and 'requires' not in requirement_key and 'optional' not in requirement_key and 'proprietary' not in requirement_key %}
{%         for requirement in requirement_data %}
{%           set _ = requirement_list.append(requirement) %}
{%         endfor %}
{%       endif %}
{%     endfor %}
# venv to build {{ role_name }}
#   * packages within the {{ role_name }} venv: {{ requirement_list }}
{%     set _ = os_group.update({role_name: requirement_list}) %}
{%   endif %}
{% endfor %}

PID=()
# Run the venv create. This will loop over all of the os_group role packages and create a python virtual env.
#  Venv creation is done parallel at a count of the known "ansible_processor_count" or using a default of 5.
#  This loop will enter the venv build directory and create tagged venvs in a distribution directory
#  If the venv archive already exists the creation process will be skipped
pushd "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}"
{% for key, value in os_group.items() %}

{% set venvwithindex = [] %}
{% for clone_item in local_packages.results.0.item.remote_package_parts %}
{%   if clone_item['name'] == (key | replace('os_', '')) and 'venvwithindex=True' in clone_item['original'] %}
{%     set _ = venvwithindex.append(true) %}
{%   endif %}
{% endfor %}
{% if venvwithindex %}
ROLE_VENV_WITH_INDEX=true
{% else %}
ROLE_VENV_WITH_INDEX=false
{% endif %}
ROLE_VENV_PATH="{{ repo_build_venv_build_dir }}/venvs/{{ key | replace('os_', '') }}"
ROLE_VENV_FILE="{{ key | replace('os_', '') }}-{{ repo_build_release_tag }}-{{ ansible_architecture | lower }}"
if [ ! -f "${ROLE_VENV_FILE}.tgz" ];then
    venv_create "${ROLE_VENV_PATH}" "${ROLE_VENV_FILE}" "${ROLE_VENV_WITH_INDEX}" "{{ value | join(' ') }}" &
    pid[{{ loop.index }}]=$!
fi
unset ROLE_VENV_PATH
unset ROLE_VENV_FILE
unset ROLE_VENV_WITH_INDEX
{% if loop.index is divisibleby(repo_build_concurrency | int) or loop.last %}
for job_pid in ${!pid[@]}; do
    wait ${pid[$job_pid]} || exit 99
done

{% endif %}
{% endfor %}
popd