From 6b94b21e18181eab1eddb1f3040c430051871a20 Mon Sep 17 00:00:00 2001 From: Jesse Pretorius Date: Fri, 23 Feb 2018 17:34:41 +0000 Subject: [PATCH] Isolate the Ansible bootstrap As we move forward with the python build/install simplification, more of the config is being applied in the pip conf, rather than being given as CLI options in each ansible task. This is good for providing a consistent experience, but has side effects for things like the Ansible bootstrap. When bootstrapping Ansible, we want to make use of a specific set of constraints which is not polluted by additional config on the host. We do not want the constraints from the repo server because they happen to include a constraint for Ansible which clashes with our requirement. As such, we only want the native OpenStack upper-constraints. In this patch we use the --isolated CLI option to ignore the host's pip configuration. We also introduce the ability to set extra pip install options by using the PIP_OPTS environment variable. This can be used to set the installation to use an alternative pypi mirror, or for extra links, etc. Change-Id: Ic966bafd04c4c01b3d93851a0e3ec2c1f3312f28 Closes-Bug: #1751316 --- scripts/bootstrap-ansible.sh | 28 +++++++++++++++------------- scripts/gate-check-commit.sh | 13 +++++++++++++ scripts/scripts-library.sh | 15 ++++++++------- 3 files changed, 36 insertions(+), 20 deletions(-) diff --git a/scripts/bootstrap-ansible.sh b/scripts/bootstrap-ansible.sh index 7f750d04f6..daf4997e60 100755 --- a/scripts/bootstrap-ansible.sh +++ b/scripts/bootstrap-ansible.sh @@ -27,6 +27,10 @@ export ANSIBLE_ROLE_FILE=${ANSIBLE_ROLE_FILE:-"ansible-role-requirements.yml"} export SSH_DIR=${SSH_DIR:-"/root/.ssh"} export DEBIAN_FRONTEND=${DEBIAN_FRONTEND:-"noninteractive"} +# Use pip opts to add options to the pip install command. +# This can be used to tell it which index to use, etc. +export PIP_OPTS=${PIP_OPTS:-""} + # Set the role fetch mode to any option [galaxy, git-clone] export ANSIBLE_ROLE_FETCH_MODE=${ANSIBLE_ROLE_FETCH_MODE:-git-clone} @@ -96,12 +100,11 @@ case ${DISTRO_ID} in esac # Ensure we use the HTTPS/HTTP proxy with pip if it is specified -PIP_OPTS="" if [ -n "$HTTPS_PROXY" ]; then - PIP_OPTS="--proxy $HTTPS_PROXY" + PIP_OPTS+="--proxy $HTTPS_PROXY" elif [ -n "$HTTP_PROXY" ]; then - PIP_OPTS="--proxy $HTTP_PROXY" + PIP_OPTS+="--proxy $HTTP_PROXY" fi # Figure out the version of python is being used @@ -134,11 +137,9 @@ if [[ "${VIRTUALENV_VERSION}" -lt "13" ]]; then pip install ${PIP_OPTS} \ --constraint ${UPPER_CONSTRAINTS_FILE} \ - virtualenv \ - || pip install ${PIP_OPTS} \ - --constraint ${UPPER_CONSTRAINTS_FILE} \ - --isolated \ - virtualenv + --isolated \ + virtualenv + # Ensure that our shell knows about the new pip hash -r virtualenv fi @@ -163,13 +164,14 @@ PIP_COMMAND="/opt/ansible-runtime/bin/pip" PIP_OPTS+=" --constraint global-requirement-pins.txt" PIP_OPTS+=" --constraint ${UPPER_CONSTRAINTS_FILE}" -# When upgrading there will already be a pip.conf file locking pip down to the -# repo server, in such cases it may be necessary to use --isolated because the -# repo server does not meet the specified requirements. +# When executing the installation, we want to specify all our options on the CLI, +# making sure to completely ignore any config already on the host. This is to +# prevent the repo server's extra constraints being applied, which include +# a different version of Ansible to the one we want to install. As such, we +# use --isolated so that the config file is ignored. # Install ansible and the other required packages -${PIP_COMMAND} install ${PIP_OPTS} -r requirements.txt ${ANSIBLE_PACKAGE} \ - || ${PIP_COMMAND} install --isolated ${PIP_OPTS} -r requirements.txt ${ANSIBLE_PACKAGE} +${PIP_COMMAND} install --isolated ${PIP_OPTS} -r requirements.txt ${ANSIBLE_PACKAGE} # Install our osa_toolkit code from the current checkout $PIP_COMMAND install -e . diff --git a/scripts/gate-check-commit.sh b/scripts/gate-check-commit.sh index 194e26e3aa..dcdd18f24c 100755 --- a/scripts/gate-check-commit.sh +++ b/scripts/gate-check-commit.sh @@ -90,6 +90,12 @@ if [ -n "${DATA_DISK_DEVICE}" ]; then export BOOTSTRAP_OPTS="${BOOTSTRAP_OPTS} bootstrap_host_data_disk_device=${DATA_DISK_DEVICE}" fi +# If in OpenStack-Infra, set some vars to use the mirror when bootstrapping Ansible +if [[ -e /etc/ci/mirror_info.sh ]]; then + source /etc/ci/mirror_info.sh + export PIP_OPTS="--index-url ${NODEPOOL_PYPI_MIRROR} --trusted-host ${NODEPOOL_MIRROR_HOST} --extra-index-url ${NODEPOOL_WHEEL_MIRROR}" +fi + # Bootstrap Ansible source "${OSA_CLONE_DIR}/scripts/bootstrap-ansible.sh" @@ -206,6 +212,13 @@ if [[ "${ACTION}" == "upgrade" ]]; then # requirements to be installed. unset ANSIBLE_PACKAGE unset UPPER_CONSTRAINTS_FILE + unset PIP_OPTS + + # If in OpenStack-Infra, set some vars to use the mirror when bootstrapping Ansible + if [[ -e /etc/ci/mirror_info.sh ]]; then + source /etc/ci/mirror_info.sh + export PIP_OPTS="--index-url ${NODEPOOL_PYPI_MIRROR} --trusted-host ${NODEPOOL_MIRROR_HOST} --extra-index-url ${NODEPOOL_WHEEL_MIRROR}" + fi # Source the current scripts-library.sh functions source "${OSA_CLONE_DIR}/scripts/scripts-library.sh" diff --git a/scripts/scripts-library.sh b/scripts/scripts-library.sh index e539cf7b66..d39accf391 100755 --- a/scripts/scripts-library.sh +++ b/scripts/scripts-library.sh @@ -321,6 +321,10 @@ function get_instance_info { function get_pip { + # Use pip opts to add options to the pip install command. + # This can be used to tell it which index to use, etc. + PIP_OPTS=${PIP_OPTS:-""} + # The python executable to use when executing get-pip is passed # as a parameter to this function. GETPIP_PYTHON_EXEC_PATH="${1:-$(which python)}" @@ -337,13 +341,10 @@ function get_pip { || ${GETPIP_CMD} https://raw.githubusercontent.com/pypa/get-pip/master/get-pip.py > ${GETPIP_FILE} fi - ${GETPIP_PYTHON_EXEC_PATH} ${GETPIP_FILE} \ - pip setuptools wheel \ - --constraint global-requirement-pins.txt \ - || ${GETPIP_PYTHON_EXEC_PATH} ${GETPIP_FILE} \ - pip setuptools wheel \ - --constraint global-requirement-pins.txt \ - --isolated + ${GETPIP_PYTHON_EXEC_PATH} ${GETPIP_FILE} ${PIP_OPTS} \ + pip setuptools wheel \ + --constraint global-requirement-pins.txt \ + --isolated } function get_bowling_ball_tests {