From cfc62b0e7635836d6c98127715d490b174292514 Mon Sep 17 00:00:00 2001
From: Jesse Pretorius <jesse.pretorius@rackspace.co.uk>
Date: Mon, 17 Jul 2017 17:57:35 +0100
Subject: [PATCH] Reduce ansible bootstrap packages

The current distro package install set is quite
broad, especially for xenial. This reduces the
set.

The get pip process is also simplified and the
virtualenv creation is normalised across python
versions.

The virtualenv creation process is also normalised
across distributions now that localhost tasks now
specify the host python executable instead of the
runtime ansible python venv.

Conflicts:
>------scripts/bootstrap-ansible.sh

Includes:
https://review.openstack.org/484797

Change-Id: I27a41d583d0ea9f7499f51e69686d1b865958e2e
(cherry picked from commit 82da9b630532feb20a9101ef6e74c37d78873f7a)
---
 scripts/bootstrap-ansible.sh | 42 ++++++++++++++++++------------------
 scripts/scripts-library.sh   | 22 +++++++------------
 2 files changed, 29 insertions(+), 35 deletions(-)

diff --git a/scripts/bootstrap-ansible.sh b/scripts/bootstrap-ansible.sh
index 7954ea1d70..2075488be9 100755
--- a/scripts/bootstrap-ansible.sh
+++ b/scripts/bootstrap-ansible.sh
@@ -30,9 +30,6 @@ export DEBIAN_FRONTEND=${DEBIAN_FRONTEND:-"noninteractive"}
 # Set the role fetch mode to any option [galaxy, git-clone]
 export ANSIBLE_ROLE_FETCH_MODE=${ANSIBLE_ROLE_FETCH_MODE:-galaxy}
 
-# virtualenv vars
-VIRTUALENV_OPTIONS="--always-copy"
-
 # This script should be executed from the root directory of the cloned repo
 cd "$(dirname "${0}")/.."
 
@@ -60,30 +57,22 @@ determine_distro
 # Install the base packages
 case ${DISTRO_ID} in
     centos|rhel)
-        yum -y install git python2 curl autoconf gcc-c++ \
-          python2-devel gcc libffi-devel nc openssl-devel \
-          python-pyasn1 pyOpenSSL python-ndg_httpsclient \
-          python-netaddr python-prettytable python-crypto PyYAML \
-          python-virtualenv
-          VIRTUALENV_OPTIONS=""
+        yum -y install \
+          git curl autoconf gcc gcc-c++ nc \
+          python2 python2-devel \
+          openssl-devel libffi-devel \
+          libselinux-python
         ;;
     ubuntu)
         apt-get update
         DEBIAN_FRONTEND=noninteractive apt-get -y install \
-          git python-all python-dev curl python2.7-dev build-essential \
-          libssl-dev libffi-dev netcat python-requests python-openssl python-pyasn1 \
-          python-netaddr python-prettytable python-crypto python-yaml \
-          python-virtualenv
+          git-core curl gcc netcat \
+          python2.7 python2.7-dev \
+          libssl-dev libffi-dev \
+          python-apt
         ;;
 esac
 
-# NOTE(mhayden): Ubuntu 16.04 needs python-ndg-httpsclient for SSL SNI support.
-#                This package is not needed in Ubuntu 14.04 and isn't available
-#                there as a package.
-if [[ "${DISTRO_ID}" == 'ubuntu' ]] && [[ "${DISTRO_VERSION_ID}" == '16.04' ]]; then
-  DEBIAN_FRONTEND=noninteractive apt-get -y install python-ndg-httpsclient
-fi
-
 # Install pip
 get_pip
 
@@ -105,8 +94,19 @@ UPPER_CONSTRAINTS_PROTO=$([ "$PYTHON_VERSION" == $(echo -e "$PYTHON_VERSION\n2.7
 # Set the location of the constraints to use for all pip installations
 export UPPER_CONSTRAINTS_FILE=${UPPER_CONSTRAINTS_FILE:-"$UPPER_CONSTRAINTS_PROTO://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?id=$(awk '/requirements_git_install_branch:/ {print $2}' playbooks/defaults/repo_packages/openstack_services.yml)"}
 
+# Make sure that host requirements are installed
+pip install ${PIP_OPTS} \
+  --requirement requirements.txt \
+  --constraint ${UPPER_CONSTRAINTS_FILE} \
+  || pip install ${PIP_OPTS} \
+       --requirement requirements.txt \
+       --constraint ${UPPER_CONSTRAINTS_FILE} \
+       --isolated
+
 # Create a Virtualenv for the Ansible runtime
-virtualenv --clear ${VIRTUALENV_OPTIONS} --python="${PYTHON_EXEC_PATH}" /opt/ansible-runtime
+virtualenv --python="${PYTHON_EXEC_PATH}" \
+           --clear \
+           /opt/ansible-runtime
 
 # The vars used to prepare the Ansible runtime venv
 PIP_OPTS+=" --upgrade"
diff --git a/scripts/scripts-library.sh b/scripts/scripts-library.sh
index 85ea6667c7..c6d1feb1e1 100755
--- a/scripts/scripts-library.sh
+++ b/scripts/scripts-library.sh
@@ -241,24 +241,18 @@ function get_pip {
   # when pip is not installed, install it
   else
 
+    # Download the get-pip script using the primary or secondary URL
+    GETPIP_CMD="curl --silent --show-error --retry 5"
+    GETPIP_FILE="/opt/get-pip.py"
     # If GET_PIP_URL is set, then just use it
     if [ -n "${GET_PIP_URL:-}" ]; then
-      curl --silent ${GET_PIP_URL} > /opt/get-pip.py
-      if head -n 1 /opt/get-pip.py | grep python; then
-        python /opt/get-pip.py ${PIP_INSTALL_OPTIONS}
-        return
-      fi
+      ${CURL_CMD} ${GET_PIP_URL} > ${OUTPUT_FILE}
+    else
+      # Otherwise, try the two standard URL's
+      ${CURL_CMD} https://bootstrap.pypa.io/get-pip.py > ${OUTPUT_FILE}\
+        || ${CURL_CMD} https://raw.githubusercontent.com/pypa/get-pip/master/get-pip.py > ${OUTPUT_FILE}
     fi
 
-    # Try getting pip from bootstrap.pypa.io as a primary source
-    curl --silent https://bootstrap.pypa.io/get-pip.py > /opt/get-pip.py
-    if head -n 1 /opt/get-pip.py | grep python; then
-      python /opt/get-pip.py ${PIP_INSTALL_OPTIONS}
-      return
-    fi
-
-    # Try the get-pip.py from the github repository as a primary source
-    curl --silent https://raw.githubusercontent.com/pypa/get-pip/master/get-pip.py > /opt/get-pip.py
     if head -n 1 /opt/get-pip.py | grep python; then
       python /opt/get-pip.py ${PIP_INSTALL_OPTIONS}
       return