From ddc3839bdc7ff73f6224273605db10fd88cd60df Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 7 May 2015 21:06:24 +0000 Subject: [PATCH] Enable optional Python 3 support Add USE_PYTHON3 and PYTHON3_VERSION variables to allow services to use python 3 if they indicate support in their python package metadata. Tested in Heat here -> I837c2fba682ab430d50e9f43913f2fed20325a7a. Project config change to add a dedicated job to Heat is here -> I0837e62d6ccc66397a5e409f0961edd4be31f467 Change-Id: I079e18b58b214bf8362945c253d6d894ca8b1a6b --- inc/python | 47 +++++++++++++++++++++++++++++++++++++--- lib/stack | 1 + stackrc | 11 ++++++++++ tools/install_pip.sh | 8 +++++++ tools/install_prereqs.sh | 3 +++ 5 files changed, 67 insertions(+), 3 deletions(-) diff --git a/inc/python b/inc/python index 59668a2c6a..c157604699 100644 --- a/inc/python +++ b/inc/python @@ -28,10 +28,13 @@ declare -A PROJECT_VENV # Get the path to the pip command. # get_pip_command function get_pip_command { - which pip || which pip-python + local version="$1" + # NOTE(dhellmann): I don't know if we actually get a pip3.4-python + # under any circumstances. + which pip${version} || which pip${version}-python if [ $? -ne 0 ]; then - die $LINENO "Unable to find pip; cannot continue" + die $LINENO "Unable to find pip${version}; cannot continue" fi } @@ -66,6 +69,13 @@ function pip_install_gr { pip_install $clean_name } +# Determine the python versions supported by a package +function get_python_versions_for_package { + local name=$1 + cd $name && python setup.py --classifiers \ + | grep 'Language' | cut -f5 -d: | grep '\.' | tr '\n' ' ' +} + # Wrapper for ``pip install`` to set cache and proxy environment variables # Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``, # ``PIP_UPGRADE``, ``TRACK_DEPENDS``, ``*_proxy``, @@ -104,8 +114,22 @@ function pip_install { local sudo_pip="env" else local cmd_pip - cmd_pip=$(get_pip_command) + cmd_pip=$(get_pip_command $PYTHON2_VERSION) local sudo_pip="sudo -H" + if python3_enabled; then + # Look at the package classifiers to find the python + # versions supported, and if we find the version of + # python3 we've been told to use, use that instead of the + # default pip + local package_dir=${!#} + local python_versions + if [[ -d "$package_dir" ]]; then + python_versions=$(get_python_versions_for_package $package_dir) + if [[ $python_versions =~ $PYTHON3_VERSION ]]; then + cmd_pip=$(get_pip_command $PYTHON3_VERSION) + fi + fi + fi fi fi @@ -113,6 +137,8 @@ function pip_install { # Always apply constraints cmd_pip="$cmd_pip -c $REQUIREMENTS_DIR/upper-constraints.txt" + # FIXME(dhellmann): Need to force multiple versions of pip for + # packages like setuptools? local pip_version pip_version=$(python -c "import pip; \ print(pip.__version__.strip('.')[0])") @@ -276,6 +302,21 @@ function setup_package { fi } +# Report whether python 3 should be used +function python3_enabled { + if [[ $USE_PYTHON3 == "True" ]]; then + return 0 + else + return 1 + fi +} + +# Install python3 packages +function install_python3 { + if is_ubuntu; then + apt_get install python3.4 python3.4-dev + fi +} # Restore xtrace $INC_PY_TRACE diff --git a/lib/stack b/lib/stack index 7d98604b82..f09ddcee85 100644 --- a/lib/stack +++ b/lib/stack @@ -19,6 +19,7 @@ function stack_install_service { local service=$1 if type install_${service} >/dev/null 2>&1; then + # FIXME(dhellmann): Needs to be python3-aware at some point. if [[ ${USE_VENV} = True && -n ${PROJECT_VENV[$service]:-} ]]; then rm -rf ${PROJECT_VENV[$service]} source $TOP_DIR/tools/build_venv.sh ${PROJECT_VENV[$service]} ${ADDITIONAL_VENV_PACKAGES//,/ } diff --git a/stackrc b/stackrc index 5dd109c0a4..f949ccbad9 100644 --- a/stackrc +++ b/stackrc @@ -118,6 +118,17 @@ if [[ -r $RC_DIR/.localrc.password ]]; then source $RC_DIR/.localrc.password fi +# Control whether Python 3 should be used. +export USE_PYTHON3=${USE_PYTHON3:-False} + +# When Python 3 is supported by an application, adding the specific +# version of Python 3 to this variable will install the app using that +# version of the interpreter instead of 2.7. +export PYTHON3_VERSION=${PYTHON3_VERSION:-3.4} + +# Just to be more explicit on the Python 2 version to use. +export PYTHON2_VERSION=${PYTHON2_VERSION:-2.7} + # allow local overrides of env variables, including repo config if [[ -f $RC_DIR/localrc ]]; then # Old-style user-supplied config diff --git a/tools/install_pip.sh b/tools/install_pip.sh index ab5efb2e77..f239c7bb16 100755 --- a/tools/install_pip.sh +++ b/tools/install_pip.sh @@ -8,6 +8,7 @@ # Assumptions: # - update pip to $INSTALL_PIP_VERSION +# - if USE_PYTHON3=True, PYTHON3_VERSION refers to a version already installed set -o errexit set -o xtrace @@ -31,6 +32,8 @@ GetDistro echo "Distro: $DISTRO" function get_versions { + # FIXME(dhellmann): Deal with multiple python versions here? This + # is just used for reporting, so maybe not? PIP=$(which pip 2>/dev/null || which pip-python 2>/dev/null || true) if [[ -n $PIP ]]; then PIP_VERSION=$($PIP --version | awk '{ print $2}') @@ -75,6 +78,9 @@ function install_get_pip { touch $LOCAL_PIP.downloaded fi sudo -H -E python $LOCAL_PIP + if python3_enabled; then + sudo -H -E python${PYTHON3_VERSION} $LOCAL_PIP + fi } @@ -114,6 +120,7 @@ get_versions # python in f23 depends on the python-pip package if ! { is_fedora && [[ $DISTRO == "f23" ]]; }; then uninstall_package python-pip + uninstall_package python3-pip fi install_get_pip @@ -122,6 +129,7 @@ if [[ -n $PYPI_ALTERNATIVE_URL ]]; then configure_pypi_alternative_url fi +set -x pip_install -U setuptools get_versions diff --git a/tools/install_prereqs.sh b/tools/install_prereqs.sh index 38452cd90f..031f8a8eca 100755 --- a/tools/install_prereqs.sh +++ b/tools/install_prereqs.sh @@ -81,6 +81,9 @@ if [[ -n "$SYSLOG" && "$SYSLOG" != "False" ]]; then fi fi +if python3_enabled; then + install_python3 +fi # Mark end of run # ---------------