devstack/inc/python

516 lines
16 KiB
Bash

#!/bin/bash
#
# **inc/python** - Python-related functions
#
# Support for pip/setuptools interfaces and virtual environments
#
# External functions used:
# - GetOSVersion
# - is_fedora
# - safe_chown
# Save trace setting
INC_PY_TRACE=$(set +o | grep xtrace)
set +o xtrace
# Global Config Variables
# PROJECT_VENV contains the name of the virtual environment for each
# project. A null value installs to the system Python directories.
declare -A -g PROJECT_VENV
# Utility Functions
# =================
# Joins bash array of extras with commas as expected by other functions
function join_extras {
local IFS=","
echo "$*"
}
# Python Functions
# ================
# Setup the global devstack virtualenvs and the associated environment
# updates.
function setup_devstack_virtualenv {
# We run devstack out of a global virtualenv.
if [[ ! -d $DEVSTACK_VENV ]] ; then
# Using system site packages to enable nova to use libguestfs.
# This package is currently installed via the distro and not
# available on pypi.
python$PYTHON3_VERSION -m venv --system-site-packages $DEVSTACK_VENV
pip_install -U pip
fi
if [[ ":$PATH:" != *":$DEVSTACK_VENV/bin:"* ]] ; then
export PATH="$DEVSTACK_VENV/bin:$PATH"
export PYTHON="$DEVSTACK_VENV/bin/python3"
fi
}
# Get the path to the pip command.
# get_pip_command
function get_pip_command {
local version="$1"
if [ -z "$version" ]; then
die $LINENO "pip python version is not set."
fi
# NOTE(dhellmann): I don't know if we actually get a pip3.4-python
# under any circumstances.
which pip${version} || which pip${version}-python
if [ $? -ne 0 ]; then
die $LINENO "Unable to find pip${version}; cannot continue"
fi
}
# Get the path to the directory where python executables are installed.
# get_python_exec_prefix
function get_python_exec_prefix {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
if [[ -z "$os_PACKAGE" ]]; then
GetOSVersion
fi
$xtrace
if [[ "$GLOBAL_VENV" == "True" ]] ; then
echo "$DEVSTACK_VENV/bin"
else
echo "/usr/local/bin"
fi
}
# Wrapper for ``pip install`` that only installs versions of libraries
# from the global-requirements specification.
#
# Uses globals ``REQUIREMENTS_DIR``
#
# pip_install_gr packagename
function pip_install_gr {
local name=$1
local clean_name
clean_name=$(get_from_global_requirements $name)
pip_install $clean_name
}
# Wrapper for ``pip install`` that only installs versions of libraries
# from the global-requirements specification with extras.
#
# Uses globals ``REQUIREMENTS_DIR``
#
# pip_install_gr_extras packagename extra1,extra2,...
function pip_install_gr_extras {
local name=$1
local extras=$2
local version_constraints
version_constraints=$(get_version_constraints_from_global_requirements $name)
pip_install $name[$extras]$version_constraints
}
# enable_python3_package() -- no-op for backwards compatibility
#
# enable_python3_package dir [dir ...]
function enable_python3_package {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
echo "It is no longer necessary to call enable_python3_package()."
$xtrace
}
# disable_python3_package() -- no-op for backwards compatibility
#
# disable_python3_package dir [dir ...]
function disable_python3_package {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
echo "It is no longer possible to call disable_python3_package()."
$xtrace
}
# Wrapper for ``pip install`` to set cache and proxy environment variables
# Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``,
# ``PIP_UPGRADE``, ``*_proxy``,
# Usage:
# pip_install pip_arguments
function pip_install {
local xtrace result
xtrace=$(set +o | grep xtrace)
set +o xtrace
local upgrade=""
local offline=${OFFLINE:-False}
if [[ "$offline" == "True" || -z "$@" ]]; then
$xtrace
return
fi
time_start "pip_install"
PIP_UPGRADE=$(trueorfalse False PIP_UPGRADE)
if [[ "$PIP_UPGRADE" = "True" ]] ; then
upgrade="--upgrade"
fi
if [[ -z "$os_PACKAGE" ]]; then
GetOSVersion
fi
# Try to extract the path of the package we are installing into
# package_dir. We need this to check for test-requirements.txt,
# at least.
#
# ${!#} expands to the last positional argument to this function.
# With "extras" syntax included, our arguments might be something
# like:
# -e /path/to/fooproject[extra]
# Thus this magic line grabs just the path without extras
#
# Note that this makes no sense if this is a pypi (rather than
# local path) install; ergo you must check this path exists before
# use. Also, if we had multiple or mixed installs, we would also
# likely break. But for historical reasons, it's basically only
# the other wrapper functions in here calling this to install
# local packages, and they do so with single call per install. So
# this works (for now...)
local package_dir=${!#%\[*\]}
if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
local sudo_pip="env"
elif [[ "${GLOBAL_VENV}" == "True" && -d ${DEVSTACK_VENV} ]] ; then
# We have to check that the DEVSTACK_VENV exists because early
# devstack boostrapping needs to operate in a system context
# too bootstrap pip. Once pip is bootstrapped we create the
# global venv and can start to use it.
local cmd_pip=$DEVSTACK_VENV/bin/pip
local sudo_pip="env"
echo "Using python $PYTHON3_VERSION to install $package_dir"
else
local cmd_pip="python$PYTHON3_VERSION -m pip"
# See
# https://github.com/pypa/setuptools/issues/2232
# http://lists.openstack.org/pipermail/openstack-discuss/2020-August/016905.html
# this makes setuptools >=50 use the platform distutils.
# We only want to do this on global pip installs, not if
# installing in a virtualenv
local sudo_pip="sudo -H LC_ALL=en_US.UTF-8 SETUPTOOLS_USE_DISTUTILS=stdlib "
echo "Using python $PYTHON3_VERSION to install $package_dir"
fi
cmd_pip="$cmd_pip install"
# Always apply constraints
cmd_pip="$cmd_pip -c $REQUIREMENTS_DIR/upper-constraints.txt"
$xtrace
$sudo_pip \
http_proxy="${http_proxy:-}" \
https_proxy="${https_proxy:-}" \
no_proxy="${no_proxy:-}" \
PIP_FIND_LINKS=$PIP_FIND_LINKS \
$cmd_pip $upgrade \
$@
result=$?
time_stop "pip_install"
return $result
}
function pip_uninstall {
# Skip uninstall if offline
[[ "${OFFLINE}" = "True" ]] && return
local name=$1
if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
local sudo_pip="env"
else
local cmd_pip="python$PYTHON3_VERSION -m pip"
local sudo_pip="sudo -H LC_ALL=en_US.UTF-8"
fi
# don't error if we can't uninstall, it might not be there
$sudo_pip $cmd_pip uninstall -y $name || /bin/true
}
# get version of a package from global requirements file
# get_from_global_requirements <package>
function get_from_global_requirements {
local package=$1
local required_pkg
required_pkg=$(grep -i -h ^${package} $REQUIREMENTS_DIR/global-requirements.txt | cut -d\# -f1)
if [[ $required_pkg == "" ]]; then
die $LINENO "Can't find package $package in requirements"
fi
echo $required_pkg
}
# get only version constraints of a package from global requirements file
# get_version_constraints_from_global_requirements <package>
function get_version_constraints_from_global_requirements {
local package=$1
local required_pkg_version_constraint
# drop the package name from output (\K)
required_pkg_version_constraint=$(grep -i -h -o -P "^${package}\K.*" $REQUIREMENTS_DIR/global-requirements.txt | cut -d\# -f1)
if [[ $required_pkg_version_constraint == "" ]]; then
die $LINENO "Can't find package $package in requirements"
fi
echo $required_pkg_version_constraint
}
# should we use this library from their git repo, or should we let it
# get pulled in via pip dependencies.
function use_library_from_git {
local name=$1
local enabled=1
[[ ${LIBS_FROM_GIT} = 'ALL' ]] || [[ ,${LIBS_FROM_GIT}, =~ ,${name}, ]] && enabled=0
return $enabled
}
# determine if a package was installed from git
function lib_installed_from_git {
local name=$1
local safe_name
safe_name=$(python -c "from pkg_resources import safe_name; \
print(safe_name('${name}'))")
# Note "pip freeze" doesn't always work here, because it tries to
# be smart about finding the remote of the git repo the package
# was installed from. This doesn't work with zuul which clones
# repos with no remote.
#
# The best option seems to be to use "pip list" which will tell
# you the path an editable install was installed from; for example
# in response to something like
# pip install -e 'git+https://opendev.org/openstack/bashate#egg=bashate'
# pip list --format columns shows
# bashate 0.5.2.dev19 /tmp/env/src/bashate
# Thus we check the third column to see if we're installed from
# some local place.
[[ -n $(pip list --format=columns 2>/dev/null | awk "/^$safe_name/ {print \$3}") ]]
}
# setup a library by name. If we are trying to use the library from
# git, we'll do a git based install, otherwise we'll punt and the
# library should be installed by a requirements pull from another
# project.
function setup_lib {
local name=$1
local dir=${GITDIR[$name]}
setup_install $dir
}
# setup a library by name in editable mode. If we are trying to use
# the library from git, we'll do a git based install, otherwise we'll
# punt and the library should be installed by a requirements pull from
# another project.
#
# use this for non namespaced libraries
#
# setup_dev_lib [-bindep] <name> [<extras>]
function setup_dev_lib {
local bindep
if [[ $1 == -bindep* ]]; then
bindep="${1}"
shift
fi
local name=$1
local dir=${GITDIR[$name]}
local extras=$2
setup_develop $bindep $dir $extras
}
# this should be used if you want to install globally, all libraries should
# use this, especially *oslo* ones
#
# setup_install project_dir [extras]
# project_dir: directory of project repo (e.g., /opt/stack/keystone)
# extras: comma-separated list of optional dependencies to install
# (e.g., ldap,memcache).
# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
# bindep: Set "-bindep" as first argument to install bindep.txt packages
# The command is like "pip install <project_dir>[<extras>]"
function setup_install {
local bindep
if [[ $1 == -bindep* ]]; then
bindep="${1}"
shift
fi
local project_dir=$1
local extras=$2
_setup_package_with_constraints_edit $bindep $project_dir "" $extras
}
# this should be used for projects which run services, like all services
#
# setup_develop project_dir [extras]
# project_dir: directory of project repo (e.g., /opt/stack/keystone)
# extras: comma-separated list of optional dependencies to install
# (e.g., ldap,memcache).
# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
# The command is like "pip install -e <project_dir>[<extras>]"
function setup_develop {
local bindep
if [[ $1 == -bindep* ]]; then
bindep="${1}"
shift
fi
local project_dir=$1
local extras=$2
_setup_package_with_constraints_edit $bindep $project_dir -e $extras
}
# ``pip install -e`` the package, which processes the dependencies
# using pip before running `setup.py develop`
#
# Updates the constraints from REQUIREMENTS_DIR to reflect the
# future installed state of this package. This ensures when we
# install this package we get the from source version.
#
# Uses globals ``REQUIREMENTS_DIR``
# _setup_package_with_constraints_edit project_dir flags [extras]
# project_dir: directory of project repo (e.g., /opt/stack/keystone)
# flags: pip CLI options/flags
# extras: comma-separated list of optional dependencies to install
# (e.g., ldap,memcache).
# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
# The command is like "pip install <flags> <project_dir>[<extras>]"
function _setup_package_with_constraints_edit {
local bindep
if [[ $1 == -bindep* ]]; then
bindep="${1}"
shift
fi
local project_dir=$1
local flags=$2
local extras=$3
# Normalize the directory name to avoid
# "installation from path or url cannot be constrained to a version"
# error.
# REVISIT(yamamoto): Remove this when fixed in pip.
# https://github.com/pypa/pip/pull/3582
project_dir=$(cd $project_dir && pwd)
if [ -n "$REQUIREMENTS_DIR" ]; then
# Remove this package from constraints before we install it.
# That way, later installs won't "downgrade" the install from
# source we are about to do.
local name
name=$(awk '/^name.*=/ {print $3}' $project_dir/setup.cfg)
if [ -z $name ]; then
name=$(awk '/^name =/ {gsub(/"/, "", $3); print $3}' $project_dir/pyproject.toml)
fi
$REQUIREMENTS_DIR/.venv/bin/edit-constraints \
$REQUIREMENTS_DIR/upper-constraints.txt -- $name
fi
setup_package $bindep $project_dir "$flags" $extras
# If this project is in LIBS_FROM_GIT, verify it was actually installed
# correctly. This helps catch errors caused by constraints mismatches.
if use_library_from_git "$project_dir"; then
if ! lib_installed_from_git "$project_dir"; then
die $LINENO "The following LIBS_FROM_GIT was not installed correctly: $project_dir"
fi
fi
}
# ``pip install -e`` the package, which processes the dependencies
# using pip before running `setup.py develop`. The command is like
# "pip install <flags> <project_dir>[<extras>]"
#
# Uses globals ``STACK_USER``
#
# Usage:
# setup_package [-bindep[=profile,profile]] <project_dir> <flags> [extras]
#
# -bindep : Use bindep to install dependencies; select extra profiles
# as comma separated arguments after "="
# project_dir : directory of project repo (e.g., /opt/stack/keystone)
# flags : pip CLI options/flags
# extras : comma-separated list of optional dependencies to install
# (e.g., ldap,memcache).
# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
function setup_package {
local bindep=0
local bindep_flag=""
local bindep_profiles=""
if [[ $1 == -bindep* ]]; then
bindep=1
IFS="=" read bindep_flag bindep_profiles <<< ${1}
shift
fi
local project_dir=$1
local flags=$2
local extras=$3
# if the flags variable exists, and it doesn't look like a flag,
# assume it's actually the extras list.
if [[ -n "$flags" && -z "$extras" && ! "$flags" =~ ^-.* ]]; then
extras=$flags
flags=""
fi
if [[ ! -z "$extras" ]]; then
extras="[$extras]"
fi
# install any bindep packages
if [[ $bindep == 1 ]]; then
install_bindep $project_dir/bindep.txt $bindep_profiles
fi
pip_install $flags "$project_dir$extras"
# ensure that further actions can do things like setup.py sdist
if [[ "$flags" == "-e" && "$GLOBAL_VENV" == "False" ]]; then
safe_chown -R $STACK_USER $1/*.egg-info
fi
}
# Report whether python 3 should be used
# TODO(frickler): drop this once all legacy uses are removed
function python3_enabled {
return 0
}
# Provide requested python version and sets PYTHON variable
function install_python {
install_python3
export PYTHON=$(which python${PYTHON3_VERSION} 2>/dev/null)
}
# Install python3 packages
function install_python3 {
if is_ubuntu; then
apt_get install python${PYTHON3_VERSION} python${PYTHON3_VERSION}-dev
elif is_fedora; then
if [ "$os_VENDOR" = "Fedora" ]; then
install_package python${PYTHON3_VERSION//.}
else
install_package python${PYTHON3_VERSION//.} python${PYTHON3_VERSION//.}-devel
fi
fi
}
function install_devstack_tools {
# intentionally old to ensure devstack-gate has control
local dstools_version=${DSTOOLS_VERSION:-0.1.2}
install_python3
sudo pip3 install -U devstack-tools==${dstools_version}
}
# Restore xtrace
$INC_PY_TRACE
# Local variables:
# mode: shell-script
# End: