Add Constraints support

Adding constraints support to libraries is slightly more complex than
services as the libraries themselves are listed in upper-constraints.txt
which leads to errors that you can't install a specific version and a
constrained version.

This change adds constraints support by also adding a helper script to
edit the constraints to remove sahara-dashboard.

As sahara-dashboard also uses run_tests.sh I've modified that code in
line with the horizon change I9bf85adb5aa2c85d8c0f0f22ac9efe5cac3646c0

Change-Id: Ieb8afdef1e9649b18a4394620e52b9f6ac3d997b
This commit is contained in:
Tony Breeds 2016-12-21 13:36:40 +11:00
parent 694a85c71c
commit 978718ab96
6 changed files with 109 additions and 107 deletions

View File

@ -53,7 +53,6 @@ function usage {
#
root=`pwd -P`
venv=$root/.venv
venv_env_version=$venv/environments
with_venv=tools/with_venv.sh
included_dirs="sahara_dashboard"
@ -85,6 +84,14 @@ check_only=0
pseudo=0
manage=0
# NOTE(tonyb): the release team will automatically update tox.ini to point at
# the correct requirements branch when creating stable/* from master. So go to
# a little effort to get the deault from there to avoid dift and having to
# update this when branching
_default_uc=$(sed -n 's/^.*{env:UPPER_CONSTRAINTS_FILE\:\([^}]*\)}.*$/\1/p' \
tox.ini | head -n1)
# Jenkins sets a "JOB_NAME" variable, if it's not set, we'll make it "default"
[ "$JOB_NAME" ] || JOB_NAME="default"
@ -211,39 +218,6 @@ function destroy_venv {
echo "Virtualenv removed."
}
function environment_check {
echo "Checking environment."
if [ -f $venv_env_version ]; then
set +o errexit
cat requirements.txt test-requirements.txt | cmp $venv_env_version - > /dev/null
local env_check_result=$?
set -o errexit
if [ $env_check_result -eq 0 ]; then
# If the environment exists and is up-to-date then set our variables
command_wrapper="${root}/${with_venv}"
echo "Environment is up to date."
return 0
fi
fi
if [ $always_venv -eq 1 ]; then
install_venv
else
if [ ! -e ${venv} ]; then
echo -e "Environment not found. Install? (Y/n) \c"
else
echo -e "Your environment appears to be out of date. Update? (Y/n) \c"
fi
read update_env
if [ "x$update_env" = "xY" -o "x$update_env" = "x" -o "x$update_env" = "xy" ]; then
install_venv
else
# Set our command wrapper anyway.
command_wrapper="${root}/${with_venv}"
fi
fi
}
function sanity_check {
# Anything that should be determined prior to running the tests, server, etc.
# Don't sanity-check anything environment-related in -N flag is set
@ -293,6 +267,7 @@ function restore_environment {
function install_venv {
# Install with install_venv.py
export UPPER_CONSTRAINTS_FILE=${UPPER_CONSTRAINTS_FILE:-$_default_uc}
export PIP_DOWNLOAD_CACHE=${PIP_DOWNLOAD_CACHE-/tmp/.pip_download_cache}
export PIP_USE_MIRRORS=true
if [ $quiet -eq 1 ]; then
@ -305,7 +280,6 @@ function install_venv {
# Make sure it worked and record the environment version
sanity_check
chmod -R 754 $venv
cat requirements.txt test-requirements.txt > $venv_env_version
}
function run_tests {
@ -473,8 +447,8 @@ if [ $never_venv -eq 0 ]; then
destroy_venv
fi
# Then check if it's up-to-date
environment_check
# Create or update venv.
install_venv
# Create a backup of the up-to-date environment if desired
if [ $backup_env -eq 1 ]; then

View File

@ -32,7 +32,7 @@ VENV = os.path.join(ROOT, '.venv')
WITH_VENV = os.path.join(ROOT, 'tools', 'with_venv.sh')
PIP_REQUIRES = os.path.join(ROOT, 'requirements.txt')
TEST_REQUIRES = os.path.join(ROOT, 'test-requirements.txt')
PIP_INSTALL_WRAPPER = os.path.join(ROOT, 'tools', 'pip_install.sh')
PIP_INSTALL_WRAPPER = os.path.join(ROOT, 'tools', 'tox_install.sh')
def die(message, *args):
@ -110,12 +110,15 @@ def create_virtualenv(venv=VENV):
def pip_install(*args):
args = [WITH_VENV, 'pip', 'install', '--upgrade'] + list(args)
constraints_url = os.environ.get('UPPER_CONSTRAINTS_FILE')
args = [WITH_VENV, 'pip', 'install', '-c', constraints_url,
'--upgrade'] + list(args)
run_command(args, redirect_output=False)
def pip_install_with_horizon(*args):
args = [WITH_VENV, PIP_INSTALL_WRAPPER, 'unconstrained'] + list(args)
constraints_url = os.environ.get('UPPER_CONSTRAINTS_FILE')
args = [WITH_VENV, PIP_INSTALL_WRAPPER, constraints_url] + list(args)
run_command(args, redirect_output=False)

View File

@ -1,66 +0,0 @@
#!/bin/sh
# The original script is borrowed from neutron-* repos.
# Many of horizon's repos suffer from the problem of depending on horizon,
# but it not existing on pypi.
# This wrapper for tox's package installer will use the existing package
# if it exists, else use zuul-cloner if that program exists, else grab it
# from horizon master via a hard-coded URL. That last case should only
# happen with devs running unit tests locally.
# From the tox.ini config page:
# install_command=ARGV
# default:
# pip install {opts} {packages}
ZUUL_CLONER=/usr/zuul-env/bin/zuul-cloner
BRANCH_NAME=master
horizon_installed=$(echo "import horizon" | python 2>/dev/null ; echo $?)
set -e
zuul_install () {
if [ -x "$ZUUL_CLONER" ]; then
export ZUUL_BRANCH=${ZUUL_BRANCH-$BRANCH}
echo "ZUUL CLONER" > /tmp/tox_install.txt
cwd=$(/bin/pwd)
cd /tmp
$ZUUL_CLONER --cache-dir \
/opt/git \
--branch $BRANCH_NAME \
git://git.openstack.org \
openstack/horizon
cd openstack/horizon
$install_cmd -e .
cd "$cwd"
else
return 1
fi
}
usual_install () {
echo "PIP HARDCODE" > /tmp/tox_install.txt
if [ -z "$HORIZON_PIP_LOCATION" ]; then
HORIZON_PIP_LOCATION="git+https://git.openstack.org/openstack/horizon@$BRANCH_NAME#egg=horizon"
fi
$install_cmd -U -e ${HORIZON_PIP_LOCATION}
}
install_cmd="pip install"
if [ "$1" = "constrained" ]; then
install_cmd="$install_cmd $2"
shift
fi
shift
if [ $horizon_installed -eq 0 ]; then
echo "ALREADY INSTALLED" > /tmp/tox_install.txt
echo "Horizon already installed; using existing package"
else
zuul_install || usual_install
fi
$install_cmd -U $*
exit $?

85
tools/tox_install.sh Executable file
View File

@ -0,0 +1,85 @@
#!/usr/bin/env bash
# Client constraint file contains this client version pin that is in conflict
# with installing the client from source. We should remove the version pin in
# the constraints file before applying it for from-source installation.
# The script also has a secondary purpose to install certain special
# dependencies directly from git.
# Wrapper for pip install that always uses constraints.
function pip_install() {
pip install -c"$localfile" -U "$@"
}
# Grab the library from git using either zuul-cloner or pip. The former is
# there to a take advantage of the setup done by the gate infrastructure
# and honour any/all Depends-On headers in the commit message
function install_from_git() {
ZUUL_CLONER=/usr/zuul-env/bin/zuul-cloner
GIT_HOST=git.openstack.org
PROJ=$1
EGG=$2
edit-constraints "$localfile" -- "$EGG"
if [ -x "$ZUUL_CLONER" ]; then
SRC_DIR="$VIRTUAL_ENV/src"
mkdir -p "$SRC_DIR"
cd "$SRC_DIR" >/dev/null
ZUUL_CACHE_DIR=${ZUUL_CACHE_DIR:-/opt/git} $ZUUL_CLONER \
--branch "$BRANCH_NAME" \
"git://$GIT_HOST" "$PROJ"
pip_install -e "$PROJ/."
cd - >/dev/null
else
pip_install -e"git+https://$GIT_HOST/$PROJ@$BRANCH_NAME#egg=${EGG}"
fi
}
CONSTRAINTS_FILE="$1"
shift 1
# This script will either complete with a return code of 0 or the return code
# of whatever failed.
set -e
# NOTE(tonyb): Place this in the tox environment's log dir so it will get
# published to logs.openstack.org for easy debugging.
mkdir -p "$VIRTUAL_ENV/log/"
localfile="$VIRTUAL_ENV/log/upper-constraints.txt"
if [[ "$CONSTRAINTS_FILE" != http* ]]; then
CONSTRAINTS_FILE="file://$CONSTRAINTS_FILE"
fi
# NOTE(tonyb): need to add curl to bindep.txt if the project supports bindep
curl "$CONSTRAINTS_FILE" --insecure --progress-bar --output "$localfile"
pip_install openstack-requirements
# This is the main purpose of the script: Allow local installation of
# the current repo. It is listed in constraints file and thus any
# install will be constrained and we need to unconstrain it.
edit-constraints "$localfile" -- "$CLIENT_NAME"
declare -a passthrough_args
while [ $# -gt 0 ] ; do
case "$1" in
# If we have any special os:<repo_name:<egg_name> deps then process them
os:*)
declare -a pkg_spec
IFS=: pkg_spec=($1)
install_from_git "${pkg_spec[1]}" "${pkg_spec[2]}"
;;
# Otherwise just pass the other deps through to the constrained pip install
*)
passthrough_args+=("$1")
;;
esac
shift 1
done
# If *only* had special args then then isn't any need to run pip.
if [ -n "$passthrough_args" ] ; then
pip_install "${passthrough_args[@]}"
fi

View File

@ -4,4 +4,7 @@ VENV_PATH=${VENV_PATH:-${TOOLS_PATH}}
VENV_DIR=${VENV_NAME:-/../.venv}
TOOLS=${TOOLS_PATH}
VENV=${VENV:-${VENV_PATH}/${VENV_DIR}}
# tools/tox_install.sh looks for VIRTUAL_ENV evrythign else looks for VENV
# so for now make them the same.
VIRTUAL_ENV=${VENV:-${VENV_PATH}/${VENV_DIR}}
source ${VENV}/bin/activate && "$@"

View File

@ -5,11 +5,14 @@ skipsdist = True
[testenv]
usedevelop = True
install_command = {toxinidir}/tools/pip_install.sh unconstrained {opts} {packages}
install_command = {toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
setenv =
VIRTUAL_ENV={envdir}
BRANCH_NAME=master
CLIENT_NAME=sahara-dashboard
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
os:openstack/horizon:horizon
whitelist_externals = /bin/bash
commands = /bin/bash run_tests.sh -N --no-pep8 {posargs}