Add Constraints support
Adding constraints support to libraries is slightly more complex than
services as the libraries themselves are listed in upper-constraints.txt
which leads to errors that you can't install a specific version and a
constrained version.
This change adds constraints support by also adding a helper script to
edit the constraints to remove sahara-dashboard.
As sahara-dashboard also uses run_tests.sh I've modified that code in
line with the horizon change I9bf85adb5aa2c85d8c0f0f22ac9efe5cac3646c0
Change-Id: Ieb8afdef1e9649b18a4394620e52b9f6ac3d997b
(cherry picked from commit 978718ab96)
This commit is contained in:
Tony Breeds
Dirk Mueller
parent
e8c6017ef0
commit
42db19abde
48
run_tests.sh
48
run_tests.sh
@ -53,7 +53,6 @@ function usage {
|
||||
#
|
||||
root=`pwd -P`
|
||||
venv=$root/.venv
|
||||
venv_env_version=$venv/environments
|
||||
with_venv=tools/with_venv.sh
|
||||
included_dirs="sahara_dashboard"
|
||||
|
||||
@ -85,6 +84,14 @@ check_only=0
|
||||
pseudo=0
|
||||
manage=0
|
||||
|
||||
# NOTE(tonyb): the release team will automatically update tox.ini to point at
|
||||
# the correct requirements branch when creating stable/* from master. So go to
|
||||
# a little effort to get the deault from there to avoid dift and having to
|
||||
# update this when branching
|
||||
_default_uc=$(sed -n 's/^.*{env:UPPER_CONSTRAINTS_FILE\:\([^}]*\)}.*$/\1/p' \
|
||||
tox.ini | head -n1)
|
||||
|
||||
|
||||
# Jenkins sets a "JOB_NAME" variable, if it's not set, we'll make it "default"
|
||||
[ "$JOB_NAME" ] || JOB_NAME="default"
|
||||
|
||||
@ -211,39 +218,6 @@ function destroy_venv {
|
||||
echo "Virtualenv removed."
|
||||
}
|
||||
|
||||
function environment_check {
|
||||
echo "Checking environment."
|
||||
if [ -f $venv_env_version ]; then
|
||||
set +o errexit
|
||||
cat requirements.txt test-requirements.txt | cmp $venv_env_version - > /dev/null
|
||||
local env_check_result=$?
|
||||
set -o errexit
|
||||
if [ $env_check_result -eq 0 ]; then
|
||||
# If the environment exists and is up-to-date then set our variables
|
||||
command_wrapper="${root}/${with_venv}"
|
||||
echo "Environment is up to date."
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $always_venv -eq 1 ]; then
|
||||
install_venv
|
||||
else
|
||||
if [ ! -e ${venv} ]; then
|
||||
echo -e "Environment not found. Install? (Y/n) \c"
|
||||
else
|
||||
echo -e "Your environment appears to be out of date. Update? (Y/n) \c"
|
||||
fi
|
||||
read update_env
|
||||
if [ "x$update_env" = "xY" -o "x$update_env" = "x" -o "x$update_env" = "xy" ]; then
|
||||
install_venv
|
||||
else
|
||||
# Set our command wrapper anyway.
|
||||
command_wrapper="${root}/${with_venv}"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function sanity_check {
|
||||
# Anything that should be determined prior to running the tests, server, etc.
|
||||
# Don't sanity-check anything environment-related in -N flag is set
|
||||
@ -293,6 +267,7 @@ function restore_environment {
|
||||
|
||||
function install_venv {
|
||||
# Install with install_venv.py
|
||||
export UPPER_CONSTRAINTS_FILE=${UPPER_CONSTRAINTS_FILE:-$_default_uc}
|
||||
export PIP_DOWNLOAD_CACHE=${PIP_DOWNLOAD_CACHE-/tmp/.pip_download_cache}
|
||||
export PIP_USE_MIRRORS=true
|
||||
if [ $quiet -eq 1 ]; then
|
||||
@ -305,7 +280,6 @@ function install_venv {
|
||||
# Make sure it worked and record the environment version
|
||||
sanity_check
|
||||
chmod -R 754 $venv
|
||||
cat requirements.txt test-requirements.txt > $venv_env_version
|
||||
}
|
||||
|
||||
function run_tests {
|
||||
@ -477,8 +451,8 @@ if [ $never_venv -eq 0 ]; then
|
||||
destroy_venv
|
||||
fi
|
||||
|
||||
# Then check if it's up-to-date
|
||||
environment_check
|
||||
# Create or update venv.
|
||||
install_venv
|
||||
|
||||
# Create a backup of the up-to-date environment if desired
|
||||
if [ $backup_env -eq 1 ]; then
|
||||
|
||||
@ -32,7 +32,7 @@ VENV = os.path.join(ROOT, '.venv')
|
||||
WITH_VENV = os.path.join(ROOT, 'tools', 'with_venv.sh')
|
||||
PIP_REQUIRES = os.path.join(ROOT, 'requirements.txt')
|
||||
TEST_REQUIRES = os.path.join(ROOT, 'test-requirements.txt')
|
||||
PIP_INSTALL_WRAPPER = os.path.join(ROOT, 'tools', 'pip_install.sh')
|
||||
PIP_INSTALL_WRAPPER = os.path.join(ROOT, 'tools', 'tox_install.sh')
|
||||
|
||||
|
||||
def die(message, *args):
|
||||
@ -110,12 +110,15 @@ def create_virtualenv(venv=VENV):
|
||||
|
||||
|
||||
def pip_install(*args):
|
||||
args = [WITH_VENV, 'pip', 'install', '--upgrade'] + list(args)
|
||||
constraints_url = os.environ.get('UPPER_CONSTRAINTS_FILE')
|
||||
args = [WITH_VENV, 'pip', 'install', '-c', constraints_url,
|
||||
'--upgrade'] + list(args)
|
||||
run_command(args, redirect_output=False)
|
||||
|
||||
|
||||
def pip_install_with_horizon(*args):
|
||||
args = [WITH_VENV, PIP_INSTALL_WRAPPER, 'unconstrained'] + list(args)
|
||||
constraints_url = os.environ.get('UPPER_CONSTRAINTS_FILE')
|
||||
args = [WITH_VENV, PIP_INSTALL_WRAPPER, constraints_url] + list(args)
|
||||
run_command(args, redirect_output=False)
|
||||
|
||||
|
||||
|
||||
@ -1,66 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
# The original script is borrowed from neutron-* repos.
|
||||
|
||||
# Many of horizon's repos suffer from the problem of depending on horizon,
|
||||
# but it not existing on pypi.
|
||||
|
||||
# This wrapper for tox's package installer will use the existing package
|
||||
# if it exists, else use zuul-cloner if that program exists, else grab it
|
||||
# from horizon master via a hard-coded URL. That last case should only
|
||||
# happen with devs running unit tests locally.
|
||||
|
||||
# From the tox.ini config page:
|
||||
# install_command=ARGV
|
||||
# default:
|
||||
# pip install {opts} {packages}
|
||||
|
||||
ZUUL_CLONER=/usr/zuul-env/bin/zuul-cloner
|
||||
BRANCH_NAME=master
|
||||
horizon_installed=$(echo "import horizon" | python 2>/dev/null ; echo $?)
|
||||
|
||||
set -e
|
||||
|
||||
zuul_install () {
|
||||
if [ -x "$ZUUL_CLONER" ]; then
|
||||
export ZUUL_BRANCH=${ZUUL_BRANCH-$BRANCH}
|
||||
echo "ZUUL CLONER" > /tmp/tox_install.txt
|
||||
cwd=$(/bin/pwd)
|
||||
cd /tmp
|
||||
$ZUUL_CLONER --cache-dir \
|
||||
/opt/git \
|
||||
--branch $BRANCH_NAME \
|
||||
git://git.openstack.org \
|
||||
openstack/horizon
|
||||
cd openstack/horizon
|
||||
$install_cmd -e .
|
||||
cd "$cwd"
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
usual_install () {
|
||||
echo "PIP HARDCODE" > /tmp/tox_install.txt
|
||||
if [ -z "$HORIZON_PIP_LOCATION" ]; then
|
||||
HORIZON_PIP_LOCATION="git+https://git.openstack.org/openstack/horizon@$BRANCH_NAME#egg=horizon"
|
||||
fi
|
||||
$install_cmd -U -e ${HORIZON_PIP_LOCATION}
|
||||
}
|
||||
|
||||
install_cmd="pip install"
|
||||
if [ "$1" = "constrained" ]; then
|
||||
install_cmd="$install_cmd $2"
|
||||
shift
|
||||
fi
|
||||
shift
|
||||
|
||||
if [ $horizon_installed -eq 0 ]; then
|
||||
echo "ALREADY INSTALLED" > /tmp/tox_install.txt
|
||||
echo "Horizon already installed; using existing package"
|
||||
else
|
||||
zuul_install || usual_install
|
||||
fi
|
||||
|
||||
$install_cmd -U $*
|
||||
exit $?
|
||||
85
tools/tox_install.sh
Executable file
85
tools/tox_install.sh
Executable file
@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Client constraint file contains this client version pin that is in conflict
|
||||
# with installing the client from source. We should remove the version pin in
|
||||
# the constraints file before applying it for from-source installation.
|
||||
# The script also has a secondary purpose to install certain special
|
||||
# dependencies directly from git.
|
||||
|
||||
# Wrapper for pip install that always uses constraints.
|
||||
function pip_install() {
|
||||
pip install -c"$localfile" -U "$@"
|
||||
}
|
||||
|
||||
# Grab the library from git using either zuul-cloner or pip. The former is
|
||||
# there to a take advantage of the setup done by the gate infrastructure
|
||||
# and honour any/all Depends-On headers in the commit message
|
||||
function install_from_git() {
|
||||
ZUUL_CLONER=/usr/zuul-env/bin/zuul-cloner
|
||||
GIT_HOST=git.openstack.org
|
||||
PROJ=$1
|
||||
EGG=$2
|
||||
|
||||
edit-constraints "$localfile" -- "$EGG"
|
||||
if [ -x "$ZUUL_CLONER" ]; then
|
||||
SRC_DIR="$VIRTUAL_ENV/src"
|
||||
mkdir -p "$SRC_DIR"
|
||||
cd "$SRC_DIR" >/dev/null
|
||||
ZUUL_CACHE_DIR=${ZUUL_CACHE_DIR:-/opt/git} $ZUUL_CLONER \
|
||||
--branch "$BRANCH_NAME" \
|
||||
"git://$GIT_HOST" "$PROJ"
|
||||
pip_install -e "$PROJ/."
|
||||
cd - >/dev/null
|
||||
else
|
||||
pip_install -e"git+https://$GIT_HOST/$PROJ@$BRANCH_NAME#egg=${EGG}"
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
|
||||
CONSTRAINTS_FILE="$1"
|
||||
shift 1
|
||||
|
||||
# This script will either complete with a return code of 0 or the return code
|
||||
# of whatever failed.
|
||||
set -e
|
||||
|
||||
# NOTE(tonyb): Place this in the tox environment's log dir so it will get
|
||||
# published to logs.openstack.org for easy debugging.
|
||||
mkdir -p "$VIRTUAL_ENV/log/"
|
||||
localfile="$VIRTUAL_ENV/log/upper-constraints.txt"
|
||||
|
||||
if [[ "$CONSTRAINTS_FILE" != http* ]]; then
|
||||
CONSTRAINTS_FILE="file://$CONSTRAINTS_FILE"
|
||||
fi
|
||||
# NOTE(tonyb): need to add curl to bindep.txt if the project supports bindep
|
||||
curl "$CONSTRAINTS_FILE" --insecure --progress-bar --output "$localfile"
|
||||
|
||||
pip_install openstack-requirements
|
||||
|
||||
# This is the main purpose of the script: Allow local installation of
|
||||
# the current repo. It is listed in constraints file and thus any
|
||||
# install will be constrained and we need to unconstrain it.
|
||||
edit-constraints "$localfile" -- "$CLIENT_NAME"
|
||||
|
||||
declare -a passthrough_args
|
||||
while [ $# -gt 0 ] ; do
|
||||
case "$1" in
|
||||
# If we have any special os:<repo_name:<egg_name> deps then process them
|
||||
os:*)
|
||||
declare -a pkg_spec
|
||||
IFS=: pkg_spec=($1)
|
||||
install_from_git "${pkg_spec[1]}" "${pkg_spec[2]}"
|
||||
;;
|
||||
# Otherwise just pass the other deps through to the constrained pip install
|
||||
*)
|
||||
passthrough_args+=("$1")
|
||||
;;
|
||||
esac
|
||||
shift 1
|
||||
done
|
||||
|
||||
# If *only* had special args then then isn't any need to run pip.
|
||||
if [ -n "$passthrough_args" ] ; then
|
||||
pip_install "${passthrough_args[@]}"
|
||||
fi
|
||||
@ -4,4 +4,7 @@ VENV_PATH=${VENV_PATH:-${TOOLS_PATH}}
|
||||
VENV_DIR=${VENV_NAME:-/../.venv}
|
||||
TOOLS=${TOOLS_PATH}
|
||||
VENV=${VENV:-${VENV_PATH}/${VENV_DIR}}
|
||||
# tools/tox_install.sh looks for VIRTUAL_ENV evrythign else looks for VENV
|
||||
# so for now make them the same.
|
||||
VIRTUAL_ENV=${VENV:-${VENV_PATH}/${VENV_DIR}}
|
||||
source ${VENV}/bin/activate && "$@"
|
||||
|
||||
5
tox.ini
5
tox.ini
@ -5,11 +5,14 @@ skipsdist = True
|
||||
|
||||
[testenv]
|
||||
usedevelop = True
|
||||
install_command = {toxinidir}/tools/pip_install.sh unconstrained {opts} {packages}
|
||||
install_command = {toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=stable/newton} {opts} {packages}
|
||||
setenv =
|
||||
VIRTUAL_ENV={envdir}
|
||||
BRANCH_NAME=stable/newton
|
||||
CLIENT_NAME=sahara-dashboard
|
||||
deps = -r{toxinidir}/requirements.txt
|
||||
-r{toxinidir}/test-requirements.txt
|
||||
os:openstack/horizon:horizon
|
||||
whitelist_externals = /bin/bash
|
||||
commands = /bin/bash run_tests.sh -N --no-pep8 {posargs}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user