Add Constraints support

Adding constraints support to libraries is slightly more complex than
services as the libraries themselves are listed in upper-constraints.txt
which leads to errors that you can't install a specific version and a
constrained version.

This change adds constraints support by also adding a helper script to
edit the constraints to remove cliff and at the same time pull any required
libraries from git.

Also the egg name fro python-neutronclient is wrong resulting in
warnings in the logs[1] so correct that wheil re-writing the deps in
tox.ini

[1] http://logs.openstack.org/42/412742/1/check/gate-cliff-tox-neutronclient-tip-ubuntu-xenial/e1ce5c0/console.html#_2016-12-20_04_40_50_638986

Change-Id: Ifa5cc8446cfbfec0c8b48b293feb76ce66a34d92

integration-tests/neutronclient-tip.sh

@@ -7,7 +7,10 @@ envdir=$1
 # The source for the client library is checked out by pip because of
 # the deps listed in tox.ini, so we just need to move into that
 # directory.
-cd $envdir/src/neutronclient/
+# NOTE(tonyb): tools/tox_install.sh will place the code in 1 of 2 paths
+# depending on whether zuul-cloner is used, so try each possible location
+cd $envdir/src/python-neutronclient || \
+    cd $envdir/src/openstack/python-neutronclient
 
 pip install -r test-requirements.txt
 

integration-tests/openstackclient-tip.sh

@@ -7,7 +7,10 @@ envdir=$1
 # The source for the client library is checked out by pip because of
 # the deps listed in tox.ini, so we just need to move into that
 # directory.
-cd $envdir/src/openstackclient/
+# NOTE(tonyb): tools/tox_install.sh will place the code in 1 of 2 paths
+# depending on whether zuul-cloner is used, so try each possible location
+cd $envdir/src/python-openstackclient/ || \
+    cd $envdir/src/openstack/python-openstackclient/
 
 pip install -r test-requirements.txt
 

tools/tox_install.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+# Client constraint file contains this client version pin that is in conflict
+# with installing the client from source. We should remove the version pin in
+# the constraints file before applying it for from-source installation.
+# The script also has a secondary purpose to install certain special
+# dependencies directly from git.
+
+# Wrapper for pip install that always uses constraints.
+function pip_install() {
+    pip install -c"$localfile" -U "$@"
+}
+
+# Grab the library from git using either zuul-cloner or pip.  The former is
+# there to a take advantage of the setup done by the gate infrastructure
+# and honour any/all Depends-On headers in the commit message
+function install_from_git() {
+    ZUUL_CLONER=/usr/zuul-env/bin/zuul-cloner
+    GIT_HOST=git.openstack.org
+    PROJ=$1
+    EGG=$2
+
+    edit-constraints "$localfile" -- "$EGG"
+    if [ -x "$ZUUL_CLONER" ]; then
+        SRC_DIR="$VIRTUAL_ENV/src"
+        mkdir -p "$SRC_DIR"
+        cd "$SRC_DIR" >/dev/null
+        ZUUL_CACHE_DIR=${ZUUL_CACHE_DIR:-/opt/git} $ZUUL_CLONER \
+            --branch "$BRANCH_NAME" \
+            "git://$GIT_HOST" "$PROJ"
+        pip_install -e "$PROJ/."
+        cd - >/dev/null
+    else
+        pip_install -e"git+https://$GIT_HOST/$PROJ@$BRANCH_NAME#egg=${EGG}"
+    fi
+}
+
+
+
+CONSTRAINTS_FILE="$1"
+shift 1
+
+# This script will either complete with a return code of 0 or the return code
+# of whatever failed.
+set -e
+
+# NOTE(tonyb): Place this in the tox environment's log dir so it will get
+# published to logs.openstack.org for easy debugging.
+mkdir -p "$VIRTUAL_ENV/log/"
+localfile="$VIRTUAL_ENV/log/upper-constraints.txt"
+
+if [[ "$CONSTRAINTS_FILE" != http* ]]; then
+    CONSTRAINTS_FILE="file://$CONSTRAINTS_FILE"
+fi
+# NOTE(tonyb): need to add curl to bindep.txt if the project supports bindep
+curl "$CONSTRAINTS_FILE" --insecure --progress-bar --output "$localfile"
+
+pip_install openstack-requirements
+
+# This is the main purpose of the script: Allow local installation of
+# the current repo. It is listed in constraints file and thus any
+# install will be constrained and we need to unconstrain it.
+edit-constraints "$localfile" -- "$CLIENT_NAME"
+
+declare -a passthrough_args
+while [ $# -gt 0 ] ; do
+    case "$1" in
+    # If we have any special os:<repo_name:<egg_name> deps then process them
+    os:*)
+        declare -a pkg_spec
+        IFS=: pkg_spec=($1)
+        install_from_git "${pkg_spec[1]}" "${pkg_spec[2]}"
+    ;;
+    # Otherwise just pass the other deps through to the constrained pip install
+    *)
+        passthrough_args+=("$1")
+    ;;
+    esac
+    shift 1
+done
+
+# If *only* had special args then then isn't any need to run pip.
+if [ -n "$passthrough_args" ] ; then
+    pip_install "${passthrough_args[@]}"
+fi

tox.ini

@@ -1,9 +1,16 @@
 [tox]
+minversion = 2.0
 envlist = py35,py34,py27,pypy,pep8
 
 [testenv]
+setenv =
+    VIRTUAL_ENV={envdir}
+    BRANCH_NAME=master
+    CLIENT_NAME=cliff
+passenv =
+    ZUUL_CACHE_DIR
 distribute = False
-install_command = pip install -U {opts} {packages}
+install_command = {toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
 commands = nosetests -d --with-coverage --cover-inclusive --cover-package cliff []
 deps = -r{toxinidir}/test-requirements.txt
 
@@ -16,12 +23,12 @@ commands = {posargs}
 
 [testenv:neutronclient-tip]
 basepython = python2.7
-deps = -egit+https://git.openstack.org/openstack/python-neutronclient#egg=neutronclient
+deps = os:openstack/python-neutronclient:python-neutronclient
 commands = {toxinidir}/integration-tests/neutronclient-tip.sh {envdir}
 
 [testenv:openstackclient-tip]
 basepython = python2.7
-deps = -egit+https://git.openstack.org/openstack/python-openstackclient#egg=openstackclient
+deps = os:openstack/python-openstackclient:python-openstackclient
 commands = {toxinidir}/integration-tests/openstackclient-tip.sh {envdir}
 
 [testenv:docs]