Add constraints support to run_tests.sh

Currently developers need to manually build a constratined venv if they
want to  match the versions that are tested in the gate.  This change
adds logic to run_tests.sh and install_venv*.py to default to the same
constraints file used in the gate.

With constraints enabled the 'churn' in the environment should be
minimised so remove the developer optimisation for detecting when
packaged versions change and always just install/update.  This has the
down side of making the script preamble slightly longer but we always
print_help() so I don't think it'll be terribly inconvenient.

Change-Id: I9bf85adb5aa2c85d8c0f0f22ac9efe5cac3646c0
Closes-Bug: 1644697

run_tests.sh

@@ -61,7 +61,6 @@ function usage {
 #
 root=`pwd -P`
 venv=$root/.venv
-venv_env_version=$venv/environments
 with_venv=tools/with_venv.sh
 included_dirs="openstack_dashboard horizon"
 
@@ -96,6 +95,13 @@ check_only=0
 pseudo=0
 manage=0
 
+# NOTE(tonyb): the release team will automatically update tox.ini to point at
+# the correct requirements branch when creating stable/* from master.  So go to
+# a little effort to get the deault from there to avoid dift and having to
+# update this when branching
+_default_uc=$(sed -n 's/^.*{env:UPPER_CONSTRAINTS_FILE\:\([^}]*\)}.*$/\1/p' \
+              tox.ini | head -n1)
+
 # Jenkins sets a "JOB_NAME" variable, if it's not set, we'll make it "default"
 [ "$JOB_NAME" ] || JOB_NAME="default"
 
@@ -241,39 +247,6 @@ function destroy_venv {
   echo "Virtualenv removed."
 }
 
-function environment_check {
-  echo "Checking environment."
-  if [ -f $venv_env_version ]; then
-    set +o errexit
-    cat requirements.txt test-requirements.txt | cmp $venv_env_version - > /dev/null
-    local env_check_result=$?
-    set -o errexit
-    if [ $env_check_result -eq 0 ]; then
-      # If the environment exists and is up-to-date then set our variables
-      command_wrapper="${root}/${with_venv}"
-      echo "Environment is up to date."
-      return 0
-    fi
-  fi
-
-  if [ $always_venv -eq 1 ]; then
-    install_venv
-  else
-    if [ ! -e ${venv} ]; then
-      echo -e "Environment not found. Install? (Y/n) \c"
-    else
-      echo -e "Your environment appears to be out of date. Update? (Y/n) \c"
-    fi
-    read update_env
-    if [ "x$update_env" = "xY" -o "x$update_env" = "x" -o "x$update_env" = "xy" ]; then
-      install_venv
-    else
-      # Set our command wrapper anyway.
-      command_wrapper="${root}/${with_venv}"
-    fi
-  fi
-}
-
 function sanity_check {
   # Anything that should be determined prior to running the tests, server, etc.
   # Don't sanity-check anything environment-related in -N flag is set
@@ -322,6 +295,7 @@ function restore_environment {
 
 function install_venv {
   # Install with install_venv.py
+  export UPPER_CONSTRAINTS_FILE=${UPPER_CONSTRAINTS_FILE:-$_default_uc}
   export PIP_DOWNLOAD_CACHE=${PIP_DOWNLOAD_CACHE-/tmp/.pip_download_cache}
   export PIP_USE_MIRRORS=true
   if [ $quiet -eq 1 ]; then
@@ -334,7 +308,6 @@ function install_venv {
   # Make sure it worked and record the environment version
   sanity_check
   chmod -R 754 $venv
-  cat requirements.txt test-requirements.txt > $venv_env_version
 }
 
 function run_tests {
@@ -530,8 +503,8 @@ if [ $never_venv -eq 0 ]; then
     destroy_venv
   fi
 
-  # Then check if it's up-to-date
-  environment_check
+  # Create or update venv.
+  install_venv
 
   # Create a backup of the up-to-date environment if desired
   if [ $backup_env -eq 1 ]; then

tools/install_venv.py

@@ -58,8 +58,10 @@ def main(argv):
     test_requires = os.path.join(root, 'test-requirements.txt')
     py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1])
     project = 'OpenStack'
+    constraints = os.environ.get('UPPER_CONSTRAINTS_FILE')
     install = install_venv.InstallVenv(root, venv, pip_requires, test_requires,
-                                       py_version, project)
+                                       py_version, project,
+                                       constraints=constraints)
     options = install.parse_args(argv)
     install.check_python_version()
     install.check_dependencies()

tools/install_venv_common.py

@@ -34,9 +34,10 @@ class InstallVenv(object):
 
     def __init__(self, root, venv, requirements,
                  test_requirements, py_version,
-                 project):
+                 project, constraints=None):
         self.root = root
         self.venv = venv
+        self.constraints = constraints
         self.requirements = requirements
         self.test_requirements = test_requirements
         self.py_version = py_version
@@ -104,17 +105,19 @@ class InstallVenv(object):
             pass
 
     def pip_install(self, *args):
-        self.run_command(['tools/with_venv.sh',
-                         'pip', 'install', '--upgrade'] + list(args),
-                         redirect_output=False)
+        cmd = ['tools/with_venv.sh', 'pip', 'install', '--upgrade']
+        if self.constraints:
+            cmd += ['-c', self.constraints]
+        self.run_command(cmd + list(args), redirect_output=False)
 
     def install_dependencies(self):
         print('Installing dependencies with pip (this can take a while)...')
 
         # First things first, make sure our venv has the latest pip and
         # setuptools and pbr
-        self.pip_install('pip>=1.4')
+        self.pip_install('pip>=7.1.0')
         self.pip_install('setuptools')
+        self.pip_install('wheel')
         self.pip_install('pbr')
 
         self.pip_install('-r', self.requirements, '-r', self.test_requirements)