Update packages from requirements.txt with 'system_tests.sh -N'

Default behaviour of the system_tests.sh is unchanged,
but with -N it will update requirements on each run (without
--upgrade).
If fuel-devops has incorrect version or not installed, then
script will fail with -N option (until fuel-devops is not packed
to PyPi repository).

- Removed dependency on fuel-devops located on github.
  This dependency splitted into two requirements:

  requirements-devops.txt : will be used with -N option to
      controll the installed fuel-devops version;

  requirements-devops-source.txt : will be used for tox tests
      and update python virtual environments on CI.

- Update system_tests.sh with two new options: -R and -N:
  REQUIREMENTS_DIR or -R : name of the dir where requirements.txt
  and constraints.txt are located (fuelweb_test by default)
  UPDATE_REQUIREMENTS or -N : 'yes' or 'no' (-N == 'yes').
  Perform update PyPi packages from requirements.txt.
  Default : 'no'

- remove unused variables and code that was used for build ISO

Change-Id: Ib9efd2136226fb43b506db390079f1b72a06069d
Related-Bug:#1575065

fuelweb_test/requirements-devops-source.txt

@@ -0,0 +1,4 @@
+# TODO(ddmitriev): remove this requirement when fuel-devops
+# will be available as a PyPi package.
+# Check virtual environment update scripts on CI first.
+git+git://github.com/openstack/fuel-devops.git@2.9.20

fuelweb_test/requirements-devops.txt

@@ -0,0 +1,3 @@
+# TODO(ddmitriev): move this requirement to requirements.txt
+# when fuel-devops will be available as a PyPi package
+fuel-devops>=2.9.20

fuelweb_test/requirements.txt

@@ -1,5 +1,4 @@
 nose==1.2.1
-git+git://github.com/openstack/fuel-devops.git@2.9.20
 anyjson>=0.3.3  # BSD
 paramiko>=1.16.0  # LGPL
 proboscis==1.2.6.0

tox.ini

@@ -9,7 +9,9 @@ envlist = pep8, py27, pylint, docs, pep8-py{34,35}, pylint-py{27}-{fuelweb,syste
 skip_missing_interpreters = True
 
 [testenv]
-deps = -r{toxinidir}/fuelweb_test/requirements.txt
+deps =
+    -r{toxinidir}/fuelweb_test/requirements-devops-source.txt
+    -r{toxinidir}/fuelweb_test/requirements.txt
 usedevelop = False
 commands =
     ./run_system_test.py show-all-groups
@@ -44,7 +46,7 @@ count = True
 
 [testenv:pylint]
 deps=
-    -r{toxinidir}/fuelweb_test/requirements.txt
+    {[testenv]deps}
     pylint
 commands=
     pylint --rcfile=.pylintrc_gerrit fuelweb_test system_test gates_tests fuel_tests
@@ -52,25 +54,25 @@ commands=
 
 [testenv:pylint-py27-fuelweb]
 deps=
-    -r{toxinidir}/fuelweb_test/requirements.txt
+    {[testenv]deps}
     pylint
 commands=pylint fuelweb_test
 
 [testenv:pylint-py27-system]
 deps=
-    -r{toxinidir}/fuelweb_test/requirements.txt
+    {[testenv]deps}
     pylint
 commands=pylint system_test
 
 [testenv:pylint-py27-gates]
 deps=
-    -r{toxinidir}/fuelweb_test/requirements.txt
+    {[testenv]deps}
     pylint
 commands=pylint gates_tests
 
 [testenv:pylint-py27-fuel]
 deps=
-    -r{toxinidir}/fuelweb_test/requirements.txt
+    {[testenv]deps}
     pylint
 commands=pylint fuel_tests
 
@@ -78,6 +80,6 @@ commands=pylint fuel_tests
 [testenv:docs]
 changedir = doc
 deps =
-    -r{toxinidir}/fuelweb_test/requirements.txt
+    {[testenv]deps}
     sphinx
 commands = sphinx-build -b html -d _build/doctrees . _build/html

utils/jenkins/system_tests.sh

@@ -40,20 +40,14 @@ if you do need to override them.
 -V (dir)    - Path to python virtual environment
 -i (file)   - Full path to ISO file to build or use for tests.
               Made from iso dir and name if not set.
--t (name)   - Name of task this script should perform. Should be one of defined ones.
-              Taken from Jenkins' job's suffix if not set.
 -o (str)    - Allows you any extra command line option to run test job if you
               want to use some parameters.
 -a (str)    - Allows you to path NOSE_ATTR to the test job if you want
               to use some parameters.
 -A (str)    - Allows you to path  NOSE_EVAL_ATTR if you want to enter attributes
               as python expressions.
--m (name)   - Use this mirror to build ISO from.
-              Uses 'srt' if not set.
 -U          - ISO URL for tests.
               Null by default.
--r (yes/no) - Should built ISO file be places with build number tag and
-              symlinked to the last build or just copied over the last file.
 -b (num)    - Allows you to override Jenkins' build number if you need to.
 -l (dir)    - Path to logs directory. Can be set by LOGS_DIR environment variable.
               Uses WORKSPACE/logs if not set.
@@ -63,6 +57,9 @@ if you do need to override them.
               Useful for debugging.
 -k          - Keep previously created test environment before tests run
 -K          - Keep test environment after tests are finished
+-R (name)   - Name of the package where requirements.txt is located. For use with the option -N only.
+              Uses 'fuelweb_test' if option is not set.
+-N          - Install PyPi packages from 'requirements.txt'.
 -h          - Show this help page
 
 Most variables uses guessing from Jenkins' job name but can be overridden
@@ -115,28 +112,22 @@ GlobalVariables() {
     ISO_PATH="${ISO_DIR}/${ISO_NAME}"
   fi
 
-  # what task should be ran
-  # it's taken from jenkins job name suffix if not set by options
-  if [ -z "${TASK_NAME}" ]; then
-    TASK_NAME="${JOB_NAME##*.}"
-  fi
-
-  # do we want to keep iso's for each build or just copy over single file
-  ROTATE_ISO="${ROTATE_ISO:=yes}"
-
-  # choose mirror to build iso from. Default is 'srt' for Saratov's mirror
-  # you can change mirror by exporting USE_MIRROR variable before running this script
-  USE_MIRROR="${USE_MIRROR:=srt}"
-
   # only show what commands would be executed but do nothing
   # this feature is useful if you want to debug this script's behaviour
   DRY_RUN="${DRY_RUN:=no}"
 
   VENV="${VENV:=yes}"
+
+  # Path to the directory where requirements.txt is placed.
+  # Default place is ./fuelweb_test/requirements.txt
+  REQUIREMENTS_DIR="${REQUIREMENTS_DIR:=fuelweb_test}"
+
+  # Perform requirements update from the requirements.txt file. Default = no.
+  UPDATE_REQUIREMENTS="${UPDATE_REQUIREMENTS:=no}"
 }
 
 GetoptsVariables() {
-  while getopts ":w:j:i:t:o:a:A:m:U:r:b:V:l:LdkKe:v:h" opt; do
+  while getopts ":w:j:i:t:o:a:A:m:U:r:b:V:l:LdkKNe:v:R:h" opt; do
     case ${opt} in
       w)
         WORKSPACE="${OPTARG}"
@@ -148,7 +139,7 @@ GetoptsVariables() {
         ISO_PATH="${OPTARG}"
         ;;
       t)
-        TASK_NAME="${OPTARG}"
+        echo "Option 'TASK_NAME' deprecated."
         ;;
       o)
         TEST_OPTIONS="${TEST_OPTIONS} ${OPTARG}"
@@ -160,13 +151,13 @@ GetoptsVariables() {
         NOSE_EVAL_ATTR="${OPTARG}"
         ;;
       m)
-        USE_MIRROR="${OPTARG}"
+        echo "Option 'USE_MIRROR' deprecated."
         ;;
       U)
         ISO_URL="${OPTARG}"
         ;;
       r)
-        ROTATE_ISO="${OPTARG}"
+        echo "Option 'ROTATE_ISO' deprecated."
         ;;
       b)
         BUILD_NUMBER="${OPTARG}"
@@ -195,6 +186,12 @@ GetoptsVariables() {
       v)
         VENV="no"
         ;;
+      R)
+        REQUIREMENTS_DIR="${OPTARG}"
+        ;;
+      N)
+        UPDATE_REQUIREMENTS="yes"
+        ;;
       h)
         ShowHelp
         exit 0
@@ -225,113 +222,12 @@ CheckVariables() {
     exit ${NOISOPATH_ERR}
   fi
 
-  if [ -z "${TASK_NAME}" ]; then
-    echo "Error! TASK_NAME is not set!"
-    exit ${NOTASKNAME_ERR}
-  fi
-
   if [ -z "${WORKSPACE}" ]; then
     echo "Error! WORKSPACE is not set!"
     exit ${NOWORKSPACE_ERR}
   fi
 }
 
-MakeISO() {
-  # Create iso file to be used in tests
-
-  # clean previous garbage
-  if [ "${DRY_RUN}" = "yes" ]; then
-    echo make deep_clean
-  else
-    make deep_clean
-  fi
-  ec="${?}"
-
-  if [ "${ec}" -gt "0" ]; then
-    echo "Error! Deep clean failed!"
-    exit ${DEEPCLEAN_ERR}
-  fi
-
-  # create ISO file
-  export USE_MIRROR
-  if [ "${DRY_RUN}" = "yes" ]; then
-    echo make iso
-  else
-    make iso
-  fi
-  ec=$?
-
-  if [ "${ec}" -gt "0" ]; then
-    echo "Error making ISO!"
-    exit ${MAKEISO_ERR}
-  fi
-
-  if [ "${DRY_RUN}" = "yes" ]; then
-    ISO="${WORKSPACE}/build/iso/fuel.iso"
-  else
-    ISO="`ls ${WORKSPACE}/build/iso/*.iso | head -n 1`"
-    # check that ISO file exists
-    if [ ! -f "${ISO}" ]; then
-      echo "Error! ISO file not found!"
-      exit ${NOISOFOUND_ERR}
-    fi
-  fi
-
-  # copy ISO file to storage dir
-  # if rotation is enabled and build number is available
-  # save iso to tagged file and symlink to the last build
-  # if rotation is not enabled just copy iso to iso_dir
-
-  if [ "${ROTATE_ISO}" = "yes" -a "${BUILD_NUMBER}" != "" ]; then
-    # copy iso file to shared dir with revision tagged name
-    NEW_BUILD_ISO_PATH="${ISO_PATH#.iso}_${BUILD_NUMBER}.iso"
-    if [ "${DRY_RUN}" = "yes" ]; then
-      echo cp "${ISO}" "${NEW_BUILD_ISO_PATH}"
-    else
-      cp "${ISO}" "${NEW_BUILD_ISO_PATH}"
-    fi
-    ec=$?
-
-    if [ "${ec}" -gt "0" ]; then
-      echo "Error! Copy ${ISO} to ${NEW_BUILD_ISO_PATH} failed!"
-      exit ${COPYISO_ERR}
-    fi
-
-    # create symlink to the last built ISO file
-    if [ "${DRY_RUN}" = "yes" ]; then
-      echo ln -sf "${NEW_BUILD_ISO_PATH}" "${ISO_PATH}"
-    else
-      ln -sf "${NEW_BUILD_ISO_PATH}" "${ISO_PATH}"
-    fi
-    ec=$?
-
-    if [ "${ec}" -gt "0" ]; then
-      echo "Error! Create symlink from ${NEW_BUILD_ISO_PATH} to ${ISO_PATH} failed!"
-      exit ${SYMLINKISO_ERR}
-    fi
-  else
-    # just copy file to shared dir
-    if [ "${DRY_RUN}" = "yes" ]; then
-      echo cp "${ISO}" "${ISO_PATH}"
-    else
-      cp "${ISO}" "${ISO_PATH}"
-    fi
-    ec=$?
-
-    if [ "${ec}" -gt "0" ]; then
-      echo "Error! Copy ${ISO} to ${ISO_PATH} failed!"
-      exit ${COPYISO_ERR}
-    fi
-  fi
-
-  if [ "${ec}" -gt "0" ]; then
-    echo "Error! Copy ISO from ${ISO} to ${ISO_PATH} failed!"
-    exit ${COPYISO_ERR}
-  fi
-  echo "Finished building ISO: ${ISO_PATH}"
-  exit 0
-}
-
 CdWorkSpace() {
     # chdir into workspace or fail if could not
     if [ "${DRY_RUN}" != "yes" ]; then
@@ -347,6 +243,40 @@ CdWorkSpace() {
     fi
 }
 
+CheckRequirements() {
+    REQUIREMENTS_PATH="${WORKSPACE}/${REQUIREMENTS_DIR}"
+
+    if [ "${UPDATE_REQUIREMENTS}" = "yes" ]; then
+        if [ -f "${REQUIREMENTS_PATH}/requirements.txt" ]; then
+            # Install packages from requirements.txt
+            pip install -r "${REQUIREMENTS_PATH}/requirements.txt"
+        fi
+
+        if [ -f "${REQUIREMENTS_PATH}/requirements-devops.txt" ]; then
+            # Try to install fuel-devops as a package, to controll that
+            # required version of fuel-devops is already installed.
+            # Installation will fail if fuel-devops is not installed or
+            # installed with correct version (until it is not a PyPi package)
+            pip install -r "${REQUIREMENTS_PATH}/requirements-devops.txt"
+        fi
+    fi
+}
+
+ActivateVirtualenv() {
+    if [ -z "${VENV_PATH}" ]; then
+        VENV_PATH="/home/jenkins/venv-nailgun-tests"
+    fi
+
+    # run python virtualenv
+    if [ "${VENV}" = "yes" ]; then
+        if [ "${DRY_RUN}" = "yes" ]; then
+            echo . ${VENV_PATH}/bin/activate
+        else
+            . ${VENV_PATH}/bin/activate
+        fi
+    fi
+}
+
 RunTest() {
     # Run test selected by task name
 
@@ -370,19 +300,6 @@ RunTest() {
         fi
     fi
 
-    if [ -z "${VENV_PATH}" ]; then
-        VENV_PATH="/home/jenkins/venv-nailgun-tests"
-    fi
-
-    # run python virtualenv
-    if [ "${VENV}" = "yes" ]; then
-        if [ "${DRY_RUN}" = "yes" ]; then
-            echo . ${VENV_PATH}/bin/activate
-        else
-            . ${VENV_PATH}/bin/activate
-        fi
-    fi
-
     if [ "${ENV_NAME}" = "" ]; then
       ENV_NAME="${JOB_NAME}_system_test"
     fi
@@ -454,26 +371,6 @@ RunTest() {
     exit "${ec}"
 }
 
-RouteTasks() {
-  # this selector defines task names that are recognised by this script
-  # and runs corresponding jobs for them
-  # running any jobs should exit this script
-
-  case "${TASK_NAME}" in
-  test)
-    RunTest
-    ;;
-  iso)
-    MakeISO
-    ;;
-  *)
-    echo "Unknown task: ${TASK_NAME}!"
-    exit ${INVALIDTASK_ERR}
-    ;;
-  esac
-  exit 0
-}
-
 # MAIN
 
 # first we want to get variable from command line options
@@ -488,5 +385,11 @@ CheckVariables
 # first we chdir into our working directory unless we dry run
 CdWorkSpace
 
-# finally we can choose what to do according to TASK_NAME
+# Activate python virtual environment
-RouteTasks
+ActivateVirtualenv
+
+# Check/update PyPi requirements
+CheckRequirements
+
+# Run the test
+RunTest