From 5a62e85194d09f3f8629d89d30ac7bb0b01b8d00 Mon Sep 17 00:00:00 2001 From: Dennis Dmitriev Date: Mon, 27 Jun 2016 14:57:38 +0300 Subject: [PATCH] Update packages from requirements.txt with 'system_tests.sh -N' Default behaviour of the system_tests.sh is unchanged, but with -N it will update requirements on each run (without --upgrade). If fuel-devops has incorrect version or not installed, then script will fail with -N option (until fuel-devops is not packed to PyPi repository). - Removed dependency on fuel-devops located on github. This dependency splitted into two requirements: requirements-devops.txt : will be used with -N option to controll the installed fuel-devops version; requirements-devops-source.txt : will be used for tox tests and update python virtual environments on CI. - Update system_tests.sh with two new options: -R and -N: REQUIREMENTS_DIR or -R : name of the dir where requirements.txt and constraints.txt are located (fuelweb_test by default) UPDATE_REQUIREMENTS or -N : 'yes' or 'no' (-N == 'yes'). Perform update PyPi packages from requirements.txt. Default : 'no' - remove unused variables and code that was used for build ISO Change-Id: Ib9efd2136226fb43b506db390079f1b72a06069d Related-Bug:#1575065 --- fuelweb_test/requirements-devops-source.txt | 4 + fuelweb_test/requirements-devops.txt | 3 + fuelweb_test/requirements.txt | 3 +- tox.ini | 16 +- utils/jenkins/system_tests.sh | 221 ++++++-------------- 5 files changed, 79 insertions(+), 168 deletions(-) create mode 100644 fuelweb_test/requirements-devops-source.txt create mode 100644 fuelweb_test/requirements-devops.txt diff --git a/fuelweb_test/requirements-devops-source.txt b/fuelweb_test/requirements-devops-source.txt new file mode 100644 index 000000000..3d8518013 --- /dev/null +++ b/fuelweb_test/requirements-devops-source.txt @@ -0,0 +1,4 @@ +# TODO(ddmitriev): remove this requirement when fuel-devops +# will be available as a PyPi package. +# Check virtual environment update scripts on CI first. +git+git://github.com/openstack/fuel-devops.git@2.9.20 diff --git a/fuelweb_test/requirements-devops.txt b/fuelweb_test/requirements-devops.txt new file mode 100644 index 000000000..08278b083 --- /dev/null +++ b/fuelweb_test/requirements-devops.txt @@ -0,0 +1,3 @@ +# TODO(ddmitriev): move this requirement to requirements.txt +# when fuel-devops will be available as a PyPi package +fuel-devops>=2.9.20 diff --git a/fuelweb_test/requirements.txt b/fuelweb_test/requirements.txt index 873e56f6e..bd4b0fa16 100644 --- a/fuelweb_test/requirements.txt +++ b/fuelweb_test/requirements.txt @@ -1,5 +1,4 @@ nose==1.2.1 -git+git://github.com/openstack/fuel-devops.git@2.9.20 anyjson>=0.3.3 # BSD paramiko>=1.16.0 # LGPL proboscis==1.2.6.0 @@ -30,4 +29,4 @@ python-jenkins>=0.4.12 pyyaml>=3.1.0 # MIT requests>=2.8.1,!=2.9.0 # Apache-2.0 tablib>=0.11.2 -xmltodict>=0.10.1 # MIT \ No newline at end of file +xmltodict>=0.10.1 # MIT diff --git a/tox.ini b/tox.ini index e07bb4fc7..be4edb947 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,9 @@ envlist = pep8, py27, pylint, docs, pep8-py{34,35}, pylint-py{27}-{fuelweb,syste skip_missing_interpreters = True [testenv] -deps = -r{toxinidir}/fuelweb_test/requirements.txt +deps = + -r{toxinidir}/fuelweb_test/requirements-devops-source.txt + -r{toxinidir}/fuelweb_test/requirements.txt usedevelop = False commands = ./run_system_test.py show-all-groups @@ -44,7 +46,7 @@ count = True [testenv:pylint] deps= - -r{toxinidir}/fuelweb_test/requirements.txt + {[testenv]deps} pylint commands= pylint --rcfile=.pylintrc_gerrit fuelweb_test system_test gates_tests fuel_tests @@ -52,25 +54,25 @@ commands= [testenv:pylint-py27-fuelweb] deps= - -r{toxinidir}/fuelweb_test/requirements.txt + {[testenv]deps} pylint commands=pylint fuelweb_test [testenv:pylint-py27-system] deps= - -r{toxinidir}/fuelweb_test/requirements.txt + {[testenv]deps} pylint commands=pylint system_test [testenv:pylint-py27-gates] deps= - -r{toxinidir}/fuelweb_test/requirements.txt + {[testenv]deps} pylint commands=pylint gates_tests [testenv:pylint-py27-fuel] deps= - -r{toxinidir}/fuelweb_test/requirements.txt + {[testenv]deps} pylint commands=pylint fuel_tests @@ -78,6 +80,6 @@ commands=pylint fuel_tests [testenv:docs] changedir = doc deps = - -r{toxinidir}/fuelweb_test/requirements.txt + {[testenv]deps} sphinx commands = sphinx-build -b html -d _build/doctrees . _build/html diff --git a/utils/jenkins/system_tests.sh b/utils/jenkins/system_tests.sh index 6c8083545..6e79da075 100755 --- a/utils/jenkins/system_tests.sh +++ b/utils/jenkins/system_tests.sh @@ -40,20 +40,14 @@ if you do need to override them. -V (dir) - Path to python virtual environment -i (file) - Full path to ISO file to build or use for tests. Made from iso dir and name if not set. --t (name) - Name of task this script should perform. Should be one of defined ones. - Taken from Jenkins' job's suffix if not set. -o (str) - Allows you any extra command line option to run test job if you want to use some parameters. -a (str) - Allows you to path NOSE_ATTR to the test job if you want to use some parameters. -A (str) - Allows you to path NOSE_EVAL_ATTR if you want to enter attributes as python expressions. --m (name) - Use this mirror to build ISO from. - Uses 'srt' if not set. -U - ISO URL for tests. Null by default. --r (yes/no) - Should built ISO file be places with build number tag and - symlinked to the last build or just copied over the last file. -b (num) - Allows you to override Jenkins' build number if you need to. -l (dir) - Path to logs directory. Can be set by LOGS_DIR environment variable. Uses WORKSPACE/logs if not set. @@ -63,6 +57,9 @@ if you do need to override them. Useful for debugging. -k - Keep previously created test environment before tests run -K - Keep test environment after tests are finished +-R (name) - Name of the package where requirements.txt is located. For use with the option -N only. + Uses 'fuelweb_test' if option is not set. +-N - Install PyPi packages from 'requirements.txt'. -h - Show this help page Most variables uses guessing from Jenkins' job name but can be overridden @@ -115,28 +112,22 @@ GlobalVariables() { ISO_PATH="${ISO_DIR}/${ISO_NAME}" fi - # what task should be ran - # it's taken from jenkins job name suffix if not set by options - if [ -z "${TASK_NAME}" ]; then - TASK_NAME="${JOB_NAME##*.}" - fi - - # do we want to keep iso's for each build or just copy over single file - ROTATE_ISO="${ROTATE_ISO:=yes}" - - # choose mirror to build iso from. Default is 'srt' for Saratov's mirror - # you can change mirror by exporting USE_MIRROR variable before running this script - USE_MIRROR="${USE_MIRROR:=srt}" - # only show what commands would be executed but do nothing # this feature is useful if you want to debug this script's behaviour DRY_RUN="${DRY_RUN:=no}" VENV="${VENV:=yes}" + + # Path to the directory where requirements.txt is placed. + # Default place is ./fuelweb_test/requirements.txt + REQUIREMENTS_DIR="${REQUIREMENTS_DIR:=fuelweb_test}" + + # Perform requirements update from the requirements.txt file. Default = no. + UPDATE_REQUIREMENTS="${UPDATE_REQUIREMENTS:=no}" } GetoptsVariables() { - while getopts ":w:j:i:t:o:a:A:m:U:r:b:V:l:LdkKe:v:h" opt; do + while getopts ":w:j:i:t:o:a:A:m:U:r:b:V:l:LdkKNe:v:R:h" opt; do case ${opt} in w) WORKSPACE="${OPTARG}" @@ -148,7 +139,7 @@ GetoptsVariables() { ISO_PATH="${OPTARG}" ;; t) - TASK_NAME="${OPTARG}" + echo "Option 'TASK_NAME' deprecated." ;; o) TEST_OPTIONS="${TEST_OPTIONS} ${OPTARG}" @@ -160,13 +151,13 @@ GetoptsVariables() { NOSE_EVAL_ATTR="${OPTARG}" ;; m) - USE_MIRROR="${OPTARG}" + echo "Option 'USE_MIRROR' deprecated." ;; U) ISO_URL="${OPTARG}" ;; r) - ROTATE_ISO="${OPTARG}" + echo "Option 'ROTATE_ISO' deprecated." ;; b) BUILD_NUMBER="${OPTARG}" @@ -195,6 +186,12 @@ GetoptsVariables() { v) VENV="no" ;; + R) + REQUIREMENTS_DIR="${OPTARG}" + ;; + N) + UPDATE_REQUIREMENTS="yes" + ;; h) ShowHelp exit 0 @@ -225,113 +222,12 @@ CheckVariables() { exit ${NOISOPATH_ERR} fi - if [ -z "${TASK_NAME}" ]; then - echo "Error! TASK_NAME is not set!" - exit ${NOTASKNAME_ERR} - fi - if [ -z "${WORKSPACE}" ]; then echo "Error! WORKSPACE is not set!" exit ${NOWORKSPACE_ERR} fi } -MakeISO() { - # Create iso file to be used in tests - - # clean previous garbage - if [ "${DRY_RUN}" = "yes" ]; then - echo make deep_clean - else - make deep_clean - fi - ec="${?}" - - if [ "${ec}" -gt "0" ]; then - echo "Error! Deep clean failed!" - exit ${DEEPCLEAN_ERR} - fi - - # create ISO file - export USE_MIRROR - if [ "${DRY_RUN}" = "yes" ]; then - echo make iso - else - make iso - fi - ec=$? - - if [ "${ec}" -gt "0" ]; then - echo "Error making ISO!" - exit ${MAKEISO_ERR} - fi - - if [ "${DRY_RUN}" = "yes" ]; then - ISO="${WORKSPACE}/build/iso/fuel.iso" - else - ISO="`ls ${WORKSPACE}/build/iso/*.iso | head -n 1`" - # check that ISO file exists - if [ ! -f "${ISO}" ]; then - echo "Error! ISO file not found!" - exit ${NOISOFOUND_ERR} - fi - fi - - # copy ISO file to storage dir - # if rotation is enabled and build number is available - # save iso to tagged file and symlink to the last build - # if rotation is not enabled just copy iso to iso_dir - - if [ "${ROTATE_ISO}" = "yes" -a "${BUILD_NUMBER}" != "" ]; then - # copy iso file to shared dir with revision tagged name - NEW_BUILD_ISO_PATH="${ISO_PATH#.iso}_${BUILD_NUMBER}.iso" - if [ "${DRY_RUN}" = "yes" ]; then - echo cp "${ISO}" "${NEW_BUILD_ISO_PATH}" - else - cp "${ISO}" "${NEW_BUILD_ISO_PATH}" - fi - ec=$? - - if [ "${ec}" -gt "0" ]; then - echo "Error! Copy ${ISO} to ${NEW_BUILD_ISO_PATH} failed!" - exit ${COPYISO_ERR} - fi - - # create symlink to the last built ISO file - if [ "${DRY_RUN}" = "yes" ]; then - echo ln -sf "${NEW_BUILD_ISO_PATH}" "${ISO_PATH}" - else - ln -sf "${NEW_BUILD_ISO_PATH}" "${ISO_PATH}" - fi - ec=$? - - if [ "${ec}" -gt "0" ]; then - echo "Error! Create symlink from ${NEW_BUILD_ISO_PATH} to ${ISO_PATH} failed!" - exit ${SYMLINKISO_ERR} - fi - else - # just copy file to shared dir - if [ "${DRY_RUN}" = "yes" ]; then - echo cp "${ISO}" "${ISO_PATH}" - else - cp "${ISO}" "${ISO_PATH}" - fi - ec=$? - - if [ "${ec}" -gt "0" ]; then - echo "Error! Copy ${ISO} to ${ISO_PATH} failed!" - exit ${COPYISO_ERR} - fi - fi - - if [ "${ec}" -gt "0" ]; then - echo "Error! Copy ISO from ${ISO} to ${ISO_PATH} failed!" - exit ${COPYISO_ERR} - fi - echo "Finished building ISO: ${ISO_PATH}" - exit 0 -} - CdWorkSpace() { # chdir into workspace or fail if could not if [ "${DRY_RUN}" != "yes" ]; then @@ -347,6 +243,40 @@ CdWorkSpace() { fi } +CheckRequirements() { + REQUIREMENTS_PATH="${WORKSPACE}/${REQUIREMENTS_DIR}" + + if [ "${UPDATE_REQUIREMENTS}" = "yes" ]; then + if [ -f "${REQUIREMENTS_PATH}/requirements.txt" ]; then + # Install packages from requirements.txt + pip install -r "${REQUIREMENTS_PATH}/requirements.txt" + fi + + if [ -f "${REQUIREMENTS_PATH}/requirements-devops.txt" ]; then + # Try to install fuel-devops as a package, to controll that + # required version of fuel-devops is already installed. + # Installation will fail if fuel-devops is not installed or + # installed with correct version (until it is not a PyPi package) + pip install -r "${REQUIREMENTS_PATH}/requirements-devops.txt" + fi + fi +} + +ActivateVirtualenv() { + if [ -z "${VENV_PATH}" ]; then + VENV_PATH="/home/jenkins/venv-nailgun-tests" + fi + + # run python virtualenv + if [ "${VENV}" = "yes" ]; then + if [ "${DRY_RUN}" = "yes" ]; then + echo . ${VENV_PATH}/bin/activate + else + . ${VENV_PATH}/bin/activate + fi + fi +} + RunTest() { # Run test selected by task name @@ -370,19 +300,6 @@ RunTest() { fi fi - if [ -z "${VENV_PATH}" ]; then - VENV_PATH="/home/jenkins/venv-nailgun-tests" - fi - - # run python virtualenv - if [ "${VENV}" = "yes" ]; then - if [ "${DRY_RUN}" = "yes" ]; then - echo . ${VENV_PATH}/bin/activate - else - . ${VENV_PATH}/bin/activate - fi - fi - if [ "${ENV_NAME}" = "" ]; then ENV_NAME="${JOB_NAME}_system_test" fi @@ -454,26 +371,6 @@ RunTest() { exit "${ec}" } -RouteTasks() { - # this selector defines task names that are recognised by this script - # and runs corresponding jobs for them - # running any jobs should exit this script - - case "${TASK_NAME}" in - test) - RunTest - ;; - iso) - MakeISO - ;; - *) - echo "Unknown task: ${TASK_NAME}!" - exit ${INVALIDTASK_ERR} - ;; - esac - exit 0 -} - # MAIN # first we want to get variable from command line options @@ -488,5 +385,11 @@ CheckVariables # first we chdir into our working directory unless we dry run CdWorkSpace -# finally we can choose what to do according to TASK_NAME -RouteTasks +# Activate python virtual environment +ActivateVirtualenv + +# Check/update PyPi requirements +CheckRequirements + +# Run the test +RunTest