Update packages from requirements.txt with 'system_tests.sh -N'
Default behaviour of the system_tests.sh is unchanged, but with -N it will update requirements on each run (without --upgrade). If fuel-devops has incorrect version or not installed, then script will fail with -N option (until fuel-devops is not packed to PyPi repository). - Removed dependency on fuel-devops located on github. This dependency splitted into two requirements: requirements-devops.txt : will be used with -N option to controll the installed fuel-devops version; requirements-devops-source.txt : will be used for tox tests and update python virtual environments on CI. - Update system_tests.sh with two new options: -R and -N: REQUIREMENTS_DIR or -R : name of the dir where requirements.txt and constraints.txt are located (fuelweb_test by default) UPDATE_REQUIREMENTS or -N : 'yes' or 'no' (-N == 'yes'). Perform update PyPi packages from requirements.txt. Default : 'no' - remove unused variables and code that was used for build ISO Change-Id: Ib9efd2136226fb43b506db390079f1b72a06069d Related-Bug:#1575065
This commit is contained in:
parent
92006f3204
commit
5a62e85194
4
fuelweb_test/requirements-devops-source.txt
Normal file
4
fuelweb_test/requirements-devops-source.txt
Normal file
@ -0,0 +1,4 @@
|
||||
# TODO(ddmitriev): remove this requirement when fuel-devops
|
||||
# will be available as a PyPi package.
|
||||
# Check virtual environment update scripts on CI first.
|
||||
git+git://github.com/openstack/fuel-devops.git@2.9.20
|
3
fuelweb_test/requirements-devops.txt
Normal file
3
fuelweb_test/requirements-devops.txt
Normal file
@ -0,0 +1,3 @@
|
||||
# TODO(ddmitriev): move this requirement to requirements.txt
|
||||
# when fuel-devops will be available as a PyPi package
|
||||
fuel-devops>=2.9.20
|
@ -1,5 +1,4 @@
|
||||
nose==1.2.1
|
||||
git+git://github.com/openstack/fuel-devops.git@2.9.20
|
||||
anyjson>=0.3.3 # BSD
|
||||
paramiko>=1.16.0 # LGPL
|
||||
proboscis==1.2.6.0
|
||||
|
16
tox.ini
16
tox.ini
@ -9,7 +9,9 @@ envlist = pep8, py27, pylint, docs, pep8-py{34,35}, pylint-py{27}-{fuelweb,syste
|
||||
skip_missing_interpreters = True
|
||||
|
||||
[testenv]
|
||||
deps = -r{toxinidir}/fuelweb_test/requirements.txt
|
||||
deps =
|
||||
-r{toxinidir}/fuelweb_test/requirements-devops-source.txt
|
||||
-r{toxinidir}/fuelweb_test/requirements.txt
|
||||
usedevelop = False
|
||||
commands =
|
||||
./run_system_test.py show-all-groups
|
||||
@ -44,7 +46,7 @@ count = True
|
||||
|
||||
[testenv:pylint]
|
||||
deps=
|
||||
-r{toxinidir}/fuelweb_test/requirements.txt
|
||||
{[testenv]deps}
|
||||
pylint
|
||||
commands=
|
||||
pylint --rcfile=.pylintrc_gerrit fuelweb_test system_test gates_tests fuel_tests
|
||||
@ -52,25 +54,25 @@ commands=
|
||||
|
||||
[testenv:pylint-py27-fuelweb]
|
||||
deps=
|
||||
-r{toxinidir}/fuelweb_test/requirements.txt
|
||||
{[testenv]deps}
|
||||
pylint
|
||||
commands=pylint fuelweb_test
|
||||
|
||||
[testenv:pylint-py27-system]
|
||||
deps=
|
||||
-r{toxinidir}/fuelweb_test/requirements.txt
|
||||
{[testenv]deps}
|
||||
pylint
|
||||
commands=pylint system_test
|
||||
|
||||
[testenv:pylint-py27-gates]
|
||||
deps=
|
||||
-r{toxinidir}/fuelweb_test/requirements.txt
|
||||
{[testenv]deps}
|
||||
pylint
|
||||
commands=pylint gates_tests
|
||||
|
||||
[testenv:pylint-py27-fuel]
|
||||
deps=
|
||||
-r{toxinidir}/fuelweb_test/requirements.txt
|
||||
{[testenv]deps}
|
||||
pylint
|
||||
commands=pylint fuel_tests
|
||||
|
||||
@ -78,6 +80,6 @@ commands=pylint fuel_tests
|
||||
[testenv:docs]
|
||||
changedir = doc
|
||||
deps =
|
||||
-r{toxinidir}/fuelweb_test/requirements.txt
|
||||
{[testenv]deps}
|
||||
sphinx
|
||||
commands = sphinx-build -b html -d _build/doctrees . _build/html
|
||||
|
@ -40,20 +40,14 @@ if you do need to override them.
|
||||
-V (dir) - Path to python virtual environment
|
||||
-i (file) - Full path to ISO file to build or use for tests.
|
||||
Made from iso dir and name if not set.
|
||||
-t (name) - Name of task this script should perform. Should be one of defined ones.
|
||||
Taken from Jenkins' job's suffix if not set.
|
||||
-o (str) - Allows you any extra command line option to run test job if you
|
||||
want to use some parameters.
|
||||
-a (str) - Allows you to path NOSE_ATTR to the test job if you want
|
||||
to use some parameters.
|
||||
-A (str) - Allows you to path NOSE_EVAL_ATTR if you want to enter attributes
|
||||
as python expressions.
|
||||
-m (name) - Use this mirror to build ISO from.
|
||||
Uses 'srt' if not set.
|
||||
-U - ISO URL for tests.
|
||||
Null by default.
|
||||
-r (yes/no) - Should built ISO file be places with build number tag and
|
||||
symlinked to the last build or just copied over the last file.
|
||||
-b (num) - Allows you to override Jenkins' build number if you need to.
|
||||
-l (dir) - Path to logs directory. Can be set by LOGS_DIR environment variable.
|
||||
Uses WORKSPACE/logs if not set.
|
||||
@ -63,6 +57,9 @@ if you do need to override them.
|
||||
Useful for debugging.
|
||||
-k - Keep previously created test environment before tests run
|
||||
-K - Keep test environment after tests are finished
|
||||
-R (name) - Name of the package where requirements.txt is located. For use with the option -N only.
|
||||
Uses 'fuelweb_test' if option is not set.
|
||||
-N - Install PyPi packages from 'requirements.txt'.
|
||||
-h - Show this help page
|
||||
|
||||
Most variables uses guessing from Jenkins' job name but can be overridden
|
||||
@ -115,28 +112,22 @@ GlobalVariables() {
|
||||
ISO_PATH="${ISO_DIR}/${ISO_NAME}"
|
||||
fi
|
||||
|
||||
# what task should be ran
|
||||
# it's taken from jenkins job name suffix if not set by options
|
||||
if [ -z "${TASK_NAME}" ]; then
|
||||
TASK_NAME="${JOB_NAME##*.}"
|
||||
fi
|
||||
|
||||
# do we want to keep iso's for each build or just copy over single file
|
||||
ROTATE_ISO="${ROTATE_ISO:=yes}"
|
||||
|
||||
# choose mirror to build iso from. Default is 'srt' for Saratov's mirror
|
||||
# you can change mirror by exporting USE_MIRROR variable before running this script
|
||||
USE_MIRROR="${USE_MIRROR:=srt}"
|
||||
|
||||
# only show what commands would be executed but do nothing
|
||||
# this feature is useful if you want to debug this script's behaviour
|
||||
DRY_RUN="${DRY_RUN:=no}"
|
||||
|
||||
VENV="${VENV:=yes}"
|
||||
|
||||
# Path to the directory where requirements.txt is placed.
|
||||
# Default place is ./fuelweb_test/requirements.txt
|
||||
REQUIREMENTS_DIR="${REQUIREMENTS_DIR:=fuelweb_test}"
|
||||
|
||||
# Perform requirements update from the requirements.txt file. Default = no.
|
||||
UPDATE_REQUIREMENTS="${UPDATE_REQUIREMENTS:=no}"
|
||||
}
|
||||
|
||||
GetoptsVariables() {
|
||||
while getopts ":w:j:i:t:o:a:A:m:U:r:b:V:l:LdkKe:v:h" opt; do
|
||||
while getopts ":w:j:i:t:o:a:A:m:U:r:b:V:l:LdkKNe:v:R:h" opt; do
|
||||
case ${opt} in
|
||||
w)
|
||||
WORKSPACE="${OPTARG}"
|
||||
@ -148,7 +139,7 @@ GetoptsVariables() {
|
||||
ISO_PATH="${OPTARG}"
|
||||
;;
|
||||
t)
|
||||
TASK_NAME="${OPTARG}"
|
||||
echo "Option 'TASK_NAME' deprecated."
|
||||
;;
|
||||
o)
|
||||
TEST_OPTIONS="${TEST_OPTIONS} ${OPTARG}"
|
||||
@ -160,13 +151,13 @@ GetoptsVariables() {
|
||||
NOSE_EVAL_ATTR="${OPTARG}"
|
||||
;;
|
||||
m)
|
||||
USE_MIRROR="${OPTARG}"
|
||||
echo "Option 'USE_MIRROR' deprecated."
|
||||
;;
|
||||
U)
|
||||
ISO_URL="${OPTARG}"
|
||||
;;
|
||||
r)
|
||||
ROTATE_ISO="${OPTARG}"
|
||||
echo "Option 'ROTATE_ISO' deprecated."
|
||||
;;
|
||||
b)
|
||||
BUILD_NUMBER="${OPTARG}"
|
||||
@ -195,6 +186,12 @@ GetoptsVariables() {
|
||||
v)
|
||||
VENV="no"
|
||||
;;
|
||||
R)
|
||||
REQUIREMENTS_DIR="${OPTARG}"
|
||||
;;
|
||||
N)
|
||||
UPDATE_REQUIREMENTS="yes"
|
||||
;;
|
||||
h)
|
||||
ShowHelp
|
||||
exit 0
|
||||
@ -225,113 +222,12 @@ CheckVariables() {
|
||||
exit ${NOISOPATH_ERR}
|
||||
fi
|
||||
|
||||
if [ -z "${TASK_NAME}" ]; then
|
||||
echo "Error! TASK_NAME is not set!"
|
||||
exit ${NOTASKNAME_ERR}
|
||||
fi
|
||||
|
||||
if [ -z "${WORKSPACE}" ]; then
|
||||
echo "Error! WORKSPACE is not set!"
|
||||
exit ${NOWORKSPACE_ERR}
|
||||
fi
|
||||
}
|
||||
|
||||
MakeISO() {
|
||||
# Create iso file to be used in tests
|
||||
|
||||
# clean previous garbage
|
||||
if [ "${DRY_RUN}" = "yes" ]; then
|
||||
echo make deep_clean
|
||||
else
|
||||
make deep_clean
|
||||
fi
|
||||
ec="${?}"
|
||||
|
||||
if [ "${ec}" -gt "0" ]; then
|
||||
echo "Error! Deep clean failed!"
|
||||
exit ${DEEPCLEAN_ERR}
|
||||
fi
|
||||
|
||||
# create ISO file
|
||||
export USE_MIRROR
|
||||
if [ "${DRY_RUN}" = "yes" ]; then
|
||||
echo make iso
|
||||
else
|
||||
make iso
|
||||
fi
|
||||
ec=$?
|
||||
|
||||
if [ "${ec}" -gt "0" ]; then
|
||||
echo "Error making ISO!"
|
||||
exit ${MAKEISO_ERR}
|
||||
fi
|
||||
|
||||
if [ "${DRY_RUN}" = "yes" ]; then
|
||||
ISO="${WORKSPACE}/build/iso/fuel.iso"
|
||||
else
|
||||
ISO="`ls ${WORKSPACE}/build/iso/*.iso | head -n 1`"
|
||||
# check that ISO file exists
|
||||
if [ ! -f "${ISO}" ]; then
|
||||
echo "Error! ISO file not found!"
|
||||
exit ${NOISOFOUND_ERR}
|
||||
fi
|
||||
fi
|
||||
|
||||
# copy ISO file to storage dir
|
||||
# if rotation is enabled and build number is available
|
||||
# save iso to tagged file and symlink to the last build
|
||||
# if rotation is not enabled just copy iso to iso_dir
|
||||
|
||||
if [ "${ROTATE_ISO}" = "yes" -a "${BUILD_NUMBER}" != "" ]; then
|
||||
# copy iso file to shared dir with revision tagged name
|
||||
NEW_BUILD_ISO_PATH="${ISO_PATH#.iso}_${BUILD_NUMBER}.iso"
|
||||
if [ "${DRY_RUN}" = "yes" ]; then
|
||||
echo cp "${ISO}" "${NEW_BUILD_ISO_PATH}"
|
||||
else
|
||||
cp "${ISO}" "${NEW_BUILD_ISO_PATH}"
|
||||
fi
|
||||
ec=$?
|
||||
|
||||
if [ "${ec}" -gt "0" ]; then
|
||||
echo "Error! Copy ${ISO} to ${NEW_BUILD_ISO_PATH} failed!"
|
||||
exit ${COPYISO_ERR}
|
||||
fi
|
||||
|
||||
# create symlink to the last built ISO file
|
||||
if [ "${DRY_RUN}" = "yes" ]; then
|
||||
echo ln -sf "${NEW_BUILD_ISO_PATH}" "${ISO_PATH}"
|
||||
else
|
||||
ln -sf "${NEW_BUILD_ISO_PATH}" "${ISO_PATH}"
|
||||
fi
|
||||
ec=$?
|
||||
|
||||
if [ "${ec}" -gt "0" ]; then
|
||||
echo "Error! Create symlink from ${NEW_BUILD_ISO_PATH} to ${ISO_PATH} failed!"
|
||||
exit ${SYMLINKISO_ERR}
|
||||
fi
|
||||
else
|
||||
# just copy file to shared dir
|
||||
if [ "${DRY_RUN}" = "yes" ]; then
|
||||
echo cp "${ISO}" "${ISO_PATH}"
|
||||
else
|
||||
cp "${ISO}" "${ISO_PATH}"
|
||||
fi
|
||||
ec=$?
|
||||
|
||||
if [ "${ec}" -gt "0" ]; then
|
||||
echo "Error! Copy ${ISO} to ${ISO_PATH} failed!"
|
||||
exit ${COPYISO_ERR}
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "${ec}" -gt "0" ]; then
|
||||
echo "Error! Copy ISO from ${ISO} to ${ISO_PATH} failed!"
|
||||
exit ${COPYISO_ERR}
|
||||
fi
|
||||
echo "Finished building ISO: ${ISO_PATH}"
|
||||
exit 0
|
||||
}
|
||||
|
||||
CdWorkSpace() {
|
||||
# chdir into workspace or fail if could not
|
||||
if [ "${DRY_RUN}" != "yes" ]; then
|
||||
@ -347,6 +243,40 @@ CdWorkSpace() {
|
||||
fi
|
||||
}
|
||||
|
||||
CheckRequirements() {
|
||||
REQUIREMENTS_PATH="${WORKSPACE}/${REQUIREMENTS_DIR}"
|
||||
|
||||
if [ "${UPDATE_REQUIREMENTS}" = "yes" ]; then
|
||||
if [ -f "${REQUIREMENTS_PATH}/requirements.txt" ]; then
|
||||
# Install packages from requirements.txt
|
||||
pip install -r "${REQUIREMENTS_PATH}/requirements.txt"
|
||||
fi
|
||||
|
||||
if [ -f "${REQUIREMENTS_PATH}/requirements-devops.txt" ]; then
|
||||
# Try to install fuel-devops as a package, to controll that
|
||||
# required version of fuel-devops is already installed.
|
||||
# Installation will fail if fuel-devops is not installed or
|
||||
# installed with correct version (until it is not a PyPi package)
|
||||
pip install -r "${REQUIREMENTS_PATH}/requirements-devops.txt"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
ActivateVirtualenv() {
|
||||
if [ -z "${VENV_PATH}" ]; then
|
||||
VENV_PATH="/home/jenkins/venv-nailgun-tests"
|
||||
fi
|
||||
|
||||
# run python virtualenv
|
||||
if [ "${VENV}" = "yes" ]; then
|
||||
if [ "${DRY_RUN}" = "yes" ]; then
|
||||
echo . ${VENV_PATH}/bin/activate
|
||||
else
|
||||
. ${VENV_PATH}/bin/activate
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
RunTest() {
|
||||
# Run test selected by task name
|
||||
|
||||
@ -370,19 +300,6 @@ RunTest() {
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "${VENV_PATH}" ]; then
|
||||
VENV_PATH="/home/jenkins/venv-nailgun-tests"
|
||||
fi
|
||||
|
||||
# run python virtualenv
|
||||
if [ "${VENV}" = "yes" ]; then
|
||||
if [ "${DRY_RUN}" = "yes" ]; then
|
||||
echo . ${VENV_PATH}/bin/activate
|
||||
else
|
||||
. ${VENV_PATH}/bin/activate
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "${ENV_NAME}" = "" ]; then
|
||||
ENV_NAME="${JOB_NAME}_system_test"
|
||||
fi
|
||||
@ -454,26 +371,6 @@ RunTest() {
|
||||
exit "${ec}"
|
||||
}
|
||||
|
||||
RouteTasks() {
|
||||
# this selector defines task names that are recognised by this script
|
||||
# and runs corresponding jobs for them
|
||||
# running any jobs should exit this script
|
||||
|
||||
case "${TASK_NAME}" in
|
||||
test)
|
||||
RunTest
|
||||
;;
|
||||
iso)
|
||||
MakeISO
|
||||
;;
|
||||
*)
|
||||
echo "Unknown task: ${TASK_NAME}!"
|
||||
exit ${INVALIDTASK_ERR}
|
||||
;;
|
||||
esac
|
||||
exit 0
|
||||
}
|
||||
|
||||
# MAIN
|
||||
|
||||
# first we want to get variable from command line options
|
||||
@ -488,5 +385,11 @@ CheckVariables
|
||||
# first we chdir into our working directory unless we dry run
|
||||
CdWorkSpace
|
||||
|
||||
# finally we can choose what to do according to TASK_NAME
|
||||
RouteTasks
|
||||
# Activate python virtual environment
|
||||
ActivateVirtualenv
|
||||
|
||||
# Check/update PyPi requirements
|
||||
CheckRequirements
|
||||
|
||||
# Run the test
|
||||
RunTest
|
||||
|
Loading…
Reference in New Issue
Block a user