From 4da9f90d8491aa9af495a9f6b515a8d89ff618b6 Mon Sep 17 00:00:00 2001 From: Vladimir Kozhukalov Date: Thu, 9 Jun 2016 21:23:08 +0300 Subject: [PATCH] Use fuel-web nailgun start/stop scripts It is better to use the same code to start/stop nailgun for all functional test jobs. fuel-web tox.ini provides envs to start and stop nailgun and we'd better use them before running fuelclient functional tests. Change-Id: Ic385d37a1dced4a19bb6068562cb590d4f1db7bb --- fuelclient/tests/functional/base.py | 12 ++- run_tests.sh | 18 ---- tools/cleanup.sh | 60 ----------- tools/env.sh | 49 +++++++++ tools/env_functions.sh | 78 ++++++++++++++ tools/prepare_nailgun.sh | 155 ---------------------------- tox.ini | 58 ++++++----- 7 files changed, 168 insertions(+), 262 deletions(-) delete mode 100755 run_tests.sh delete mode 100644 tools/cleanup.sh create mode 100755 tools/env.sh create mode 100644 tools/env_functions.sh delete mode 100644 tools/prepare_nailgun.sh diff --git a/fuelclient/tests/functional/base.py b/fuelclient/tests/functional/base.py index 1933830b..33fe9182 100644 --- a/fuelclient/tests/functional/base.py +++ b/fuelclient/tests/functional/base.py @@ -51,6 +51,7 @@ class BaseTestCase(oslo_base.BaseTestCase): handler = '' nailgun_root = os.environ.get('NAILGUN_ROOT', '/tmp/fuel_web/nailgun') + fuel_web_root = os.environ.get('FUEL_WEB_ROOT', '/tmp/fuel_web') def setUp(self): super(BaseTestCase, self).setUp() @@ -82,16 +83,17 @@ class BaseTestCase(oslo_base.BaseTestCase): @classmethod def reload_nailgun_server(cls): for action in ("dropdb", "syncdb", "loaddefault"): - cmd = 'tox -evenv -- manage.py %s' % action - cls.run_command(cmd, cwd=cls.nailgun_root) + cmd = 'tox -evenv -- {0}/manage.py {1}'.format( + cls.nailgun_root, action) + cls.run_command(cmd, cwd=cls.fuel_web_root) @classmethod def load_data_to_nailgun_server(cls): file_path = os.path.join(cls.nailgun_root, 'nailgun/fixtures/sample_environment.json') - - cmd = 'tox -evenv -- manage.py loaddata %s' % file_path - cls.run_command(cmd, cwd=cls.nailgun_root) + cmd = 'tox -evenv -- {0}/manage.py loaddata {1}'.format( + cls.nailgun_root, file_path) + cls.run_command(cmd, cwd=cls.fuel_web_root) def run_cli_command(self, command_line, check_errors=True, env=os.environ.copy()): diff --git a/run_tests.sh b/run_tests.sh deleted file mode 100755 index 3d10930d..00000000 --- a/run_tests.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Copyright 2015 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# THIS FILE WILL BE REMOVED AS SOON AS FUEL-CI IS UPDATED. -tox diff --git a/tools/cleanup.sh b/tools/cleanup.sh deleted file mode 100644 index 97e09b00..00000000 --- a/tools/cleanup.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -# Copyright 2015 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -eu - -NAILGUN_CONFIG=$ARTIFACTS/test.yaml - - -# Sends SIGING to the running instance of Nailgun, if it exists -kill_server() { - echo "Stopping Nailgun and waiting $NAILGUN_START_MAX_WAIT_TIME seconds." - - local pid=$(lsof -ti tcp:$NAILGUN_PORT) - if [[ -n "$pid" ]]; then - kill $pid - sleep $NAILGUN_START_MAX_WAIT_TIME - fi -} - - -drop_database () { - echo "Dropping the database." - - if [[ -f "$NAILGUN_CONFIG" ]] && [[ -d $NAILGUN_ROOT ]]; then - pushd $NAILGUN_ROOT > /dev/null - tox -e venv -- python manage.py dropdb > /dev/null - popd > /dev/null - fi -} - - -delete_files() { - echo "Deleting the files." - rm -rf "$ARTIFACTS" - - if [[ "$FUEL_WEB_CLONE" == "yes" ]] && [[ -d "$FUEL_WEB_ROOT" ]]; then - rm -rf "$FUEL_WEB_ROOT" - fi -} - - -echo "Doing a clean up to ensure clean environment." - -kill_server -drop_database -delete_files - diff --git a/tools/env.sh b/tools/env.sh new file mode 100755 index 00000000..96dbe281 --- /dev/null +++ b/tools/env.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -eu + +. $(dirname $0)/env_functions.sh + +case $1 in + prepare_nailgun) + prepare_nailgun + ;; + cleanup_nailgun) + cleanup_nailgun + ;; + prepare_fuelclient) + prepare_fuelclient_config + ;; + cleanup_fuelclient) + cleanup_fuelclient_config + ;; + prepare_fuel_web_repo) + prepare_fuel_web_repo + ;; + cleanup_fuel_web_repo) + cleanup_fuel_web_repo + ;; + *) + echo "Not supported subcommand. Available subcommands: " + echo "cleanup_nailgun" + echo "prepare_nailgun" + echo "cleanup_fuelclient" + echo "prepare_fuelclient" + echo "cleanup_fuel_web_repo" + echo "prepare_fuel_web_repo" + exit 1 + ;; +esac diff --git a/tools/env_functions.sh b/tools/env_functions.sh new file mode 100644 index 00000000..010dacfe --- /dev/null +++ b/tools/env_functions.sh @@ -0,0 +1,78 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +function cleanup_nailgun { + echo "Cleaning up nailgun." + pushd "$FUEL_WEB_ROOT" > /dev/null + tox -e stop || echo "Error while stopping nailgun." + popd > /dev/null + + pushd "$FUEL_WEB_ROOT" > /dev/null + tox -e cleanup + popd > /dev/null +} + +function prepare_nailgun { + echo "Preparing nailgun." + pushd "$FUEL_WEB_ROOT" > /dev/null + tox -e start + popd > /dev/null +} + +function prepare_fuelclient_config { + echo "Creating fuelclient config file $FUELCLIENT_CUSTOM_SETTINGS" + mkdir -p $(dirname $FUELCLIENT_CUSTOM_SETTINGS) + cat > $FUELCLIENT_CUSTOM_SETTINGS < /dev/null + + if [[ -n $FUEL_WEB_FETCH_REPO ]]; then + git fetch "$FUEL_WEB_FETCH_REPO" "$FUEL_WEB_FETCH_REFSPEC" || \ + { echo "Failed to fetch $FUEL_WEB_FETCH_REPO"; exit 1; } + fi + + git checkout "$FUEL_WEB_COMMIT" || \ + { echo "Failed to checkout to $FUEL_WEB_COMMIT"; exit 1; } + + popd > /dev/null + +} + +function cleanup_fuel_web_repo { + echo "Removing $FUEL_WEB_ROOT directory." + + if [[ "$FUEL_WEB_CLONE" == "yes" ]]; then + rm -rf $FUEL_WEB_ROOT + fi +} diff --git a/tools/prepare_nailgun.sh b/tools/prepare_nailgun.sh deleted file mode 100644 index 5834ea13..00000000 --- a/tools/prepare_nailgun.sh +++ /dev/null @@ -1,155 +0,0 @@ -#!/bin/bash - -# Copyright 2015 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -eu - -NAILGUN_CONFIG=$ARTIFACTS/test.yaml - -err() { - printf "%s\n" "$1" >&2 -} - -msg() { - printf "%s\n" "$1" -} - -# Synchronizes the schema of the database and loads default data. -setup_db() { - msg "Setting up database." - - pushd $NAILGUN_ROOT > /dev/null - tox -e venv -- python manage.py syncdb > /dev/null - tox -e venv -- python manage.py loaddefault > /dev/null - - popd > /dev/null -} - - -# Returns: a server pid, that you have to close manually -run_server() { - - local run_server_cmd="\ - python manage.py run \ - --port=$NAILGUN_PORT \ - --config=$NAILGUN_CONFIG \ - --fake-tasks \ - --fake-tasks-tick-count=80 \ - --fake-tasks-tick-interval=1" - local server_log=$(mktemp $ARTIFACTS/test_nailgun_cli_server.XXXX) - local check_url="http://0.0.0.0:${NAILGUN_PORT}${NAILGUN_CHECK_PATH}" - - # run new server instance - pushd $NAILGUN_ROOT > /dev/null - tox -e venv -- $run_server_cmd >> $server_log 2>&1 & - popd > /dev/null - - # Wait for server's availability - which curl > /dev/null - - if [[ $? -eq 0 ]]; then - local num_retries=$((NAILGUN_START_MAX_WAIT_TIME * 10)) - local i=0 - - while true; do - # Fail if number of retries exeeded - if [[ $i -gt $((num_retries + 1)) ]]; then return 1; fi - - local http_code=$(curl -s -w %{http_code} -o /dev/null $check_url) - - if [[ "$http_code" != "000" ]]; then return 0; fi - - sleep 0.1 - i=$((i + 1)) - done - else - err "Failed to start Nailgun in ${NAILGUN_START_MAX_WAIT_TIME} seconds." - exit 1 - fi -} - - -# Set up test environment -prepare_env() { - mkdir -p $ARTIFACTS -} - - -# Generates server configuration file -create_settings_yaml() { - cat > $NAILGUN_CONFIG < $FUELCLIENT_CUSTOM_SETTINGS < /dev/null - - echo $FETCH_REPO - - if [[ -n $FETCH_REPO ]]; then - err "Fetching changes from $FETCH_REPO $FETCH_REFSPEC" - git fetch "$FETCH_REPO" "$FETCH_REFSPEC" || \ - { err "Failed to pull changes"; exit 1; } - fi - - git checkout "$FUEL_COMMIT" || \ - { err "Failed to checkout to $FUEL_COMMIT"; exit 1; } - - popd > /dev/null -} - - -prepare_env -obtain_nailgun -create_settings_yaml -setup_db -run_server diff --git a/tox.ini b/tox.ini index 4c8cbfc8..da9bf49c 100644 --- a/tox.ini +++ b/tox.ini @@ -3,45 +3,55 @@ minversion = 2.1 skipsdist = True envlist = py27,pep8,functional,cleanup +[base] +ARTS = {env:ARTS:test_run} + +[common] +changedir={toxinidir}/nailgun +setenv = + NAILGUN_DB={env:TEST_NAILGUN_DB:nailgun} + NAILGUN_DB_USER={env:NAILGUN_DB_USER:nailgun} + NAILGUN_DB_USERPW={env:NAILGUN_DB_USERPW:nailgun} + NAILGUN_DB_PREPARE={env:NAILGUN_DB_PREPARE:no} + + FUEL_WEB_CLONE={env:FUEL_WEB_CLONE:yes} + FUEL_WEB_REPO={env:FUEL_WEB_REPO:https://github.com/openstack/fuel-web.git} + FUEL_WEB_COMMIT={env:FUEL_COMMIT:stable/mitaka} + FUEL_WEB_ROOT={env:FUEL_WEB_ROOT:/tmp/fuel_web} + FUEL_WEB_FETCH_REPO={env:FUEL_WEB_FETCH_REPO:} + FUEL_WEB_FETCH_REFSPEC={env:FUEL_WEB_FETCH_REFSPEC:} + + NAILGUN_PORT={env:NAILGUN_PORT:8000} + NAILGUN_ROOT={env:FUEL_WEB_ROOT}/nailgun + FUELCLIENT_CUSTOM_SETTINGS={[base]ARTS}/fuelclient_custom_settings.yaml + [testenv] usedevelop = True whitelist_externals = bash python setenv = VIRTUAL_ENV={envdir} - ARTIFACTS={toxinidir}/{env:ARTIFACTS:test_run} - FUELCLIENT_JUNIT={env:FUELCLIENT_JUNIT:fuelclient}-{envname}.xml - FUELCLIENT_CUSTOM_SETTINGS={env:ARTIFACTS:test_run}/fuel_client_config.yaml - - # Functional env settings - FUEL_WEB_CLONE={env:FUEL_WEB_CLONE:yes} - FUEL_WEB_REPO={env:FUEL_WEB_REPO:https://github.com/openstack/fuel-web.git} - FUEL_WEB_ROOT={env:FUEL_WEB_ROOT:/tmp/fuel_web} - FETCH_REPO={env:FETCH_REPO:} - FETCH_REFSPEC={env:FETCH_REFSPEC:} - FUEL_COMMIT={env:FUEL_COMMIT:stable/mitaka} - NAILGUN_ROOT={env:FUEL_WEB_ROOT:/tmp/fuel_web}/nailgun - - # Nailgun server parameters - NAILGUN_PORT={env:NAILGUN_PORT:8003} - NAILGUN_CHECK_PATH={env:NAILGUN_CHECK_PATH:/api/version} - NAILGUN_START_MAX_WAIT_TIME={env:NAILGUN_START_MAX_WAIT_TIME:20} - TEST_NAILGUN_DB={env:TEST_NAILGUN_DB:nailgun} + {[common]setenv} +passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt - -commands = ostestr --serial {posargs} +commands = + ostestr --serial {posargs} [testenv:functional] -setenv = {[testenv]setenv} +setenv = VIRTUAL_ENV={envdir} + {[common]setenv} OS_TEST_PATH={toxinidir}/fuelclient/tests/functional commands = - bash {toxinidir}/tools/cleanup.sh - bash {toxinidir}/tools/prepare_nailgun.sh + bash -c "{toxinidir}/tools/env.sh prepare_fuel_web_repo" + bash -c "{toxinidir}/tools/env.sh prepare_nailgun" + bash -c "{toxinidir}/tools/env.sh prepare_fuelclient" ostestr --serial {posargs} [testenv:cleanup] commands = - bash {toxinidir}/tools/cleanup.sh + bash -c "{toxinidir}/tools/env.sh cleanup_nailgun" + bash -c "{toxinidir}/tools/env.sh cleanup_fuel_web_repo" + bash -c "{toxinidir}/tools/env.sh cleanup_fuelclient" bash -c "find {toxinidir} -name \"*.pyc\" -delete" [testenv:pep8]