From 5df748b2eda5fcf1db4d64d7b19138aba07ba1ea Mon Sep 17 00:00:00 2001 From: Adam Spiers Date: Mon, 19 Aug 2019 11:31:59 +0100 Subject: [PATCH] Make it easier to run a selection of tests relevant to ongoing work During development of a new git commit, locally running a whole unit or functional test suite to check every minor code change is prohibitively expensive. For maximum developer productivity and happiness, it's generally desirable to make the feedback loop of the traditional red/green cycle as quick as possible. So add run-tests-for-diff.sh and run-tests.py to the tools/ subdirectory, using a few tricks as explained below to help with this. run-tests.py takes a list of files on STDIN, filters the list for tests which can be run in the current tox virtualenv, and then runs them with the correct stestr options. run-tests-for-diff.sh is a simple wrapper around run-tests.py which determines which tests to run using output from "git diff". This allows running only the test files changed/added in the working tree: tools/run-tests-for-diff.sh or by a single commit: tools/run-tests-for-diff.sh mybranch^! or a range of commits, e.g. a branch containing a whole patch series for a blueprint: tools/run-tests-for-diff.sh gerrit/master..bp/my-blueprint It supports the same "-HEAD" invocation syntax as flake8wrap.sh (as used by the "fast8" tox environment): tools/run-tests-for-diff.sh -HEAD run-tests.py uses two tricks to make test runs as quick as possible: 1. It's (already) possible to speed up running of tests by source'ing the "activate" file for the desired tox virtualenv, e.g. source .tox/py36/bin/activate and then running stestr directly. This saves a few seconds by skipping the overhead introduced by running tox. 2. When only one test file needs to be run, specifying the -n option to stestr will skip the costly test discovery phase, saving several more valuable seconds. Future commits could build on top of this work, harnessing a framework such as watchdog / watchmedo[0] or Guard[1] in order to automatically run relevant tests every time your editor saves changes to a .py file. [0] https://github.com/gorakhargosh/watchdog - Python-based [1] https://guardgem.org - probably best in class, but Ruby-based so maybe unacceptable for use within Nova. Change-Id: I9a9bda5d29bbb8d8d77f769cd1abf7c42a18c36b --- HACKING.rst | 24 +++++++ doc/source/contributor/testing.rst | 7 ++ tools/run-tests-for-diff.sh | 25 +++++++ tools/run-tests.py | 107 +++++++++++++++++++++++++++++ 4 files changed, 163 insertions(+) create mode 100755 tools/run-tests-for-diff.sh create mode 100755 tools/run-tests.py diff --git a/HACKING.rst b/HACKING.rst index 16664f02fd88..f47c8d32f51b 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -117,6 +117,30 @@ command directly. Running ``stestr run`` will run the entire test suite. tests in parallel). More information about stestr can be found at: http://stestr.readthedocs.io/ +Since when testing locally, running the entire test suite on a regular +basis is prohibitively expensive, the ``tools/run-tests-for-diff.sh`` +script is provided as a convenient way to run selected tests using +output from ``git diff``. For example, this allows running only the +test files changed/added in the working tree:: + + tools/run-tests-for-diff.sh + +However since it passes its arguments directly to ``git diff``, tests +can be selected in lots of other interesting ways, e.g. it can run all +tests affected by a single commit at the tip of a given branch:: + + tools/run-tests-for-diff.sh mybranch^! + +or all those affected by a range of commits, e.g. a branch containing +a whole patch series for a blueprint:: + + tools/run-tests-for-diff.sh gerrit/master..bp/my-blueprint + +It supports the same ``-HEAD`` invocation syntax as ``flake8wrap.sh`` +(as used by the ``fast8`` tox environment):: + + tools/run-tests-for-diff.sh -HEAD + By default tests log at ``INFO`` level. It is possible to make them log at ``DEBUG`` level by exporting the ``OS_DEBUG`` environment variable to ``True``. diff --git a/doc/source/contributor/testing.rst b/doc/source/contributor/testing.rst index 9f67b0308f1f..d64ae30b58ad 100644 --- a/doc/source/contributor/testing.rst +++ b/doc/source/contributor/testing.rst @@ -32,6 +32,13 @@ For details on plans to report the current test coverage, refer to Running tests and reporting results =================================== +Running tests locally +--------------------- + +Please see +https://opendev.org/openstack/nova/src/branch/master/HACKING.rst#running-tests + + Voting in Gerrit ---------------- diff --git a/tools/run-tests-for-diff.sh b/tools/run-tests-for-diff.sh new file mode 100755 index 000000000000..4dc4897964f6 --- /dev/null +++ b/tools/run-tests-for-diff.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +here=`dirname $0` + +if test -z "$1"; then + set -- HEAD +elif test "x$1" = "x-HEAD"; then + # Simulate behaviour from flake8wrap.sh + shift + set -- HEAD~1 +fi + +git diff --name-only "$@" | $here/run-tests.py diff --git a/tools/run-tests.py b/tools/run-tests.py new file mode 100755 index 000000000000..34bc960b5ab6 --- /dev/null +++ b/tools/run-tests.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# +# Copyright (c) 2019 SUSE +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + + +# Filters list of files on STDIN for test files (can be more than one +# per line), and runs any which can be run in the current tox +# virtualenv. Automatically detects which stestr options to use, +# e.g. --test-path=./nova/tests/functional if we're in a virtualenv +# for functional tests. + +import os +import re +import subprocess +import sys + + +def get_tox_env(): + if os.getenv('VIRTUAL_ENV', '').find('/.tox/') == -1: + sys.stderr.write( + """%s should be run within a tox virtualenv. +Please first activate the tox virtualenv you want to use, e.g. + + source .tox/py36/bin/activate +""" % + os.path.realpath(__file__)) + sys.exit(1) + + return os.getenv('VIRTUAL_ENV') + + +def tox_env_is_functional(): + return get_tox_env().find('.tox/functional') != -1 + + +def get_stestr_opts(): + opts = sys.argv[1:] + + if tox_env_is_functional(): + opts = ['--test-path=./nova/tests/functional'] + opts + + return opts + + +def get_test_files(): + test_files = [] + functional = tox_env_is_functional() + + for line in sys.stdin: + files = line.strip().split() + for f in files: + if not re.match(r'^nova/tests/.*\.py$', f): + # In the future we could get really clever and + # map source files to their corresponding tests, + # as is typically done by Guardfile in projects + # which use Guard: https://guardgem.org + continue + + functional_re = r'^nova/tests/functional/' + if functional: + if not re.match(functional_re, f): + continue + else: + if re.match(functional_re, f): + continue + + test_files.append(f[:-3].replace('/', '.')) + + return test_files + + +def main(): + stestr_opts = get_stestr_opts() + test_files = get_test_files() + + if not test_files: + print("No test files found to run") + sys.exit(0) + + if len(test_files) == 1: + # If there's only one module to run (or test therein), we can + # skip discovery which will chop quite a few seconds off the + # runtime. + stestr_opts = ['-n'] + stestr_opts + + cmd = ['stestr', 'run', *stestr_opts] + test_files + print(' '.join(cmd)) + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + print("\nstestr returned non-zero exit code %d\n" % e.returncode) + + +if __name__ == '__main__': + main()