From 7453dd6f4f5bb2f01a6784a5d7ba84818001babb Mon Sep 17 00:00:00 2001 From: Federico Ressi Date: Fri, 25 Mar 2022 11:46:17 +0100 Subject: [PATCH] Fix test runner - disable forked execution (by default) - accept string as test path - raise FileNotFoundError when test case files are not found - create a new test result instance when any is given - raise RunTestCasesFailed after test case errors or failures - use 'spawn' context to create workers pool Change-Id: Ifde3d43d023e7508ca099bee0a9e9bab0c639789 --- tobiko/run/__init__.py | 5 +++ tobiko/run/_config.py | 2 +- tobiko/run/_discover.py | 2 +- tobiko/run/_find.py | 26 ++++++++----- tobiko/run/_run.py | 48 +++++++++--------------- tobiko/run/_worker.py | 12 +++--- tobiko/tests/functional/run/test_run.py | 50 +++++++++++++++++++++++++ 7 files changed, 99 insertions(+), 46 deletions(-) create mode 100644 tobiko/tests/functional/run/test_run.py diff --git a/tobiko/run/__init__.py b/tobiko/run/__init__.py index cff92ad67..da6b283f0 100644 --- a/tobiko/run/__init__.py +++ b/tobiko/run/__init__.py @@ -17,9 +17,14 @@ from __future__ import absolute_import from tobiko.run import _discover from tobiko.run import _find +from tobiko.run import _run + discover_test_ids = _discover.discover_test_ids find_test_ids = _discover.find_test_ids forked_discover_test_ids = _discover.forked_discover_test_ids find_test_files = _find.find_test_files + +run_tests = _run.run_tests +run_test_ids = _run.run_test_ids diff --git a/tobiko/run/_config.py b/tobiko/run/_config.py index b2285e9ec..c768513b7 100644 --- a/tobiko/run/_config.py +++ b/tobiko/run/_config.py @@ -36,7 +36,7 @@ class RunConfigFixture(tobiko.SharedFixture): @property def forked(self) -> bool: - return self.workers_count != 1 + return self.workers_count is not None and self.workers_count != 1 def run_confing(obj=None) -> RunConfigFixture: diff --git a/tobiko/run/_discover.py b/tobiko/run/_discover.py index 87654c8a4..9ee1f9fcf 100644 --- a/tobiko/run/_discover.py +++ b/tobiko/run/_discover.py @@ -32,7 +32,7 @@ from tobiko.run import _worker LOG = log.getLogger(__name__) -def find_test_ids(test_path: typing.Iterable[str], +def find_test_ids(test_path: typing.Union[str, typing.Iterable[str]], test_filename: str = None, python_path: typing.Iterable[str] = None, forked: bool = None, diff --git a/tobiko/run/_find.py b/tobiko/run/_find.py index a96a6f69d..cfc6d2b92 100644 --- a/tobiko/run/_find.py +++ b/tobiko/run/_find.py @@ -28,15 +28,17 @@ from tobiko.run import _config LOG = log.getLogger(__name__) -def find_test_files(test_path: typing.Iterable[str] = None, +def find_test_files(test_path: typing.Union[str, typing.Iterable[str]] = None, test_filename: str = None, config: _config.RunConfigFixture = None) \ -> typing.List[str]: config = _config.run_confing(config) - if test_path: - test_path = list(test_path) - if not test_path: + if test_path is None: test_path = config.test_path + elif isinstance(test_path, str): + test_path = [test_path] + else: + test_path = list(test_path) if not test_filename: test_filename = config.test_filename test_files: typing.List[str] = [] @@ -56,14 +58,20 @@ def find_test_files(test_path: typing.Iterable[str] = None, LOG.debug("Find test files...\n" f" dir: '{find_dir}'\n" - f" name: '{find_name}'\n") - output = subprocess.check_output( - ['find', find_dir, '-name', find_name], - universal_newlines=True) + f" name: '{find_name}'") + try: + output = subprocess.check_output( + ['find', find_dir, '-name', find_name], + universal_newlines=True) + except subprocess.CalledProcessError as ex: + LOG.exception("Test files not found.") + raise FileNotFoundError('Test files not found: \n' + f" dir: '{find_dir}'\n" + f" name: '{find_name}'") from ex + for line in output.splitlines(): line = line.strip() if line: - assert os.path.isfile(line) test_files.append(line) LOG.debug("Found test file(s):\n" diff --git a/tobiko/run/_run.py b/tobiko/run/_run.py index 0e0f3e90e..1cea98526 100644 --- a/tobiko/run/_run.py +++ b/tobiko/run/_run.py @@ -25,32 +25,31 @@ from oslo_log import log import tobiko from tobiko.run import _config from tobiko.run import _discover -from tobiko.run import _result -from tobiko.run import _worker LOG = log.getLogger(__name__) -def run_tests(test_path: typing.Iterable[str], +def run_tests(test_path: typing.Union[str, typing.Iterable[str]], test_filename: str = None, python_path: typing.Iterable[str] = None, - forked: bool = None, - config: _config.RunConfigFixture = None): + config: _config.RunConfigFixture = None, + result: unittest.TestResult = None): test_ids = _discover.find_test_ids(test_path=test_path, test_filename=test_filename, python_path=python_path, - forked=forked, config=config) - if forked: - forked_run_test_ids(test_ids=test_ids) - else: - run_test_ids(test_ids=test_ids) + return run_test_ids(test_ids=test_ids, result=result) -def run_test_ids(test_ids: typing.Iterable[str]) -> int: +def run_test_ids(test_ids: typing.List[str], + result: unittest.TestResult = None) \ + -> int: test_classes: typing.Dict[str, typing.List[str]] = \ collections.defaultdict(list) + # run the test suite + if result is None: + result = unittest.TestResult() # regroup test ids my test class keeping test names order test_ids = list(test_ids) @@ -66,41 +65,30 @@ def run_test_ids(test_ids: typing.Iterable[str]) -> int: test = test_class(test_name) suite.addTest(test) - # run the test suite - result = tobiko.setup_fixture(_result.TestResultFixture()).result LOG.info(f'Run {len(test_ids)} test(s)') suite.run(result) LOG.info(f'{result.testsRun} test(s) run') + if result.testsRun and (result.errors or result.failures): + raise RunTestCasesFailed( + errors='\n'.join(str(e) for e in result.errors), + failures='\n'.join(str(e) for e in result.failures)) return result.testsRun -def forked_run_test_ids(test_ids: typing.Iterable[str]) -> int: - test_classes: typing.Dict[str, typing.List[str]] = \ - collections.defaultdict(list) - test_ids = list(test_ids) - LOG.info(f'Run {len(test_ids)} test(s)') - for test_id in test_ids: - test_class_id, _ = test_id.rsplit('.', 1) - test_classes[test_class_id].append(test_id) - results = [_worker.call_async(run_test_ids, test_ids=grouped_ids) - for _, grouped_ids in sorted(test_classes.items())] - count = 0 - for result in results: - count += result.get() - LOG.info(f'{count} test(s) run') - return count +class RunTestCasesFailed(tobiko.TobikoException): + message = ('Test case execution failed:\n' + '{errors}\n' + '{failures}\n') def main(test_path: typing.Iterable[str] = None, test_filename: str = None, - forked: bool = False, python_path: typing.Iterable[str] = None): if test_path is None: test_path = sys.argv[1:] try: run_tests(test_path=test_path, test_filename=test_filename, - forked=forked, python_path=python_path) except Exception: LOG.exception("Error running test cases") diff --git a/tobiko/run/_worker.py b/tobiko/run/_worker.py index 17ae4a92b..07ce353d1 100644 --- a/tobiko/run/_worker.py +++ b/tobiko/run/_worker.py @@ -15,7 +15,8 @@ # under the License. from __future__ import absolute_import -import multiprocessing.pool +import multiprocessing +from multiprocessing import pool import typing import tobiko @@ -26,7 +27,7 @@ class WorkersPoolFixture(tobiko.SharedFixture): config = tobiko.required_fixture(_config.RunConfigFixture) - pool: multiprocessing.pool.Pool + pool: pool.Pool workers_count: int = 0 def __init__(self, workers_count: int = None): @@ -39,14 +40,15 @@ class WorkersPoolFixture(tobiko.SharedFixture): if not workers_count: workers_count = self.config.workers_count self.workers_count = workers_count or 0 - self.pool = multiprocessing.pool.Pool(processes=workers_count or None) + context = multiprocessing.get_context('spawn') + self.pool = context.Pool(processes=workers_count or None) -def workers_pool() -> multiprocessing.pool.Pool: +def workers_pool() -> pool.Pool: return tobiko.setup_fixture(WorkersPoolFixture).pool def call_async(func: typing.Callable, *args, - **kwargs) -> multiprocessing.pool.AsyncResult: + **kwargs): return workers_pool().apply_async(func, args=args, kwds=kwargs) diff --git a/tobiko/tests/functional/run/test_run.py b/tobiko/tests/functional/run/test_run.py new file mode 100644 index 000000000..fa6f0b90b --- /dev/null +++ b/tobiko/tests/functional/run/test_run.py @@ -0,0 +1,50 @@ +# Copyright (c) 2022 Red Hat, Inc. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import functools +import os +import typing +import unittest + +import testtools + +from tobiko import run + + +def nested_test_case(test_method: typing.Callable[[testtools.TestCase], None]): + + @functools.wraps(test_method) + def wrapper(self: unittest.TestCase): + nested_counter = int(os.environ.get('NESTED_TEST_CASE', 0)) + if not nested_counter: + os.environ['NESTED_TEST_CASE'] = str(nested_counter + 1) + try: + test_method(self) + finally: + if nested_counter: + os.environ['NESTED_TEST_CASE'] = str(nested_counter) + else: + os.environ.pop('NESTED_TEST_CASE') + return wrapper + + +class RunTestsTest(unittest.TestCase): + + @nested_test_case + def test_run_tests(self): + result = run.run_tests(__file__) + self.assertGreater(result, 0)