diff --git a/infrared_plugin/plugin.spec b/infrared_plugin/plugin.spec index 3234a9917..8c8425e91 100644 --- a/infrared_plugin/plugin.spec +++ b/infrared_plugin/plugin.spec @@ -117,6 +117,10 @@ subparsers: type: Value help: Test case timeout in seconds ansible_variable: test_case_timeout + test-runner-timeout: + type: Value + help: Test runner timeout in seconds + ansible_variable: test_runner_timeout undercloud_host: type: Value help: inventory hostname of the undercloud host diff --git a/roles/tobiko-configure/defaults/main.yaml b/roles/tobiko-configure/defaults/main.yaml index 6035b0a75..f021d2ee5 100644 --- a/roles/tobiko-configure/defaults/main.yaml +++ b/roles/tobiko-configure/defaults/main.yaml @@ -8,13 +8,15 @@ test_default_conf: testcase: timeout: "{{ test_case_timeout }}" + test_runner_timeout: "{{ test_runner_timeout }}" tripleo: undercloud_ssh_hostname: "{{ undercloud_ssh_hostname }}" test_log_debug: false -test_case_timeout: 7200. +test_case_timeout: 1800. +test_runner_timeout: 14400. # OpenStack client credentials stackrc_file: '{{ ansible_user_dir }}/overcloudrc' diff --git a/roles/tobiko-tox/defaults/main.yaml b/roles/tobiko-tox/defaults/main.yaml index 7d1354063..1cafd17f3 100644 --- a/roles/tobiko-tox/defaults/main.yaml +++ b/roles/tobiko-tox/defaults/main.yaml @@ -15,7 +15,7 @@ tox_step_index: 0 tox_report_name: "{{ test_report_name }}{% if tox_step_index %}_{{ '{:02d}'.format(tox_step_index | int) }}{% endif %}{% if tox_step_name %}_{{ tox_step_name }}{% endif %}{% if tox_envlist %}_{{ tox_envlist }}{% endif %}" -tox_run_tests_timeout: 14400 # 4 hours +tox_run_tests_timeout: 18000 # 5 hours tox_constraints_file: '{{ remote_constraints_file }}' diff --git a/tobiko/common/_testcase.py b/tobiko/common/_testcase.py index 2aa56c42f..93d95a811 100644 --- a/tobiko/common/_testcase.py +++ b/tobiko/common/_testcase.py @@ -15,33 +15,51 @@ from __future__ import absolute_import import os import sys -import typing # noqa +import typing +from oslo_log import log import testtools from tobiko.common import _exception +from tobiko.common import _time +LOG = log.getLogger(__name__) + os.environ.setdefault('PYTHON', sys.executable) +class TestCaseEntry(typing.NamedTuple): + test_case: testtools.TestCase + start_time: float + + class TestCasesManager(object): + start_time: _time.Seconds = None + def __init__(self): - self._test_cases: typing.List[testtools.TestCase] = [] + self._test_cases: typing.List[TestCaseEntry] = [] def get_test_case(self) -> testtools.TestCase: try: - return self._test_cases[-1] + return self._test_cases[-1].test_case except IndexError: return DUMMY_TEST_CASE def pop_test_case(self) -> testtools.TestCase: - return self._test_cases.pop() + entry = self._test_cases.pop() + elapsed_time = _time.time() - entry.start_time + LOG.debug(f"Exit test case '{entry.test_case.id()}' after " + f"{elapsed_time} seconds") + return entry.test_case def push_test_case(self, test_case: testtools.TestCase): _exception.check_valid_type(test_case, testtools.TestCase) - self._test_cases.append(test_case) + entry = TestCaseEntry(test_case=test_case, + start_time=_time.time()) + self._test_cases.append(entry) + LOG.debug(f"Enter test case '{test_case.id()}'") TEST_CASES = TestCasesManager() diff --git a/tobiko/config.py b/tobiko/config.py index 6ce12c08f..e30f801a1 100644 --- a/tobiko/config.py +++ b/tobiko/config.py @@ -66,7 +66,11 @@ TESTCASE_OPTIONS = [ cfg.FloatOpt('timeout', default=None, help=("Timeout (in seconds) used for interrupting test case " - "execution"))] + "execution")), + cfg.FloatOpt('test_runner_timeout', + default=None, + help=("Timeout (in seconds) used for interrupting test " + "runner execution"))] def workspace_config_files(project=None, prog=None): diff --git a/tobiko/tests/conftest.py b/tobiko/tests/conftest.py index a0ad67f9f..29c013622 100644 --- a/tobiko/tests/conftest.py +++ b/tobiko/tests/conftest.py @@ -22,6 +22,7 @@ from oslo_log import log from py.xml import html # pylint: disable=no-name-in-module,import-error import pytest +import tobiko LOG = log.getLogger(__name__) @@ -47,7 +48,6 @@ def configure_metadata(config): def configure_caplog(config): - import tobiko tobiko_config = tobiko.tobiko_config() if tobiko_config.logging.capture_log is True: @@ -93,8 +93,40 @@ def set_default_inicfg(config, key, default): LOG.debug(f"Keep existing inicfg: {key} = {value}") +class TestRunnerTimeoutManager(tobiko.SharedFixture): + timeout: tobiko.Seconds = None + deadline: tobiko.Seconds = None + + def setup_fixture(self): + tobiko_config = tobiko.tobiko_config() + self.timeout = tobiko_config.testcase.test_runner_timeout + if self.timeout is None: + LOG.info('Test runner timeout is disabled') + else: + LOG.info('Test runner timeout is enabled: ' + f'timeout is {self.timeout} seconds') + self.deadline = tobiko.time() + self.timeout + + @classmethod + def check_test_runner_timeout(cls): + self = tobiko.setup_fixture(cls) + if self.deadline is not None: + time_left = self.deadline - tobiko.time() + if time_left <= 0.: + pytest.skip( + f"Test runner execution timed out after {self.timeout} " + f"seconds", + allow_module_level=True) + else: + LOG.debug('Test runner timeout is enabled: ' + f'{time_left} seconds left') + + +def check_test_runner_timeout(): + TestRunnerTimeoutManager.check_test_runner_timeout() + + def configure_timeout(config): - import tobiko tobiko_config = tobiko.tobiko_config() default = tobiko_config.testcase.timeout if default is not None and default > 0.: @@ -128,7 +160,7 @@ def pytest_html_report_title(report): @pytest.hookimpl(hookwrapper=True) def pytest_runtest_call(item): # pylint: disable=protected-access - import tobiko + check_test_runner_timeout() tobiko.push_test_case(item._testcase) try: yield