From 61a63321859b37ffb9132db5789aaf3eade6eef6 Mon Sep 17 00:00:00 2001 From: Federico Ressi Date: Fri, 9 Jul 2021 11:34:50 +0200 Subject: [PATCH] Add option to skip all remaining test cases after a given timeout is expired After some given seconds of timeout test runner execution self-interrupt to avoid hitting jobs timeout. The problem is that when it occurs all remaining test cases that didn't started yet are missing from the reports. This changes things by marking such test cases as skipped, so that we can see it from test results. Change-Id: Idd0064812fc5a2da295712914c4531bdef466c24 --- infrared_plugin/plugin.spec | 4 +++ roles/tobiko-configure/defaults/main.yaml | 4 ++- roles/tobiko-tox/defaults/main.yaml | 2 +- tobiko/common/_testcase.py | 28 ++++++++++++++--- tobiko/config.py | 6 +++- tobiko/tests/conftest.py | 38 +++++++++++++++++++++-- 6 files changed, 71 insertions(+), 11 deletions(-) diff --git a/infrared_plugin/plugin.spec b/infrared_plugin/plugin.spec index 3234a9917..8c8425e91 100644 --- a/infrared_plugin/plugin.spec +++ b/infrared_plugin/plugin.spec @@ -117,6 +117,10 @@ subparsers: type: Value help: Test case timeout in seconds ansible_variable: test_case_timeout + test-runner-timeout: + type: Value + help: Test runner timeout in seconds + ansible_variable: test_runner_timeout undercloud_host: type: Value help: inventory hostname of the undercloud host diff --git a/roles/tobiko-configure/defaults/main.yaml b/roles/tobiko-configure/defaults/main.yaml index 6035b0a75..f021d2ee5 100644 --- a/roles/tobiko-configure/defaults/main.yaml +++ b/roles/tobiko-configure/defaults/main.yaml @@ -8,13 +8,15 @@ test_default_conf: testcase: timeout: "{{ test_case_timeout }}" + test_runner_timeout: "{{ test_runner_timeout }}" tripleo: undercloud_ssh_hostname: "{{ undercloud_ssh_hostname }}" test_log_debug: false -test_case_timeout: 7200. +test_case_timeout: 1800. +test_runner_timeout: 14400. # OpenStack client credentials stackrc_file: '{{ ansible_user_dir }}/overcloudrc' diff --git a/roles/tobiko-tox/defaults/main.yaml b/roles/tobiko-tox/defaults/main.yaml index 7d1354063..1cafd17f3 100644 --- a/roles/tobiko-tox/defaults/main.yaml +++ b/roles/tobiko-tox/defaults/main.yaml @@ -15,7 +15,7 @@ tox_step_index: 0 tox_report_name: "{{ test_report_name }}{% if tox_step_index %}_{{ '{:02d}'.format(tox_step_index | int) }}{% endif %}{% if tox_step_name %}_{{ tox_step_name }}{% endif %}{% if tox_envlist %}_{{ tox_envlist }}{% endif %}" -tox_run_tests_timeout: 14400 # 4 hours +tox_run_tests_timeout: 18000 # 5 hours tox_constraints_file: '{{ remote_constraints_file }}' diff --git a/tobiko/common/_testcase.py b/tobiko/common/_testcase.py index 2aa56c42f..93d95a811 100644 --- a/tobiko/common/_testcase.py +++ b/tobiko/common/_testcase.py @@ -15,33 +15,51 @@ from __future__ import absolute_import import os import sys -import typing # noqa +import typing +from oslo_log import log import testtools from tobiko.common import _exception +from tobiko.common import _time +LOG = log.getLogger(__name__) + os.environ.setdefault('PYTHON', sys.executable) +class TestCaseEntry(typing.NamedTuple): + test_case: testtools.TestCase + start_time: float + + class TestCasesManager(object): + start_time: _time.Seconds = None + def __init__(self): - self._test_cases: typing.List[testtools.TestCase] = [] + self._test_cases: typing.List[TestCaseEntry] = [] def get_test_case(self) -> testtools.TestCase: try: - return self._test_cases[-1] + return self._test_cases[-1].test_case except IndexError: return DUMMY_TEST_CASE def pop_test_case(self) -> testtools.TestCase: - return self._test_cases.pop() + entry = self._test_cases.pop() + elapsed_time = _time.time() - entry.start_time + LOG.debug(f"Exit test case '{entry.test_case.id()}' after " + f"{elapsed_time} seconds") + return entry.test_case def push_test_case(self, test_case: testtools.TestCase): _exception.check_valid_type(test_case, testtools.TestCase) - self._test_cases.append(test_case) + entry = TestCaseEntry(test_case=test_case, + start_time=_time.time()) + self._test_cases.append(entry) + LOG.debug(f"Enter test case '{test_case.id()}'") TEST_CASES = TestCasesManager() diff --git a/tobiko/config.py b/tobiko/config.py index 6ce12c08f..e30f801a1 100644 --- a/tobiko/config.py +++ b/tobiko/config.py @@ -66,7 +66,11 @@ TESTCASE_OPTIONS = [ cfg.FloatOpt('timeout', default=None, help=("Timeout (in seconds) used for interrupting test case " - "execution"))] + "execution")), + cfg.FloatOpt('test_runner_timeout', + default=None, + help=("Timeout (in seconds) used for interrupting test " + "runner execution"))] def workspace_config_files(project=None, prog=None): diff --git a/tobiko/tests/conftest.py b/tobiko/tests/conftest.py index a0ad67f9f..29c013622 100644 --- a/tobiko/tests/conftest.py +++ b/tobiko/tests/conftest.py @@ -22,6 +22,7 @@ from oslo_log import log from py.xml import html # pylint: disable=no-name-in-module,import-error import pytest +import tobiko LOG = log.getLogger(__name__) @@ -47,7 +48,6 @@ def configure_metadata(config): def configure_caplog(config): - import tobiko tobiko_config = tobiko.tobiko_config() if tobiko_config.logging.capture_log is True: @@ -93,8 +93,40 @@ def set_default_inicfg(config, key, default): LOG.debug(f"Keep existing inicfg: {key} = {value}") +class TestRunnerTimeoutManager(tobiko.SharedFixture): + timeout: tobiko.Seconds = None + deadline: tobiko.Seconds = None + + def setup_fixture(self): + tobiko_config = tobiko.tobiko_config() + self.timeout = tobiko_config.testcase.test_runner_timeout + if self.timeout is None: + LOG.info('Test runner timeout is disabled') + else: + LOG.info('Test runner timeout is enabled: ' + f'timeout is {self.timeout} seconds') + self.deadline = tobiko.time() + self.timeout + + @classmethod + def check_test_runner_timeout(cls): + self = tobiko.setup_fixture(cls) + if self.deadline is not None: + time_left = self.deadline - tobiko.time() + if time_left <= 0.: + pytest.skip( + f"Test runner execution timed out after {self.timeout} " + f"seconds", + allow_module_level=True) + else: + LOG.debug('Test runner timeout is enabled: ' + f'{time_left} seconds left') + + +def check_test_runner_timeout(): + TestRunnerTimeoutManager.check_test_runner_timeout() + + def configure_timeout(config): - import tobiko tobiko_config = tobiko.tobiko_config() default = tobiko_config.testcase.timeout if default is not None and default > 0.: @@ -128,7 +160,7 @@ def pytest_html_report_title(report): @pytest.hookimpl(hookwrapper=True) def pytest_runtest_call(item): # pylint: disable=protected-access - import tobiko + check_test_runner_timeout() tobiko.push_test_case(item._testcase) try: yield