Merge "Add option to skip all remaining test cases after a given timeout is expired"

infrared_plugin/plugin.spec

@@ -117,6 +117,10 @@ subparsers:
             type: Value
             help: Test case timeout in seconds
             ansible_variable: test_case_timeout
+          test-runner-timeout:
+            type: Value
+            help: Test runner timeout in seconds
+            ansible_variable: test_runner_timeout
           undercloud_host:
             type: Value
             help: inventory hostname of the undercloud host

roles/tobiko-configure/defaults/main.yaml

@@ -8,13 +8,15 @@ test_default_conf:
 
   testcase:
     timeout: "{{ test_case_timeout }}"
+    test_runner_timeout: "{{ test_runner_timeout }}"
 
   tripleo:
     undercloud_ssh_hostname: "{{ undercloud_ssh_hostname }}"
 
 test_log_debug: false
 
-test_case_timeout: 7200.
+test_case_timeout: 1800.
+test_runner_timeout: 14400.
 
 # OpenStack client credentials
 stackrc_file: '{{ ansible_user_dir }}/overcloudrc'

roles/tobiko-tox/defaults/main.yaml

@@ -15,7 +15,7 @@ tox_step_index: 0
 tox_report_name:
   "{{ test_report_name }}{% if tox_step_index %}_{{ '{:02d}'.format(tox_step_index | int) }}{% endif %}{% if tox_step_name %}_{{ tox_step_name }}{% endif %}{% if tox_envlist %}_{{ tox_envlist }}{% endif %}"
 
-tox_run_tests_timeout: 14400  # 4 hours
+tox_run_tests_timeout: 18000  # 5 hours
 
 tox_constraints_file: '{{ remote_constraints_file }}'
 

tobiko/common/_testcase.py

@@ -15,33 +15,51 @@ from __future__ import absolute_import
 
 import os
 import sys
-import typing  # noqa
+import typing
 
+from oslo_log import log
 import testtools
 
 from tobiko.common import _exception
+from tobiko.common import _time
 
 
+LOG = log.getLogger(__name__)
+
 os.environ.setdefault('PYTHON', sys.executable)
 
 
+class TestCaseEntry(typing.NamedTuple):
+    test_case: testtools.TestCase
+    start_time: float
+
+
 class TestCasesManager(object):
 
+    start_time: _time.Seconds = None
+
     def __init__(self):
-        self._test_cases: typing.List[testtools.TestCase] = []
+        self._test_cases: typing.List[TestCaseEntry] = []
 
     def get_test_case(self) -> testtools.TestCase:
         try:
-            return self._test_cases[-1]
+            return self._test_cases[-1].test_case
         except IndexError:
             return DUMMY_TEST_CASE
 
     def pop_test_case(self) -> testtools.TestCase:
-        return self._test_cases.pop()
+        entry = self._test_cases.pop()
+        elapsed_time = _time.time() - entry.start_time
+        LOG.debug(f"Exit test case '{entry.test_case.id()}' after "
+                  f"{elapsed_time} seconds")
+        return entry.test_case
 
     def push_test_case(self, test_case: testtools.TestCase):
         _exception.check_valid_type(test_case, testtools.TestCase)
-        self._test_cases.append(test_case)
+        entry = TestCaseEntry(test_case=test_case,
+                              start_time=_time.time())
+        self._test_cases.append(entry)
+        LOG.debug(f"Enter test case '{test_case.id()}'")
 
 
 TEST_CASES = TestCasesManager()

tobiko/config.py

@@ -66,7 +66,11 @@ TESTCASE_OPTIONS = [
     cfg.FloatOpt('timeout',
                  default=None,
                  help=("Timeout (in seconds) used for interrupting test case "
-                       "execution"))]
+                       "execution")),
+    cfg.FloatOpt('test_runner_timeout',
+                 default=None,
+                 help=("Timeout (in seconds) used for interrupting test "
+                       "runner execution"))]
 
 
 def workspace_config_files(project=None, prog=None):

tobiko/tests/conftest.py

@@ -22,6 +22,7 @@ from oslo_log import log
 from py.xml import html  # pylint: disable=no-name-in-module,import-error
 import pytest
 
+import tobiko
 
 LOG = log.getLogger(__name__)
 
@@ -47,7 +48,6 @@ def configure_metadata(config):
 
 
 def configure_caplog(config):
-    import tobiko
     tobiko_config = tobiko.tobiko_config()
 
     if tobiko_config.logging.capture_log is True:
@@ -93,8 +93,40 @@ def set_default_inicfg(config, key, default):
         LOG.debug(f"Keep existing inicfg: {key} = {value}")
 
 
+class TestRunnerTimeoutManager(tobiko.SharedFixture):
+    timeout: tobiko.Seconds = None
+    deadline: tobiko.Seconds = None
+
+    def setup_fixture(self):
+        tobiko_config = tobiko.tobiko_config()
+        self.timeout = tobiko_config.testcase.test_runner_timeout
+        if self.timeout is None:
+            LOG.info('Test runner timeout is disabled')
+        else:
+            LOG.info('Test runner timeout is enabled: '
+                     f'timeout is {self.timeout} seconds')
+            self.deadline = tobiko.time() + self.timeout
+
+    @classmethod
+    def check_test_runner_timeout(cls):
+        self = tobiko.setup_fixture(cls)
+        if self.deadline is not None:
+            time_left = self.deadline - tobiko.time()
+            if time_left <= 0.:
+                pytest.skip(
+                    f"Test runner execution timed out after {self.timeout} "
+                    f"seconds",
+                    allow_module_level=True)
+            else:
+                LOG.debug('Test runner timeout is enabled: '
+                          f'{time_left} seconds left')
+
+
+def check_test_runner_timeout():
+    TestRunnerTimeoutManager.check_test_runner_timeout()
+
+
 def configure_timeout(config):
-    import tobiko
     tobiko_config = tobiko.tobiko_config()
     default = tobiko_config.testcase.timeout
     if default is not None and default > 0.:
@@ -128,7 +160,7 @@ def pytest_html_report_title(report):
 @pytest.hookimpl(hookwrapper=True)
 def pytest_runtest_call(item):
     # pylint: disable=protected-access
-    import tobiko
+    check_test_runner_timeout()
     tobiko.push_test_case(item._testcase)
     try:
         yield