Add option to skip all remaining test cases after a given timeout is expired

After some given seconds of timeout test runner execution self-interrupt
to avoid hitting jobs timeout.

The problem is that when it occurs all remaining test cases that didn't started
yet are missing from the reports.

This changes things by marking such test cases as skipped, so that we
can see it from test results.

Change-Id: Idd0064812fc5a2da295712914c4531bdef466c24

infrared_plugin/plugin.spec

@@ -117,6 +117,10 @@ subparsers:
             type: Value
             help: Test case timeout in seconds
             ansible_variable: test_case_timeout
+          test-runner-timeout:
+            type: Value
+            help: Test runner timeout in seconds
+            ansible_variable: test_runner_timeout
           undercloud_host:
             type: Value
             help: inventory hostname of the undercloud host

roles/tobiko-configure/defaults/main.yaml

@@ -8,13 +8,15 @@ test_default_conf:
 
   testcase:
     timeout: "{{ test_case_timeout }}"
+    test_runner_timeout: "{{ test_runner_timeout }}"
 
   tripleo:
     undercloud_ssh_hostname: "{{ undercloud_ssh_hostname }}"
 
 test_log_debug: false
 
-test_case_timeout: 7200.
+test_case_timeout: 1800.
+test_runner_timeout: 14400.
 
 # OpenStack client credentials
 stackrc_file: '{{ ansible_user_dir }}/overcloudrc'

roles/tobiko-tox/defaults/main.yaml

@@ -15,7 +15,7 @@ tox_step_index: 0
 tox_report_name:
   "{{ test_report_name }}{% if tox_step_index %}_{{ '{:02d}'.format(tox_step_index | int) }}{% endif %}{% if tox_step_name %}_{{ tox_step_name }}{% endif %}{% if tox_envlist %}_{{ tox_envlist }}{% endif %}"
 
-tox_run_tests_timeout: 14400  # 4 hours
+tox_run_tests_timeout: 18000  # 5 hours
 
 tox_constraints_file: '{{ remote_constraints_file }}'
 

tobiko/common/_testcase.py

@@ -15,33 +15,51 @@ from __future__ import absolute_import
 
 import os
 import sys
-import typing  # noqa
+import typing
 
+from oslo_log import log
 import testtools
 
 from tobiko.common import _exception
+from tobiko.common import _time
 
 
+LOG = log.getLogger(__name__)
+
 os.environ.setdefault('PYTHON', sys.executable)
 
 
+class TestCaseEntry(typing.NamedTuple):
+    test_case: testtools.TestCase
+    start_time: float
+
+
 class TestCasesManager(object):
 
+    start_time: _time.Seconds = None
+
     def __init__(self):
-        self._test_cases: typing.List[testtools.TestCase] = []
+        self._test_cases: typing.List[TestCaseEntry] = []
 
     def get_test_case(self) -> testtools.TestCase:
         try:
-            return self._test_cases[-1]
+            return self._test_cases[-1].test_case
         except IndexError:
             return DUMMY_TEST_CASE
 
     def pop_test_case(self) -> testtools.TestCase:
-        return self._test_cases.pop()
+        entry = self._test_cases.pop()
+        elapsed_time = _time.time() - entry.start_time
+        LOG.debug(f"Exit test case '{entry.test_case.id()}' after "
+                  f"{elapsed_time} seconds")
+        return entry.test_case
 
     def push_test_case(self, test_case: testtools.TestCase):
         _exception.check_valid_type(test_case, testtools.TestCase)
-        self._test_cases.append(test_case)
+        entry = TestCaseEntry(test_case=test_case,
+                              start_time=_time.time())
+        self._test_cases.append(entry)
+        LOG.debug(f"Enter test case '{test_case.id()}'")
 
 
 TEST_CASES = TestCasesManager()

tobiko/config.py

@@ -66,7 +66,11 @@ TESTCASE_OPTIONS = [
     cfg.FloatOpt('timeout',
                  default=None,
                  help=("Timeout (in seconds) used for interrupting test case "
-                       "execution"))]
+                       "execution")),
+    cfg.FloatOpt('test_runner_timeout',
+                 default=None,
+                 help=("Timeout (in seconds) used for interrupting test "
+                       "runner execution"))]
 
 
 def workspace_config_files(project=None, prog=None):

tobiko/tests/conftest.py

@@ -22,6 +22,7 @@ from oslo_log import log
 from py.xml import html  # pylint: disable=no-name-in-module,import-error
 import pytest
 
+import tobiko
 
 LOG = log.getLogger(__name__)
 
@@ -47,7 +48,6 @@ def configure_metadata(config):
 
 
 def configure_caplog(config):
-    import tobiko
     tobiko_config = tobiko.tobiko_config()
 
     if tobiko_config.logging.capture_log is True:
@@ -93,8 +93,40 @@ def set_default_inicfg(config, key, default):
         LOG.debug(f"Keep existing inicfg: {key} = {value}")
 
 
+class TestRunnerTimeoutManager(tobiko.SharedFixture):
+    timeout: tobiko.Seconds = None
+    deadline: tobiko.Seconds = None
+
+    def setup_fixture(self):
+        tobiko_config = tobiko.tobiko_config()
+        self.timeout = tobiko_config.testcase.test_runner_timeout
+        if self.timeout is None:
+            LOG.info('Test runner timeout is disabled')
+        else:
+            LOG.info('Test runner timeout is enabled: '
+                     f'timeout is {self.timeout} seconds')
+            self.deadline = tobiko.time() + self.timeout
+
+    @classmethod
+    def check_test_runner_timeout(cls):
+        self = tobiko.setup_fixture(cls)
+        if self.deadline is not None:
+            time_left = self.deadline - tobiko.time()
+            if time_left <= 0.:
+                pytest.skip(
+                    f"Test runner execution timed out after {self.timeout} "
+                    f"seconds",
+                    allow_module_level=True)
+            else:
+                LOG.debug('Test runner timeout is enabled: '
+                          f'{time_left} seconds left')
+
+
+def check_test_runner_timeout():
+    TestRunnerTimeoutManager.check_test_runner_timeout()
+
+
 def configure_timeout(config):
-    import tobiko
     tobiko_config = tobiko.tobiko_config()
     default = tobiko_config.testcase.timeout
     if default is not None and default > 0.:
@@ -128,7 +160,7 @@ def pytest_html_report_title(report):
 @pytest.hookimpl(hookwrapper=True)
 def pytest_runtest_call(item):
     # pylint: disable=protected-access
-    import tobiko
+    check_test_runner_timeout()
     tobiko.push_test_case(item._testcase)
     try:
         yield