Merge "Add option to skip all remaining test cases after a given timeout is expired"
This commit is contained in:
commit
88fbfe6329
|
@ -117,6 +117,10 @@ subparsers:
|
||||||
type: Value
|
type: Value
|
||||||
help: Test case timeout in seconds
|
help: Test case timeout in seconds
|
||||||
ansible_variable: test_case_timeout
|
ansible_variable: test_case_timeout
|
||||||
|
test-runner-timeout:
|
||||||
|
type: Value
|
||||||
|
help: Test runner timeout in seconds
|
||||||
|
ansible_variable: test_runner_timeout
|
||||||
undercloud_host:
|
undercloud_host:
|
||||||
type: Value
|
type: Value
|
||||||
help: inventory hostname of the undercloud host
|
help: inventory hostname of the undercloud host
|
||||||
|
|
|
@ -8,13 +8,15 @@ test_default_conf:
|
||||||
|
|
||||||
testcase:
|
testcase:
|
||||||
timeout: "{{ test_case_timeout }}"
|
timeout: "{{ test_case_timeout }}"
|
||||||
|
test_runner_timeout: "{{ test_runner_timeout }}"
|
||||||
|
|
||||||
tripleo:
|
tripleo:
|
||||||
undercloud_ssh_hostname: "{{ undercloud_ssh_hostname }}"
|
undercloud_ssh_hostname: "{{ undercloud_ssh_hostname }}"
|
||||||
|
|
||||||
test_log_debug: false
|
test_log_debug: false
|
||||||
|
|
||||||
test_case_timeout: 7200.
|
test_case_timeout: 1800.
|
||||||
|
test_runner_timeout: 14400.
|
||||||
|
|
||||||
# OpenStack client credentials
|
# OpenStack client credentials
|
||||||
stackrc_file: '{{ ansible_user_dir }}/overcloudrc'
|
stackrc_file: '{{ ansible_user_dir }}/overcloudrc'
|
||||||
|
|
|
@ -15,7 +15,7 @@ tox_step_index: 0
|
||||||
tox_report_name:
|
tox_report_name:
|
||||||
"{{ test_report_name }}{% if tox_step_index %}_{{ '{:02d}'.format(tox_step_index | int) }}{% endif %}{% if tox_step_name %}_{{ tox_step_name }}{% endif %}{% if tox_envlist %}_{{ tox_envlist }}{% endif %}"
|
"{{ test_report_name }}{% if tox_step_index %}_{{ '{:02d}'.format(tox_step_index | int) }}{% endif %}{% if tox_step_name %}_{{ tox_step_name }}{% endif %}{% if tox_envlist %}_{{ tox_envlist }}{% endif %}"
|
||||||
|
|
||||||
tox_run_tests_timeout: 14400 # 4 hours
|
tox_run_tests_timeout: 18000 # 5 hours
|
||||||
|
|
||||||
tox_constraints_file: '{{ remote_constraints_file }}'
|
tox_constraints_file: '{{ remote_constraints_file }}'
|
||||||
|
|
||||||
|
|
|
@ -15,33 +15,51 @@ from __future__ import absolute_import
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import typing # noqa
|
import typing
|
||||||
|
|
||||||
|
from oslo_log import log
|
||||||
import testtools
|
import testtools
|
||||||
|
|
||||||
from tobiko.common import _exception
|
from tobiko.common import _exception
|
||||||
|
from tobiko.common import _time
|
||||||
|
|
||||||
|
|
||||||
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
os.environ.setdefault('PYTHON', sys.executable)
|
os.environ.setdefault('PYTHON', sys.executable)
|
||||||
|
|
||||||
|
|
||||||
|
class TestCaseEntry(typing.NamedTuple):
|
||||||
|
test_case: testtools.TestCase
|
||||||
|
start_time: float
|
||||||
|
|
||||||
|
|
||||||
class TestCasesManager(object):
|
class TestCasesManager(object):
|
||||||
|
|
||||||
|
start_time: _time.Seconds = None
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self._test_cases: typing.List[testtools.TestCase] = []
|
self._test_cases: typing.List[TestCaseEntry] = []
|
||||||
|
|
||||||
def get_test_case(self) -> testtools.TestCase:
|
def get_test_case(self) -> testtools.TestCase:
|
||||||
try:
|
try:
|
||||||
return self._test_cases[-1]
|
return self._test_cases[-1].test_case
|
||||||
except IndexError:
|
except IndexError:
|
||||||
return DUMMY_TEST_CASE
|
return DUMMY_TEST_CASE
|
||||||
|
|
||||||
def pop_test_case(self) -> testtools.TestCase:
|
def pop_test_case(self) -> testtools.TestCase:
|
||||||
return self._test_cases.pop()
|
entry = self._test_cases.pop()
|
||||||
|
elapsed_time = _time.time() - entry.start_time
|
||||||
|
LOG.debug(f"Exit test case '{entry.test_case.id()}' after "
|
||||||
|
f"{elapsed_time} seconds")
|
||||||
|
return entry.test_case
|
||||||
|
|
||||||
def push_test_case(self, test_case: testtools.TestCase):
|
def push_test_case(self, test_case: testtools.TestCase):
|
||||||
_exception.check_valid_type(test_case, testtools.TestCase)
|
_exception.check_valid_type(test_case, testtools.TestCase)
|
||||||
self._test_cases.append(test_case)
|
entry = TestCaseEntry(test_case=test_case,
|
||||||
|
start_time=_time.time())
|
||||||
|
self._test_cases.append(entry)
|
||||||
|
LOG.debug(f"Enter test case '{test_case.id()}'")
|
||||||
|
|
||||||
|
|
||||||
TEST_CASES = TestCasesManager()
|
TEST_CASES = TestCasesManager()
|
||||||
|
|
|
@ -66,7 +66,11 @@ TESTCASE_OPTIONS = [
|
||||||
cfg.FloatOpt('timeout',
|
cfg.FloatOpt('timeout',
|
||||||
default=None,
|
default=None,
|
||||||
help=("Timeout (in seconds) used for interrupting test case "
|
help=("Timeout (in seconds) used for interrupting test case "
|
||||||
"execution"))]
|
"execution")),
|
||||||
|
cfg.FloatOpt('test_runner_timeout',
|
||||||
|
default=None,
|
||||||
|
help=("Timeout (in seconds) used for interrupting test "
|
||||||
|
"runner execution"))]
|
||||||
|
|
||||||
|
|
||||||
def workspace_config_files(project=None, prog=None):
|
def workspace_config_files(project=None, prog=None):
|
||||||
|
|
|
@ -22,6 +22,7 @@ from oslo_log import log
|
||||||
from py.xml import html # pylint: disable=no-name-in-module,import-error
|
from py.xml import html # pylint: disable=no-name-in-module,import-error
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
import tobiko
|
||||||
|
|
||||||
LOG = log.getLogger(__name__)
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
|
@ -47,7 +48,6 @@ def configure_metadata(config):
|
||||||
|
|
||||||
|
|
||||||
def configure_caplog(config):
|
def configure_caplog(config):
|
||||||
import tobiko
|
|
||||||
tobiko_config = tobiko.tobiko_config()
|
tobiko_config = tobiko.tobiko_config()
|
||||||
|
|
||||||
if tobiko_config.logging.capture_log is True:
|
if tobiko_config.logging.capture_log is True:
|
||||||
|
@ -93,8 +93,40 @@ def set_default_inicfg(config, key, default):
|
||||||
LOG.debug(f"Keep existing inicfg: {key} = {value}")
|
LOG.debug(f"Keep existing inicfg: {key} = {value}")
|
||||||
|
|
||||||
|
|
||||||
|
class TestRunnerTimeoutManager(tobiko.SharedFixture):
|
||||||
|
timeout: tobiko.Seconds = None
|
||||||
|
deadline: tobiko.Seconds = None
|
||||||
|
|
||||||
|
def setup_fixture(self):
|
||||||
|
tobiko_config = tobiko.tobiko_config()
|
||||||
|
self.timeout = tobiko_config.testcase.test_runner_timeout
|
||||||
|
if self.timeout is None:
|
||||||
|
LOG.info('Test runner timeout is disabled')
|
||||||
|
else:
|
||||||
|
LOG.info('Test runner timeout is enabled: '
|
||||||
|
f'timeout is {self.timeout} seconds')
|
||||||
|
self.deadline = tobiko.time() + self.timeout
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def check_test_runner_timeout(cls):
|
||||||
|
self = tobiko.setup_fixture(cls)
|
||||||
|
if self.deadline is not None:
|
||||||
|
time_left = self.deadline - tobiko.time()
|
||||||
|
if time_left <= 0.:
|
||||||
|
pytest.skip(
|
||||||
|
f"Test runner execution timed out after {self.timeout} "
|
||||||
|
f"seconds",
|
||||||
|
allow_module_level=True)
|
||||||
|
else:
|
||||||
|
LOG.debug('Test runner timeout is enabled: '
|
||||||
|
f'{time_left} seconds left')
|
||||||
|
|
||||||
|
|
||||||
|
def check_test_runner_timeout():
|
||||||
|
TestRunnerTimeoutManager.check_test_runner_timeout()
|
||||||
|
|
||||||
|
|
||||||
def configure_timeout(config):
|
def configure_timeout(config):
|
||||||
import tobiko
|
|
||||||
tobiko_config = tobiko.tobiko_config()
|
tobiko_config = tobiko.tobiko_config()
|
||||||
default = tobiko_config.testcase.timeout
|
default = tobiko_config.testcase.timeout
|
||||||
if default is not None and default > 0.:
|
if default is not None and default > 0.:
|
||||||
|
@ -128,7 +160,7 @@ def pytest_html_report_title(report):
|
||||||
@pytest.hookimpl(hookwrapper=True)
|
@pytest.hookimpl(hookwrapper=True)
|
||||||
def pytest_runtest_call(item):
|
def pytest_runtest_call(item):
|
||||||
# pylint: disable=protected-access
|
# pylint: disable=protected-access
|
||||||
import tobiko
|
check_test_runner_timeout()
|
||||||
tobiko.push_test_case(item._testcase)
|
tobiko.push_test_case(item._testcase)
|
||||||
try:
|
try:
|
||||||
yield
|
yield
|
||||||
|
|
Loading…
Reference in New Issue