diff --git a/infrared_plugin/plugin.spec b/infrared_plugin/plugin.spec index 3c147e049..c9f59405b 100644 --- a/infrared_plugin/plugin.spec +++ b/infrared_plugin/plugin.spec @@ -160,6 +160,10 @@ subparsers: type: Value help: Python interpreter to be used for executing test cases ansible_variable: tox_python + run-tests-timeout: + type: Value + help: Timeout (in seconds) to interrupt test cases execution + ansible_variable: tox_run_tests_timeout test-report-dir: type: Value help: directory where to store test report files diff --git a/roles/tobiko-tox/defaults/main.yaml b/roles/tobiko-tox/defaults/main.yaml index e6cc87a8e..7d1354063 100644 --- a/roles/tobiko-tox/defaults/main.yaml +++ b/roles/tobiko-tox/defaults/main.yaml @@ -15,12 +15,15 @@ tox_step_index: 0 tox_report_name: "{{ test_report_name }}{% if tox_step_index %}_{{ '{:02d}'.format(tox_step_index | int) }}{% endif %}{% if tox_step_name %}_{{ tox_step_name }}{% endif %}{% if tox_envlist %}_{{ tox_envlist }}{% endif %}" +tox_run_tests_timeout: 14400 # 4 hours + tox_constraints_file: '{{ remote_constraints_file }}' tox_constrain_env: TOX_REPORT_DIR: '{{ tox_report_dir }}' TOX_REPORT_NAME: '{{ tox_report_name }}' TOX_CONSTRAINTS_FILE: '{{ tox_constraints_file }}' + TOX_RUN_TESTS_TIMEOUT: '{{ tox_run_tests_timeout | float }}' tox_succeeded_rc: 0 tox_failed_rc: 1 diff --git a/test-requirements.txt b/test-requirements.txt index 53ab2aee5..7c5f6fd9e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,5 +4,6 @@ coverage>=4.5.0 # Apache-2.0 junitxml>=0.7.0 # LGPL-3 mock>=2.0.0 # BSD os-testr>=1.0.0 # Apache-2.0 +psutil>=5.7.2 # BSD python-subunit>=1.4.0 # Apache-2.0 stestr>=3.0.0 # Apache-2.0 diff --git a/tools/run_tests.py b/tools/run_tests.py index 67f57002b..248379e9e 100755 --- a/tools/run_tests.py +++ b/tools/run_tests.py @@ -15,10 +15,14 @@ from __future__ import absolute_import import os +import psutil +import signal import sys import subprocess + + TOP_DIR = os.path.dirname(os.path.dirname(__file__)) if TOP_DIR not in sys.path: sys.path.insert(0, TOP_DIR) @@ -47,6 +51,8 @@ TOX_REPORT_HTML = os.environ.get( TOX_REPORT_XML = os.environ.get( 'TOX_REPORT_XML', TOX_REPORT_PREFIX + '.xml') +TOX_RUN_TESTS_TIMEOUT = float(os.environ.get('TOX_RUN_TESTS_TIMEOUT') or 0.) + def main(): common.setup_logging() @@ -65,13 +71,17 @@ def main(): def run_tests(): + setup_timeout() cleanup_report_dir() log_environ() succeeded = True try: run_test_cases() - except subprocess.CalledProcessError: + except (subprocess.CalledProcessError, + subprocess.TimeoutExpired, + ProcessLookupError) as ex: + LOG.error(f"Error while running test cases.\n{ex}") succeeded = False try: @@ -91,6 +101,35 @@ def run_tests(): return succeeded +def setup_timeout(): + + if TOX_RUN_TESTS_TIMEOUT > 0.: + + def handle_timeout(_signum, _frame): + LOG.error( + f"run_tests.py timeout out after {TOX_RUN_TESTS_TIMEOUT} " + "seconds") + terminate_childs() + raise subprocess.TimeoutExpired("run_tests.py", + TOX_RUN_TESTS_TIMEOUT) + + signal.setitimer(signal.ITIMER_REAL, TOX_RUN_TESTS_TIMEOUT) + signal.signal(signal.SIGALRM, handle_timeout) + LOG.debug(f'Run tests timeout set as {TOX_RUN_TESTS_TIMEOUT} seconds') + + +def terminate_childs(): + current_process = psutil.Process() + children = current_process.children(recursive=False) + for child in children: + LOG.debug(f"Interrupt child process execution (pid={child.pid})") + os.kill(child.pid, signal.SIGINT) + for child in children: + LOG.debug("Wait for top-child process termination " + f"(pid={child.pid})...") + os.waitpid(child.pid, 0) + + def cleanup_report_dir(): for report_file in [TOX_REPORT_LOG, TOX_REPORT_SUBUNIT, TOX_REPORT_HTML, TOX_REPORT_XML]: diff --git a/zuul.d/devstack.yaml b/zuul.d/devstack.yaml index 37d17956f..dd496e0b2 100644 --- a/zuul.d/devstack.yaml +++ b/zuul.d/devstack.yaml @@ -70,8 +70,9 @@ yaml: true yml: true test_log_debug: true - test_case_timeout: 1800 + test_case_timeout: 1200 tobiko_dir: '/opt/stack/tobiko' + tox_run_tests_timeout: 3600 upper_constraints_file: '{{ ansible_user_dir }}/src/opendev.org/openstack/requirements/upper-constraints.txt' pre-run: playbooks/devstack/pre.yaml