Add a timeout for workflow stages

Change-Id: I9938a8bbd71ab131c8b6a2210e5d1a903ef21efd

infrared_plugin/plugin.spec

@@ -160,6 +160,10 @@ subparsers:
             type: Value
             help: Python interpreter to be used for executing test cases
             ansible_variable: tox_python
+          run-tests-timeout:
+            type: Value
+            help: Timeout (in seconds) to interrupt test cases execution
+            ansible_variable: tox_run_tests_timeout
           test-report-dir:
             type: Value
             help: directory where to store test report files

roles/tobiko-tox/defaults/main.yaml

@@ -15,12 +15,15 @@ tox_step_index: 0
 tox_report_name:
   "{{ test_report_name }}{% if tox_step_index %}_{{ '{:02d}'.format(tox_step_index | int) }}{% endif %}{% if tox_step_name %}_{{ tox_step_name }}{% endif %}{% if tox_envlist %}_{{ tox_envlist }}{% endif %}"
 
+tox_run_tests_timeout: 14400  # 4 hours
+
 tox_constraints_file: '{{ remote_constraints_file }}'
 
 tox_constrain_env:
   TOX_REPORT_DIR: '{{ tox_report_dir }}'
   TOX_REPORT_NAME: '{{ tox_report_name }}'
   TOX_CONSTRAINTS_FILE: '{{ tox_constraints_file }}'
+  TOX_RUN_TESTS_TIMEOUT: '{{ tox_run_tests_timeout | float }}'
 
 tox_succeeded_rc: 0
 tox_failed_rc: 1

test-requirements.txt

@@ -4,5 +4,6 @@ coverage>=4.5.0        # Apache-2.0
 junitxml>=0.7.0        # LGPL-3
 mock>=2.0.0            # BSD
 os-testr>=1.0.0        # Apache-2.0
+psutil>=5.7.2          # BSD
 python-subunit>=1.4.0  # Apache-2.0
 stestr>=3.0.0          # Apache-2.0

tools/run_tests.py

@@ -15,10 +15,14 @@
 from __future__ import absolute_import
 
 import os
+import psutil
+import signal
 import sys
 import subprocess
 
 
+
+
 TOP_DIR = os.path.dirname(os.path.dirname(__file__))
 if TOP_DIR not in sys.path:
     sys.path.insert(0, TOP_DIR)
@@ -47,6 +51,8 @@ TOX_REPORT_HTML = os.environ.get(
 TOX_REPORT_XML = os.environ.get(
     'TOX_REPORT_XML', TOX_REPORT_PREFIX + '.xml')
 
+TOX_RUN_TESTS_TIMEOUT = float(os.environ.get('TOX_RUN_TESTS_TIMEOUT') or 0.)
+
 
 def main():
     common.setup_logging()
@@ -65,13 +71,17 @@ def main():
 
 
 def run_tests():
+    setup_timeout()
     cleanup_report_dir()
     log_environ()
 
     succeeded = True
     try:
         run_test_cases()
-    except subprocess.CalledProcessError:
+    except (subprocess.CalledProcessError,
+            subprocess.TimeoutExpired,
+            ProcessLookupError) as ex:
+        LOG.error(f"Error while running test cases.\n{ex}")
         succeeded = False
 
     try:
@@ -91,6 +101,35 @@ def run_tests():
     return succeeded
 
 
+def setup_timeout():
+
+    if TOX_RUN_TESTS_TIMEOUT > 0.:
+
+        def handle_timeout(_signum, _frame):
+            LOG.error(
+                f"run_tests.py timeout out after {TOX_RUN_TESTS_TIMEOUT} "
+                "seconds")
+            terminate_childs()
+            raise subprocess.TimeoutExpired("run_tests.py",
+                                            TOX_RUN_TESTS_TIMEOUT)
+
+        signal.setitimer(signal.ITIMER_REAL, TOX_RUN_TESTS_TIMEOUT)
+        signal.signal(signal.SIGALRM, handle_timeout)
+        LOG.debug(f'Run tests timeout set as {TOX_RUN_TESTS_TIMEOUT} seconds')
+
+
+def terminate_childs():
+    current_process = psutil.Process()
+    children = current_process.children(recursive=False)
+    for child in children:
+        LOG.debug(f"Interrupt child process execution (pid={child.pid})")
+        os.kill(child.pid, signal.SIGINT)
+    for child in children:
+        LOG.debug("Wait for top-child process termination "
+                  f"(pid={child.pid})...")
+        os.waitpid(child.pid, 0)
+
+
 def cleanup_report_dir():
     for report_file in [TOX_REPORT_LOG, TOX_REPORT_SUBUNIT, TOX_REPORT_HTML,
                         TOX_REPORT_XML]:

zuul.d/devstack.yaml

@@ -70,8 +70,9 @@
         yaml: true
         yml: true
       test_log_debug: true
-      test_case_timeout: 1800
+      test_case_timeout: 1200
       tobiko_dir: '/opt/stack/tobiko'
+      tox_run_tests_timeout: 3600
       upper_constraints_file: '{{ ansible_user_dir }}/src/opendev.org/openstack/requirements/upper-constraints.txt'
 
     pre-run: playbooks/devstack/pre.yaml