Add a timeout for workflow stages

Change-Id: I9938a8bbd71ab131c8b6a2210e5d1a903ef21efd
This commit is contained in:
Federico Ressi 2020-09-28 10:41:19 +02:00
parent ac9b3a44f3
commit 31a9a6e8e6
5 changed files with 50 additions and 2 deletions

View File

@ -160,6 +160,10 @@ subparsers:
type: Value
help: Python interpreter to be used for executing test cases
ansible_variable: tox_python
run-tests-timeout:
type: Value
help: Timeout (in seconds) to interrupt test cases execution
ansible_variable: tox_run_tests_timeout
test-report-dir:
type: Value
help: directory where to store test report files

View File

@ -15,12 +15,15 @@ tox_step_index: 0
tox_report_name:
"{{ test_report_name }}{% if tox_step_index %}_{{ '{:02d}'.format(tox_step_index | int) }}{% endif %}{% if tox_step_name %}_{{ tox_step_name }}{% endif %}{% if tox_envlist %}_{{ tox_envlist }}{% endif %}"
tox_run_tests_timeout: 14400 # 4 hours
tox_constraints_file: '{{ remote_constraints_file }}'
tox_constrain_env:
TOX_REPORT_DIR: '{{ tox_report_dir }}'
TOX_REPORT_NAME: '{{ tox_report_name }}'
TOX_CONSTRAINTS_FILE: '{{ tox_constraints_file }}'
TOX_RUN_TESTS_TIMEOUT: '{{ tox_run_tests_timeout | float }}'
tox_succeeded_rc: 0
tox_failed_rc: 1

View File

@ -4,5 +4,6 @@ coverage>=4.5.0 # Apache-2.0
junitxml>=0.7.0 # LGPL-3
mock>=2.0.0 # BSD
os-testr>=1.0.0 # Apache-2.0
psutil>=5.7.2 # BSD
python-subunit>=1.4.0 # Apache-2.0
stestr>=3.0.0 # Apache-2.0

View File

@ -15,10 +15,14 @@
from __future__ import absolute_import
import os
import psutil
import signal
import sys
import subprocess
TOP_DIR = os.path.dirname(os.path.dirname(__file__))
if TOP_DIR not in sys.path:
sys.path.insert(0, TOP_DIR)
@ -47,6 +51,8 @@ TOX_REPORT_HTML = os.environ.get(
TOX_REPORT_XML = os.environ.get(
'TOX_REPORT_XML', TOX_REPORT_PREFIX + '.xml')
TOX_RUN_TESTS_TIMEOUT = float(os.environ.get('TOX_RUN_TESTS_TIMEOUT') or 0.)
def main():
common.setup_logging()
@ -65,13 +71,17 @@ def main():
def run_tests():
setup_timeout()
cleanup_report_dir()
log_environ()
succeeded = True
try:
run_test_cases()
except subprocess.CalledProcessError:
except (subprocess.CalledProcessError,
subprocess.TimeoutExpired,
ProcessLookupError) as ex:
LOG.error(f"Error while running test cases.\n{ex}")
succeeded = False
try:
@ -91,6 +101,35 @@ def run_tests():
return succeeded
def setup_timeout():
if TOX_RUN_TESTS_TIMEOUT > 0.:
def handle_timeout(_signum, _frame):
LOG.error(
f"run_tests.py timeout out after {TOX_RUN_TESTS_TIMEOUT} "
"seconds")
terminate_childs()
raise subprocess.TimeoutExpired("run_tests.py",
TOX_RUN_TESTS_TIMEOUT)
signal.setitimer(signal.ITIMER_REAL, TOX_RUN_TESTS_TIMEOUT)
signal.signal(signal.SIGALRM, handle_timeout)
LOG.debug(f'Run tests timeout set as {TOX_RUN_TESTS_TIMEOUT} seconds')
def terminate_childs():
current_process = psutil.Process()
children = current_process.children(recursive=False)
for child in children:
LOG.debug(f"Interrupt child process execution (pid={child.pid})")
os.kill(child.pid, signal.SIGINT)
for child in children:
LOG.debug("Wait for top-child process termination "
f"(pid={child.pid})...")
os.waitpid(child.pid, 0)
def cleanup_report_dir():
for report_file in [TOX_REPORT_LOG, TOX_REPORT_SUBUNIT, TOX_REPORT_HTML,
TOX_REPORT_XML]:

View File

@ -70,8 +70,9 @@
yaml: true
yml: true
test_log_debug: true
test_case_timeout: 1800
test_case_timeout: 1200
tobiko_dir: '/opt/stack/tobiko'
tox_run_tests_timeout: 3600
upper_constraints_file: '{{ ansible_user_dir }}/src/opendev.org/openstack/requirements/upper-constraints.txt'
pre-run: playbooks/devstack/pre.yaml