From b12445569796b4353aa21d1c973396e622fec256 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Tue, 6 Feb 2018 15:00:55 -0800 Subject: [PATCH] Make timeout value apply to entire job Rather than have the job timeout apply to each of the three run steps (pre, run, and post) individualy convert the timeout to a cummulative timeout that applies across all three steps. This is more intuitive to users and allows us to continue to have long job timeouts without multiplying their effect when things go wrong. Change-Id: I8f46736e578c8ea3e9f2fba25f53ab47715101af --- zuul/executor/server.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/zuul/executor/server.py b/zuul/executor/server.py index a2a9b42cdd..1766a884c1 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -835,6 +835,13 @@ class AnsibleJob(object): repo.checkout(selected_ref) return selected_ref + def getAnsibleTimeout(self, start, timeout): + if timeout is not None: + now = time.time() + elapsed = now - start + timeout = timeout - elapsed + return timeout + def runPlaybooks(self, args): result = None @@ -852,10 +859,15 @@ class AnsibleJob(object): pre_failed = False success = False self.started = True + time_started = time.time() + # timeout value is total job timeout or put another way + # the cummulative time that pre, run, and post can consume. + job_timeout = args['timeout'] for index, playbook in enumerate(self.jobdir.pre_playbooks): # TODOv3(pabelanger): Implement pre-run timeout setting. + ansible_timeout = self.getAnsibleTimeout(time_started, job_timeout) pre_status, pre_code = self.runAnsiblePlaybook( - playbook, args['timeout'], phase='pre', index=index) + playbook, ansible_timeout, phase='pre', index=index) if pre_status != self.RESULT_NORMAL or pre_code != 0: # These should really never fail, so return None and have # zuul try again @@ -863,8 +875,9 @@ class AnsibleJob(object): break if not pre_failed: + ansible_timeout = self.getAnsibleTimeout(time_started, job_timeout) job_status, job_code = self.runAnsiblePlaybook( - self.jobdir.playbook, args['timeout'], phase='run') + self.jobdir.playbook, ansible_timeout, phase='run') if job_status == self.RESULT_ABORTED: return 'ABORTED' elif job_status == self.RESULT_TIMED_OUT: @@ -885,8 +898,9 @@ class AnsibleJob(object): for index, playbook in enumerate(self.jobdir.post_playbooks): # TODOv3(pabelanger): Implement post-run timeout setting. + ansible_timeout = self.getAnsibleTimeout(time_started, job_timeout) post_status, post_code = self.runAnsiblePlaybook( - playbook, args['timeout'], success, phase='post', index=index) + playbook, ansible_timeout, success, phase='post', index=index) if post_status == self.RESULT_ABORTED: return 'ABORTED' if post_status != self.RESULT_NORMAL or post_code != 0: