zuul
/
zuul
Code Issues Proposed changes

Fix stuck job caused by exception during repo update 

If we hit an exception while updating repo the job is not marked as
complete. This causes the job that waits for it to be stuck
forever. This situation can only be solved by forcing a dequeue of the
change or restarting the executor.

Change-Id: Ia52cc96c3ce39c50bb9d7ff8162d0aa0f7f13c77
This commit is contained in:
Tobias Henkel 2018-08-09 09:13:37 +02:00
parent 8a58a358d1
commit 393d66b04d
1 changed files with 26 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
zuul/executor/server.py
View File

@ -492,6 +492,7 @@ class UpdateTask(object):
self.branches = None self.branches = None
self.refs = None self.refs = None
self.event = threading.Event() self.event = threading.Event()
self.success = False
def __eq__(self, other): def __eq__(self, other):
if (other and other.connection_name == self.connection_name and if (other and other.connection_name == self.connection_name and
@ -802,6 +803,11 @@ class AnsibleJob(object):
for task in tasks: for task in tasks:
task.wait() task.wait()
if not task.success:
raise ExecutorError(
'Failed to update project %s' % task.canonical_name)
self.project_info[task.canonical_name] = { self.project_info[task.canonical_name] = {
'refs': task.refs, 'refs': task.refs,
'branches': task.branches, 'branches': task.branches,
@ -2304,19 +2310,26 @@ class ExecutorServer(object):
if task is None: if task is None:
# We are asked to stop # We are asked to stop
raise StopException() raise StopException()
with self.merger_lock: try:
self.log.info("Updating repo %s/%s" % ( with self.merger_lock:
task.connection_name, task.project_name)) self.log.info("Updating repo %s/%s",
self.merger.updateRepo(task.connection_name, task.project_name) task.connection_name, task.project_name)
repo = self.merger.getRepo(task.connection_name, task.project_name) self.merger.updateRepo(task.connection_name, task.project_name)
source = self.connections.getSource(task.connection_name) repo = self.merger.getRepo(
project = source.getProject(task.project_name) task.connection_name, task.project_name)
task.canonical_name = project.canonical_name source = self.connections.getSource(task.connection_name)
task.branches = repo.getBranches() project = source.getProject(task.project_name)
task.refs = [r.name for r in repo.getRefs()] task.canonical_name = project.canonical_name
self.log.debug("Finished updating repo %s/%s" % task.branches = repo.getBranches()
(task.connection_name, task.project_name)) task.refs = [r.name for r in repo.getRefs()]
task.setComplete() self.log.debug("Finished updating repo %s/%s",
task.connection_name, task.project_name)
task.success = True
except Exception:
self.log.exception('Got exception while updating repo %s/%s',
task.connection_name, task.project_name)
finally:
task.setComplete()
def update(self, connection_name, project_name): def update(self, connection_name, project_name):
# Update a repository in the main merger # Update a repository in the main merger