zuul
/
zuul
Code Issues Proposed changes

Fix stuck job caused by exception during repo update 

If we hit an exception while updating repo the job is not marked as
complete. This causes the job that waits for it to be stuck
forever. This situation can only be solved by forcing a dequeue of the
change or restarting the executor.

Change-Id: Ia52cc96c3ce39c50bb9d7ff8162d0aa0f7f13c77
This commit is contained in:
Tobias Henkel 2018-08-09 09:13:37 +02:00
parent 8a58a358d1
commit 393d66b04d
1 changed files with 26 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
zuul/executor/server.py
View File

@ -492,6 +492,7 @@ class UpdateTask(object):
self.branches = None
self.refs = None
self.event = threading.Event()
self.success = False
def __eq__(self, other):
if (other and other.connection_name == self.connection_name and
@ -802,6 +803,11 @@ class AnsibleJob(object):
for task in tasks:
task.wait()
if not task.success:
raise ExecutorError(
'Failed to update project %s' % task.canonical_name)
self.project_info[task.canonical_name] = {
'refs': task.refs,
'branches': task.branches,
@ -2304,19 +2310,26 @@ class ExecutorServer(object):
if task is None:
# We are asked to stop
raise StopException()
with self.merger_lock:
self.log.info("Updating repo %s/%s" % (
task.connection_name, task.project_name))
self.merger.updateRepo(task.connection_name, task.project_name)
repo = self.merger.getRepo(task.connection_name, task.project_name)
source = self.connections.getSource(task.connection_name)
project = source.getProject(task.project_name)
task.canonical_name = project.canonical_name
task.branches = repo.getBranches()
task.refs = [r.name for r in repo.getRefs()]
self.log.debug("Finished updating repo %s/%s" %
(task.connection_name, task.project_name))
task.setComplete()
try:
with self.merger_lock:
self.log.info("Updating repo %s/%s",
task.connection_name, task.project_name)
self.merger.updateRepo(task.connection_name, task.project_name)
repo = self.merger.getRepo(
task.connection_name, task.project_name)
source = self.connections.getSource(task.connection_name)
project = source.getProject(task.project_name)
task.canonical_name = project.canonical_name
task.branches = repo.getBranches()
task.refs = [r.name for r in repo.getRefs()]
self.log.debug("Finished updating repo %s/%s",
task.connection_name, task.project_name)
task.success = True
except Exception:
self.log.exception('Got exception while updating repo %s/%s',
task.connection_name, task.project_name)
finally:
task.setComplete()
def update(self, connection_name, project_name):
# Update a repository in the main merger