Browse Source

Fix stuck job caused by exception during repo update

If we hit an exception while updating repo the job is not marked as
complete. This causes the job that waits for it to be stuck
forever. This situation can only be solved by forcing a dequeue of the
change or restarting the executor.

Change-Id: Ia52cc96c3ce39c50bb9d7ff8162d0aa0f7f13c77
changes/97/590697/2
Tobias Henkel 3 years ago
parent
commit
393d66b04d
1 changed files with 26 additions and 13 deletions
  1. +26
    -13
      zuul/executor/server.py

+ 26
- 13
zuul/executor/server.py View File

@ -492,6 +492,7 @@ class UpdateTask(object):
self.branches = None
self.refs = None
self.event = threading.Event()
self.success = False
def __eq__(self, other):
if (other and other.connection_name == self.connection_name and
@ -802,6 +803,11 @@ class AnsibleJob(object):
for task in tasks:
task.wait()
if not task.success:
raise ExecutorError(
'Failed to update project %s' % task.canonical_name)
self.project_info[task.canonical_name] = {
'refs': task.refs,
'branches': task.branches,
@ -2304,19 +2310,26 @@ class ExecutorServer(object):
if task is None:
# We are asked to stop
raise StopException()
with self.merger_lock:
self.log.info("Updating repo %s/%s" % (
task.connection_name, task.project_name))
self.merger.updateRepo(task.connection_name, task.project_name)
repo = self.merger.getRepo(task.connection_name, task.project_name)
source = self.connections.getSource(task.connection_name)
project = source.getProject(task.project_name)
task.canonical_name = project.canonical_name
task.branches = repo.getBranches()
task.refs = [r.name for r in repo.getRefs()]
self.log.debug("Finished updating repo %s/%s" %
(task.connection_name, task.project_name))
task.setComplete()
try:
with self.merger_lock:
self.log.info("Updating repo %s/%s",
task.connection_name, task.project_name)
self.merger.updateRepo(task.connection_name, task.project_name)
repo = self.merger.getRepo(
task.connection_name, task.project_name)
source = self.connections.getSource(task.connection_name)
project = source.getProject(task.project_name)
task.canonical_name = project.canonical_name
task.branches = repo.getBranches()
task.refs = [r.name for r in repo.getRefs()]
self.log.debug("Finished updating repo %s/%s",
task.connection_name, task.project_name)
task.success = True
except Exception:
self.log.exception('Got exception while updating repo %s/%s',
task.connection_name, task.project_name)
finally:
task.setComplete()
def update(self, connection_name, project_name):
# Update a repository in the main merger


Loading…
Cancel
Save