Browse Source

Contain pipeline exceptions

This will continue processing other pipelines (and other tenants)
if one pipeline has an exception.  This shouldn't happen except in
the case of a Zuul bug, but if it does, it can contain the fallout.

Story: 2007761
Change-Id: Idecffb71b1897cd2269a9c7edc40d7d7e62614b9
changes/17/733917/2
James E. Blair 1 year ago
parent
commit
ec18f479e5
  1. 7
      zuul/model.py
  2. 15
      zuul/scheduler.py

7
zuul/model.py

@ -245,6 +245,9 @@ class Pipeline(object):
Reporter
Communicates success and failure results somewhere
"""
STATE_NORMAL = 'normal'
STATE_ERROR = 'error'
def __init__(self, name, tenant):
self.name = name
# Note that pipelines are not portable across tenants (new
@ -287,6 +290,7 @@ class Pipeline(object):
self.window_increase_factor = None
self.window_decrease_type = None
self.window_decrease_factor = None
self.state = self.STATE_NORMAL
@property
def actions(self):
@ -355,7 +359,8 @@ class Pipeline(object):
def formatStatusJSON(self, websocket_url=None):
j_pipeline = dict(name=self.name,
description=self.description)
description=self.description,
state=self.state)
j_queues = []
j_pipeline['change_queues'] = j_queues
for queue in self.queues:

15
zuul/scheduler.py

@ -1232,10 +1232,17 @@ class Scheduler(threading.Thread):
for tenant in self.abide.tenants.values():
for pipeline in tenant.layout.pipelines.values():
while (pipeline.manager.processQueue() and
not self._stopped):
pass
try:
while (pipeline.manager.processQueue() and
not self._stopped):
pass
except Exception:
self.log.exception(
"Exception in pipeline processing:")
pipeline.state = pipeline.STATE_ERROR
# Continue processing other pipelines+tenants
else:
pipeline.state = pipeline.STATE_NORMAL
except Exception:
self.log.exception("Exception in run handler:")
# There may still be more events to process

Loading…
Cancel
Save