diff --git a/zuul/model.py b/zuul/model.py index 377b4ba40a..34f358b0ed 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -245,6 +245,9 @@ class Pipeline(object): Reporter Communicates success and failure results somewhere """ + STATE_NORMAL = 'normal' + STATE_ERROR = 'error' + def __init__(self, name, tenant): self.name = name # Note that pipelines are not portable across tenants (new @@ -287,6 +290,7 @@ class Pipeline(object): self.window_increase_factor = None self.window_decrease_type = None self.window_decrease_factor = None + self.state = self.STATE_NORMAL @property def actions(self): @@ -355,7 +359,8 @@ class Pipeline(object): def formatStatusJSON(self, websocket_url=None): j_pipeline = dict(name=self.name, - description=self.description) + description=self.description, + state=self.state) j_queues = [] j_pipeline['change_queues'] = j_queues for queue in self.queues: diff --git a/zuul/scheduler.py b/zuul/scheduler.py index cfcc64485c..897c16610a 100644 --- a/zuul/scheduler.py +++ b/zuul/scheduler.py @@ -1232,10 +1232,17 @@ class Scheduler(threading.Thread): for tenant in self.abide.tenants.values(): for pipeline in tenant.layout.pipelines.values(): - while (pipeline.manager.processQueue() and - not self._stopped): - pass - + try: + while (pipeline.manager.processQueue() and + not self._stopped): + pass + except Exception: + self.log.exception( + "Exception in pipeline processing:") + pipeline.state = pipeline.STATE_ERROR + # Continue processing other pipelines+tenants + else: + pipeline.state = pipeline.STATE_NORMAL except Exception: self.log.exception("Exception in run handler:") # There may still be more events to process