From ec18f479e5652a7025e141af0f6bb6c637105a7c Mon Sep 17 00:00:00 2001
From: "James E. Blair" <jeblair@redhat.com>
Date: Fri, 5 Jun 2020 13:57:42 -0700
Subject: [PATCH] Contain pipeline exceptions

This will continue processing other pipelines (and other tenants)
if one pipeline has an exception.  This shouldn't happen except in
the case of a Zuul bug, but if it does, it can contain the fallout.

Story: 2007761
Change-Id: Idecffb71b1897cd2269a9c7edc40d7d7e62614b9
---
 zuul/model.py     |  7 ++++++-
 zuul/scheduler.py | 15 +++++++++++----
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/zuul/model.py b/zuul/model.py
index 377b4ba40a..34f358b0ed 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -245,6 +245,9 @@ class Pipeline(object):
     Reporter
         Communicates success and failure results somewhere
     """
+    STATE_NORMAL = 'normal'
+    STATE_ERROR = 'error'
+
     def __init__(self, name, tenant):
         self.name = name
         # Note that pipelines are not portable across tenants (new
@@ -287,6 +290,7 @@ class Pipeline(object):
         self.window_increase_factor = None
         self.window_decrease_type = None
         self.window_decrease_factor = None
+        self.state = self.STATE_NORMAL
 
     @property
     def actions(self):
@@ -355,7 +359,8 @@ class Pipeline(object):
 
     def formatStatusJSON(self, websocket_url=None):
         j_pipeline = dict(name=self.name,
-                          description=self.description)
+                          description=self.description,
+                          state=self.state)
         j_queues = []
         j_pipeline['change_queues'] = j_queues
         for queue in self.queues:
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index cfcc64485c..897c16610a 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -1232,10 +1232,17 @@ class Scheduler(threading.Thread):
 
                 for tenant in self.abide.tenants.values():
                     for pipeline in tenant.layout.pipelines.values():
-                        while (pipeline.manager.processQueue() and
-                               not self._stopped):
-                            pass
-
+                        try:
+                            while (pipeline.manager.processQueue() and
+                                   not self._stopped):
+                                pass
+                        except Exception:
+                            self.log.exception(
+                                "Exception in pipeline processing:")
+                            pipeline.state = pipeline.STATE_ERROR
+                            # Continue processing other pipelines+tenants
+                        else:
+                            pipeline.state = pipeline.STATE_NORMAL
             except Exception:
                 self.log.exception("Exception in run handler:")
                 # There may still be more events to process