From 186792d5cdb304c6b6dd8e1dc43e428532b9ef46 Mon Sep 17 00:00:00 2001
From: "James E. Blair" <jim@acmegating.com>
Date: Thu, 13 Jan 2022 15:30:59 -0800
Subject: [PATCH] Handle missing config files in zuul-web

The following sequence is possible:

1) New project added to tenant config file
2) Scheduler begins smart reconfiguration
3) Scheduler encounters problem accessing the branch list for the
   new repo.  This is treated as a ConfigurationError and added
   to the layout error list but proceeds.
?) If at some point a branch listing for the new project has succeeded,
   there will be an entry for the branch in the branch cache.
4) A zuul-web starts (or attempts a reconfiguration), sees the branch
   in the branch cache, attempts to load the files from the config
   cache, and fails to load the layout.

The scheduler and web show different behaviors because web is unable
to fetch files via mergers.  To bring them closer to the same behavior,
treat missing files from the config cache as layout errors.

Change-Id: I76d659f558cc3ed95a6ba7259d11b457ca57976c
---
 tests/unit/test_web.py | 49 ++++++++++++++++++++++++++++++------------
 zuul/configloader.py   |  8 +++++--
 2 files changed, 41 insertions(+), 16 deletions(-)

diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py
index 1045127dfd..a34680f84b 100644
--- a/tests/unit/test_web.py
+++ b/tests/unit/test_web.py
@@ -46,13 +46,10 @@ class FakeConfig(object):
         return self.config.get(section, {}).get(option)
 
 
-class BaseTestWeb(ZuulTestCase):
-    tenant_config_file = 'config/single-tenant/main.yaml'
+class BaseWithWeb(ZuulTestCase):
     config_ini_data = {}
 
-    def setUp(self):
-        super(BaseTestWeb, self).setUp()
-
+    def startWebServer(self):
         self.zuul_ini_config = FakeConfig(self.config_ini_data)
         # Start the web server
         self.web = self.useFixture(
@@ -78,15 +75,6 @@ class BaseTestWeb(ZuulTestCase):
         self.base_url = "http://{host}:{port}".format(
             host=self.host, port=self.port)
 
-    def add_base_changes(self):
-        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
-        A.addApproval('Code-Review', 2)
-        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
-        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
-        B.addApproval('Code-Review', 2)
-        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
-        self.waitUntilSettled()
-
     def get_url(self, url, *args, **kwargs):
         return requests.get(
             urllib.parse.urljoin(self.base_url, url), *args, **kwargs)
@@ -103,6 +91,23 @@ class BaseTestWeb(ZuulTestCase):
         return requests.options(
             urllib.parse.urljoin(self.base_url, url), *args, **kwargs)
 
+
+class BaseTestWeb(BaseWithWeb):
+    tenant_config_file = 'config/single-tenant/main.yaml'
+
+    def setUp(self):
+        super(BaseTestWeb, self).setUp()
+        self.startWebServer()
+
+    def add_base_changes(self):
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        A.addApproval('Code-Review', 2)
+        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
+        B.addApproval('Code-Review', 2)
+        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
+        self.waitUntilSettled()
+
     def tearDown(self):
         self.executor_server.hold_jobs_in_build = False
         self.executor_server.release()
@@ -1387,6 +1392,22 @@ class TestTenantInfoConfigBroken(BaseTestWeb):
         self.assertEqual(404, resp.status_code)
 
 
+class TestBrokenConfigCache(BaseWithWeb):
+    tenant_config_file = 'config/single-tenant/main.yaml'
+
+    def test_broken_config_cache(self):
+        # Delete the cached config files from ZK to simulate a
+        # scheduler encountering an error in reconfiguration.
+        path = '/zuul/config/cache/review.example.com%2Forg%2Fproject'
+        self.assertIsNotNone(self.zk_client.client.exists(path))
+        self.zk_client.client.delete(path, recursive=True)
+        self.startWebServer()
+        config_errors = self.get_url(
+            "api/tenant/tenant-one/config-errors").json()
+        self.assertIn('Configuration files missing',
+                      config_errors[0]['error'])
+
+
 class TestWebSocketInfo(TestInfo):
 
     config_ini_data = {
diff --git a/zuul/configloader.py b/zuul/configloader.py
index efefd0976c..58650b1daf 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -1827,8 +1827,12 @@ class TenantParser(object):
                 extra_config_files = abide.getExtraConfigFiles(project.name)
                 extra_config_dirs = abide.getExtraConfigDirs(project.name)
                 if not self.merger:
-                    raise RuntimeError(
-                        "Cannot load config files without a merger client.")
+                    with project_configuration_exceptions(source_context,
+                                                          loading_errors):
+                        raise Exception(
+                            "Configuration files missing from cache. "
+                            "Check Zuul scheduler logs for more information.")
+                    continue
                 ltime = self.zk_client.getCurrentLtime()
                 job = self.merger.getFiles(
                     project.source.connection.connection_name,