Clear branch cache on full reconfiguration

The full reconfiguration event is primarily used to correct invalid
states which may result from missed events, such as a branch creation.
However, it did not invalidate or update the actual data in the branch
cache.  Correct this by completely clearing the branch cache of every
connection when starting a full reconfiguration.  This will cause the
connections to query their upstream sources the next time that we ask
for a list of branches.

Change-Id: I1529fe1dd6092de612a94c14ec0527ab76f62e47

tests/fixtures/config/one-project/git/common-config/playbooks/base.yaml

@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []

tests/fixtures/config/one-project/git/common-config/zuul.yaml

@@ -0,0 +1,41 @@
+- pipeline:
+    name: check
+    manager: independent
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- pipeline:
+    name: gate
+    manager: dependent
+    success-message: Build succeeded (gate).
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - Approved: 1
+    success:
+      gerrit:
+        Verified: 2
+        submit: true
+    failure:
+      gerrit:
+        Verified: -2
+    start:
+      gerrit:
+        Verified: 0
+    precedence: high
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml
+
+- job:
+    name: test-job

tests/fixtures/config/one-project/git/org_project/zuul.yaml

@@ -0,0 +1,8 @@
+- project:
+    name: org/project
+    check:
+      jobs:
+        - test-job
+    gate:
+      jobs:
+        - test-job

tests/fixtures/config/one-project/main.yaml

@@ -0,0 +1,9 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project
+

tests/unit/test_v3.py

@@ -388,6 +388,32 @@ class TestFinal(ZuulTestCase):
         self.assertIn('Unable to modify final job', A.messages[0])
 
 
+class TestBranchCreation(ZuulTestCase):
+    tenant_config_file = 'config/one-project/main.yaml'
+
+    def test_missed_branch_create(self):
+        # Test that if we miss a branch creation event, we can recover
+        # by issuing a full-reconfiguration.
+        self.create_branch('org/project', 'stable/yoga')
+        # We do not emit the gerrit event, thus simulating a missed event;
+        # verify that nothing happens
+        A = self.fake_gerrit.addFakeChange('org/project', 'stable/yoga', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertEqual(A.reported, 0)
+        self.assertHistory([])
+
+        # Correct the situation with a full reconfiguration
+        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
+        self.waitUntilSettled()
+
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertEqual(A.reported, 1)
+        self.assertHistory([
+            dict(name='test-job', result='SUCCESS', changes='1,1')])
+
+
 class TestBranchDeletion(ZuulTestCase):
     tenant_config_file = 'config/branch-deletion/main.yaml'
 

zuul/configloader.py

@@ -1660,9 +1660,15 @@ class TenantParser(object):
         tpc.shadow_projects = frozenset(shadow_projects)
 
     def _getProjectBranches(self, tenant, tpc, branch_cache_min_ltimes=None):
-        if branch_cache_min_ltimes:
-            min_ltime = branch_cache_min_ltimes.get(
-                tpc.project.source.connection.connection_name, -1)
+        if branch_cache_min_ltimes is not None:
+            # Use try/except here instead of .get in order to allow
+            # defaultdict to supply a default other than our default
+            # of -1.
+            try:
+                min_ltime = branch_cache_min_ltimes[
+                    tpc.project.source.connection.connection_name]
+            except KeyError:
+                min_ltime = -1
         else:
             min_ltime = -1
         branches = sorted(tpc.project.source.getProjectBranches(

zuul/connection/__init__.py

@@ -315,6 +315,15 @@ class ZKBranchCacheMixin:
             # again.
             event.branch_protected = True
 
+    def clearBranchCache(self):
+        """Clear the branch cache
+
+        In case the branch cache gets out of sync with the source,
+        this method can be called to clear it and force querying the
+        source the next time the cache is used.
+        """
+        self._branch_cache.clear()
+
 
 class ZKChangeCacheMixin:
     # Expected to be defined by the connection and to be an instance

zuul/scheduler.py

@@ -1303,6 +1303,25 @@ class Scheduler(threading.Thread):
             loader.loadTPCs(self.abide, self.unparsed_abide)
             loader.loadAdminRules(self.abide, self.unparsed_abide)
 
+            if event.smart:
+                # Consider caches always valid
+                min_ltimes = defaultdict(
+                    lambda: defaultdict(lambda: -1))
+                # Consider all project branch caches valid.
+                branch_cache_min_ltimes = defaultdict(lambda: -1)
+            else:
+                # Consider caches valid if the cache ltime >= event ltime
+                min_ltimes = defaultdict(
+                    lambda: defaultdict(lambda: event.zuul_event_ltime))
+                # Invalidate the branch cache for all connections
+                for connection in self.connections.connections.values():
+                    if hasattr(connection, 'clearBranchCache'):
+                        connection.clearBranchCache()
+                ltime = self.zk_client.getCurrentLtime()
+                # Consider the branch cache valid only after we
+                # cleared it
+                branch_cache_min_ltimes = defaultdict(lambda: ltime)
+
             for tenant_name in tenant_names:
                 if event.smart:
                     old_tenant = old_unparsed_abide.tenants.get(tenant_name)
@@ -1313,17 +1332,6 @@ class Scheduler(threading.Thread):
                     continue
 
                 old_tenant = self.abide.tenants.get(tenant_name)
-                if event.smart:
-                    # Consider caches always valid
-                    min_ltimes = defaultdict(
-                        lambda: defaultdict(lambda: -1))
-                else:
-                    # Consider caches valid if the cache ltime >= event ltime
-                    min_ltimes = defaultdict(
-                        lambda: defaultdict(lambda: event.zuul_event_ltime))
-
-                # Consider all project branch caches valid.
-                branch_cache_min_ltimes = defaultdict(lambda: -1)
 
                 stats_key = f'zuul.tenant.{tenant_name}'
                 with tenant_write_lock(

zuul/zk/branch_cache.py

@@ -104,6 +104,13 @@ class BranchCache:
                 self.cache = BranchCacheZKObject.new(
                     self.zk_context, _path=data_path)
 
+    def clear(self):
+        """Clear the cache"""
+        with locked(self.wlock):
+            with self.cache.activeContext(self.zk_context):
+                self.cache.protected.clear()
+                self.cache.remainder.clear()
+
     def getProjectBranches(self, project_name, exclude_unprotected,
                            min_ltime=-1):
         """Get the branch names for the given project.