Fix branch deletion after failed reconfig

Normally when a branch is deleted, we delete the cached data for
that branch and perform a partial ("tenant") reconfiguration.  However,
there are two caches of configuration for each project -- the full
cache, and the per-branch cache.  We were not deleting the per-branch
cache, so if the tenant reconfiguration failed, the per-branch cache
would still contain the deleted branch.

Also, on full reconfiguration, we update the per-branch cache, however,
we never deleted branches from the cache in that case, so even a full
reconfiguration would be unable to clear that cache.  This is fixed
as well.

Change-Id: Ia9c17238ef0e42609e3de304af9c0c4588c72995

tests/base.py

@@ -529,6 +529,24 @@ class FakeGerritConnection(gerritconnection.GerritConnection):
         }
         return event
 
+    def getFakeBranchDeletedEvent(self, project, branch):
+        oldrev = '4abd38457c2da2a72d4d030219ab180ecdb04bf0'
+        newrev = 40 * '0'
+
+        event = {
+            "type": "ref-updated",
+            "submitter": {
+                "name": "User Name",
+            },
+            "refUpdate": {
+                "oldRev": oldrev,
+                "newRev": newrev,
+                "refName": 'refs/heads/' + branch,
+                "project": project,
+            }
+        }
+        return event
+
     def review(self, project, changeid, message, action):
         number, ps = changeid.split(',')
         change = self.changes[int(number)]

tests/fixtures/config/branch-deletion/git/common-config/playbooks/base.yaml

@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []

tests/fixtures/config/branch-deletion/git/common-config/zuul.yaml

@@ -0,0 +1,17 @@
+- pipeline:
+    name: check
+    manager: independent
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml

tests/fixtures/config/branch-deletion/git/org_project/zuul.yaml

@@ -0,0 +1,4 @@
+- project:
+    name: org/project
+    check:
+      jobs: []

tests/fixtures/config/branch-deletion/git/org_project1/zuul.yaml

@@ -0,0 +1,3 @@
+- project:
+    check:
+      jobs: []

tests/fixtures/config/branch-deletion/main.yaml

@@ -0,0 +1,10 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project
+          - org/project1
+

tests/unit/test_v3.py

@@ -301,6 +301,106 @@ class TestFinal(ZuulTestCase):
         self.assertIn('Unable to modify final job', A.messages[0])
 
 
+class TestBranchDeletion(ZuulTestCase):
+    tenant_config_file = 'config/branch-deletion/main.yaml'
+
+    def test_branch_delete(self):
+        # This tests a tenant reconfiguration on deleting a branch
+        # *after* an earlier failed tenant reconfiguration.  This
+        # ensures that cached data are appropriately removed, even if
+        # we are recovering from an invalid config.
+        self.create_branch('org/project', 'stable/queens')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project', 'stable/queens'))
+        self.waitUntilSettled()
+
+        in_repo_conf = textwrap.dedent(
+            """
+            - project:
+                check:
+                  jobs:
+                    - nonexistent-job
+            """)
+
+        file_dict = {'zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project', 'stable/queens', 'A',
+                                           files=file_dict)
+        A.setMerged()
+        self.fake_gerrit.addEvent(A.getChangeMergedEvent())
+        self.waitUntilSettled()
+
+        self.delete_branch('org/project', 'stable/queens')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchDeletedEvent(
+                'org/project', 'stable/queens'))
+        self.waitUntilSettled()
+
+        in_repo_conf = textwrap.dedent(
+            """
+            - project:
+                check:
+                  jobs:
+                    - base
+            """)
+
+        file_dict = {'zuul.yaml': in_repo_conf}
+        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertEqual(B.reported, 1)
+        self.assertHistory([
+            dict(name='base', result='SUCCESS', changes='2,1')])
+
+    def test_branch_delete_full_reconfiguration(self):
+        # This tests a full configuration after deleting a branch
+        # *after* an earlier failed tenant reconfiguration.  This
+        # ensures that cached data are appropriately removed, even if
+        # we are recovering from an invalid config.
+        self.create_branch('org/project', 'stable/queens')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project', 'stable/queens'))
+        self.waitUntilSettled()
+
+        in_repo_conf = textwrap.dedent(
+            """
+            - project:
+                check:
+                  jobs:
+                    - nonexistent-job
+            """)
+
+        file_dict = {'zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project', 'stable/queens', 'A',
+                                           files=file_dict)
+        A.setMerged()
+        self.fake_gerrit.addEvent(A.getChangeMergedEvent())
+        self.waitUntilSettled()
+
+        self.delete_branch('org/project', 'stable/queens')
+        self.sched.reconfigure(self.config)
+        self.waitUntilSettled()
+
+        in_repo_conf = textwrap.dedent(
+            """
+            - project:
+                check:
+                  jobs:
+                    - base
+            """)
+
+        file_dict = {'zuul.yaml': in_repo_conf}
+        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertEqual(B.reported, 1)
+        self.assertHistory([
+            dict(name='base', result='SUCCESS', changes='2,1')])
+
+
 class TestBranchTag(ZuulTestCase):
     tenant_config_file = 'config/branch-tag/main.yaml'
 

zuul/configloader.py

@@ -1545,8 +1545,7 @@ class TenantParser(object):
             project.unparsed_config = data
         for project, branch_config in \
             new_project_unparsed_branch_config.items():
-            for branch, data in branch_config.items():
-                project.unparsed_branch_config[branch] = data
+            project.unparsed_branch_config = branch_config
         return config_projects_config, untrusted_projects_config
 
     @staticmethod

zuul/scheduler.py

@@ -570,6 +570,7 @@ class Scheduler(threading.Thread):
             # config before reconfiguring.
             for project in event.projects:
                 project.unparsed_config = None
+                project.unparsed_branch_config = {}
             old_tenant = self.abide.tenants[event.tenant_name]
             loader = configloader.ConfigLoader()
             abide = loader.reloadTenant(