Merge "Refresh cached branches in timer driver"

tests/unit/test_scheduler.py

@@ -23,6 +23,7 @@ import threading
 import time
 from collections import namedtuple
 from unittest import mock, skip
+from uuid import uuid4
 from kazoo.exceptions import NoNodeError
 
 import git
@@ -4241,6 +4242,17 @@ class TestScheduler(ZuulTestCase):
         self.commitConfigUpdate('common-config', config_file)
         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
 
+        # Collect the currently cached branches in order to later check,
+        # that the timer driver refreshes the cache.
+        cached_versions = {}
+        tenant = self.scheds.first.sched.abide.tenants['tenant-one']
+        for project_name in tenant.layout.project_configs:
+            _, project = tenant.getProject('org/project')
+            for branch in project.source.getProjectBranches(project, tenant):
+                event = self._create_dummy_event(project, branch)
+                change = project.source.getChange(event)
+                cached_versions[branch] = change.cache_version
+
         # The pipeline triggers every second, so we should have seen
         # several by now.
         for _ in iterate_timeout(60, 'jobs started'):
@@ -4282,6 +4294,26 @@ class TestScheduler(ZuulTestCase):
             self.assertEqual(job.name, 'project-bitrot')
             self.assertIn(job.ref, ('refs/heads/stable', 'refs/heads/master'))
 
+        for project_name in tenant.layout.project_configs:
+            _, project = tenant.getProject('org/project')
+            for branch in project.source.getProjectBranches(project, tenant):
+                event = self._create_dummy_event(project, branch)
+                change = project.source.getChange(event)
+                # Make sure the timer driver refreshed the cache
+                self.assertGreater(change.cache_version,
+                                   cached_versions[branch])
+
+    def _create_dummy_event(self, project, branch):
+        event = zuul.model.TriggerEvent()
+        event.type = 'test'
+        event.project_hostname = project.canonical_hostname
+        event.project_name = project.name
+        event.ref = f'refs/heads/{branch}'
+        event.branch = branch
+        event.zuul_event_id = str(uuid4().hex)
+        event.timestamp = time.time()
+        return event
+
     def test_timer(self):
         "Test that a periodic job is triggered"
         self._test_timer('layouts/timer.yaml')

zuul/driver/gerrit/gerritconnection.py

@@ -748,19 +748,20 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
             change = self._getChange(event.change_number, event.patch_number,
                                      refresh=refresh)
         elif event.ref and event.ref.startswith('refs/tags/'):
-            change = self._getTag(event)
+            change = self._getTag(event, refresh=refresh)
         elif event.ref and not event.ref.startswith('refs/'):
             # Pre 2.13 Gerrit ref-updated events don't have branch prefixes.
             change = self._getBranch(event, branch=event.ref,
-                                     ref=f'refs/heads/{event.ref}')
+                                     ref=f'refs/heads/{event.ref}',
+                                     refresh=refresh)
         elif event.ref and event.ref.startswith('refs/heads/'):
             # From the timer trigger or Post 2.13 Gerrit
             change = self._getBranch(event,
                                      branch=event.ref[len('refs/heads/'):],
-                                     ref=event.ref)
+                                     ref=event.ref, refresh=refresh)
         elif event.ref:
             # catch-all ref (ie, not a branch or head)
-            change = self._getRef(event)
+            change = self._getRef(event, refresh=refresh)
         else:
             self.log.warning("Unable to get change for %s" % (event,))
             change = None
@@ -782,12 +783,15 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
             change.patchset = patchset
         return self._updateChange(key, change, event, history)
 
-    def _getTag(self, event):
+    def _getTag(self, event, refresh=False):
         tag = event.ref[len('refs/tags/'):]
         key = ChangeKey(self.connection_name, event.project_name,
                         'Tag', tag, event.newrev)
         change = self._change_cache.get(key)
         if change:
+            if refresh:
+                self._change_cache.updateChangeWithRetry(
+                    key, change, lambda c: None)
             return change
         project = self.source.getProject(event.project_name)
         change = Tag(project)
@@ -802,11 +806,14 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
             change = self._change_cache.get(key)
         return change
 
-    def _getBranch(self, event, branch, ref):
+    def _getBranch(self, event, branch, ref, refresh=False):
         key = ChangeKey(self.connection_name, event.project_name,
                         'Branch', branch, event.newrev)
         change = self._change_cache.get(key)
         if change:
+            if refresh:
+                self._change_cache.updateChangeWithRetry(
+                    key, change, lambda c: None)
             return change
         project = self.source.getProject(event.project_name)
         change = Branch(project)
@@ -821,11 +828,14 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
             change = self._change_cache.get(key)
         return change
 
-    def _getRef(self, event):
+    def _getRef(self, event, refresh=False):
         key = ChangeKey(self.connection_name, event.project_name,
                         'Ref', event.ref, event.newrev)
         change = self._change_cache.get(key)
         if change:
+            if refresh:
+                self._change_cache.updateChangeWithRetry(
+                    key, change, lambda c: None)
             return change
         project = self.source.getProject(event.project_name)
         change = Ref(project)

zuul/driver/github/githubconnection.py

@@ -1306,11 +1306,11 @@ class GithubConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
                                      event=event)
         else:
             if event.ref and event.ref.startswith('refs/tags/'):
-                change = self._getTag(project, event)
+                change = self._getTag(project, event, refresh=refresh)
             elif event.ref and event.ref.startswith('refs/heads/'):
-                change = self._getBranch(project, event)
+                change = self._getBranch(project, event, refresh=refresh)
             else:
-                change = self._getRef(project, event)
+                change = self._getRef(project, event, refresh=refresh)
         return change
 
     def _getChange(self, project, number, patchset=None, refresh=False,
@@ -1363,12 +1363,15 @@ class GithubConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
                 log.debug('Finished updating change %s', change)
         return change
 
-    def _getTag(self, project, event):
+    def _getTag(self, project, event, refresh=False):
         tag = event.ref[len('refs/tags/'):]
         key = ChangeKey(self.connection_name, project.name,
                         'Tag', tag, event.newrev)
         change = self._change_cache.get(key)
         if change:
+            if refresh:
+                self._change_cache.updateChangeWithRetry(
+                    key, change, lambda c: None)
             return change
         change = Tag(project)
         change.tag = tag
@@ -1385,12 +1388,15 @@ class GithubConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
             change = self._change_cache.get(key)
         return change
 
-    def _getBranch(self, project, event):
+    def _getBranch(self, project, event, refresh=False):
         branch = event.ref[len('refs/heads/'):]
         key = ChangeKey(self.connection_name, project.name,
                         'Branch', branch, event.newrev)
         change = self._change_cache.get(key)
         if change:
+            if refresh:
+                self._change_cache.updateChangeWithRetry(
+                    key, change, lambda c: None)
             return change
         change = Branch(project)
         change.branch = branch
@@ -1406,11 +1412,14 @@ class GithubConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
             change = self._change_cache.get(key)
         return change
 
-    def _getRef(self, project, event):
+    def _getRef(self, project, event, refresh=False):
         key = ChangeKey(self.connection_name, project.name,
                         'Ref', event.ref, event.newrev)
         change = self._change_cache.get(key)
         if change:
+            if refresh:
+                self._change_cache.updateChangeWithRetry(
+                    key, change, lambda c: None)
             return change
         change = Ref(project)
         change.ref = event.ref

zuul/driver/gitlab/gitlabconnection.py

@@ -632,11 +632,14 @@ class GitlabConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
         log.info("Updated change from Gitlab %s" % change)
         return change
 
-    def _getNonMRRef(self, project, event):
+    def _getNonMRRef(self, project, event, refresh=False):
         key = ChangeKey(self.connection_name, project.name,
                         'Ref', event.ref, event.newrev)
         change = self._change_cache.get(key)
         if change:
+            if refresh:
+                self._change_cache.updateChangeWithRetry(
+                    key, change, lambda c: None)
             return change
         if event.ref and event.ref.startswith('refs/tags/'):
             change = Tag(project)

zuul/driver/pagure/pagureconnection.py

@@ -611,7 +611,7 @@ class PagureConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
         else:
             self.log.info("Getting change for %s ref:%s" % (
                 project, event.ref))
-            change = self._getNonPRRef(project, event)
+            change = self._getNonPRRef(project, event, refresh=refresh)
         return change
 
     def _getChange(self, project, number, patchset=None,
@@ -646,11 +646,14 @@ class PagureConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
                                                           _update_change)
         return change
 
-    def _getNonPRRef(self, project, event):
+    def _getNonPRRef(self, project, event, refresh=False):
         key = ChangeKey(self.connection_name, project.name,
                         'Ref', event.ref, event.newrev)
         change = self._change_cache.get(key)
         if change:
+            if refresh:
+                self._change_cache.updateChangeWithRetry(
+                    key, change, lambda: None)
             return change
         if event.ref and event.ref.startswith('refs/tags/'):
             change = Tag(project)

zuul/driver/timer/__init__.py

@@ -173,6 +173,9 @@ class TimerDriver(Driver, TriggerInterface):
                 event.branch = branch
                 event.zuul_event_id = str(uuid4().hex)
                 event.timestamp = time.time()
+                # Refresh the branch in order to update the item in the
+                # change cache.
+                project.source.getChange(event, refresh=True)
                 log = get_annotated_logger(self.log, event)
                 log.debug("Adding event")
                 self.sched.addTriggerEvent(self.name, event)