zuul
/
zuul
Code Issues Proposed changes

Merge "Refresh cached branches in timer driver"

This commit is contained in:
Zuul 2022-01-27 21:09:05 +00:00 committed by Gerrit Code Review
parent 4c0fef8c1e 3d62dc862d
commit 0279c3116e
6 changed files with 76 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
tests/unit/test_scheduler.py
View File

@ -23,6 +23,7 @@ import threading
import time
from collections import namedtuple
from unittest import mock, skip
from uuid import uuid4
from kazoo.exceptions import NoNodeError
import git
@ -4241,6 +4242,17 @@ class TestScheduler(ZuulTestCase):
self.commitConfigUpdate('common-config', config_file)
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
# Collect the currently cached branches in order to later check,
# that the timer driver refreshes the cache.
cached_versions = {}
tenant = self.scheds.first.sched.abide.tenants['tenant-one']
for project_name in tenant.layout.project_configs:
_, project = tenant.getProject('org/project')
for branch in project.source.getProjectBranches(project, tenant):
event = self._create_dummy_event(project, branch)
change = project.source.getChange(event)
cached_versions[branch] = change.cache_version
# The pipeline triggers every second, so we should have seen
# several by now.
for _ in iterate_timeout(60, 'jobs started'):
@ -4282,6 +4294,26 @@ class TestScheduler(ZuulTestCase):
self.assertEqual(job.name, 'project-bitrot')
self.assertIn(job.ref, ('refs/heads/stable', 'refs/heads/master'))
for project_name in tenant.layout.project_configs:
_, project = tenant.getProject('org/project')
for branch in project.source.getProjectBranches(project, tenant):
event = self._create_dummy_event(project, branch)
change = project.source.getChange(event)
# Make sure the timer driver refreshed the cache
self.assertGreater(change.cache_version,
cached_versions[branch])
def _create_dummy_event(self, project, branch):
event = zuul.model.TriggerEvent()
event.type = 'test'
event.project_hostname = project.canonical_hostname
event.project_name = project.name
event.ref = f'refs/heads/{branch}'
event.branch = branch
event.zuul_event_id = str(uuid4().hex)
event.timestamp = time.time()
return event
def test_timer(self):
"Test that a periodic job is triggered"
self._test_timer('layouts/timer.yaml')

24
zuul/driver/gerrit/gerritconnection.py
View File

@ -748,19 +748,20 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
change = self._getChange(event.change_number, event.patch_number,
refresh=refresh)
elif event.ref and event.ref.startswith('refs/tags/'):
change = self._getTag(event)
change = self._getTag(event, refresh=refresh)
elif event.ref and not event.ref.startswith('refs/'):
# Pre 2.13 Gerrit ref-updated events don't have branch prefixes.
change = self._getBranch(event, branch=event.ref,
ref=f'refs/heads/{event.ref}')
ref=f'refs/heads/{event.ref}',
refresh=refresh)
elif event.ref and event.ref.startswith('refs/heads/'):
# From the timer trigger or Post 2.13 Gerrit
change = self._getBranch(event,
branch=event.ref[len('refs/heads/'):],
ref=event.ref)
ref=event.ref, refresh=refresh)
elif event.ref:
# catch-all ref (ie, not a branch or head)
change = self._getRef(event)
change = self._getRef(event, refresh=refresh)
else:
self.log.warning("Unable to get change for %s" % (event,))
change = None
@ -782,12 +783,15 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
change.patchset = patchset
return self._updateChange(key, change, event, history)
def _getTag(self, event):
def _getTag(self, event, refresh=False):
tag = event.ref[len('refs/tags/'):]
key = ChangeKey(self.connection_name, event.project_name,
'Tag', tag, event.newrev)
change = self._change_cache.get(key)
if change:
if refresh:
self._change_cache.updateChangeWithRetry(
key, change, lambda c: None)
return change
project = self.source.getProject(event.project_name)
change = Tag(project)
@ -802,11 +806,14 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
change = self._change_cache.get(key)
return change
def _getBranch(self, event, branch, ref):
def _getBranch(self, event, branch, ref, refresh=False):
key = ChangeKey(self.connection_name, event.project_name,
'Branch', branch, event.newrev)
change = self._change_cache.get(key)
if change:
if refresh:
self._change_cache.updateChangeWithRetry(
key, change, lambda c: None)
return change
project = self.source.getProject(event.project_name)
change = Branch(project)
@ -821,11 +828,14 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
change = self._change_cache.get(key)
return change
def _getRef(self, event):
def _getRef(self, event, refresh=False):
key = ChangeKey(self.connection_name, event.project_name,
'Ref', event.ref, event.newrev)
change = self._change_cache.get(key)
if change:
if refresh:
self._change_cache.updateChangeWithRetry(
key, change, lambda c: None)
return change
project = self.source.getProject(event.project_name)
change = Ref(project)

21
zuul/driver/github/githubconnection.py
View File

@ -1306,11 +1306,11 @@ class GithubConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
event=event)
else:
if event.ref and event.ref.startswith('refs/tags/'):
change = self._getTag(project, event)
change = self._getTag(project, event, refresh=refresh)
elif event.ref and event.ref.startswith('refs/heads/'):
change = self._getBranch(project, event)
change = self._getBranch(project, event, refresh=refresh)
else:
change = self._getRef(project, event)
change = self._getRef(project, event, refresh=refresh)
return change
def _getChange(self, project, number, patchset=None, refresh=False,
@ -1363,12 +1363,15 @@ class GithubConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
log.debug('Finished updating change %s', change)
return change
def _getTag(self, project, event):
def _getTag(self, project, event, refresh=False):
tag = event.ref[len('refs/tags/'):]
key = ChangeKey(self.connection_name, project.name,
'Tag', tag, event.newrev)
change = self._change_cache.get(key)
if change:
if refresh:
self._change_cache.updateChangeWithRetry(
key, change, lambda c: None)
return change
change = Tag(project)
change.tag = tag
@ -1385,12 +1388,15 @@ class GithubConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
change = self._change_cache.get(key)
return change
def _getBranch(self, project, event):
def _getBranch(self, project, event, refresh=False):
branch = event.ref[len('refs/heads/'):]
key = ChangeKey(self.connection_name, project.name,
'Branch', branch, event.newrev)
change = self._change_cache.get(key)
if change:
if refresh:
self._change_cache.updateChangeWithRetry(
key, change, lambda c: None)
return change
change = Branch(project)
change.branch = branch
@ -1406,11 +1412,14 @@ class GithubConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
change = self._change_cache.get(key)
return change
def _getRef(self, project, event):
def _getRef(self, project, event, refresh=False):
key = ChangeKey(self.connection_name, project.name,
'Ref', event.ref, event.newrev)
change = self._change_cache.get(key)
if change:
if refresh:
self._change_cache.updateChangeWithRetry(
key, change, lambda c: None)
return change
change = Ref(project)
change.ref = event.ref

5
zuul/driver/gitlab/gitlabconnection.py
View File

@ -632,11 +632,14 @@ class GitlabConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
log.info("Updated change from Gitlab %s" % change)
return change
def _getNonMRRef(self, project, event):
def _getNonMRRef(self, project, event, refresh=False):
key = ChangeKey(self.connection_name, project.name,
'Ref', event.ref, event.newrev)
change = self._change_cache.get(key)
if change:
if refresh:
self._change_cache.updateChangeWithRetry(
key, change, lambda c: None)
return change
if event.ref and event.ref.startswith('refs/tags/'):
change = Tag(project)

7
zuul/driver/pagure/pagureconnection.py
View File

@ -611,7 +611,7 @@ class PagureConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
else:
self.log.info("Getting change for %s ref:%s" % (
project, event.ref))
change = self._getNonPRRef(project, event)
change = self._getNonPRRef(project, event, refresh=refresh)
return change
def _getChange(self, project, number, patchset=None,
@ -646,11 +646,14 @@ class PagureConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
_update_change)
return change
def _getNonPRRef(self, project, event):
def _getNonPRRef(self, project, event, refresh=False):
key = ChangeKey(self.connection_name, project.name,
'Ref', event.ref, event.newrev)
change = self._change_cache.get(key)
if change:
if refresh:
self._change_cache.updateChangeWithRetry(
key, change, lambda: None)
return change
if event.ref and event.ref.startswith('refs/tags/'):
change = Tag(project)

3
zuul/driver/timer/__init__.py
View File

@ -173,6 +173,9 @@ class TimerDriver(Driver, TriggerInterface):
event.branch = branch
event.zuul_event_id = str(uuid4().hex)
event.timestamp = time.time()
# Refresh the branch in order to update the item in the
# change cache.
project.source.getChange(event, refresh=True)
log = get_annotated_logger(self.log, event)
log.debug("Adding event")
self.sched.addTriggerEvent(self.name, event)