Don't clear connection caches during full reconfig

Clearing of connection caches doesn't seem to be needed in practice and
is removed as it can't be easily coordinated between multiple
schedulers.

Change-Id: I663fa1f1752d0dd599af06a7275a14235faaff2f
This commit is contained in:
Simon Westphahl 2021-07-08 16:25:01 +02:00
parent 32f3d5fe35
commit ce0e348eb6
12 changed files with 54 additions and 44 deletions

View File

@ -1388,15 +1388,21 @@ class TestGithubCircularDependencies(ZuulTestCase):
'master', True) 'master', True)
github.repo_from_project('gh/project')._set_branch_protection( github.repo_from_project('gh/project')._set_branch_protection(
'stable/foo', True) 'stable/foo', True)
pevent = self.fake_github.getPushEvent(project='gh/project',
ref='refs/heads/stable/foo')
self.fake_github.emitEvent(pevent)
self.create_branch('gh/project1', 'stable/bar') self.create_branch('gh/project1', 'stable/bar')
github.repo_from_project('gh/project1')._set_branch_protection( github.repo_from_project('gh/project1')._set_branch_protection(
'master', True) 'master', True)
github.repo_from_project('gh/project1')._set_branch_protection( github.repo_from_project('gh/project1')._set_branch_protection(
'stable/bar', True) 'stable/bar', True)
pevent = self.fake_github.getPushEvent(project='gh/project',
ref='refs/heads/stable/bar')
self.fake_github.emitEvent(pevent)
# Reconfigure to pick up branch protection settings # Wait until push events are processed to pick up branch
self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) # protection settings
self.waitUntilSettled() self.waitUntilSettled()
A = self.fake_github.openFakePullRequest( A = self.fake_github.openFakePullRequest(

View File

@ -434,8 +434,13 @@ class TestTenantConfigBranches(ZuulTestCase):
self.log.debug('Creating branches') self.log.debug('Creating branches')
self.create_branch('common-config', 'stable') self.create_branch('common-config', 'stable')
self.create_branch('common-config', 'feat_x') self.create_branch('common-config', 'feat_x')
self.fake_gerrit.addEvent(
self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.fake_gerrit.getFakeBranchCreatedEvent(
'common-config', 'stable'))
self.fake_gerrit.addEvent(
self.fake_gerrit.getFakeBranchCreatedEvent(
'common-config', 'feat_x'))
self.waitUntilSettled()
# Job must be defined in master # Job must be defined in master
self._validate_job(common_job, 'master') self._validate_job(common_job, 'master')

View File

@ -1339,7 +1339,10 @@ class TestGithubUnprotectedBranches(ZuulTestCase):
github = self.fake_github.getGithubClient() github = self.fake_github.getGithubClient()
repo = github.repo_from_project('org/project2') repo = github.repo_from_project('org/project2')
repo._set_branch_protection('master', True) repo._set_branch_protection('master', True)
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
pevent = self.fake_github.getPushEvent(project='org/project2',
ref='refs/heads/master')
self.fake_github.emitEvent(pevent)
self.waitUntilSettled() self.waitUntilSettled()
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
@ -1470,6 +1473,9 @@ class TestGithubUnprotectedBranches(ZuulTestCase):
github = self.fake_github.getGithubClient() github = self.fake_github.getGithubClient()
repo = github.repo_from_project('org/project2') repo = github.repo_from_project('org/project2')
repo._set_branch_protection('master', True) repo._set_branch_protection('master', True)
self.fake_github.emitEvent(
self.fake_github.getPushEvent(
project='org/project2', ref='refs/heads/master'))
A = self.fake_github.openFakePullRequest('org/project2', 'master', 'A') A = self.fake_github.openFakePullRequest('org/project2', 'master', 'A')
A.setMerged("merging A") A.setMerged("merging A")
@ -1477,8 +1483,10 @@ class TestGithubUnprotectedBranches(ZuulTestCase):
# add a spare branch so that the project is not empty after master gets # add a spare branch so that the project is not empty after master gets
# deleted. # deleted.
repo._create_branch('feat-x') repo._create_branch('feat-x')
self.fake_github.emitEvent(
self.fake_github.getPushEvent(
project='org/project2', ref='refs/heads/feat-x'))
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
self.waitUntilSettled() self.waitUntilSettled()
# record previous tenant reconfiguration time, which may not be set # record previous tenant reconfiguration time, which may not be set

View File

@ -726,7 +726,8 @@ class TestGitlabUnprotectedBranches(ZuulTestCase):
# now enable branch protection and trigger reload # now enable branch protection and trigger reload
self.fake_gitlab.protectBranch('org', 'project2', 'master') self.fake_gitlab.protectBranch('org', 'project2', 'master')
self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) pevent = self.fake_gitlab.getPushEvent(project='org/project2')
self.fake_gitlab.emitEvent(pevent)
self.waitUntilSettled() self.waitUntilSettled()
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
@ -869,6 +870,9 @@ class TestGitlabUnprotectedBranches(ZuulTestCase):
# Prepare repo with an initial commit and enable branch protection # Prepare repo with an initial commit and enable branch protection
self.fake_gitlab.protectBranch('org', 'project2', 'master') self.fake_gitlab.protectBranch('org', 'project2', 'master')
self.fake_gitlab.emitEvent(
self.fake_gitlab.getPushEvent(
project='org/project2', branch='refs/heads/master'))
A = self.fake_gitlab.openFakeMergeRequest('org/project2', 'master', A = self.fake_gitlab.openFakeMergeRequest('org/project2', 'master',
'A') 'A')
@ -879,8 +883,9 @@ class TestGitlabUnprotectedBranches(ZuulTestCase):
self.create_branch('org/project2', 'feat-x') self.create_branch('org/project2', 'feat-x')
self.fake_gitlab.protectBranch('org', 'project2', 'feat-x', self.fake_gitlab.protectBranch('org', 'project2', 'feat-x',
protected=False) protected=False)
self.fake_gitlab.emitEvent(
self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.fake_gitlab.getPushEvent(
project='org/project2', branch='refs/heads/feat-x'))
self.waitUntilSettled() self.waitUntilSettled()
# record previous tenant reconfiguration time, which may not be set # record previous tenant reconfiguration time, which may not be set

View File

@ -4072,6 +4072,9 @@ class TestScheduler(ZuulTestCase):
# timer-triggered job so that we have an opportunity to set # timer-triggered job so that we have an opportunity to set
# the hold flag before the first job. # the hold flag before the first job.
self.create_branch('org/project', 'stable') self.create_branch('org/project', 'stable')
self.fake_gerrit.addEvent(
self.fake_gerrit.getFakeBranchCreatedEvent(
'org/project', 'stable'))
self.executor_server.hold_jobs_in_build = True self.executor_server.hold_jobs_in_build = True
self.commitConfigUpdate('common-config', 'layouts/timer-template.yaml') self.commitConfigUpdate('common-config', 'layouts/timer-template.yaml')
self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
@ -4122,6 +4125,9 @@ class TestScheduler(ZuulTestCase):
# timer-triggered job so that we have an opportunity to set # timer-triggered job so that we have an opportunity to set
# the hold flag before the first job. # the hold flag before the first job.
self.create_branch('org/project', 'stable') self.create_branch('org/project', 'stable')
self.fake_gerrit.addEvent(
self.fake_gerrit.getFakeBranchCreatedEvent(
'org/project', 'stable'))
self.executor_server.hold_jobs_in_build = True self.executor_server.hold_jobs_in_build = True
self.commitConfigUpdate('common-config', config_file) self.commitConfigUpdate('common-config', config_file)
self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

View File

@ -2542,6 +2542,8 @@ class TestGlobalRepoState(AnsibleZuulTestCase):
github.repo_from_project( github.repo_from_project(
'org/requiringproject-github')._set_branch_protection( 'org/requiringproject-github')._set_branch_protection(
'master', True) 'master', True)
self.fake_github.emitEvent(self.fake_github.getPushEvent(
'org/requiringproject-github', ref='refs/heads/master'))
# Create unprotected branch feat-x. This branch will be the target # Create unprotected branch feat-x. This branch will be the target
# of override-checkout # of override-checkout
@ -2549,9 +2551,11 @@ class TestGlobalRepoState(AnsibleZuulTestCase):
repo._set_branch_protection('master', True) repo._set_branch_protection('master', True)
repo._create_branch('feat-x') repo._create_branch('feat-x')
self.create_branch('org/requiredproject-github', 'feat-x') self.create_branch('org/requiredproject-github', 'feat-x')
self.fake_github.emitEvent(self.fake_github.getPushEvent(
'org/requiredproject-github', ref='refs/heads/feat-x'))
# Reconfigure to ensure zuul knows about the branch protection # Wait until Zuul has processed the push events and knows about
self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) # the branch protection
self.waitUntilSettled() self.waitUntilSettled()
A = self.fake_github.openFakePullRequest( A = self.fake_github.openFakePullRequest(

View File

@ -1794,6 +1794,9 @@ class TestTenantScopedWebApi(BaseTestWeb):
"""Test that the admin web interface can dequeue a change""" """Test that the admin web interface can dequeue a change"""
start_builds = len(self.builds) start_builds = len(self.builds)
self.create_branch('org/project', 'stable') self.create_branch('org/project', 'stable')
self.fake_gerrit.addEvent(
self.fake_gerrit.getFakeBranchCreatedEvent(
'org/project', 'stable'))
self.executor_server.hold_jobs_in_build = True self.executor_server.hold_jobs_in_build = True
self.commitConfigUpdate('common-config', 'layouts/timer.yaml') self.commitConfigUpdate('common-config', 'layouts/timer.yaml')
self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
@ -2538,6 +2541,9 @@ class TestCLIViaWebApi(BaseTestWeb):
"""Test that the CLI can dequeue a change via REST""" """Test that the CLI can dequeue a change via REST"""
start_builds = len(self.builds) start_builds = len(self.builds)
self.create_branch('org/project', 'stable') self.create_branch('org/project', 'stable')
self.fake_gerrit.addEvent(
self.fake_gerrit.getFakeBranchCreatedEvent(
'org/project', 'stable'))
self.executor_server.hold_jobs_in_build = True self.executor_server.hold_jobs_in_build = True
self.commitConfigUpdate('common-config', 'layouts/timer.yaml') self.commitConfigUpdate('common-config', 'layouts/timer.yaml')
self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

View File

@ -291,6 +291,9 @@ class TestZuulClientAdmin(BaseTestWeb):
self.executor_server.hold_jobs_in_build = True self.executor_server.hold_jobs_in_build = True
start_builds = len(self.builds) start_builds = len(self.builds)
self.create_branch('org/project', 'stable') self.create_branch('org/project', 'stable')
self.fake_gerrit.addEvent(
self.fake_gerrit.getFakeBranchCreatedEvent(
'org/project', 'stable'))
self.executor_server.hold_jobs_in_build = True self.executor_server.hold_jobs_in_build = True
self.commitConfigUpdate('common-config', 'layouts/timer.yaml') self.commitConfigUpdate('common-config', 'layouts/timer.yaml')
self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

View File

@ -77,17 +77,6 @@ class BaseConnection(object, metaclass=abc.ABCMeta):
def registerScheduler(self, sched) -> None: def registerScheduler(self, sched) -> None:
self.sched = sched self.sched = sched
def clearCache(self):
"""Clear the cache for this connection.
This is called immediately prior to performing a full
reconfiguration. The cache should be cleared so that a
full reconfiguration can be used to correct any errors in
cached data.
"""
pass
def maintainCache(self, relevant): def maintainCache(self, relevant):
"""Make cache contain relevant changes. """Make cache contain relevant changes.
@ -236,17 +225,6 @@ class CachedBranchConnection(BaseConnection):
cache[project.name] = branches cache[project.name] = branches
return branches return branches
def clearCache(self) -> None:
"""Clear the connection cache for all projects.
This method is called by the scheduler in order to perform a full
reconfiguration.
"""
self.log.debug("Clearing branch cache for all branches: %s",
self.connection_name)
self._project_branch_cache_exclude_unprotected = {}
self._project_branch_cache_include_unprotected = {}
def clearProjectCache(self, project: Project) -> None: def clearProjectCache(self, project: Project) -> None:
"""Clear the connection cache for this project. """Clear the connection cache for this project.
""" """

View File

@ -721,9 +721,6 @@ class GerritConnection(BaseConnection):
def addProject(self, project: Project) -> None: def addProject(self, project: Project) -> None:
self.projects[project.name] = project self.projects[project.name] = project
def clearCache(self):
self._project_branch_cache = {}
def _clearBranchCache(self, project): def _clearBranchCache(self, project):
try: try:
del self._project_branch_cache[project.name] del self._project_branch_cache[project.name]

View File

@ -534,9 +534,6 @@ class PagureConnection(BaseConnection):
for key in remove: for key in remove:
del self._change_cache[key] del self._change_cache[key]
def clearCache(self):
self.project_branch_cache = {}
def getWebController(self, zuul_web): def getWebController(self, zuul_web):
return PagureWebController(zuul_web, self) return PagureWebController(zuul_web, self)

View File

@ -881,11 +881,6 @@ class Scheduler(threading.Thread):
self.ansible_manager = AnsibleManager( self.ansible_manager = AnsibleManager(
default_version=default_ansible_version) default_version=default_ansible_version)
if not event.smart:
for connection in self.connections.connections.values():
self.log.debug("Clear cache for: %s" % connection)
connection.clearCache()
loader = configloader.ConfigLoader( loader = configloader.ConfigLoader(
self.connections, self, self.merger, self.keystore) self.connections, self, self.merger, self.keystore)
tenant_config, script = self._checkTenantSourceConf(self.config) tenant_config, script = self._checkTenantSourceConf(self.config)