Don't clear connection caches during full reconfig
Clearing of connection caches doesn't seem to be needed in practice and is removed as it can't be easily coordinated between multiple schedulers. Change-Id: I663fa1f1752d0dd599af06a7275a14235faaff2f
This commit is contained in:
parent
32f3d5fe35
commit
ce0e348eb6
|
@ -1388,15 +1388,21 @@ class TestGithubCircularDependencies(ZuulTestCase):
|
||||||
'master', True)
|
'master', True)
|
||||||
github.repo_from_project('gh/project')._set_branch_protection(
|
github.repo_from_project('gh/project')._set_branch_protection(
|
||||||
'stable/foo', True)
|
'stable/foo', True)
|
||||||
|
pevent = self.fake_github.getPushEvent(project='gh/project',
|
||||||
|
ref='refs/heads/stable/foo')
|
||||||
|
self.fake_github.emitEvent(pevent)
|
||||||
|
|
||||||
self.create_branch('gh/project1', 'stable/bar')
|
self.create_branch('gh/project1', 'stable/bar')
|
||||||
github.repo_from_project('gh/project1')._set_branch_protection(
|
github.repo_from_project('gh/project1')._set_branch_protection(
|
||||||
'master', True)
|
'master', True)
|
||||||
github.repo_from_project('gh/project1')._set_branch_protection(
|
github.repo_from_project('gh/project1')._set_branch_protection(
|
||||||
'stable/bar', True)
|
'stable/bar', True)
|
||||||
|
pevent = self.fake_github.getPushEvent(project='gh/project',
|
||||||
|
ref='refs/heads/stable/bar')
|
||||||
|
self.fake_github.emitEvent(pevent)
|
||||||
|
|
||||||
# Reconfigure to pick up branch protection settings
|
# Wait until push events are processed to pick up branch
|
||||||
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
# protection settings
|
||||||
self.waitUntilSettled()
|
self.waitUntilSettled()
|
||||||
|
|
||||||
A = self.fake_github.openFakePullRequest(
|
A = self.fake_github.openFakePullRequest(
|
||||||
|
|
|
@ -434,8 +434,13 @@ class TestTenantConfigBranches(ZuulTestCase):
|
||||||
self.log.debug('Creating branches')
|
self.log.debug('Creating branches')
|
||||||
self.create_branch('common-config', 'stable')
|
self.create_branch('common-config', 'stable')
|
||||||
self.create_branch('common-config', 'feat_x')
|
self.create_branch('common-config', 'feat_x')
|
||||||
|
self.fake_gerrit.addEvent(
|
||||||
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
self.fake_gerrit.getFakeBranchCreatedEvent(
|
||||||
|
'common-config', 'stable'))
|
||||||
|
self.fake_gerrit.addEvent(
|
||||||
|
self.fake_gerrit.getFakeBranchCreatedEvent(
|
||||||
|
'common-config', 'feat_x'))
|
||||||
|
self.waitUntilSettled()
|
||||||
|
|
||||||
# Job must be defined in master
|
# Job must be defined in master
|
||||||
self._validate_job(common_job, 'master')
|
self._validate_job(common_job, 'master')
|
||||||
|
|
|
@ -1339,7 +1339,10 @@ class TestGithubUnprotectedBranches(ZuulTestCase):
|
||||||
github = self.fake_github.getGithubClient()
|
github = self.fake_github.getGithubClient()
|
||||||
repo = github.repo_from_project('org/project2')
|
repo = github.repo_from_project('org/project2')
|
||||||
repo._set_branch_protection('master', True)
|
repo._set_branch_protection('master', True)
|
||||||
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
|
||||||
|
pevent = self.fake_github.getPushEvent(project='org/project2',
|
||||||
|
ref='refs/heads/master')
|
||||||
|
self.fake_github.emitEvent(pevent)
|
||||||
self.waitUntilSettled()
|
self.waitUntilSettled()
|
||||||
|
|
||||||
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
||||||
|
@ -1470,6 +1473,9 @@ class TestGithubUnprotectedBranches(ZuulTestCase):
|
||||||
github = self.fake_github.getGithubClient()
|
github = self.fake_github.getGithubClient()
|
||||||
repo = github.repo_from_project('org/project2')
|
repo = github.repo_from_project('org/project2')
|
||||||
repo._set_branch_protection('master', True)
|
repo._set_branch_protection('master', True)
|
||||||
|
self.fake_github.emitEvent(
|
||||||
|
self.fake_github.getPushEvent(
|
||||||
|
project='org/project2', ref='refs/heads/master'))
|
||||||
|
|
||||||
A = self.fake_github.openFakePullRequest('org/project2', 'master', 'A')
|
A = self.fake_github.openFakePullRequest('org/project2', 'master', 'A')
|
||||||
A.setMerged("merging A")
|
A.setMerged("merging A")
|
||||||
|
@ -1477,8 +1483,10 @@ class TestGithubUnprotectedBranches(ZuulTestCase):
|
||||||
# add a spare branch so that the project is not empty after master gets
|
# add a spare branch so that the project is not empty after master gets
|
||||||
# deleted.
|
# deleted.
|
||||||
repo._create_branch('feat-x')
|
repo._create_branch('feat-x')
|
||||||
|
self.fake_github.emitEvent(
|
||||||
|
self.fake_github.getPushEvent(
|
||||||
|
project='org/project2', ref='refs/heads/feat-x'))
|
||||||
|
|
||||||
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
|
||||||
self.waitUntilSettled()
|
self.waitUntilSettled()
|
||||||
|
|
||||||
# record previous tenant reconfiguration time, which may not be set
|
# record previous tenant reconfiguration time, which may not be set
|
||||||
|
|
|
@ -726,7 +726,8 @@ class TestGitlabUnprotectedBranches(ZuulTestCase):
|
||||||
|
|
||||||
# now enable branch protection and trigger reload
|
# now enable branch protection and trigger reload
|
||||||
self.fake_gitlab.protectBranch('org', 'project2', 'master')
|
self.fake_gitlab.protectBranch('org', 'project2', 'master')
|
||||||
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
pevent = self.fake_gitlab.getPushEvent(project='org/project2')
|
||||||
|
self.fake_gitlab.emitEvent(pevent)
|
||||||
self.waitUntilSettled()
|
self.waitUntilSettled()
|
||||||
|
|
||||||
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
||||||
|
@ -869,6 +870,9 @@ class TestGitlabUnprotectedBranches(ZuulTestCase):
|
||||||
|
|
||||||
# Prepare repo with an initial commit and enable branch protection
|
# Prepare repo with an initial commit and enable branch protection
|
||||||
self.fake_gitlab.protectBranch('org', 'project2', 'master')
|
self.fake_gitlab.protectBranch('org', 'project2', 'master')
|
||||||
|
self.fake_gitlab.emitEvent(
|
||||||
|
self.fake_gitlab.getPushEvent(
|
||||||
|
project='org/project2', branch='refs/heads/master'))
|
||||||
|
|
||||||
A = self.fake_gitlab.openFakeMergeRequest('org/project2', 'master',
|
A = self.fake_gitlab.openFakeMergeRequest('org/project2', 'master',
|
||||||
'A')
|
'A')
|
||||||
|
@ -879,8 +883,9 @@ class TestGitlabUnprotectedBranches(ZuulTestCase):
|
||||||
self.create_branch('org/project2', 'feat-x')
|
self.create_branch('org/project2', 'feat-x')
|
||||||
self.fake_gitlab.protectBranch('org', 'project2', 'feat-x',
|
self.fake_gitlab.protectBranch('org', 'project2', 'feat-x',
|
||||||
protected=False)
|
protected=False)
|
||||||
|
self.fake_gitlab.emitEvent(
|
||||||
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
self.fake_gitlab.getPushEvent(
|
||||||
|
project='org/project2', branch='refs/heads/feat-x'))
|
||||||
self.waitUntilSettled()
|
self.waitUntilSettled()
|
||||||
|
|
||||||
# record previous tenant reconfiguration time, which may not be set
|
# record previous tenant reconfiguration time, which may not be set
|
||||||
|
|
|
@ -4072,6 +4072,9 @@ class TestScheduler(ZuulTestCase):
|
||||||
# timer-triggered job so that we have an opportunity to set
|
# timer-triggered job so that we have an opportunity to set
|
||||||
# the hold flag before the first job.
|
# the hold flag before the first job.
|
||||||
self.create_branch('org/project', 'stable')
|
self.create_branch('org/project', 'stable')
|
||||||
|
self.fake_gerrit.addEvent(
|
||||||
|
self.fake_gerrit.getFakeBranchCreatedEvent(
|
||||||
|
'org/project', 'stable'))
|
||||||
self.executor_server.hold_jobs_in_build = True
|
self.executor_server.hold_jobs_in_build = True
|
||||||
self.commitConfigUpdate('common-config', 'layouts/timer-template.yaml')
|
self.commitConfigUpdate('common-config', 'layouts/timer-template.yaml')
|
||||||
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
||||||
|
@ -4122,6 +4125,9 @@ class TestScheduler(ZuulTestCase):
|
||||||
# timer-triggered job so that we have an opportunity to set
|
# timer-triggered job so that we have an opportunity to set
|
||||||
# the hold flag before the first job.
|
# the hold flag before the first job.
|
||||||
self.create_branch('org/project', 'stable')
|
self.create_branch('org/project', 'stable')
|
||||||
|
self.fake_gerrit.addEvent(
|
||||||
|
self.fake_gerrit.getFakeBranchCreatedEvent(
|
||||||
|
'org/project', 'stable'))
|
||||||
self.executor_server.hold_jobs_in_build = True
|
self.executor_server.hold_jobs_in_build = True
|
||||||
self.commitConfigUpdate('common-config', config_file)
|
self.commitConfigUpdate('common-config', config_file)
|
||||||
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
||||||
|
|
|
@ -2542,6 +2542,8 @@ class TestGlobalRepoState(AnsibleZuulTestCase):
|
||||||
github.repo_from_project(
|
github.repo_from_project(
|
||||||
'org/requiringproject-github')._set_branch_protection(
|
'org/requiringproject-github')._set_branch_protection(
|
||||||
'master', True)
|
'master', True)
|
||||||
|
self.fake_github.emitEvent(self.fake_github.getPushEvent(
|
||||||
|
'org/requiringproject-github', ref='refs/heads/master'))
|
||||||
|
|
||||||
# Create unprotected branch feat-x. This branch will be the target
|
# Create unprotected branch feat-x. This branch will be the target
|
||||||
# of override-checkout
|
# of override-checkout
|
||||||
|
@ -2549,9 +2551,11 @@ class TestGlobalRepoState(AnsibleZuulTestCase):
|
||||||
repo._set_branch_protection('master', True)
|
repo._set_branch_protection('master', True)
|
||||||
repo._create_branch('feat-x')
|
repo._create_branch('feat-x')
|
||||||
self.create_branch('org/requiredproject-github', 'feat-x')
|
self.create_branch('org/requiredproject-github', 'feat-x')
|
||||||
|
self.fake_github.emitEvent(self.fake_github.getPushEvent(
|
||||||
|
'org/requiredproject-github', ref='refs/heads/feat-x'))
|
||||||
|
|
||||||
# Reconfigure to ensure zuul knows about the branch protection
|
# Wait until Zuul has processed the push events and knows about
|
||||||
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
# the branch protection
|
||||||
self.waitUntilSettled()
|
self.waitUntilSettled()
|
||||||
|
|
||||||
A = self.fake_github.openFakePullRequest(
|
A = self.fake_github.openFakePullRequest(
|
||||||
|
|
|
@ -1794,6 +1794,9 @@ class TestTenantScopedWebApi(BaseTestWeb):
|
||||||
"""Test that the admin web interface can dequeue a change"""
|
"""Test that the admin web interface can dequeue a change"""
|
||||||
start_builds = len(self.builds)
|
start_builds = len(self.builds)
|
||||||
self.create_branch('org/project', 'stable')
|
self.create_branch('org/project', 'stable')
|
||||||
|
self.fake_gerrit.addEvent(
|
||||||
|
self.fake_gerrit.getFakeBranchCreatedEvent(
|
||||||
|
'org/project', 'stable'))
|
||||||
self.executor_server.hold_jobs_in_build = True
|
self.executor_server.hold_jobs_in_build = True
|
||||||
self.commitConfigUpdate('common-config', 'layouts/timer.yaml')
|
self.commitConfigUpdate('common-config', 'layouts/timer.yaml')
|
||||||
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
||||||
|
@ -2538,6 +2541,9 @@ class TestCLIViaWebApi(BaseTestWeb):
|
||||||
"""Test that the CLI can dequeue a change via REST"""
|
"""Test that the CLI can dequeue a change via REST"""
|
||||||
start_builds = len(self.builds)
|
start_builds = len(self.builds)
|
||||||
self.create_branch('org/project', 'stable')
|
self.create_branch('org/project', 'stable')
|
||||||
|
self.fake_gerrit.addEvent(
|
||||||
|
self.fake_gerrit.getFakeBranchCreatedEvent(
|
||||||
|
'org/project', 'stable'))
|
||||||
self.executor_server.hold_jobs_in_build = True
|
self.executor_server.hold_jobs_in_build = True
|
||||||
self.commitConfigUpdate('common-config', 'layouts/timer.yaml')
|
self.commitConfigUpdate('common-config', 'layouts/timer.yaml')
|
||||||
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
||||||
|
|
|
@ -291,6 +291,9 @@ class TestZuulClientAdmin(BaseTestWeb):
|
||||||
self.executor_server.hold_jobs_in_build = True
|
self.executor_server.hold_jobs_in_build = True
|
||||||
start_builds = len(self.builds)
|
start_builds = len(self.builds)
|
||||||
self.create_branch('org/project', 'stable')
|
self.create_branch('org/project', 'stable')
|
||||||
|
self.fake_gerrit.addEvent(
|
||||||
|
self.fake_gerrit.getFakeBranchCreatedEvent(
|
||||||
|
'org/project', 'stable'))
|
||||||
self.executor_server.hold_jobs_in_build = True
|
self.executor_server.hold_jobs_in_build = True
|
||||||
self.commitConfigUpdate('common-config', 'layouts/timer.yaml')
|
self.commitConfigUpdate('common-config', 'layouts/timer.yaml')
|
||||||
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
||||||
|
|
|
@ -77,17 +77,6 @@ class BaseConnection(object, metaclass=abc.ABCMeta):
|
||||||
def registerScheduler(self, sched) -> None:
|
def registerScheduler(self, sched) -> None:
|
||||||
self.sched = sched
|
self.sched = sched
|
||||||
|
|
||||||
def clearCache(self):
|
|
||||||
"""Clear the cache for this connection.
|
|
||||||
|
|
||||||
This is called immediately prior to performing a full
|
|
||||||
reconfiguration. The cache should be cleared so that a
|
|
||||||
full reconfiguration can be used to correct any errors in
|
|
||||||
cached data.
|
|
||||||
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def maintainCache(self, relevant):
|
def maintainCache(self, relevant):
|
||||||
|
|
||||||
"""Make cache contain relevant changes.
|
"""Make cache contain relevant changes.
|
||||||
|
@ -236,17 +225,6 @@ class CachedBranchConnection(BaseConnection):
|
||||||
cache[project.name] = branches
|
cache[project.name] = branches
|
||||||
return branches
|
return branches
|
||||||
|
|
||||||
def clearCache(self) -> None:
|
|
||||||
"""Clear the connection cache for all projects.
|
|
||||||
|
|
||||||
This method is called by the scheduler in order to perform a full
|
|
||||||
reconfiguration.
|
|
||||||
"""
|
|
||||||
self.log.debug("Clearing branch cache for all branches: %s",
|
|
||||||
self.connection_name)
|
|
||||||
self._project_branch_cache_exclude_unprotected = {}
|
|
||||||
self._project_branch_cache_include_unprotected = {}
|
|
||||||
|
|
||||||
def clearProjectCache(self, project: Project) -> None:
|
def clearProjectCache(self, project: Project) -> None:
|
||||||
"""Clear the connection cache for this project.
|
"""Clear the connection cache for this project.
|
||||||
"""
|
"""
|
||||||
|
|
|
@ -721,9 +721,6 @@ class GerritConnection(BaseConnection):
|
||||||
def addProject(self, project: Project) -> None:
|
def addProject(self, project: Project) -> None:
|
||||||
self.projects[project.name] = project
|
self.projects[project.name] = project
|
||||||
|
|
||||||
def clearCache(self):
|
|
||||||
self._project_branch_cache = {}
|
|
||||||
|
|
||||||
def _clearBranchCache(self, project):
|
def _clearBranchCache(self, project):
|
||||||
try:
|
try:
|
||||||
del self._project_branch_cache[project.name]
|
del self._project_branch_cache[project.name]
|
||||||
|
|
|
@ -534,9 +534,6 @@ class PagureConnection(BaseConnection):
|
||||||
for key in remove:
|
for key in remove:
|
||||||
del self._change_cache[key]
|
del self._change_cache[key]
|
||||||
|
|
||||||
def clearCache(self):
|
|
||||||
self.project_branch_cache = {}
|
|
||||||
|
|
||||||
def getWebController(self, zuul_web):
|
def getWebController(self, zuul_web):
|
||||||
return PagureWebController(zuul_web, self)
|
return PagureWebController(zuul_web, self)
|
||||||
|
|
||||||
|
|
|
@ -881,11 +881,6 @@ class Scheduler(threading.Thread):
|
||||||
self.ansible_manager = AnsibleManager(
|
self.ansible_manager = AnsibleManager(
|
||||||
default_version=default_ansible_version)
|
default_version=default_ansible_version)
|
||||||
|
|
||||||
if not event.smart:
|
|
||||||
for connection in self.connections.connections.values():
|
|
||||||
self.log.debug("Clear cache for: %s" % connection)
|
|
||||||
connection.clearCache()
|
|
||||||
|
|
||||||
loader = configloader.ConfigLoader(
|
loader = configloader.ConfigLoader(
|
||||||
self.connections, self, self.merger, self.keystore)
|
self.connections, self, self.merger, self.keystore)
|
||||||
tenant_config, script = self._checkTenantSourceConf(self.config)
|
tenant_config, script = self._checkTenantSourceConf(self.config)
|
||||||
|
|
Loading…
Reference in New Issue