Browse Source

Merge "Don't clear connection caches during full reconfig"

changes/82/800282/1
Zuul 2 months ago
committed by Gerrit Code Review
parent
commit
0e43823138
  1. 10
      tests/unit/test_circular_dependencies.py
  2. 9
      tests/unit/test_configloader.py
  3. 12
      tests/unit/test_github_driver.py
  4. 11
      tests/unit/test_gitlab_driver.py
  5. 6
      tests/unit/test_scheduler.py
  6. 8
      tests/unit/test_v3.py
  7. 6
      tests/unit/test_web.py
  8. 3
      tests/zuul_client/test_zuulclient.py
  9. 22
      zuul/connection/__init__.py
  10. 3
      zuul/driver/gerrit/gerritconnection.py
  11. 3
      zuul/driver/pagure/pagureconnection.py
  12. 5
      zuul/scheduler.py

10
tests/unit/test_circular_dependencies.py

@ -1388,15 +1388,21 @@ class TestGithubCircularDependencies(ZuulTestCase):
'master', True)
github.repo_from_project('gh/project')._set_branch_protection(
'stable/foo', True)
pevent = self.fake_github.getPushEvent(project='gh/project',
ref='refs/heads/stable/foo')
self.fake_github.emitEvent(pevent)
self.create_branch('gh/project1', 'stable/bar')
github.repo_from_project('gh/project1')._set_branch_protection(
'master', True)
github.repo_from_project('gh/project1')._set_branch_protection(
'stable/bar', True)
pevent = self.fake_github.getPushEvent(project='gh/project',
ref='refs/heads/stable/bar')
self.fake_github.emitEvent(pevent)
# Reconfigure to pick up branch protection settings
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
# Wait until push events are processed to pick up branch
# protection settings
self.waitUntilSettled()
A = self.fake_github.openFakePullRequest(

9
tests/unit/test_configloader.py

@ -434,8 +434,13 @@ class TestTenantConfigBranches(ZuulTestCase):
self.log.debug('Creating branches')
self.create_branch('common-config', 'stable')
self.create_branch('common-config', 'feat_x')
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
self.fake_gerrit.addEvent(
self.fake_gerrit.getFakeBranchCreatedEvent(
'common-config', 'stable'))
self.fake_gerrit.addEvent(
self.fake_gerrit.getFakeBranchCreatedEvent(
'common-config', 'feat_x'))
self.waitUntilSettled()
# Job must be defined in master
self._validate_job(common_job, 'master')

12
tests/unit/test_github_driver.py

@ -1339,7 +1339,10 @@ class TestGithubUnprotectedBranches(ZuulTestCase):
github = self.fake_github.getGithubClient()
repo = github.repo_from_project('org/project2')
repo._set_branch_protection('master', True)
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
pevent = self.fake_github.getPushEvent(project='org/project2',
ref='refs/heads/master')
self.fake_github.emitEvent(pevent)
self.waitUntilSettled()
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
@ -1470,6 +1473,9 @@ class TestGithubUnprotectedBranches(ZuulTestCase):
github = self.fake_github.getGithubClient()
repo = github.repo_from_project('org/project2')
repo._set_branch_protection('master', True)
self.fake_github.emitEvent(
self.fake_github.getPushEvent(
project='org/project2', ref='refs/heads/master'))
A = self.fake_github.openFakePullRequest('org/project2', 'master', 'A')
A.setMerged("merging A")
@ -1477,8 +1483,10 @@ class TestGithubUnprotectedBranches(ZuulTestCase):
# add a spare branch so that the project is not empty after master gets
# deleted.
repo._create_branch('feat-x')
self.fake_github.emitEvent(
self.fake_github.getPushEvent(
project='org/project2', ref='refs/heads/feat-x'))
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
self.waitUntilSettled()
# record previous tenant reconfiguration time, which may not be set

11
tests/unit/test_gitlab_driver.py

@ -726,7 +726,8 @@ class TestGitlabUnprotectedBranches(ZuulTestCase):
# now enable branch protection and trigger reload
self.fake_gitlab.protectBranch('org', 'project2', 'master')
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
pevent = self.fake_gitlab.getPushEvent(project='org/project2')
self.fake_gitlab.emitEvent(pevent)
self.waitUntilSettled()
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
@ -869,6 +870,9 @@ class TestGitlabUnprotectedBranches(ZuulTestCase):
# Prepare repo with an initial commit and enable branch protection
self.fake_gitlab.protectBranch('org', 'project2', 'master')
self.fake_gitlab.emitEvent(
self.fake_gitlab.getPushEvent(
project='org/project2', branch='refs/heads/master'))
A = self.fake_gitlab.openFakeMergeRequest('org/project2', 'master',
'A')
@ -879,8 +883,9 @@ class TestGitlabUnprotectedBranches(ZuulTestCase):
self.create_branch('org/project2', 'feat-x')
self.fake_gitlab.protectBranch('org', 'project2', 'feat-x',
protected=False)
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
self.fake_gitlab.emitEvent(
self.fake_gitlab.getPushEvent(
project='org/project2', branch='refs/heads/feat-x'))
self.waitUntilSettled()
# record previous tenant reconfiguration time, which may not be set

6
tests/unit/test_scheduler.py

@ -4072,6 +4072,9 @@ class TestScheduler(ZuulTestCase):
# timer-triggered job so that we have an opportunity to set
# the hold flag before the first job.
self.create_branch('org/project', 'stable')
self.fake_gerrit.addEvent(
self.fake_gerrit.getFakeBranchCreatedEvent(
'org/project', 'stable'))
self.executor_server.hold_jobs_in_build = True
self.commitConfigUpdate('common-config', 'layouts/timer-template.yaml')
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
@ -4122,6 +4125,9 @@ class TestScheduler(ZuulTestCase):
# timer-triggered job so that we have an opportunity to set
# the hold flag before the first job.
self.create_branch('org/project', 'stable')
self.fake_gerrit.addEvent(
self.fake_gerrit.getFakeBranchCreatedEvent(
'org/project', 'stable'))
self.executor_server.hold_jobs_in_build = True
self.commitConfigUpdate('common-config', config_file)
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

8
tests/unit/test_v3.py

@ -2541,6 +2541,8 @@ class TestGlobalRepoState(AnsibleZuulTestCase):
github.repo_from_project(
'org/requiringproject-github')._set_branch_protection(
'master', True)
self.fake_github.emitEvent(self.fake_github.getPushEvent(
'org/requiringproject-github', ref='refs/heads/master'))
# Create unprotected branch feat-x. This branch will be the target
# of override-checkout
@ -2548,9 +2550,11 @@ class TestGlobalRepoState(AnsibleZuulTestCase):
repo._set_branch_protection('master', True)
repo._create_branch('feat-x')
self.create_branch('org/requiredproject-github', 'feat-x')
self.fake_github.emitEvent(self.fake_github.getPushEvent(
'org/requiredproject-github', ref='refs/heads/feat-x'))
# Reconfigure to ensure zuul knows about the branch protection
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
# Wait until Zuul has processed the push events and knows about
# the branch protection
self.waitUntilSettled()
A = self.fake_github.openFakePullRequest(

6
tests/unit/test_web.py

@ -1794,6 +1794,9 @@ class TestTenantScopedWebApi(BaseTestWeb):
"""Test that the admin web interface can dequeue a change"""
start_builds = len(self.builds)
self.create_branch('org/project', 'stable')
self.fake_gerrit.addEvent(
self.fake_gerrit.getFakeBranchCreatedEvent(
'org/project', 'stable'))
self.executor_server.hold_jobs_in_build = True
self.commitConfigUpdate('common-config', 'layouts/timer.yaml')
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
@ -2538,6 +2541,9 @@ class TestCLIViaWebApi(BaseTestWeb):
"""Test that the CLI can dequeue a change via REST"""
start_builds = len(self.builds)
self.create_branch('org/project', 'stable')
self.fake_gerrit.addEvent(
self.fake_gerrit.getFakeBranchCreatedEvent(
'org/project', 'stable'))
self.executor_server.hold_jobs_in_build = True
self.commitConfigUpdate('common-config', 'layouts/timer.yaml')
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

3
tests/zuul_client/test_zuulclient.py

@ -291,6 +291,9 @@ class TestZuulClientAdmin(BaseTestWeb):
self.executor_server.hold_jobs_in_build = True
start_builds = len(self.builds)
self.create_branch('org/project', 'stable')
self.fake_gerrit.addEvent(
self.fake_gerrit.getFakeBranchCreatedEvent(
'org/project', 'stable'))
self.executor_server.hold_jobs_in_build = True
self.commitConfigUpdate('common-config', 'layouts/timer.yaml')
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))

22
zuul/connection/__init__.py

@ -77,17 +77,6 @@ class BaseConnection(object, metaclass=abc.ABCMeta):
def registerScheduler(self, sched) -> None:
self.sched = sched
def clearCache(self):
"""Clear the cache for this connection.
This is called immediately prior to performing a full
reconfiguration. The cache should be cleared so that a
full reconfiguration can be used to correct any errors in
cached data.
"""
pass
def maintainCache(self, relevant):
"""Make cache contain relevant changes.
@ -236,17 +225,6 @@ class CachedBranchConnection(BaseConnection):
cache[project.name] = branches
return branches
def clearCache(self) -> None:
"""Clear the connection cache for all projects.
This method is called by the scheduler in order to perform a full
reconfiguration.
"""
self.log.debug("Clearing branch cache for all branches: %s",
self.connection_name)
self._project_branch_cache_exclude_unprotected = {}
self._project_branch_cache_include_unprotected = {}
def clearProjectCache(self, project: Project) -> None:
"""Clear the connection cache for this project.
"""

3
zuul/driver/gerrit/gerritconnection.py

@ -721,9 +721,6 @@ class GerritConnection(BaseConnection):
def addProject(self, project: Project) -> None:
self.projects[project.name] = project
def clearCache(self):
self._project_branch_cache = {}
def _clearBranchCache(self, project):
try:
del self._project_branch_cache[project.name]

3
zuul/driver/pagure/pagureconnection.py

@ -534,9 +534,6 @@ class PagureConnection(BaseConnection):
for key in remove:
del self._change_cache[key]
def clearCache(self):
self.project_branch_cache = {}
def getWebController(self, zuul_web):
return PagureWebController(zuul_web, self)

5
zuul/scheduler.py

@ -881,11 +881,6 @@ class Scheduler(threading.Thread):
self.ansible_manager = AnsibleManager(
default_version=default_ansible_version)
if not event.smart:
for connection in self.connections.connections.values():
self.log.debug("Clear cache for: %s" % connection)
connection.clearCache()
loader = configloader.ConfigLoader(
self.connections, self, self.merger, self.keystore)
tenant_config, script = self._checkTenantSourceConf(self.config)

Loading…
Cancel
Save