Use branch cache min. ltime when loading a tenant

The dictionary with the minimum ltimes for the connection's project
branch caches is used when loading a tenant.

This allows us to detect if we need to refresh the branch cache from
Zookeeper.

Change-Id: Id4eb7c7cd0af008abe3356f1d70b6617bbb58819
This commit is contained in:
Simon Westphahl 2021-11-04 14:05:56 +01:00
parent 982aebe4c8
commit 9cf8b6f7d0
2 changed files with 51 additions and 19 deletions

View File

@ -1518,7 +1518,7 @@ class TenantParser(object):
return vs.Schema(tenant)
def fromYaml(self, abide, conf, ansible_manager, min_ltimes=None,
layout_uuid=None):
layout_uuid=None, branch_cache_min_ltimes=None):
self.getSchema()(conf)
tenant = model.Tenant(conf['name'])
pcontext = ParseContext(self.connections, self.scheduler,
@ -1560,7 +1560,7 @@ class TenantParser(object):
tpc.project.canonical_name, tpc.project.name)
with project_configuration_exceptions(source_context,
loading_errors):
self._getProjectBranches(tenant, tpc)
self._getProjectBranches(tenant, tpc, branch_cache_min_ltimes)
self._resolveShadowProjects(tenant, tpc)
# Set default ansible version
@ -1618,9 +1618,14 @@ class TenantParser(object):
shadow_projects.append(project.canonical_name)
tpc.shadow_projects = frozenset(shadow_projects)
def _getProjectBranches(self, tenant, tpc):
def _getProjectBranches(self, tenant, tpc, branch_cache_min_ltimes=None):
if branch_cache_min_ltimes:
min_ltime = branch_cache_min_ltimes.get(
tpc.project.source.connection.connection_name, -1)
else:
min_ltime = -1
branches = sorted(tpc.project.source.getProjectBranches(
tpc.project, tenant))
tpc.project, tenant, min_ltime))
if 'master' in branches:
branches.remove('master')
branches = ['master'] + branches
@ -2281,7 +2286,8 @@ class ConfigLoader(object):
abide.addUntrustedTPC(tenant_name, tpc)
def loadTenant(self, abide, tenant_name, ansible_manager, unparsed_abide,
min_ltimes=None, layout_uuid=None):
min_ltimes=None, layout_uuid=None,
branch_cache_min_ltimes=None):
"""(Re-)load a single tenant.
Description of cache stages:
@ -2357,7 +2363,8 @@ class ConfigLoader(object):
unparsed_config = unparsed_abide.tenants[tenant_name]
new_tenant = self.tenant_parser.fromYaml(
abide, unparsed_config, ansible_manager, min_ltimes, layout_uuid)
abide, unparsed_config, ansible_manager, min_ltimes, layout_uuid,
branch_cache_min_ltimes)
# Copy tenants dictionary to not break concurrent iterations.
tenants = abide.tenants.copy()
tenants[tenant_name] = new_tenant

View File

@ -771,10 +771,19 @@ class Scheduler(threading.Thread):
layout_state = self.tenant_layout_state.get(tenant_name)
layout_uuid = layout_state and layout_state.uuid
if layout_state:
branch_cache_min_ltimes = (
layout_state.branch_cache_min_ltimes)
else:
# Consider all project branch caches valid if
# we don't have a layout state.
branch_cache_min_ltimes = defaultdict(lambda: -1)
tenant = loader.loadTenant(
self.abide, tenant_name, self.ansible_manager,
self.unparsed_abide, min_ltimes=min_ltimes,
layout_uuid=layout_uuid)
layout_uuid=layout_uuid,
branch_cache_min_ltimes=branch_cache_min_ltimes)
if layout_state is None:
# Reconfigure only tenants w/o an existing layout state
@ -970,11 +979,16 @@ class Scheduler(threading.Thread):
self.log.debug("Updating local layout of tenant %s ", tenant_name)
layout_state = self.tenant_layout_state.get(tenant_name)
layout_uuid = layout_state and layout_state.uuid
tenant = loader.loadTenant(self.abide, tenant_name,
self.ansible_manager,
self.unparsed_abide,
min_ltimes=min_ltimes,
layout_uuid=layout_uuid)
if layout_state:
branch_cache_min_ltimes = layout_state.branch_cache_min_ltimes
else:
# We don't need the cache ltimes as the tenant was deleted
branch_cache_min_ltimes = None
tenant = loader.loadTenant(
self.abide, tenant_name, self.ansible_manager,
self.unparsed_abide, min_ltimes=min_ltimes,
layout_uuid=layout_uuid,
branch_cache_min_ltimes=branch_cache_min_ltimes)
if tenant is not None:
self.local_layout_state[tenant_name] = layout_state
self.connections.reconfigureDrivers(tenant)
@ -1103,11 +1117,15 @@ class Scheduler(threading.Thread):
# Consider caches valid if the cache ltime >= event ltime
min_ltimes = defaultdict(
lambda: defaultdict(lambda: event.zuul_event_ltime))
# Consider all project branch caches valid.
branch_cache_min_ltimes = defaultdict(lambda: -1)
with tenant_write_lock(self.zk_client, tenant_name) as lock:
tenant = loader.loadTenant(self.abide, tenant_name,
self.ansible_manager,
self.unparsed_abide,
min_ltimes=min_ltimes)
tenant = loader.loadTenant(
self.abide, tenant_name, self.ansible_manager,
self.unparsed_abide, min_ltimes=min_ltimes,
branch_cache_min_ltimes=branch_cache_min_ltimes)
reconfigured_tenants.append(tenant_name)
ctx = self.createZKContext(lock, self.log)
if tenant is not None:
@ -1144,6 +1162,12 @@ class Scheduler(threading.Thread):
branch_name
] = event.zuul_event_ltime
# Consider all branch caches valid except for the ones where
# the events provides a minimum ltime.
branch_cache_min_ltimes = defaultdict(lambda: -1)
for connection_name, ltime in event.branch_cache_ltimes.items():
branch_cache_min_ltimes[connection_name] = ltime
loader = configloader.ConfigLoader(
self.connections, self, self.merger, self.keystore)
old_tenant = self.abide.tenants.get(event.tenant_name)
@ -1151,9 +1175,10 @@ class Scheduler(threading.Thread):
[event.tenant_name])
with tenant_write_lock(self.zk_client, event.tenant_name) as lock:
loader.loadTenant(self.abide, event.tenant_name,
self.ansible_manager, self.unparsed_abide,
min_ltimes=min_ltimes)
loader.loadTenant(
self.abide, event.tenant_name, self.ansible_manager,
self.unparsed_abide, min_ltimes=min_ltimes,
branch_cache_min_ltimes=branch_cache_min_ltimes)
tenant = self.abide.tenants[event.tenant_name]
ctx = self.createZKContext(lock, self.log)
self._reconfigureTenant(ctx, tenant, old_tenant)