Use branch cache min. ltime when loading a tenant
The dictionary with the minimum ltimes for the connection's project branch caches is used when loading a tenant. This allows us to detect if we need to refresh the branch cache from Zookeeper. Change-Id: Id4eb7c7cd0af008abe3356f1d70b6617bbb58819
This commit is contained in:
parent
982aebe4c8
commit
9cf8b6f7d0
|
@ -1518,7 +1518,7 @@ class TenantParser(object):
|
|||
return vs.Schema(tenant)
|
||||
|
||||
def fromYaml(self, abide, conf, ansible_manager, min_ltimes=None,
|
||||
layout_uuid=None):
|
||||
layout_uuid=None, branch_cache_min_ltimes=None):
|
||||
self.getSchema()(conf)
|
||||
tenant = model.Tenant(conf['name'])
|
||||
pcontext = ParseContext(self.connections, self.scheduler,
|
||||
|
@ -1560,7 +1560,7 @@ class TenantParser(object):
|
|||
tpc.project.canonical_name, tpc.project.name)
|
||||
with project_configuration_exceptions(source_context,
|
||||
loading_errors):
|
||||
self._getProjectBranches(tenant, tpc)
|
||||
self._getProjectBranches(tenant, tpc, branch_cache_min_ltimes)
|
||||
self._resolveShadowProjects(tenant, tpc)
|
||||
|
||||
# Set default ansible version
|
||||
|
@ -1618,9 +1618,14 @@ class TenantParser(object):
|
|||
shadow_projects.append(project.canonical_name)
|
||||
tpc.shadow_projects = frozenset(shadow_projects)
|
||||
|
||||
def _getProjectBranches(self, tenant, tpc):
|
||||
def _getProjectBranches(self, tenant, tpc, branch_cache_min_ltimes=None):
|
||||
if branch_cache_min_ltimes:
|
||||
min_ltime = branch_cache_min_ltimes.get(
|
||||
tpc.project.source.connection.connection_name, -1)
|
||||
else:
|
||||
min_ltime = -1
|
||||
branches = sorted(tpc.project.source.getProjectBranches(
|
||||
tpc.project, tenant))
|
||||
tpc.project, tenant, min_ltime))
|
||||
if 'master' in branches:
|
||||
branches.remove('master')
|
||||
branches = ['master'] + branches
|
||||
|
@ -2281,7 +2286,8 @@ class ConfigLoader(object):
|
|||
abide.addUntrustedTPC(tenant_name, tpc)
|
||||
|
||||
def loadTenant(self, abide, tenant_name, ansible_manager, unparsed_abide,
|
||||
min_ltimes=None, layout_uuid=None):
|
||||
min_ltimes=None, layout_uuid=None,
|
||||
branch_cache_min_ltimes=None):
|
||||
"""(Re-)load a single tenant.
|
||||
|
||||
Description of cache stages:
|
||||
|
@ -2357,7 +2363,8 @@ class ConfigLoader(object):
|
|||
|
||||
unparsed_config = unparsed_abide.tenants[tenant_name]
|
||||
new_tenant = self.tenant_parser.fromYaml(
|
||||
abide, unparsed_config, ansible_manager, min_ltimes, layout_uuid)
|
||||
abide, unparsed_config, ansible_manager, min_ltimes, layout_uuid,
|
||||
branch_cache_min_ltimes)
|
||||
# Copy tenants dictionary to not break concurrent iterations.
|
||||
tenants = abide.tenants.copy()
|
||||
tenants[tenant_name] = new_tenant
|
||||
|
|
|
@ -771,10 +771,19 @@ class Scheduler(threading.Thread):
|
|||
layout_state = self.tenant_layout_state.get(tenant_name)
|
||||
layout_uuid = layout_state and layout_state.uuid
|
||||
|
||||
if layout_state:
|
||||
branch_cache_min_ltimes = (
|
||||
layout_state.branch_cache_min_ltimes)
|
||||
else:
|
||||
# Consider all project branch caches valid if
|
||||
# we don't have a layout state.
|
||||
branch_cache_min_ltimes = defaultdict(lambda: -1)
|
||||
|
||||
tenant = loader.loadTenant(
|
||||
self.abide, tenant_name, self.ansible_manager,
|
||||
self.unparsed_abide, min_ltimes=min_ltimes,
|
||||
layout_uuid=layout_uuid)
|
||||
layout_uuid=layout_uuid,
|
||||
branch_cache_min_ltimes=branch_cache_min_ltimes)
|
||||
|
||||
if layout_state is None:
|
||||
# Reconfigure only tenants w/o an existing layout state
|
||||
|
@ -970,11 +979,16 @@ class Scheduler(threading.Thread):
|
|||
self.log.debug("Updating local layout of tenant %s ", tenant_name)
|
||||
layout_state = self.tenant_layout_state.get(tenant_name)
|
||||
layout_uuid = layout_state and layout_state.uuid
|
||||
tenant = loader.loadTenant(self.abide, tenant_name,
|
||||
self.ansible_manager,
|
||||
self.unparsed_abide,
|
||||
min_ltimes=min_ltimes,
|
||||
layout_uuid=layout_uuid)
|
||||
if layout_state:
|
||||
branch_cache_min_ltimes = layout_state.branch_cache_min_ltimes
|
||||
else:
|
||||
# We don't need the cache ltimes as the tenant was deleted
|
||||
branch_cache_min_ltimes = None
|
||||
tenant = loader.loadTenant(
|
||||
self.abide, tenant_name, self.ansible_manager,
|
||||
self.unparsed_abide, min_ltimes=min_ltimes,
|
||||
layout_uuid=layout_uuid,
|
||||
branch_cache_min_ltimes=branch_cache_min_ltimes)
|
||||
if tenant is not None:
|
||||
self.local_layout_state[tenant_name] = layout_state
|
||||
self.connections.reconfigureDrivers(tenant)
|
||||
|
@ -1103,11 +1117,15 @@ class Scheduler(threading.Thread):
|
|||
# Consider caches valid if the cache ltime >= event ltime
|
||||
min_ltimes = defaultdict(
|
||||
lambda: defaultdict(lambda: event.zuul_event_ltime))
|
||||
|
||||
# Consider all project branch caches valid.
|
||||
branch_cache_min_ltimes = defaultdict(lambda: -1)
|
||||
|
||||
with tenant_write_lock(self.zk_client, tenant_name) as lock:
|
||||
tenant = loader.loadTenant(self.abide, tenant_name,
|
||||
self.ansible_manager,
|
||||
self.unparsed_abide,
|
||||
min_ltimes=min_ltimes)
|
||||
tenant = loader.loadTenant(
|
||||
self.abide, tenant_name, self.ansible_manager,
|
||||
self.unparsed_abide, min_ltimes=min_ltimes,
|
||||
branch_cache_min_ltimes=branch_cache_min_ltimes)
|
||||
reconfigured_tenants.append(tenant_name)
|
||||
ctx = self.createZKContext(lock, self.log)
|
||||
if tenant is not None:
|
||||
|
@ -1144,6 +1162,12 @@ class Scheduler(threading.Thread):
|
|||
branch_name
|
||||
] = event.zuul_event_ltime
|
||||
|
||||
# Consider all branch caches valid except for the ones where
|
||||
# the events provides a minimum ltime.
|
||||
branch_cache_min_ltimes = defaultdict(lambda: -1)
|
||||
for connection_name, ltime in event.branch_cache_ltimes.items():
|
||||
branch_cache_min_ltimes[connection_name] = ltime
|
||||
|
||||
loader = configloader.ConfigLoader(
|
||||
self.connections, self, self.merger, self.keystore)
|
||||
old_tenant = self.abide.tenants.get(event.tenant_name)
|
||||
|
@ -1151,9 +1175,10 @@ class Scheduler(threading.Thread):
|
|||
[event.tenant_name])
|
||||
|
||||
with tenant_write_lock(self.zk_client, event.tenant_name) as lock:
|
||||
loader.loadTenant(self.abide, event.tenant_name,
|
||||
self.ansible_manager, self.unparsed_abide,
|
||||
min_ltimes=min_ltimes)
|
||||
loader.loadTenant(
|
||||
self.abide, event.tenant_name, self.ansible_manager,
|
||||
self.unparsed_abide, min_ltimes=min_ltimes,
|
||||
branch_cache_min_ltimes=branch_cache_min_ltimes)
|
||||
tenant = self.abide.tenants[event.tenant_name]
|
||||
ctx = self.createZKContext(lock, self.log)
|
||||
self._reconfigureTenant(ctx, tenant, old_tenant)
|
||||
|
|
Loading…
Reference in New Issue