zuul
/
zuul
Code Issues Proposed changes

Always request all required project configuration 

In preparation for caching a project's config in Zookeeper we need a way
to request all necessary config for all tenants the project is part of
with a single merger call.

To do that, we create the list of all TPCs before we start loading any
tenant. We can then combine the list of extra config files/dirs for a
project from all tenants for the cat job.

It's worth noting that this change adds some overhead for projects using
extra config files/dirs without providing any immediate benefits.

Change-Id: If0d134f2583ba5a5a176ba29f6d9f906b46aad05
This commit is contained in:
Simon Westphahl 2021-04-30 13:29:27 +02:00
parent 8b6a887336
commit 60d01d1e58
2 changed files with 76 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
zuul/configloader.py
View File

@ -1608,9 +1608,10 @@ class TenantParser(object):
tenant.unparsed_config = conf
# tpcs is TenantProjectConfigs
config_tpcs, untrusted_tpcs = self._loadTenantProjects(conf)
config_tpcs = abide.getConfigTPCs(tenant.name)
for tpc in config_tpcs:
tenant.addConfigProject(tpc)
untrusted_tpcs = abide.getUntrustedTPCs(tenant.name)
for tpc in untrusted_tpcs:
tenant.addUntrustedProject(tpc)
@ -1774,7 +1775,7 @@ class TenantParser(object):
raise Exception("Unable to parse project %s", conf)
return projects
def _loadTenantProjects(self, conf_tenant):
def loadTenantProjects(self, conf_tenant):
config_projects = []
untrusted_projects = []
@ -1823,12 +1824,14 @@ class TenantParser(object):
# If all config classes are excluded then do not
# request any getFiles jobs.
continue
extra_config_files = abide.getExtraConfigFiles(project.name)
extra_config_dirs = abide.getExtraConfigDirs(project.name)
job = self.merger.getFiles(
project.source.connection.connection_name,
project.name, branch,
files=(['zuul.yaml', '.zuul.yaml'] +
list(tpc.extra_config_files)),
dirs=['zuul.d', '.zuul.d'] + list(tpc.extra_config_dirs))
list(extra_config_files)),
dirs=['zuul.d', '.zuul.d'] + list(extra_config_dirs))
self.log.debug("Submitting cat job %s for %s %s %s" % (
job, project.source.connection.connection_name,
project.name, branch))
@ -2258,11 +2261,22 @@ class ConfigLoader(object):
abide.admin_rules[admin_rule.name] = admin_rule
if tenants:
tenants_to_load = [unparsed_abide.tenants[t] for t in tenants]
tenants_to_load = {t: unparsed_abide.tenants[t] for t in tenants}
else:
tenants_to_load = unparsed_abide.tenants.values()
tenants_to_load = unparsed_abide.tenants
for conf_tenant in tenants_to_load:
# Pre-load TenantProjectConfigs so we can get and cache all of a
# project's config files (incl. tenant specific extra config) at once.
for tenant_name, conf_tenant in tenants_to_load.items():
config_tpcs, untrusted_tpcs = (
self.tenant_parser.loadTenantProjects(conf_tenant)
)
for tpc in config_tpcs:
abide.addConfigTPC(tenant_name, tpc)
for tpc in untrusted_tpcs:
abide.addUntrustedTPC(tenant_name, tpc)
for conf_tenant in tenants_to_load.values():
# When performing a full reload, do not use cached data.
tenant = self.tenant_parser.fromYaml(
abide, conf_tenant, ansible_manager)
@ -2284,6 +2298,8 @@ class ConfigLoader(object):
new_abide.admin_rules = abide.admin_rules.copy()
new_abide.unparsed_project_branch_cache = \
abide.unparsed_project_branch_cache
new_abide.config_tpcs = abide.config_tpcs
new_abide.untrusted_tpcs = abide.untrusted_tpcs
if unparsed_abide:
# We got a new unparsed abide so re-load the tenant completely.
@ -2297,6 +2313,17 @@ class ConfigLoader(object):
else:
unparsed_config = tenant.unparsed_config
# Pre-load TenantProjectConfig so we can get and cache all of a
# project's config files (incl. tenant specific extra config) at once.
config_tpcs, untrusted_tpcs = (
self.tenant_parser.loadTenantProjects(unparsed_config)
)
new_abide.clearTPCs(tenant.name)
for tpc in config_tpcs:
new_abide.addConfigTPC(tenant.name, tpc)
for tpc in untrusted_tpcs:
new_abide.addUntrustedTPC(tenant.name, tpc)
# When reloading a tenant only, use cached data if available.
new_tenant = self.tenant_parser.fromYaml(
new_abide, unparsed_config, ansible_manager)

42
zuul/model.py
View File

@ -5238,9 +5238,51 @@ class Abide(object):
def __init__(self):
self.admin_rules = OrderedDict()
self.tenants = OrderedDict()
# tenant -> project -> list(tpcs)
# The project TPCs are stored as a list as we don't check for
# duplicate projects here.
self.config_tpcs = defaultdict(lambda: defaultdict(list))
self.untrusted_tpcs = defaultdict(lambda: defaultdict(list))
# project -> branch -> UnparsedBranchCache
self.unparsed_project_branch_cache = {}
def addConfigTPC(self, tenant_name, tpc):
self.config_tpcs[tenant_name][tpc.project.name].append(tpc)
def getConfigTPCs(self, tenant_name):
return list(itertools.chain.from_iterable(
self.config_tpcs[tenant_name].values()))
def addUntrustedTPC(self, tenant_name, tpc):
self.untrusted_tpcs[tenant_name][tpc.project.name].append(tpc)
def getUntrustedTPCs(self, tenant_name):
return list(itertools.chain.from_iterable(
self.untrusted_tpcs[tenant_name].values()))
def clearTPCs(self, tenant_name):
self.config_tpcs[tenant_name].clear()
self.untrusted_tpcs[tenant_name].clear()
def _allProjectTPCs(self, project_name):
# Flatten the lists of a project TPCs from all tenants
return itertools.chain.from_iterable(
tenant_tpcs.get(project_name, [])
for tenant_tpcs in itertools.chain(self.config_tpcs.values(),
self.untrusted_tpcs.values()))
def getExtraConfigFiles(self, project_name):
"""Get all extra config files for a project accross tenants."""
return set(itertools.chain.from_iterable(
tpc.extra_config_files
for tpc in self._allProjectTPCs(project_name)))
def getExtraConfigDirs(self, project_name):
"""Get all extra config dirs for a project accross tenants."""
return set(itertools.chain.from_iterable(
tpc.extra_config_dirs
for tpc in self._allProjectTPCs(project_name)))
def hasUnparsedBranchCache(self, canonical_project_name, branch):
project_branch_cache = self.unparsed_project_branch_cache.setdefault(
canonical_project_name, {})