Always request all required project configuration
In preparation for caching a project's config in Zookeeper we need a way to request all necessary config for all tenants the project is part of with a single merger call. To do that, we create the list of all TPCs before we start loading any tenant. We can then combine the list of extra config files/dirs for a project from all tenants for the cat job. It's worth noting that this change adds some overhead for projects using extra config files/dirs without providing any immediate benefits. Change-Id: If0d134f2583ba5a5a176ba29f6d9f906b46aad05
This commit is contained in:
parent
8b6a887336
commit
60d01d1e58
|
@ -1608,9 +1608,10 @@ class TenantParser(object):
|
||||||
|
|
||||||
tenant.unparsed_config = conf
|
tenant.unparsed_config = conf
|
||||||
# tpcs is TenantProjectConfigs
|
# tpcs is TenantProjectConfigs
|
||||||
config_tpcs, untrusted_tpcs = self._loadTenantProjects(conf)
|
config_tpcs = abide.getConfigTPCs(tenant.name)
|
||||||
for tpc in config_tpcs:
|
for tpc in config_tpcs:
|
||||||
tenant.addConfigProject(tpc)
|
tenant.addConfigProject(tpc)
|
||||||
|
untrusted_tpcs = abide.getUntrustedTPCs(tenant.name)
|
||||||
for tpc in untrusted_tpcs:
|
for tpc in untrusted_tpcs:
|
||||||
tenant.addUntrustedProject(tpc)
|
tenant.addUntrustedProject(tpc)
|
||||||
|
|
||||||
|
@ -1774,7 +1775,7 @@ class TenantParser(object):
|
||||||
raise Exception("Unable to parse project %s", conf)
|
raise Exception("Unable to parse project %s", conf)
|
||||||
return projects
|
return projects
|
||||||
|
|
||||||
def _loadTenantProjects(self, conf_tenant):
|
def loadTenantProjects(self, conf_tenant):
|
||||||
config_projects = []
|
config_projects = []
|
||||||
untrusted_projects = []
|
untrusted_projects = []
|
||||||
|
|
||||||
|
@ -1823,12 +1824,14 @@ class TenantParser(object):
|
||||||
# If all config classes are excluded then do not
|
# If all config classes are excluded then do not
|
||||||
# request any getFiles jobs.
|
# request any getFiles jobs.
|
||||||
continue
|
continue
|
||||||
|
extra_config_files = abide.getExtraConfigFiles(project.name)
|
||||||
|
extra_config_dirs = abide.getExtraConfigDirs(project.name)
|
||||||
job = self.merger.getFiles(
|
job = self.merger.getFiles(
|
||||||
project.source.connection.connection_name,
|
project.source.connection.connection_name,
|
||||||
project.name, branch,
|
project.name, branch,
|
||||||
files=(['zuul.yaml', '.zuul.yaml'] +
|
files=(['zuul.yaml', '.zuul.yaml'] +
|
||||||
list(tpc.extra_config_files)),
|
list(extra_config_files)),
|
||||||
dirs=['zuul.d', '.zuul.d'] + list(tpc.extra_config_dirs))
|
dirs=['zuul.d', '.zuul.d'] + list(extra_config_dirs))
|
||||||
self.log.debug("Submitting cat job %s for %s %s %s" % (
|
self.log.debug("Submitting cat job %s for %s %s %s" % (
|
||||||
job, project.source.connection.connection_name,
|
job, project.source.connection.connection_name,
|
||||||
project.name, branch))
|
project.name, branch))
|
||||||
|
@ -2258,11 +2261,22 @@ class ConfigLoader(object):
|
||||||
abide.admin_rules[admin_rule.name] = admin_rule
|
abide.admin_rules[admin_rule.name] = admin_rule
|
||||||
|
|
||||||
if tenants:
|
if tenants:
|
||||||
tenants_to_load = [unparsed_abide.tenants[t] for t in tenants]
|
tenants_to_load = {t: unparsed_abide.tenants[t] for t in tenants}
|
||||||
else:
|
else:
|
||||||
tenants_to_load = unparsed_abide.tenants.values()
|
tenants_to_load = unparsed_abide.tenants
|
||||||
|
|
||||||
for conf_tenant in tenants_to_load:
|
# Pre-load TenantProjectConfigs so we can get and cache all of a
|
||||||
|
# project's config files (incl. tenant specific extra config) at once.
|
||||||
|
for tenant_name, conf_tenant in tenants_to_load.items():
|
||||||
|
config_tpcs, untrusted_tpcs = (
|
||||||
|
self.tenant_parser.loadTenantProjects(conf_tenant)
|
||||||
|
)
|
||||||
|
for tpc in config_tpcs:
|
||||||
|
abide.addConfigTPC(tenant_name, tpc)
|
||||||
|
for tpc in untrusted_tpcs:
|
||||||
|
abide.addUntrustedTPC(tenant_name, tpc)
|
||||||
|
|
||||||
|
for conf_tenant in tenants_to_load.values():
|
||||||
# When performing a full reload, do not use cached data.
|
# When performing a full reload, do not use cached data.
|
||||||
tenant = self.tenant_parser.fromYaml(
|
tenant = self.tenant_parser.fromYaml(
|
||||||
abide, conf_tenant, ansible_manager)
|
abide, conf_tenant, ansible_manager)
|
||||||
|
@ -2284,6 +2298,8 @@ class ConfigLoader(object):
|
||||||
new_abide.admin_rules = abide.admin_rules.copy()
|
new_abide.admin_rules = abide.admin_rules.copy()
|
||||||
new_abide.unparsed_project_branch_cache = \
|
new_abide.unparsed_project_branch_cache = \
|
||||||
abide.unparsed_project_branch_cache
|
abide.unparsed_project_branch_cache
|
||||||
|
new_abide.config_tpcs = abide.config_tpcs
|
||||||
|
new_abide.untrusted_tpcs = abide.untrusted_tpcs
|
||||||
|
|
||||||
if unparsed_abide:
|
if unparsed_abide:
|
||||||
# We got a new unparsed abide so re-load the tenant completely.
|
# We got a new unparsed abide so re-load the tenant completely.
|
||||||
|
@ -2297,6 +2313,17 @@ class ConfigLoader(object):
|
||||||
else:
|
else:
|
||||||
unparsed_config = tenant.unparsed_config
|
unparsed_config = tenant.unparsed_config
|
||||||
|
|
||||||
|
# Pre-load TenantProjectConfig so we can get and cache all of a
|
||||||
|
# project's config files (incl. tenant specific extra config) at once.
|
||||||
|
config_tpcs, untrusted_tpcs = (
|
||||||
|
self.tenant_parser.loadTenantProjects(unparsed_config)
|
||||||
|
)
|
||||||
|
new_abide.clearTPCs(tenant.name)
|
||||||
|
for tpc in config_tpcs:
|
||||||
|
new_abide.addConfigTPC(tenant.name, tpc)
|
||||||
|
for tpc in untrusted_tpcs:
|
||||||
|
new_abide.addUntrustedTPC(tenant.name, tpc)
|
||||||
|
|
||||||
# When reloading a tenant only, use cached data if available.
|
# When reloading a tenant only, use cached data if available.
|
||||||
new_tenant = self.tenant_parser.fromYaml(
|
new_tenant = self.tenant_parser.fromYaml(
|
||||||
new_abide, unparsed_config, ansible_manager)
|
new_abide, unparsed_config, ansible_manager)
|
||||||
|
|
|
@ -5238,9 +5238,51 @@ class Abide(object):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.admin_rules = OrderedDict()
|
self.admin_rules = OrderedDict()
|
||||||
self.tenants = OrderedDict()
|
self.tenants = OrderedDict()
|
||||||
|
# tenant -> project -> list(tpcs)
|
||||||
|
# The project TPCs are stored as a list as we don't check for
|
||||||
|
# duplicate projects here.
|
||||||
|
self.config_tpcs = defaultdict(lambda: defaultdict(list))
|
||||||
|
self.untrusted_tpcs = defaultdict(lambda: defaultdict(list))
|
||||||
# project -> branch -> UnparsedBranchCache
|
# project -> branch -> UnparsedBranchCache
|
||||||
self.unparsed_project_branch_cache = {}
|
self.unparsed_project_branch_cache = {}
|
||||||
|
|
||||||
|
def addConfigTPC(self, tenant_name, tpc):
|
||||||
|
self.config_tpcs[tenant_name][tpc.project.name].append(tpc)
|
||||||
|
|
||||||
|
def getConfigTPCs(self, tenant_name):
|
||||||
|
return list(itertools.chain.from_iterable(
|
||||||
|
self.config_tpcs[tenant_name].values()))
|
||||||
|
|
||||||
|
def addUntrustedTPC(self, tenant_name, tpc):
|
||||||
|
self.untrusted_tpcs[tenant_name][tpc.project.name].append(tpc)
|
||||||
|
|
||||||
|
def getUntrustedTPCs(self, tenant_name):
|
||||||
|
return list(itertools.chain.from_iterable(
|
||||||
|
self.untrusted_tpcs[tenant_name].values()))
|
||||||
|
|
||||||
|
def clearTPCs(self, tenant_name):
|
||||||
|
self.config_tpcs[tenant_name].clear()
|
||||||
|
self.untrusted_tpcs[tenant_name].clear()
|
||||||
|
|
||||||
|
def _allProjectTPCs(self, project_name):
|
||||||
|
# Flatten the lists of a project TPCs from all tenants
|
||||||
|
return itertools.chain.from_iterable(
|
||||||
|
tenant_tpcs.get(project_name, [])
|
||||||
|
for tenant_tpcs in itertools.chain(self.config_tpcs.values(),
|
||||||
|
self.untrusted_tpcs.values()))
|
||||||
|
|
||||||
|
def getExtraConfigFiles(self, project_name):
|
||||||
|
"""Get all extra config files for a project accross tenants."""
|
||||||
|
return set(itertools.chain.from_iterable(
|
||||||
|
tpc.extra_config_files
|
||||||
|
for tpc in self._allProjectTPCs(project_name)))
|
||||||
|
|
||||||
|
def getExtraConfigDirs(self, project_name):
|
||||||
|
"""Get all extra config dirs for a project accross tenants."""
|
||||||
|
return set(itertools.chain.from_iterable(
|
||||||
|
tpc.extra_config_dirs
|
||||||
|
for tpc in self._allProjectTPCs(project_name)))
|
||||||
|
|
||||||
def hasUnparsedBranchCache(self, canonical_project_name, branch):
|
def hasUnparsedBranchCache(self, canonical_project_name, branch):
|
||||||
project_branch_cache = self.unparsed_project_branch_cache.setdefault(
|
project_branch_cache = self.unparsed_project_branch_cache.setdefault(
|
||||||
canonical_project_name, {})
|
canonical_project_name, {})
|
||||||
|
|
Loading…
Reference in New Issue