Refactor config/tenant (re-)loading
In a multi-scheduler deployment we don't want to have a global layout lock in Zookeeper. Instead, we will use a read/write lock per tenant. This allows use to continue processing pipelines for other tenants in the meantime. In respect of that, this change refactors the config loading so that we have a single interface that performs config (re-)loading on a per-tenant basis. This is also important as later on a scheduler startup will no longer require a full reconfiguration. With the refactoring, the configloader always (re-)loads the config of a specific tenant as requested by the scheduler. If a full reconfiguration should be performed the scheduler will create a new Abide instance before loading any tenant. For the case of smart or tenant reconfigurations we will re-use the existing Abide instance instead of copying everything over from the old to the new instance. Change-Id: I6b7afd34203b74d5a6524e5277055a813147ff9e
This commit is contained in:
parent
3a0bbd205c
commit
ca0d379973
|
@ -2266,12 +2266,12 @@ class ConfigLoader(object):
|
||||||
unparsed_abide.extend(data)
|
unparsed_abide.extend(data)
|
||||||
return unparsed_abide
|
return unparsed_abide
|
||||||
|
|
||||||
def loadConfig(self, unparsed_abide, ansible_manager, tenants=None):
|
def loadAdminRules(self, abide, unparsed_abide):
|
||||||
abide = model.Abide()
|
|
||||||
for conf_admin_rule in unparsed_abide.admin_rules:
|
for conf_admin_rule in unparsed_abide.admin_rules:
|
||||||
admin_rule = self.admin_rule_parser.fromYaml(conf_admin_rule)
|
admin_rule = self.admin_rule_parser.fromYaml(conf_admin_rule)
|
||||||
abide.admin_rules[admin_rule.name] = admin_rule
|
abide.admin_rules[admin_rule.name] = admin_rule
|
||||||
|
|
||||||
|
def loadTPCs(self, abide, unparsed_abide, tenants=None):
|
||||||
if tenants:
|
if tenants:
|
||||||
tenants_to_load = {t: unparsed_abide.tenants[t] for t in tenants}
|
tenants_to_load = {t: unparsed_abide.tenants[t] for t in tenants}
|
||||||
else:
|
else:
|
||||||
|
@ -2279,76 +2279,34 @@ class ConfigLoader(object):
|
||||||
|
|
||||||
# Pre-load TenantProjectConfigs so we can get and cache all of a
|
# Pre-load TenantProjectConfigs so we can get and cache all of a
|
||||||
# project's config files (incl. tenant specific extra config) at once.
|
# project's config files (incl. tenant specific extra config) at once.
|
||||||
for tenant_name, conf_tenant in tenants_to_load.items():
|
for tenant_name, unparsed_config in tenants_to_load.items():
|
||||||
config_tpcs, untrusted_tpcs = (
|
config_tpcs, untrusted_tpcs = (
|
||||||
self.tenant_parser.loadTenantProjects(conf_tenant)
|
self.tenant_parser.loadTenantProjects(unparsed_config)
|
||||||
)
|
)
|
||||||
|
abide.clearTPCs(tenant_name)
|
||||||
for tpc in config_tpcs:
|
for tpc in config_tpcs:
|
||||||
abide.addConfigTPC(tenant_name, tpc)
|
abide.addConfigTPC(tenant_name, tpc)
|
||||||
for tpc in untrusted_tpcs:
|
for tpc in untrusted_tpcs:
|
||||||
abide.addUntrustedTPC(tenant_name, tpc)
|
abide.addUntrustedTPC(tenant_name, tpc)
|
||||||
|
|
||||||
for conf_tenant in tenants_to_load.values():
|
def loadTenant(self, abide, tenant_name, ansible_manager, unparsed_abide,
|
||||||
# When performing a full reload, do not use cached data.
|
cache_ltime=None):
|
||||||
tenant = self.tenant_parser.fromYaml(
|
if tenant_name not in unparsed_abide.tenants:
|
||||||
abide, conf_tenant, ansible_manager)
|
del abide.tenants[tenant_name]
|
||||||
abide.tenants[tenant.name] = tenant
|
return None
|
||||||
if len(tenant.layout.loading_errors):
|
|
||||||
self.log.warning(
|
|
||||||
"%s errors detected during %s tenant "
|
|
||||||
"configuration loading" % (
|
|
||||||
len(tenant.layout.loading_errors), tenant.name))
|
|
||||||
# Log accumulated errors
|
|
||||||
for err in tenant.layout.loading_errors.errors[:10]:
|
|
||||||
self.log.warning(err.error)
|
|
||||||
return abide
|
|
||||||
|
|
||||||
def reloadTenant(self, abide, tenant, ansible_manager,
|
unparsed_config = unparsed_abide.tenants[tenant_name]
|
||||||
unparsed_abide=None, cache_ltime=None):
|
|
||||||
new_abide = model.Abide()
|
|
||||||
new_abide.tenants = abide.tenants.copy()
|
|
||||||
new_abide.admin_rules = abide.admin_rules.copy()
|
|
||||||
new_abide.unparsed_project_branch_cache = \
|
|
||||||
abide.unparsed_project_branch_cache
|
|
||||||
new_abide.config_tpcs = abide.config_tpcs
|
|
||||||
new_abide.untrusted_tpcs = abide.untrusted_tpcs
|
|
||||||
|
|
||||||
if unparsed_abide:
|
|
||||||
# We got a new unparsed abide so re-load the tenant completely.
|
|
||||||
# First check if the tenant is still existing and if not remove
|
|
||||||
# from the abide.
|
|
||||||
if tenant.name not in unparsed_abide.tenants:
|
|
||||||
del new_abide.tenants[tenant.name]
|
|
||||||
return new_abide
|
|
||||||
|
|
||||||
unparsed_config = unparsed_abide.tenants[tenant.name]
|
|
||||||
else:
|
|
||||||
unparsed_config = tenant.unparsed_config
|
|
||||||
|
|
||||||
# Pre-load TenantProjectConfig so we can get and cache all of a
|
|
||||||
# project's config files (incl. tenant specific extra config) at once.
|
|
||||||
config_tpcs, untrusted_tpcs = (
|
|
||||||
self.tenant_parser.loadTenantProjects(unparsed_config)
|
|
||||||
)
|
|
||||||
new_abide.clearTPCs(tenant.name)
|
|
||||||
for tpc in config_tpcs:
|
|
||||||
new_abide.addConfigTPC(tenant.name, tpc)
|
|
||||||
for tpc in untrusted_tpcs:
|
|
||||||
new_abide.addUntrustedTPC(tenant.name, tpc)
|
|
||||||
|
|
||||||
# When reloading a tenant only, use cached data if available.
|
|
||||||
new_tenant = self.tenant_parser.fromYaml(
|
new_tenant = self.tenant_parser.fromYaml(
|
||||||
new_abide, unparsed_config, ansible_manager, cache_ltime)
|
abide, unparsed_config, ansible_manager, cache_ltime)
|
||||||
new_abide.tenants[tenant.name] = new_tenant
|
abide.tenants[tenant_name] = new_tenant
|
||||||
if len(new_tenant.layout.loading_errors):
|
if len(new_tenant.layout.loading_errors):
|
||||||
self.log.warning(
|
self.log.warning(
|
||||||
"%s errors detected during %s tenant "
|
"%s errors detected during %s tenant configuration loading",
|
||||||
"configuration re-loading" % (
|
len(new_tenant.layout.loading_errors), tenant_name)
|
||||||
len(new_tenant.layout.loading_errors), tenant.name))
|
|
||||||
# Log accumulated errors
|
# Log accumulated errors
|
||||||
for err in new_tenant.layout.loading_errors.errors[:10]:
|
for err in new_tenant.layout.loading_errors.errors[:10]:
|
||||||
self.log.warning(err.error)
|
self.log.warning(err.error)
|
||||||
return new_abide
|
return new_tenant
|
||||||
|
|
||||||
def _loadDynamicProjectData(self, config, project,
|
def _loadDynamicProjectData(self, config, project,
|
||||||
files, trusted, item, loading_errors,
|
files, trusted, item, loading_errors,
|
||||||
|
|
|
@ -63,7 +63,6 @@ from zuul.model import (
|
||||||
PromoteEvent,
|
PromoteEvent,
|
||||||
ReconfigureEvent,
|
ReconfigureEvent,
|
||||||
SmartReconfigureEvent,
|
SmartReconfigureEvent,
|
||||||
Tenant,
|
|
||||||
TenantReconfigureEvent,
|
TenantReconfigureEvent,
|
||||||
UnparsedAbideConfig,
|
UnparsedAbideConfig,
|
||||||
)
|
)
|
||||||
|
@ -779,26 +778,48 @@ class Scheduler(threading.Thread):
|
||||||
tenant_config, script = self._checkTenantSourceConf(self.config)
|
tenant_config, script = self._checkTenantSourceConf(self.config)
|
||||||
self.unparsed_abide = loader.readConfig(
|
self.unparsed_abide = loader.readConfig(
|
||||||
tenant_config, from_script=script)
|
tenant_config, from_script=script)
|
||||||
abide = loader.loadConfig(
|
|
||||||
self.unparsed_abide, self.ansible_manager,
|
tenants_to_load = list(self.unparsed_abide.tenants)
|
||||||
event.validate_tenants)
|
if event.validate_tenants is not None:
|
||||||
if event.validate_tenants is None:
|
validate_tenants = set(event.validate_tenants)
|
||||||
for tenant in abide.tenants.values():
|
if not validate_tenants.issubset(tenants_to_load):
|
||||||
self._reconfigureTenant(tenant)
|
invalid = validate_tenants.difference(tenants_to_load)
|
||||||
for old_tenant in self.abide.tenants.values():
|
raise RuntimeError(f"Invalid tenant(s) found: {invalid}")
|
||||||
if not abide.tenants.get(old_tenant.name):
|
# In case we have an empty list, we validate all tenants.
|
||||||
# We deleted a tenant
|
tenants_to_load = event.validate_tenants or tenants_to_load
|
||||||
self._reconfigureDeleteTenant(old_tenant)
|
|
||||||
self.abide = abide
|
abide = Abide()
|
||||||
else:
|
loader.loadAdminRules(abide, self.unparsed_abide)
|
||||||
|
loader.loadTPCs(abide, self.unparsed_abide)
|
||||||
|
for tenant_name in tenants_to_load:
|
||||||
|
tenant = loader.loadTenant(abide, tenant_name,
|
||||||
|
self.ansible_manager,
|
||||||
|
self.unparsed_abide,
|
||||||
|
cache_ltime=None)
|
||||||
|
if event.validate_tenants:
|
||||||
|
# We are only validating the tenant config; skip reconfig
|
||||||
|
continue
|
||||||
|
|
||||||
|
if tenant is not None:
|
||||||
|
old_tenant = self.abide.tenants.get(tenant_name)
|
||||||
|
self._reconfigureTenant(tenant, old_tenant)
|
||||||
|
|
||||||
|
for old_tenant in self.abide.tenants.values():
|
||||||
|
if old_tenant.name not in abide.tenants:
|
||||||
|
# We deleted a tenant
|
||||||
|
self._reconfigureDeleteTenant(old_tenant)
|
||||||
|
|
||||||
|
if event.validate_tenants is not None:
|
||||||
loading_errors = []
|
loading_errors = []
|
||||||
for tenant in abide.tenants.values():
|
for tenant in abide.tenants.values():
|
||||||
for error in tenant.layout.loading_errors:
|
for error in tenant.layout.loading_errors:
|
||||||
loading_errors.append(error.__repr__())
|
loading_errors.append(repr(error))
|
||||||
if loading_errors:
|
if loading_errors:
|
||||||
summary = '\n\n\n'.join(loading_errors)
|
summary = '\n\n\n'.join(loading_errors)
|
||||||
raise configloader.ConfigurationSyntaxError(
|
raise configloader.ConfigurationSyntaxError(
|
||||||
'Configuration errors: {}'.format(summary))
|
f"Configuration errors: {summary}")
|
||||||
|
|
||||||
|
self.abide = abide
|
||||||
finally:
|
finally:
|
||||||
self.layout_lock.release()
|
self.layout_lock.release()
|
||||||
|
|
||||||
|
@ -833,6 +854,14 @@ class Scheduler(threading.Thread):
|
||||||
# all tenants from the currently known and the new ones.
|
# all tenants from the currently known and the new ones.
|
||||||
tenant_names = {t for t in self.abide.tenants}
|
tenant_names = {t for t in self.abide.tenants}
|
||||||
tenant_names.update(self.unparsed_abide.tenants.keys())
|
tenant_names.update(self.unparsed_abide.tenants.keys())
|
||||||
|
|
||||||
|
# Remove TPCs of deleted tenants
|
||||||
|
deleted_tenants = tenant_names.difference(
|
||||||
|
self.unparsed_abide.tenants.keys())
|
||||||
|
for tenant_name in deleted_tenants:
|
||||||
|
self.abide.clearTPCs(tenant_name)
|
||||||
|
loader.loadTPCs(self.abide, self.unparsed_abide)
|
||||||
|
|
||||||
for tenant_name in tenant_names:
|
for tenant_name in tenant_names:
|
||||||
old_tenant = old_unparsed_abide.tenants.get(tenant_name)
|
old_tenant = old_unparsed_abide.tenants.get(tenant_name)
|
||||||
new_tenant = self.unparsed_abide.tenants.get(tenant_name)
|
new_tenant = self.unparsed_abide.tenants.get(tenant_name)
|
||||||
|
@ -841,20 +870,16 @@ class Scheduler(threading.Thread):
|
||||||
|
|
||||||
reconfigured_tenants.append(tenant_name)
|
reconfigured_tenants.append(tenant_name)
|
||||||
old_tenant = self.abide.tenants.get(tenant_name)
|
old_tenant = self.abide.tenants.get(tenant_name)
|
||||||
if old_tenant is None:
|
tenant = loader.loadTenant(self.abide, tenant_name,
|
||||||
# If there is no old tenant, use a fake tenant with the
|
self.ansible_manager,
|
||||||
# correct name
|
self.unparsed_abide,
|
||||||
old_tenant = Tenant(tenant_name)
|
cache_ltime=event.zuul_event_ltime)
|
||||||
abide = loader.reloadTenant(
|
|
||||||
self.abide, old_tenant, self.ansible_manager,
|
|
||||||
self.unparsed_abide)
|
|
||||||
|
|
||||||
tenant = abide.tenants.get(tenant_name)
|
tenant = self.abide.tenants.get(tenant_name)
|
||||||
if tenant is not None:
|
if tenant is not None:
|
||||||
self._reconfigureTenant(tenant)
|
self._reconfigureTenant(tenant, old_tenant)
|
||||||
else:
|
else:
|
||||||
self._reconfigureDeleteTenant(old_tenant)
|
self._reconfigureDeleteTenant(old_tenant)
|
||||||
self.abide = abide
|
|
||||||
duration = round(time.monotonic() - start, 3)
|
duration = round(time.monotonic() - start, 3)
|
||||||
self.log.info("Smart reconfiguration of tenants %s complete "
|
self.log.info("Smart reconfiguration of tenants %s complete "
|
||||||
"(duration: %s seconds)", reconfigured_tenants, duration)
|
"(duration: %s seconds)", reconfigured_tenants, duration)
|
||||||
|
@ -879,15 +904,16 @@ class Scheduler(threading.Thread):
|
||||||
self.unparsed_config_cache.clearCache(project_name,
|
self.unparsed_config_cache.clearCache(project_name,
|
||||||
branch_name)
|
branch_name)
|
||||||
|
|
||||||
old_tenant = self.abide.tenants[event.tenant_name]
|
|
||||||
loader = configloader.ConfigLoader(
|
loader = configloader.ConfigLoader(
|
||||||
self.connections, self, self.merger, self.keystore)
|
self.connections, self, self.merger, self.keystore)
|
||||||
abide = loader.reloadTenant(self.abide, old_tenant,
|
old_tenant = self.abide.tenants.get(event.tenant_name)
|
||||||
self.ansible_manager,
|
loader.loadTPCs(self.abide, self.unparsed_abide,
|
||||||
cache_ltime=event.zuul_event_ltime)
|
[event.tenant_name])
|
||||||
tenant = abide.tenants[event.tenant_name]
|
loader.loadTenant(self.abide, event.tenant_name,
|
||||||
self._reconfigureTenant(tenant)
|
self.ansible_manager, self.unparsed_abide,
|
||||||
self.abide = abide
|
cache_ltime=event.zuul_event_ltime)
|
||||||
|
tenant = self.abide.tenants[event.tenant_name]
|
||||||
|
self._reconfigureTenant(tenant, old_tenant)
|
||||||
finally:
|
finally:
|
||||||
self.layout_lock.release()
|
self.layout_lock.release()
|
||||||
duration = round(time.monotonic() - start, 3)
|
duration = round(time.monotonic() - start, 3)
|
||||||
|
@ -1038,11 +1064,9 @@ class Scheduler(threading.Thread):
|
||||||
if not new_pipeline:
|
if not new_pipeline:
|
||||||
self._reconfigureDeletePipeline(old_pipeline)
|
self._reconfigureDeletePipeline(old_pipeline)
|
||||||
|
|
||||||
def _reconfigureTenant(self, tenant):
|
def _reconfigureTenant(self, tenant, old_tenant=None):
|
||||||
# This is called from _doReconfigureEvent while holding the
|
# This is called from _doReconfigureEvent while holding the
|
||||||
# layout lock
|
# layout lock
|
||||||
old_tenant = self.abide.tenants.get(tenant.name)
|
|
||||||
|
|
||||||
if old_tenant:
|
if old_tenant:
|
||||||
self._reenqueueTenant(old_tenant, tenant)
|
self._reenqueueTenant(old_tenant, tenant)
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue