Add ConfigLoader initializer
Pass in some commonly used tools for the configloader to use. Also, make some TenantParser private methods public for use by the configloader. Change-Id: If08a60d5ba3576837f462c35dbdf95b1613859b3
This commit is contained in:
parent
3a58e86100
commit
d83ebfb65e
|
@ -1547,11 +1547,11 @@ class TenantParser(object):
|
||||||
project = source_context.project
|
project = source_context.project
|
||||||
branch = source_context.branch
|
branch = source_context.branch
|
||||||
if source_context.trusted:
|
if source_context.trusted:
|
||||||
incdata = self._parseConfigProjectLayout(
|
incdata = self.loadConfigProjectLayout(
|
||||||
job.files[fn], source_context, tenant)
|
job.files[fn], source_context, tenant)
|
||||||
config_projects_config.extend(incdata, tenant)
|
config_projects_config.extend(incdata, tenant)
|
||||||
else:
|
else:
|
||||||
incdata = self._parseUntrustedProjectLayout(
|
incdata = self.loadUntrustedProjectLayout(
|
||||||
job.files[fn], source_context, tenant)
|
job.files[fn], source_context, tenant)
|
||||||
untrusted_projects_config.extend(incdata, tenant)
|
untrusted_projects_config.extend(incdata, tenant)
|
||||||
new_project_unparsed_config[project].extend(
|
new_project_unparsed_config[project].extend(
|
||||||
|
@ -1569,14 +1569,14 @@ class TenantParser(object):
|
||||||
project.unparsed_branch_config = branch_config
|
project.unparsed_branch_config = branch_config
|
||||||
return config_projects_config, untrusted_projects_config
|
return config_projects_config, untrusted_projects_config
|
||||||
|
|
||||||
def _parseConfigProjectLayout(self, data, source_context, tenant):
|
def loadConfigProjectLayout(self, data, source_context, tenant):
|
||||||
# This is the top-level configuration for a tenant.
|
# This is the top-level configuration for a tenant.
|
||||||
config = model.UnparsedTenantConfig()
|
config = model.UnparsedTenantConfig()
|
||||||
with early_configuration_exceptions(source_context):
|
with early_configuration_exceptions(source_context):
|
||||||
config.extend(safe_load_yaml(data, source_context), tenant)
|
config.extend(safe_load_yaml(data, source_context), tenant)
|
||||||
return config
|
return config
|
||||||
|
|
||||||
def _parseUntrustedProjectLayout(self, data, source_context, tenant):
|
def loadUntrustedProjectLayout(self, data, source_context, tenant):
|
||||||
config = model.UnparsedTenantConfig()
|
config = model.UnparsedTenantConfig()
|
||||||
with early_configuration_exceptions(source_context):
|
with early_configuration_exceptions(source_context):
|
||||||
config.extend(safe_load_yaml(data, source_context), tenant)
|
config.extend(safe_load_yaml(data, source_context), tenant)
|
||||||
|
@ -1712,6 +1712,12 @@ class TenantParser(object):
|
||||||
class ConfigLoader(object):
|
class ConfigLoader(object):
|
||||||
log = logging.getLogger("zuul.ConfigLoader")
|
log = logging.getLogger("zuul.ConfigLoader")
|
||||||
|
|
||||||
|
def __init__(self, connections, scheduler, merger):
|
||||||
|
self.connections = connections
|
||||||
|
self.scheduler = scheduler
|
||||||
|
self.merger = merger
|
||||||
|
self.tenant_parser = TenantParser(connections, scheduler, merger)
|
||||||
|
|
||||||
def expandConfigPath(self, config_path):
|
def expandConfigPath(self, config_path):
|
||||||
if config_path:
|
if config_path:
|
||||||
config_path = os.path.expanduser(config_path)
|
config_path = os.path.expanduser(config_path)
|
||||||
|
@ -1720,8 +1726,7 @@ class ConfigLoader(object):
|
||||||
config_path)
|
config_path)
|
||||||
return config_path
|
return config_path
|
||||||
|
|
||||||
def loadConfig(self, config_path, project_key_dir, scheduler, merger,
|
def loadConfig(self, config_path, project_key_dir):
|
||||||
connections):
|
|
||||||
abide = model.Abide()
|
abide = model.Abide()
|
||||||
|
|
||||||
config_path = self.expandConfigPath(config_path)
|
config_path = self.expandConfigPath(config_path)
|
||||||
|
@ -1732,31 +1737,28 @@ class ConfigLoader(object):
|
||||||
config.extend(data)
|
config.extend(data)
|
||||||
base = os.path.dirname(os.path.realpath(config_path))
|
base = os.path.dirname(os.path.realpath(config_path))
|
||||||
|
|
||||||
tenant_parser = TenantParser(connections, scheduler, merger)
|
|
||||||
for conf_tenant in config.tenants:
|
for conf_tenant in config.tenants:
|
||||||
# When performing a full reload, do not use cached data.
|
# When performing a full reload, do not use cached data.
|
||||||
tenant = tenant_parser.fromYaml(base, project_key_dir,
|
tenant = self.tenant_parser.fromYaml(base, project_key_dir,
|
||||||
conf_tenant, old_tenant=None)
|
conf_tenant, old_tenant=None)
|
||||||
abide.tenants[tenant.name] = tenant
|
abide.tenants[tenant.name] = tenant
|
||||||
return abide
|
return abide
|
||||||
|
|
||||||
def reloadTenant(self, config_path, project_key_dir, scheduler,
|
def reloadTenant(self, config_path, project_key_dir, abide, tenant):
|
||||||
merger, connections, abide, tenant):
|
|
||||||
new_abide = model.Abide()
|
new_abide = model.Abide()
|
||||||
new_abide.tenants = abide.tenants.copy()
|
new_abide.tenants = abide.tenants.copy()
|
||||||
|
|
||||||
config_path = self.expandConfigPath(config_path)
|
config_path = self.expandConfigPath(config_path)
|
||||||
base = os.path.dirname(os.path.realpath(config_path))
|
base = os.path.dirname(os.path.realpath(config_path))
|
||||||
tenant_parser = TenantParser(connections, scheduler, merger)
|
|
||||||
|
|
||||||
# When reloading a tenant only, use cached data if available.
|
# When reloading a tenant only, use cached data if available.
|
||||||
new_tenant = tenant_parser.fromYaml(
|
new_tenant = self.tenant_parser.fromYaml(
|
||||||
base, project_key_dir,
|
base, project_key_dir,
|
||||||
tenant.unparsed_config, old_tenant=tenant)
|
tenant.unparsed_config, old_tenant=tenant)
|
||||||
new_abide.tenants[tenant.name] = new_tenant
|
new_abide.tenants[tenant.name] = new_tenant
|
||||||
return new_abide
|
return new_abide
|
||||||
|
|
||||||
def _loadDynamicProjectData(self, tenant_parser, config, project,
|
def _loadDynamicProjectData(self, config, project,
|
||||||
files, trusted, tenant):
|
files, trusted, tenant):
|
||||||
if trusted:
|
if trusted:
|
||||||
branches = ['master']
|
branches = ['master']
|
||||||
|
@ -1810,29 +1812,30 @@ class ConfigLoader(object):
|
||||||
loaded = conf_root
|
loaded = conf_root
|
||||||
|
|
||||||
if trusted:
|
if trusted:
|
||||||
incdata = tenant_parser._parseConfigProjectLayout(
|
incdata = (self.tenant_parser.
|
||||||
data, source_context, tenant)
|
loadConfigProjectLayout(
|
||||||
|
data, source_context, tenant))
|
||||||
else:
|
else:
|
||||||
incdata = tenant_parser._parseUntrustedProjectLayout(
|
incdata = (self.tenant_parser.
|
||||||
data, source_context, tenant)
|
loadUntrustedProjectLayout(
|
||||||
|
data, source_context, tenant))
|
||||||
|
|
||||||
config.extend(incdata, tenant)
|
config.extend(incdata, tenant)
|
||||||
|
|
||||||
def createDynamicLayout(self, tenant, files,
|
def createDynamicLayout(self, tenant, files,
|
||||||
include_config_projects=False,
|
include_config_projects=False,
|
||||||
scheduler=None, connections=None):
|
scheduler=None, connections=None):
|
||||||
tenant_parser = TenantParser(connections, scheduler, None)
|
|
||||||
if include_config_projects:
|
if include_config_projects:
|
||||||
config = model.UnparsedTenantConfig()
|
config = model.UnparsedTenantConfig()
|
||||||
for project in tenant.config_projects:
|
for project in tenant.config_projects:
|
||||||
self._loadDynamicProjectData(
|
self._loadDynamicProjectData(
|
||||||
tenant_parser, config, project, files, True, tenant)
|
config, project, files, True, tenant)
|
||||||
else:
|
else:
|
||||||
config = tenant.config_projects_config.copy()
|
config = tenant.config_projects_config.copy()
|
||||||
|
|
||||||
for project in tenant.untrusted_projects:
|
for project in tenant.untrusted_projects:
|
||||||
self._loadDynamicProjectData(tenant_parser, config,
|
self._loadDynamicProjectData(config, project, files,
|
||||||
project, files, False, tenant)
|
False, tenant)
|
||||||
|
|
||||||
layout = model.Layout(tenant)
|
layout = model.Layout(tenant)
|
||||||
self.log.debug("Created layout id %s", layout.uuid)
|
self.log.debug("Created layout id %s", layout.uuid)
|
||||||
|
@ -1856,8 +1859,8 @@ class ConfigLoader(object):
|
||||||
else:
|
else:
|
||||||
skip_pipelines = skip_semaphores = False
|
skip_pipelines = skip_semaphores = False
|
||||||
|
|
||||||
tenant_parser._parseLayoutItems(layout, tenant, config,
|
self.tenant_parser._parseLayoutItems(layout, tenant, config,
|
||||||
skip_pipelines=skip_pipelines,
|
skip_pipelines=skip_pipelines,
|
||||||
skip_semaphores=skip_semaphores)
|
skip_semaphores=skip_semaphores)
|
||||||
|
|
||||||
return layout
|
return layout
|
||||||
|
|
|
@ -460,7 +460,8 @@ class PipelineManager(object):
|
||||||
# Load layout
|
# Load layout
|
||||||
# Late import to break an import loop
|
# Late import to break an import loop
|
||||||
import zuul.configloader
|
import zuul.configloader
|
||||||
loader = zuul.configloader.ConfigLoader()
|
loader = zuul.configloader.ConfigLoader(
|
||||||
|
self.sched.connections, self.sched, None)
|
||||||
|
|
||||||
self.log.debug("Loading dynamic layout")
|
self.log.debug("Loading dynamic layout")
|
||||||
(trusted_updates, untrusted_updates) = item.includesConfigUpdates()
|
(trusted_updates, untrusted_updates) = item.includesConfigUpdates()
|
||||||
|
@ -476,9 +477,7 @@ class PipelineManager(object):
|
||||||
loader.createDynamicLayout(
|
loader.createDynamicLayout(
|
||||||
item.pipeline.layout.tenant,
|
item.pipeline.layout.tenant,
|
||||||
build_set.files,
|
build_set.files,
|
||||||
include_config_projects=True,
|
include_config_projects=True)
|
||||||
scheduler=self.sched,
|
|
||||||
connections=self.sched.connections)
|
|
||||||
trusted_layout_verified = True
|
trusted_layout_verified = True
|
||||||
|
|
||||||
# Then create the config a second time but without changes
|
# Then create the config a second time but without changes
|
||||||
|
|
|
@ -548,11 +548,11 @@ class Scheduler(threading.Thread):
|
||||||
self.config = event.config
|
self.config = event.config
|
||||||
try:
|
try:
|
||||||
self.log.info("Full reconfiguration beginning")
|
self.log.info("Full reconfiguration beginning")
|
||||||
loader = configloader.ConfigLoader()
|
loader = configloader.ConfigLoader(
|
||||||
|
self.connections, self, self.merger)
|
||||||
abide = loader.loadConfig(
|
abide = loader.loadConfig(
|
||||||
self.config.get('scheduler', 'tenant_config'),
|
self.config.get('scheduler', 'tenant_config'),
|
||||||
self._get_project_key_dir(),
|
self._get_project_key_dir())
|
||||||
self, self.merger, self.connections)
|
|
||||||
for tenant in abide.tenants.values():
|
for tenant in abide.tenants.values():
|
||||||
self._reconfigureTenant(tenant)
|
self._reconfigureTenant(tenant)
|
||||||
self.abide = abide
|
self.abide = abide
|
||||||
|
@ -572,11 +572,11 @@ class Scheduler(threading.Thread):
|
||||||
project.unparsed_config = None
|
project.unparsed_config = None
|
||||||
project.unparsed_branch_config = {}
|
project.unparsed_branch_config = {}
|
||||||
old_tenant = self.abide.tenants[event.tenant_name]
|
old_tenant = self.abide.tenants[event.tenant_name]
|
||||||
loader = configloader.ConfigLoader()
|
loader = configloader.ConfigLoader(
|
||||||
|
self.connections, self, self.merger)
|
||||||
abide = loader.reloadTenant(
|
abide = loader.reloadTenant(
|
||||||
self.config.get('scheduler', 'tenant_config'),
|
self.config.get('scheduler', 'tenant_config'),
|
||||||
self._get_project_key_dir(),
|
self._get_project_key_dir(),
|
||||||
self, self.merger, self.connections,
|
|
||||||
self.abide, old_tenant)
|
self.abide, old_tenant)
|
||||||
tenant = abide.tenants[event.tenant_name]
|
tenant = abide.tenants[event.tenant_name]
|
||||||
self._reconfigureTenant(tenant)
|
self._reconfigureTenant(tenant)
|
||||||
|
|
Loading…
Reference in New Issue