From d83ebfb65e08891436322901b5b8265604c652fd Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Thu, 15 Feb 2018 16:38:17 -0800 Subject: [PATCH] Add ConfigLoader initializer Pass in some commonly used tools for the configloader to use. Also, make some TenantParser private methods public for use by the configloader. Change-Id: If08a60d5ba3576837f462c35dbdf95b1613859b3 --- zuul/configloader.py | 53 +++++++++++++++++++++------------------- zuul/manager/__init__.py | 7 +++--- zuul/scheduler.py | 10 ++++---- 3 files changed, 36 insertions(+), 34 deletions(-) diff --git a/zuul/configloader.py b/zuul/configloader.py index f501322694..d2256ba9d3 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -1547,11 +1547,11 @@ class TenantParser(object): project = source_context.project branch = source_context.branch if source_context.trusted: - incdata = self._parseConfigProjectLayout( + incdata = self.loadConfigProjectLayout( job.files[fn], source_context, tenant) config_projects_config.extend(incdata, tenant) else: - incdata = self._parseUntrustedProjectLayout( + incdata = self.loadUntrustedProjectLayout( job.files[fn], source_context, tenant) untrusted_projects_config.extend(incdata, tenant) new_project_unparsed_config[project].extend( @@ -1569,14 +1569,14 @@ class TenantParser(object): project.unparsed_branch_config = branch_config return config_projects_config, untrusted_projects_config - def _parseConfigProjectLayout(self, data, source_context, tenant): + def loadConfigProjectLayout(self, data, source_context, tenant): # This is the top-level configuration for a tenant. config = model.UnparsedTenantConfig() with early_configuration_exceptions(source_context): config.extend(safe_load_yaml(data, source_context), tenant) return config - def _parseUntrustedProjectLayout(self, data, source_context, tenant): + def loadUntrustedProjectLayout(self, data, source_context, tenant): config = model.UnparsedTenantConfig() with early_configuration_exceptions(source_context): config.extend(safe_load_yaml(data, source_context), tenant) @@ -1712,6 +1712,12 @@ class TenantParser(object): class ConfigLoader(object): log = logging.getLogger("zuul.ConfigLoader") + def __init__(self, connections, scheduler, merger): + self.connections = connections + self.scheduler = scheduler + self.merger = merger + self.tenant_parser = TenantParser(connections, scheduler, merger) + def expandConfigPath(self, config_path): if config_path: config_path = os.path.expanduser(config_path) @@ -1720,8 +1726,7 @@ class ConfigLoader(object): config_path) return config_path - def loadConfig(self, config_path, project_key_dir, scheduler, merger, - connections): + def loadConfig(self, config_path, project_key_dir): abide = model.Abide() config_path = self.expandConfigPath(config_path) @@ -1732,31 +1737,28 @@ class ConfigLoader(object): config.extend(data) base = os.path.dirname(os.path.realpath(config_path)) - tenant_parser = TenantParser(connections, scheduler, merger) for conf_tenant in config.tenants: # When performing a full reload, do not use cached data. - tenant = tenant_parser.fromYaml(base, project_key_dir, - conf_tenant, old_tenant=None) + tenant = self.tenant_parser.fromYaml(base, project_key_dir, + conf_tenant, old_tenant=None) abide.tenants[tenant.name] = tenant return abide - def reloadTenant(self, config_path, project_key_dir, scheduler, - merger, connections, abide, tenant): + def reloadTenant(self, config_path, project_key_dir, abide, tenant): new_abide = model.Abide() new_abide.tenants = abide.tenants.copy() config_path = self.expandConfigPath(config_path) base = os.path.dirname(os.path.realpath(config_path)) - tenant_parser = TenantParser(connections, scheduler, merger) # When reloading a tenant only, use cached data if available. - new_tenant = tenant_parser.fromYaml( + new_tenant = self.tenant_parser.fromYaml( base, project_key_dir, tenant.unparsed_config, old_tenant=tenant) new_abide.tenants[tenant.name] = new_tenant return new_abide - def _loadDynamicProjectData(self, tenant_parser, config, project, + def _loadDynamicProjectData(self, config, project, files, trusted, tenant): if trusted: branches = ['master'] @@ -1810,29 +1812,30 @@ class ConfigLoader(object): loaded = conf_root if trusted: - incdata = tenant_parser._parseConfigProjectLayout( - data, source_context, tenant) + incdata = (self.tenant_parser. + loadConfigProjectLayout( + data, source_context, tenant)) else: - incdata = tenant_parser._parseUntrustedProjectLayout( - data, source_context, tenant) + incdata = (self.tenant_parser. + loadUntrustedProjectLayout( + data, source_context, tenant)) config.extend(incdata, tenant) def createDynamicLayout(self, tenant, files, include_config_projects=False, scheduler=None, connections=None): - tenant_parser = TenantParser(connections, scheduler, None) if include_config_projects: config = model.UnparsedTenantConfig() for project in tenant.config_projects: self._loadDynamicProjectData( - tenant_parser, config, project, files, True, tenant) + config, project, files, True, tenant) else: config = tenant.config_projects_config.copy() for project in tenant.untrusted_projects: - self._loadDynamicProjectData(tenant_parser, config, - project, files, False, tenant) + self._loadDynamicProjectData(config, project, files, + False, tenant) layout = model.Layout(tenant) self.log.debug("Created layout id %s", layout.uuid) @@ -1856,8 +1859,8 @@ class ConfigLoader(object): else: skip_pipelines = skip_semaphores = False - tenant_parser._parseLayoutItems(layout, tenant, config, - skip_pipelines=skip_pipelines, - skip_semaphores=skip_semaphores) + self.tenant_parser._parseLayoutItems(layout, tenant, config, + skip_pipelines=skip_pipelines, + skip_semaphores=skip_semaphores) return layout diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py index 88ddf7d144..506b94f708 100644 --- a/zuul/manager/__init__.py +++ b/zuul/manager/__init__.py @@ -460,7 +460,8 @@ class PipelineManager(object): # Load layout # Late import to break an import loop import zuul.configloader - loader = zuul.configloader.ConfigLoader() + loader = zuul.configloader.ConfigLoader( + self.sched.connections, self.sched, None) self.log.debug("Loading dynamic layout") (trusted_updates, untrusted_updates) = item.includesConfigUpdates() @@ -476,9 +477,7 @@ class PipelineManager(object): loader.createDynamicLayout( item.pipeline.layout.tenant, build_set.files, - include_config_projects=True, - scheduler=self.sched, - connections=self.sched.connections) + include_config_projects=True) trusted_layout_verified = True # Then create the config a second time but without changes diff --git a/zuul/scheduler.py b/zuul/scheduler.py index 7a0e28c5a0..a195b09e7a 100644 --- a/zuul/scheduler.py +++ b/zuul/scheduler.py @@ -548,11 +548,11 @@ class Scheduler(threading.Thread): self.config = event.config try: self.log.info("Full reconfiguration beginning") - loader = configloader.ConfigLoader() + loader = configloader.ConfigLoader( + self.connections, self, self.merger) abide = loader.loadConfig( self.config.get('scheduler', 'tenant_config'), - self._get_project_key_dir(), - self, self.merger, self.connections) + self._get_project_key_dir()) for tenant in abide.tenants.values(): self._reconfigureTenant(tenant) self.abide = abide @@ -572,11 +572,11 @@ class Scheduler(threading.Thread): project.unparsed_config = None project.unparsed_branch_config = {} old_tenant = self.abide.tenants[event.tenant_name] - loader = configloader.ConfigLoader() + loader = configloader.ConfigLoader( + self.connections, self, self.merger) abide = loader.reloadTenant( self.config.get('scheduler', 'tenant_config'), self._get_project_key_dir(), - self, self.merger, self.connections, self.abide, old_tenant) tenant = abide.tenants[event.tenant_name] self._reconfigureTenant(tenant)