Alter config format to lists of dictionaries

Rather than the previous dictionary of lists.

Change-Id: I0f8ffba15da489da097b14388699685f22b0579f
changes/27/260727/1
James E. Blair 7 years ago
parent fb610ce274
commit d8e778fdb0
  1. 7
      tests/fixtures/config/in-repo/common.yaml
  2. 4
      tests/fixtures/config/in-repo/main.yaml
  3. 4
      tests/fixtures/config/multi-tenant/common.yaml
  4. 8
      tests/fixtures/config/multi-tenant/main.yaml
  5. 12
      tests/fixtures/config/multi-tenant/tenant-one.yaml
  6. 12
      tests/fixtures/config/multi-tenant/tenant-two.yaml
  7. 23
      tests/fixtures/config/project-template/common.yaml
  8. 4
      tests/fixtures/config/project-template/main.yaml
  9. 33
      tests/fixtures/config/project-template/tenant-one.yaml
  10. 8
      tests/test_v3.py
  11. 218
      zuul/configloader.py
  12. 78
      zuul/model.py

@ -1,5 +1,5 @@
pipelines:
- name: check
- pipeline:
name: check
manager: independent
source:
gerrit
@ -13,7 +13,8 @@ pipelines:
gerrit:
verified: -1
- name: tenant-one-gate
- pipeline:
name: tenant-one-gate
manager: dependent
success-message: Build succeeded (tenant-one-gate).
source:

@ -1,5 +1,5 @@
tenants:
- name: tenant-one
- tenant:
name: tenant-one
include:
- common.yaml
source:

@ -1,5 +1,5 @@
pipelines:
- name: check
- pipeline:
name: check
manager: independent
source:
gerrit

@ -1,9 +1,11 @@
tenants:
- name: tenant-one
- tenant:
name: tenant-one
include:
- common.yaml
- tenant-one.yaml
- name: tenant-two
- tenant:
name: tenant-two
include:
- common.yaml
- tenant-two.yaml

@ -1,5 +1,5 @@
pipelines:
- name: tenant-one-gate
- pipeline:
name: tenant-one-gate
manager: dependent
success-message: Build succeeded (tenant-one-gate).
source:
@ -21,12 +21,12 @@ pipelines:
verified: 0
precedence: high
jobs:
- name:
- job:
name:
project1-test1
projects:
- name: org/project1
- project:
name: org/project1
check:
jobs:
- project1-test1

@ -1,5 +1,5 @@
pipelines:
- name: tenant-two-gate
- pipeline:
name: tenant-two-gate
manager: dependent
success-message: Build succeeded (tenant-two-gate).
source:
@ -21,12 +21,12 @@ pipelines:
verified: 0
precedence: high
jobs:
- name:
- job:
name:
project2-test1
projects:
- name: org/project2
- project:
name: org/project2
check:
jobs:
- project2-test1

@ -1,5 +1,5 @@
pipelines:
- name: check
- pipeline:
name: check
manager: independent
source:
gerrit
@ -13,7 +13,8 @@ pipelines:
gerrit:
verified: -1
- name: gate
- pipeline:
name: gate
manager: dependent
success-message: Build succeeded (gate).
source:
@ -35,20 +36,22 @@ pipelines:
verified: 0
precedence: high
jobs:
- name:
- job:
name:
project-test1
- name:
- job:
name:
project-test2
project-templates:
- name: test-template
- project-template:
name: test-template
gate:
jobs:
- project-test2
projects:
- name: org/project
- project:
name: org/project
templates:
- test-template
gate:

@ -1,4 +1,4 @@
tenants:
- name: tenant-one
- tenant:
name: tenant-one
include:
- common.yaml

@ -1,33 +0,0 @@
pipelines:
- name: tenant-one-gate
manager: dependent
success-message: Build succeeded (tenant-one-gate).
source:
gerrit
trigger:
gerrit:
- event: comment-added
approval:
- approved: 1
success:
gerrit:
verified: 2
submit: true
failure:
gerrit:
verified: -2
start:
gerrit:
verified: 0
precedence: high
jobs:
- name:
project1-test1
projects:
- name: org/project1
check:
- project1-test1
tenant-one-gate:
- project1-test1

@ -72,11 +72,11 @@ class TestInRepoConfig(ZuulTestCase):
def setup_repos(self):
in_repo_conf = textwrap.dedent(
"""
jobs:
- name: project-test1
- job:
name: project-test1
projects:
- name: org/project
- project:
name: org/project
tenant-one-gate:
jobs:
- project-test1

@ -37,51 +37,6 @@ def as_list(item):
return [item]
def extend_dict(a, b):
"""Extend dictionary a (which will be modified in place) with the
contents of b. This is designed for Zuul yaml files which are
typically dictionaries of lists of dictionaries, e.g.,
{'pipelines': ['name': 'gate']}. If two such dictionaries each
define a pipeline, the result will be a single dictionary with
a pipelines entry whose value is a two-element list."""
for k, v in b.items():
if k not in a:
a[k] = v
elif isinstance(v, dict) and isinstance(a[k], dict):
extend_dict(a[k], v)
elif isinstance(v, list) and isinstance(a[k], list):
a[k] += v
elif isinstance(v, list):
a[k] = [a[k]] + v
elif isinstance(a[k], list):
a[k] += [v]
else:
raise Exception("Unhandled case in extend_dict at %s" % (k,))
def deep_format(obj, paramdict):
"""Apply the paramdict via str.format() to all string objects found within
the supplied obj. Lists and dicts are traversed recursively.
Borrowed from Jenkins Job Builder project"""
if isinstance(obj, str):
ret = obj.format(**paramdict)
elif isinstance(obj, list):
ret = []
for item in obj:
ret.append(deep_format(item, paramdict))
elif isinstance(obj, dict):
ret = {}
for item in obj:
exp_item = item.format(**paramdict)
ret[exp_item] = deep_format(obj[item], paramdict)
else:
ret = obj
return ret
class JobParser(object):
@staticmethod
def getSchema():
@ -445,90 +400,102 @@ class PipelineParser(object):
return pipeline
class AbideValidator(object):
class TenantParser(object):
log = logging.getLogger("zuul.TenantParser")
tenant_source = vs.Schema({'repos': [str]})
def validateTenantSources(self, connections):
@staticmethod
def validateTenantSources(connections):
def v(value, path=[]):
if isinstance(value, dict):
for k, val in value.items():
connections.getSource(k)
self.validateTenantSource(val, path + [k])
TenantParser.validateTenantSource(val, path + [k])
else:
raise vs.Invalid("Invalid tenant source", path)
return v
def validateTenantSource(self, value, path=[]):
self.tenant_source(value)
@staticmethod
def validateTenantSource(value, path=[]):
TenantParser.tenant_source(value)
def getSchema(self, connections=None):
@staticmethod
def getSchema(connections=None):
tenant = {vs.Required('name'): str,
'include': to_list(str),
'source': self.validateTenantSources(connections)}
schema = vs.Schema({'tenants': [tenant]})
return schema
def validate(self, data, connections=None):
schema = self.getSchema(connections)
schema(data)
'source': TenantParser.validateTenantSources(connections)}
return vs.Schema(tenant)
class ConfigLoader(object):
log = logging.getLogger("zuul.ConfigLoader")
def loadConfig(self, config_path, scheduler, merger, connections):
abide = model.Abide()
if config_path:
config_path = os.path.expanduser(config_path)
if not os.path.exists(config_path):
raise Exception("Unable to read tenant config file at %s" %
config_path)
with open(config_path) as config_file:
self.log.info("Loading configuration from %s" % (config_path,))
data = yaml.load(config_file)
base = os.path.dirname(os.path.realpath(config_path))
@staticmethod
def fromYaml(base, connections, scheduler, merger, conf):
TenantParser.getSchema(connections)(conf)
tenant = model.Tenant(conf['name'])
tenant_config = model.UnparsedTenantConfig()
for fn in conf.get('include', []):
if not os.path.isabs(fn):
fn = os.path.join(base, fn)
fn = os.path.expanduser(fn)
with open(fn) as config_file:
TenantParser.log.info("Loading configuration from %s" % (fn,))
incdata = yaml.load(config_file)
tenant_config.extend(incdata)
incdata = TenantParser._loadTenantInRepoLayouts(merger, connections,
conf)
tenant_config.extend(incdata)
tenant.layout = TenantParser._parseLayout(base, tenant_config,
scheduler, connections)
return tenant
validator = AbideValidator()
validator.validate(data, connections)
@staticmethod
def _loadTenantInRepoLayouts(merger, connections, conf_tenant):
config = model.UnparsedTenantConfig()
jobs = []
for source_name, conf_source in conf_tenant.get('source', {}).items():
source = connections.getSource(source_name)
for conf_repo in conf_source.get('repos'):
project = source.getProject(conf_repo)
url = source.getGitUrl(project)
# TODOv3(jeblair): config should be branch specific
job = merger.getFiles(project.name, url, 'master',
files=['.zuul.yaml'])
job.project = project
jobs.append(job)
for job in jobs:
TenantParser.log.debug("Waiting for cat job %s" % (job,))
job.wait()
if job.files.get('.zuul.yaml'):
TenantParser.log.info(
"Loading configuration from %s/.zuul.yaml" %
(job.project,))
incdata = TenantParser._parseInRepoLayout(
job.files['.zuul.yaml'])
config.extend(incdata)
return config
for conf_tenant in data['tenants']:
tenant = model.Tenant(conf_tenant['name'])
abide.tenants[tenant.name] = tenant
tenant_config = {}
for fn in conf_tenant.get('include', []):
if not os.path.isabs(fn):
fn = os.path.join(base, fn)
fn = os.path.expanduser(fn)
with open(fn) as config_file:
self.log.info("Loading configuration from %s" % (fn,))
incdata = yaml.load(config_file)
extend_dict(tenant_config, incdata)
incdata = self._loadTenantInRepoLayouts(merger, connections,
conf_tenant)
extend_dict(tenant_config, incdata)
tenant.layout = self._parseLayout(base, tenant_config,
scheduler, connections)
return abide
@staticmethod
def _parseInRepoLayout(data):
# TODOv3(jeblair): this should implement some rules to protect
# aspects of the config that should not be changed in-repo
return yaml.load(data)
def _parseLayout(self, base, data, scheduler, connections):
@staticmethod
def _parseLayout(base, data, scheduler, connections):
layout = model.Layout()
for config_pipeline in data.get('pipelines', []):
for config_pipeline in data.pipelines:
layout.addPipeline(PipelineParser.fromYaml(layout, connections,
scheduler,
config_pipeline))
for config_job in data.get('jobs', []):
for config_job in data.jobs:
layout.addJob(JobParser.fromYaml(layout, config_job))
for config_template in data.get('project-templates', []):
for config_template in data.project_templates:
layout.addProjectTemplate(ProjectTemplateParser.fromYaml(
layout, config_template))
for config_project in data.get('projects', []):
for config_project in data.projects:
layout.addProjectConfig(ProjectParser.fromYaml(
layout, config_project))
@ -537,30 +504,27 @@ class ConfigLoader(object):
return layout
def _loadTenantInRepoLayouts(self, merger, connections, conf_tenant):
config = {}
jobs = []
for source_name, conf_source in conf_tenant.get('source', {}).items():
source = connections.getSource(source_name)
for conf_repo in conf_source.get('repos'):
project = source.getProject(conf_repo)
url = source.getGitUrl(project)
# TODOv3(jeblair): config should be branch specific
job = merger.getFiles(project.name, url, 'master',
files=['.zuul.yaml'])
job.project = project
jobs.append(job)
for job in jobs:
self.log.debug("Waiting for cat job %s" % (job,))
job.wait()
if job.files.get('.zuul.yaml'):
self.log.info("Loading configuration from %s/.zuul.yaml" %
(job.project,))
incdata = self._parseInRepoLayout(job.files['.zuul.yaml'])
extend_dict(config, incdata)
return config
def _parseInRepoLayout(self, data):
# TODOv3(jeblair): this should implement some rules to protect
# aspects of the config that should not be changed in-repo
return yaml.load(data)
class ConfigLoader(object):
log = logging.getLogger("zuul.ConfigLoader")
def loadConfig(self, config_path, scheduler, merger, connections):
abide = model.Abide()
if config_path:
config_path = os.path.expanduser(config_path)
if not os.path.exists(config_path):
raise Exception("Unable to read tenant config file at %s" %
config_path)
with open(config_path) as config_file:
self.log.info("Loading configuration from %s" % (config_path,))
data = yaml.load(config_file)
config = model.UnparsedAbideConfig()
config.extend(data)
base = os.path.dirname(os.path.realpath(config_path))
for conf_tenant in config.tenants:
tenant = TenantParser.fromYaml(base, connections, scheduler,
merger, conf_tenant)
abide.tenants[tenant.name] = tenant
return abide

@ -1382,6 +1382,84 @@ class ProjectConfig(object):
self.pipelines = {}
class UnparsedAbideConfig(object):
# A collection of yaml lists that has not yet been parsed into
# objects.
def __init__(self):
self.tenants = []
def extend(self, conf):
if isinstance(conf, UnparsedAbideConfig):
self.tenants.extend(conf.tenants)
return
if not isinstance(conf, list):
raise Exception("Configuration items must be in the form of "
"a list of dictionaries (when parsing %s)" %
(conf,))
for item in conf:
if not isinstance(item, dict):
raise Exception("Configuration items must be in the form of "
"a list of dictionaries (when parsing %s)" %
(conf,))
if len(item.keys()) > 1:
raise Exception("Configuration item dictionaries must have "
"a single key (when parsing %s)" %
(conf,))
key, value = item.items()[0]
if key == 'tenant':
self.tenants.append(value)
else:
raise Exception("Configuration item not recognized "
"(when parsing %s)" %
(conf,))
class UnparsedTenantConfig(object):
# A collection of yaml lists that has not yet been parsed into
# objects.
def __init__(self):
self.pipelines = []
self.jobs = []
self.project_templates = []
self.projects = []
def extend(self, conf):
if isinstance(conf, UnparsedTenantConfig):
self.pipelines.extend(conf.pipelines)
self.jobs.extend(conf.jobs)
self.project_templates.extend(conf.project_templates)
self.projects.extend(conf.projects)
return
if not isinstance(conf, list):
raise Exception("Configuration items must be in the form of "
"a list of dictionaries (when parsing %s)" %
(conf,))
for item in conf:
if not isinstance(item, dict):
raise Exception("Configuration items must be in the form of "
"a list of dictionaries (when parsing %s)" %
(conf,))
if len(item.keys()) > 1:
raise Exception("Configuration item dictionaries must have "
"a single key (when parsing %s)" %
(conf,))
key, value = item.items()[0]
if key == 'project':
self.projects.append(value)
elif key == 'job':
self.jobs.append(value)
elif key == 'project-template':
self.project_templates.append(value)
elif key == 'pipeline':
self.pipelines.append(value)
else:
raise Exception("Configuration item not recognized "
"(when parsing %s)" %
(conf,))
class Layout(object):
def __init__(self):
self.projects = {}

Loading…
Cancel
Save