Merge configurations with multiple layout files
Add a test for multiple tenants with a partially shared config. We might use something like this in OpenStack to define common pipelines but then have separate tenant config files for groups of projects. Change-Id: I29dc9327e3d72d5f6797eb2c366c36fe5be8ea8e
This commit is contained in:
parent
f84026c6e9
commit
96f2694b67
|
@ -8,7 +8,6 @@
|
|||
AUTHORS
|
||||
build/*
|
||||
ChangeLog
|
||||
config
|
||||
doc/build/*
|
||||
zuul/versioninfo
|
||||
dist/
|
||||
|
|
|
@ -0,0 +1,14 @@
|
|||
pipelines:
|
||||
- name: check
|
||||
manager: IndependentPipelineManager
|
||||
source:
|
||||
gerrit
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: patchset-created
|
||||
success:
|
||||
gerrit:
|
||||
verified: 1
|
||||
failure:
|
||||
gerrit:
|
||||
verified: -1
|
|
@ -0,0 +1,9 @@
|
|||
tenants:
|
||||
- name: tenant-one
|
||||
include:
|
||||
- common.yaml
|
||||
- tenant-one.yaml
|
||||
- name: tenant-two
|
||||
include:
|
||||
- common.yaml
|
||||
- tenant-two.yaml
|
|
@ -0,0 +1,29 @@
|
|||
pipelines:
|
||||
- name: tenant-one-gate
|
||||
manager: DependentPipelineManager
|
||||
success-message: Build succeeded (tenant-one-gate).
|
||||
source:
|
||||
gerrit
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: comment-added
|
||||
approval:
|
||||
- approved: 1
|
||||
success:
|
||||
gerrit:
|
||||
verified: 2
|
||||
submit: true
|
||||
failure:
|
||||
gerrit:
|
||||
verified: -2
|
||||
start:
|
||||
gerrit:
|
||||
verified: 0
|
||||
precedence: high
|
||||
|
||||
projects:
|
||||
- name: org/project1
|
||||
check:
|
||||
- project1-test1
|
||||
tenant-one-gate:
|
||||
- project1-test1
|
|
@ -0,0 +1,29 @@
|
|||
pipelines:
|
||||
- name: tenant-two-gate
|
||||
manager: DependentPipelineManager
|
||||
success-message: Build succeeded (tenant-two-gate).
|
||||
source:
|
||||
gerrit
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: comment-added
|
||||
approval:
|
||||
- approved: 1
|
||||
success:
|
||||
gerrit:
|
||||
verified: 2
|
||||
submit: true
|
||||
failure:
|
||||
gerrit:
|
||||
verified: -2
|
||||
start:
|
||||
gerrit:
|
||||
verified: 0
|
||||
precedence: high
|
||||
|
||||
projects:
|
||||
- name: org/project2
|
||||
check:
|
||||
- project2-test1
|
||||
tenant-two-gate:
|
||||
- project2-test1
|
|
@ -0,0 +1,36 @@
|
|||
[gearman]
|
||||
server=127.0.0.1
|
||||
|
||||
[zuul]
|
||||
tenant_config=tests/fixtures/config/multi-tenant/main.yaml
|
||||
url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}
|
||||
job_name_in_report=true
|
||||
|
||||
[merger]
|
||||
git_dir=/tmp/zuul-test/git
|
||||
git_user_email=zuul@example.com
|
||||
git_user_name=zuul
|
||||
zuul_url=http://zuul.example.com/p
|
||||
|
||||
[swift]
|
||||
authurl=https://identity.api.example.org/v2.0/
|
||||
user=username
|
||||
key=password
|
||||
tenant_name=" "
|
||||
|
||||
default_container=logs
|
||||
region_name=EXP
|
||||
logserver_prefix=http://logs.example.org/server.app/
|
||||
|
||||
[connection gerrit]
|
||||
driver=gerrit
|
||||
server=review.example.com
|
||||
user=jenkins
|
||||
sshkey=none
|
||||
|
||||
[connection smtp]
|
||||
driver=smtp
|
||||
server=localhost
|
||||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
|
@ -28,36 +28,37 @@ logging.basicConfig(level=logging.DEBUG,
|
|||
class TestV3(ZuulTestCase):
|
||||
# A temporary class to hold new tests while others are disabled
|
||||
|
||||
def test_jobs_launched(self):
|
||||
"Test that jobs are launched and a change is merged"
|
||||
def test_multiple_tenants(self):
|
||||
self.setup_config('config/multi-tenant/zuul.conf')
|
||||
self.sched.reconfigure(self.config)
|
||||
|
||||
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
|
||||
A.addApproval('CRVW', 2)
|
||||
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
|
||||
self.waitUntilSettled()
|
||||
self.assertEqual(self.getJobFromHistory('project-merge').result,
|
||||
'SUCCESS')
|
||||
self.assertEqual(self.getJobFromHistory('project-test1').result,
|
||||
'SUCCESS')
|
||||
self.assertEqual(self.getJobFromHistory('project-test2').result,
|
||||
self.assertEqual(self.getJobFromHistory('project1-test1').result,
|
||||
'SUCCESS')
|
||||
self.assertEqual(A.data['status'], 'MERGED')
|
||||
self.assertEqual(A.reported, 2)
|
||||
self.assertEqual(A.reported, 2,
|
||||
"A should report start and success")
|
||||
self.assertIn('tenant-one-gate', A.messages[1],
|
||||
"A should transit tenant-one gate")
|
||||
self.assertNotIn('tenant-two-gate', A.messages[1],
|
||||
"A should *not* transit tenant-two gate")
|
||||
|
||||
self.assertReportedStat('gerrit.event.comment-added', value='1|c')
|
||||
self.assertReportedStat('zuul.pipeline.gate.current_changes',
|
||||
value='1|g')
|
||||
self.assertReportedStat('zuul.pipeline.gate.job.project-merge.SUCCESS',
|
||||
kind='ms')
|
||||
self.assertReportedStat('zuul.pipeline.gate.job.project-merge.SUCCESS',
|
||||
value='1|c')
|
||||
self.assertReportedStat('zuul.pipeline.gate.resident_time', kind='ms')
|
||||
self.assertReportedStat('zuul.pipeline.gate.total_changes',
|
||||
value='1|c')
|
||||
self.assertReportedStat(
|
||||
'zuul.pipeline.gate.org.project.resident_time', kind='ms')
|
||||
self.assertReportedStat(
|
||||
'zuul.pipeline.gate.org.project.total_changes', value='1|c')
|
||||
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
|
||||
B.addApproval('CRVW', 2)
|
||||
self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
|
||||
self.waitUntilSettled()
|
||||
self.assertEqual(self.getJobFromHistory('project2-test1').result,
|
||||
'SUCCESS')
|
||||
self.assertEqual(B.data['status'], 'MERGED')
|
||||
self.assertEqual(B.reported, 2,
|
||||
"B should report start and success")
|
||||
self.assertIn('tenant-two-gate', B.messages[1],
|
||||
"B should transit tenant-two gate")
|
||||
self.assertNotIn('tenant-one-gate', B.messages[1],
|
||||
"B should *not* transit tenant-one gate")
|
||||
|
||||
for build in self.builds:
|
||||
self.assertEqual(build.parameters['ZUUL_VOTING'], '1')
|
||||
self.assertEqual(A.reported, 2, "Activity in tenant two should"
|
||||
"not affect tenant one")
|
||||
|
|
|
@ -59,6 +59,29 @@ def deep_format(obj, paramdict):
|
|||
return ret
|
||||
|
||||
|
||||
def extend_dict(a, b):
|
||||
"""Extend dictionary a (which will be modified in place) with the
|
||||
contents of b. This is designed for Zuul yaml files which are
|
||||
typically dictionaries of lists of dictionaries, e.g.,
|
||||
{'pipelines': ['name': 'gate']}. If two such dictionaries each
|
||||
define a pipeline, the result will be a single dictionary with
|
||||
a pipelines entry whose value is a two-element list."""
|
||||
|
||||
for k, v in b.items():
|
||||
if k not in a:
|
||||
a[k] = v
|
||||
elif isinstance(v, dict) and isinstance(a[k], dict):
|
||||
extend_dict(a[k], v)
|
||||
elif isinstance(v, list) and isinstance(a[k], list):
|
||||
a[k] += v
|
||||
elif isinstance(v, list):
|
||||
a[k] = [a[k]] + v
|
||||
elif isinstance(a[k], list):
|
||||
a[k] += [v]
|
||||
else:
|
||||
raise Exception("Unhandled case in extend_dict at %s" % (k,))
|
||||
|
||||
|
||||
class ManagementEvent(object):
|
||||
"""An event that should be processed within the main queue run loop"""
|
||||
def __init__(self):
|
||||
|
@ -331,6 +354,7 @@ class Scheduler(threading.Thread):
|
|||
config_path)
|
||||
with open(config_path) as config_file:
|
||||
data = yaml.load(config_file)
|
||||
base = os.path.dirname(os.path.realpath(config_path))
|
||||
|
||||
validator = layoutvalidator.ConfigValidator()
|
||||
validator.validate(data, connections)
|
||||
|
@ -338,26 +362,21 @@ class Scheduler(threading.Thread):
|
|||
for conf_tenant in data['tenants']:
|
||||
tenant = model.Tenant(conf_tenant['name'])
|
||||
abide.tenants[tenant.name] = tenant
|
||||
tenant_config = {}
|
||||
for fn in conf_tenant.get('include', []):
|
||||
if not os.path.isabs(fn):
|
||||
base = os.path.dirname(os.path.realpath(config_path))
|
||||
fn = os.path.join(base, fn)
|
||||
fn = os.path.expanduser(fn)
|
||||
tenant.layout = self._parseLayout(fn, connections)
|
||||
with open(fn) as config_file:
|
||||
incdata = yaml.load(config_file)
|
||||
extend_dict(tenant_config, incdata)
|
||||
tenant.layout = self._parseLayout(base, tenant_config, connections)
|
||||
return abide
|
||||
|
||||
def _parseLayout(self, config_path, connections):
|
||||
def _parseLayout(self, base, data, connections):
|
||||
layout = model.Layout()
|
||||
project_templates = {}
|
||||
|
||||
if config_path:
|
||||
config_path = os.path.expanduser(config_path)
|
||||
if not os.path.exists(config_path):
|
||||
raise Exception("Unable to read layout config file at %s" %
|
||||
config_path)
|
||||
with open(config_path) as config_file:
|
||||
data = yaml.load(config_file)
|
||||
|
||||
validator = layoutvalidator.LayoutValidator()
|
||||
validator.validate(data, connections)
|
||||
|
||||
|
@ -366,7 +385,6 @@ class Scheduler(threading.Thread):
|
|||
if 'python-file' in include:
|
||||
fn = include['python-file']
|
||||
if not os.path.isabs(fn):
|
||||
base = os.path.dirname(os.path.realpath(config_path))
|
||||
fn = os.path.join(base, fn)
|
||||
fn = os.path.expanduser(fn)
|
||||
execfile(fn, config_env)
|
||||
|
|
Loading…
Reference in New Issue