From 96f2694b6701ee32c25a9086538a7830e6e8c5fd Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Wed, 9 Dec 2015 10:15:59 -0800 Subject: [PATCH] Merge configurations with multiple layout files Add a test for multiple tenants with a partially shared config. We might use something like this in OpenStack to define common pipelines but then have separate tenant config files for groups of projects. Change-Id: I29dc9327e3d72d5f6797eb2c366c36fe5be8ea8e --- .gitignore | 1 - .../fixtures/config/multi-tenant/common.yaml | 14 +++++ tests/fixtures/config/multi-tenant/main.yaml | 9 ++++ .../config/multi-tenant/tenant-one.yaml | 29 +++++++++++ .../config/multi-tenant/tenant-two.yaml | 29 +++++++++++ tests/fixtures/config/multi-tenant/zuul.conf | 36 +++++++++++++ tests/test_v3.py | 51 ++++++++++--------- zuul/scheduler.py | 42 ++++++++++----- 8 files changed, 173 insertions(+), 38 deletions(-) create mode 100644 tests/fixtures/config/multi-tenant/common.yaml create mode 100644 tests/fixtures/config/multi-tenant/main.yaml create mode 100644 tests/fixtures/config/multi-tenant/tenant-one.yaml create mode 100644 tests/fixtures/config/multi-tenant/tenant-two.yaml create mode 100644 tests/fixtures/config/multi-tenant/zuul.conf diff --git a/.gitignore b/.gitignore index 9703f16c92..21a0a9fd3f 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ AUTHORS build/* ChangeLog -config doc/build/* zuul/versioninfo dist/ diff --git a/tests/fixtures/config/multi-tenant/common.yaml b/tests/fixtures/config/multi-tenant/common.yaml new file mode 100644 index 0000000000..d36448ed22 --- /dev/null +++ b/tests/fixtures/config/multi-tenant/common.yaml @@ -0,0 +1,14 @@ +pipelines: + - name: check + manager: IndependentPipelineManager + source: + gerrit + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + verified: 1 + failure: + gerrit: + verified: -1 diff --git a/tests/fixtures/config/multi-tenant/main.yaml b/tests/fixtures/config/multi-tenant/main.yaml new file mode 100644 index 0000000000..b9eaa1482a --- /dev/null +++ b/tests/fixtures/config/multi-tenant/main.yaml @@ -0,0 +1,9 @@ +tenants: + - name: tenant-one + include: + - common.yaml + - tenant-one.yaml + - name: tenant-two + include: + - common.yaml + - tenant-two.yaml diff --git a/tests/fixtures/config/multi-tenant/tenant-one.yaml b/tests/fixtures/config/multi-tenant/tenant-one.yaml new file mode 100644 index 0000000000..7b2298c6a0 --- /dev/null +++ b/tests/fixtures/config/multi-tenant/tenant-one.yaml @@ -0,0 +1,29 @@ +pipelines: + - name: tenant-one-gate + manager: DependentPipelineManager + success-message: Build succeeded (tenant-one-gate). + source: + gerrit + trigger: + gerrit: + - event: comment-added + approval: + - approved: 1 + success: + gerrit: + verified: 2 + submit: true + failure: + gerrit: + verified: -2 + start: + gerrit: + verified: 0 + precedence: high + +projects: + - name: org/project1 + check: + - project1-test1 + tenant-one-gate: + - project1-test1 diff --git a/tests/fixtures/config/multi-tenant/tenant-two.yaml b/tests/fixtures/config/multi-tenant/tenant-two.yaml new file mode 100644 index 0000000000..57ad64de07 --- /dev/null +++ b/tests/fixtures/config/multi-tenant/tenant-two.yaml @@ -0,0 +1,29 @@ +pipelines: + - name: tenant-two-gate + manager: DependentPipelineManager + success-message: Build succeeded (tenant-two-gate). + source: + gerrit + trigger: + gerrit: + - event: comment-added + approval: + - approved: 1 + success: + gerrit: + verified: 2 + submit: true + failure: + gerrit: + verified: -2 + start: + gerrit: + verified: 0 + precedence: high + +projects: + - name: org/project2 + check: + - project2-test1 + tenant-two-gate: + - project2-test1 diff --git a/tests/fixtures/config/multi-tenant/zuul.conf b/tests/fixtures/config/multi-tenant/zuul.conf new file mode 100644 index 0000000000..ceb3903d2e --- /dev/null +++ b/tests/fixtures/config/multi-tenant/zuul.conf @@ -0,0 +1,36 @@ +[gearman] +server=127.0.0.1 + +[zuul] +tenant_config=tests/fixtures/config/multi-tenant/main.yaml +url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number} +job_name_in_report=true + +[merger] +git_dir=/tmp/zuul-test/git +git_user_email=zuul@example.com +git_user_name=zuul +zuul_url=http://zuul.example.com/p + +[swift] +authurl=https://identity.api.example.org/v2.0/ +user=username +key=password +tenant_name=" " + +default_container=logs +region_name=EXP +logserver_prefix=http://logs.example.org/server.app/ + +[connection gerrit] +driver=gerrit +server=review.example.com +user=jenkins +sshkey=none + +[connection smtp] +driver=smtp +server=localhost +port=25 +default_from=zuul@example.com +default_to=you@example.com diff --git a/tests/test_v3.py b/tests/test_v3.py index 240cd6136e..2e1674203b 100644 --- a/tests/test_v3.py +++ b/tests/test_v3.py @@ -28,36 +28,37 @@ logging.basicConfig(level=logging.DEBUG, class TestV3(ZuulTestCase): # A temporary class to hold new tests while others are disabled - def test_jobs_launched(self): - "Test that jobs are launched and a change is merged" + def test_multiple_tenants(self): + self.setup_config('config/multi-tenant/zuul.conf') + self.sched.reconfigure(self.config) - A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') A.addApproval('CRVW', 2) self.fake_gerrit.addEvent(A.addApproval('APRV', 1)) self.waitUntilSettled() - self.assertEqual(self.getJobFromHistory('project-merge').result, - 'SUCCESS') - self.assertEqual(self.getJobFromHistory('project-test1').result, - 'SUCCESS') - self.assertEqual(self.getJobFromHistory('project-test2').result, + self.assertEqual(self.getJobFromHistory('project1-test1').result, 'SUCCESS') self.assertEqual(A.data['status'], 'MERGED') - self.assertEqual(A.reported, 2) + self.assertEqual(A.reported, 2, + "A should report start and success") + self.assertIn('tenant-one-gate', A.messages[1], + "A should transit tenant-one gate") + self.assertNotIn('tenant-two-gate', A.messages[1], + "A should *not* transit tenant-two gate") - self.assertReportedStat('gerrit.event.comment-added', value='1|c') - self.assertReportedStat('zuul.pipeline.gate.current_changes', - value='1|g') - self.assertReportedStat('zuul.pipeline.gate.job.project-merge.SUCCESS', - kind='ms') - self.assertReportedStat('zuul.pipeline.gate.job.project-merge.SUCCESS', - value='1|c') - self.assertReportedStat('zuul.pipeline.gate.resident_time', kind='ms') - self.assertReportedStat('zuul.pipeline.gate.total_changes', - value='1|c') - self.assertReportedStat( - 'zuul.pipeline.gate.org.project.resident_time', kind='ms') - self.assertReportedStat( - 'zuul.pipeline.gate.org.project.total_changes', value='1|c') + B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') + B.addApproval('CRVW', 2) + self.fake_gerrit.addEvent(B.addApproval('APRV', 1)) + self.waitUntilSettled() + self.assertEqual(self.getJobFromHistory('project2-test1').result, + 'SUCCESS') + self.assertEqual(B.data['status'], 'MERGED') + self.assertEqual(B.reported, 2, + "B should report start and success") + self.assertIn('tenant-two-gate', B.messages[1], + "B should transit tenant-two gate") + self.assertNotIn('tenant-one-gate', B.messages[1], + "B should *not* transit tenant-one gate") - for build in self.builds: - self.assertEqual(build.parameters['ZUUL_VOTING'], '1') + self.assertEqual(A.reported, 2, "Activity in tenant two should" + "not affect tenant one") diff --git a/zuul/scheduler.py b/zuul/scheduler.py index fab035c206..06c54cf613 100644 --- a/zuul/scheduler.py +++ b/zuul/scheduler.py @@ -59,6 +59,29 @@ def deep_format(obj, paramdict): return ret +def extend_dict(a, b): + """Extend dictionary a (which will be modified in place) with the + contents of b. This is designed for Zuul yaml files which are + typically dictionaries of lists of dictionaries, e.g., + {'pipelines': ['name': 'gate']}. If two such dictionaries each + define a pipeline, the result will be a single dictionary with + a pipelines entry whose value is a two-element list.""" + + for k, v in b.items(): + if k not in a: + a[k] = v + elif isinstance(v, dict) and isinstance(a[k], dict): + extend_dict(a[k], v) + elif isinstance(v, list) and isinstance(a[k], list): + a[k] += v + elif isinstance(v, list): + a[k] = [a[k]] + v + elif isinstance(a[k], list): + a[k] += [v] + else: + raise Exception("Unhandled case in extend_dict at %s" % (k,)) + + class ManagementEvent(object): """An event that should be processed within the main queue run loop""" def __init__(self): @@ -331,6 +354,7 @@ class Scheduler(threading.Thread): config_path) with open(config_path) as config_file: data = yaml.load(config_file) + base = os.path.dirname(os.path.realpath(config_path)) validator = layoutvalidator.ConfigValidator() validator.validate(data, connections) @@ -338,26 +362,21 @@ class Scheduler(threading.Thread): for conf_tenant in data['tenants']: tenant = model.Tenant(conf_tenant['name']) abide.tenants[tenant.name] = tenant + tenant_config = {} for fn in conf_tenant.get('include', []): if not os.path.isabs(fn): - base = os.path.dirname(os.path.realpath(config_path)) fn = os.path.join(base, fn) fn = os.path.expanduser(fn) - tenant.layout = self._parseLayout(fn, connections) + with open(fn) as config_file: + incdata = yaml.load(config_file) + extend_dict(tenant_config, incdata) + tenant.layout = self._parseLayout(base, tenant_config, connections) return abide - def _parseLayout(self, config_path, connections): + def _parseLayout(self, base, data, connections): layout = model.Layout() project_templates = {} - if config_path: - config_path = os.path.expanduser(config_path) - if not os.path.exists(config_path): - raise Exception("Unable to read layout config file at %s" % - config_path) - with open(config_path) as config_file: - data = yaml.load(config_file) - validator = layoutvalidator.LayoutValidator() validator.validate(data, connections) @@ -366,7 +385,6 @@ class Scheduler(threading.Thread): if 'python-file' in include: fn = include['python-file'] if not os.path.isabs(fn): - base = os.path.dirname(os.path.realpath(config_path)) fn = os.path.join(base, fn) fn = os.path.expanduser(fn) execfile(fn, config_env)