Merge configurations with multiple layout files

Add a test for multiple tenants with a partially shared config.

We might use something like this in OpenStack to define common
pipelines but then have separate tenant config files for groups
of projects.

Change-Id: I29dc9327e3d72d5f6797eb2c366c36fe5be8ea8e

.gitignore

@@ -8,7 +8,6 @@
 AUTHORS
 build/*
 ChangeLog
-config
 doc/build/*
 zuul/versioninfo
 dist/

tests/fixtures/config/multi-tenant/common.yaml

@@ -0,0 +1,14 @@
+pipelines:
+  - name: check
+    manager: IndependentPipelineManager
+    source:
+      gerrit
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        verified: 1
+    failure:
+      gerrit:
+        verified: -1

tests/fixtures/config/multi-tenant/main.yaml

@@ -0,0 +1,9 @@
+tenants:
+  - name: tenant-one
+    include:
+      - common.yaml
+      - tenant-one.yaml
+  - name: tenant-two
+    include:
+      - common.yaml
+      - tenant-two.yaml

tests/fixtures/config/multi-tenant/tenant-one.yaml

@@ -0,0 +1,29 @@
+pipelines:
+  - name: tenant-one-gate
+    manager: DependentPipelineManager
+    success-message: Build succeeded (tenant-one-gate).
+    source:
+      gerrit
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - approved: 1
+    success:
+      gerrit:
+        verified: 2
+        submit: true
+    failure:
+      gerrit:
+        verified: -2
+    start:
+      gerrit:
+        verified: 0
+    precedence: high
+
+projects:
+  - name: org/project1
+    check:
+      - project1-test1
+    tenant-one-gate:
+      - project1-test1

tests/fixtures/config/multi-tenant/tenant-two.yaml

@@ -0,0 +1,29 @@
+pipelines:
+  - name: tenant-two-gate
+    manager: DependentPipelineManager
+    success-message: Build succeeded (tenant-two-gate).
+    source:
+      gerrit
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - approved: 1
+    success:
+      gerrit:
+        verified: 2
+        submit: true
+    failure:
+      gerrit:
+        verified: -2
+    start:
+      gerrit:
+        verified: 0
+    precedence: high
+
+projects:
+  - name: org/project2
+    check:
+      - project2-test1
+    tenant-two-gate:
+      - project2-test1

tests/fixtures/config/multi-tenant/zuul.conf

@@ -0,0 +1,36 @@
+[gearman]
+server=127.0.0.1
+
+[zuul]
+tenant_config=tests/fixtures/config/multi-tenant/main.yaml
+url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}
+job_name_in_report=true
+
+[merger]
+git_dir=/tmp/zuul-test/git
+git_user_email=zuul@example.com
+git_user_name=zuul
+zuul_url=http://zuul.example.com/p
+
+[swift]
+authurl=https://identity.api.example.org/v2.0/
+user=username
+key=password
+tenant_name=" "
+
+default_container=logs
+region_name=EXP
+logserver_prefix=http://logs.example.org/server.app/
+
+[connection gerrit]
+driver=gerrit
+server=review.example.com
+user=jenkins
+sshkey=none
+
+[connection smtp]
+driver=smtp
+server=localhost
+port=25
+default_from=zuul@example.com
+default_to=you@example.com

tests/test_v3.py

@@ -28,36 +28,37 @@ logging.basicConfig(level=logging.DEBUG,
 class TestV3(ZuulTestCase):
     # A temporary class to hold new tests while others are disabled
 
-    def test_jobs_launched(self):
-        "Test that jobs are launched and a change is merged"
+    def test_multiple_tenants(self):
+        self.setup_config('config/multi-tenant/zuul.conf')
+        self.sched.reconfigure(self.config)
 
-        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
         A.addApproval('CRVW', 2)
         self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
         self.waitUntilSettled()
-        self.assertEqual(self.getJobFromHistory('project-merge').result,
-                         'SUCCESS')
-        self.assertEqual(self.getJobFromHistory('project-test1').result,
-                         'SUCCESS')
-        self.assertEqual(self.getJobFromHistory('project-test2').result,
+        self.assertEqual(self.getJobFromHistory('project1-test1').result,
                          'SUCCESS')
         self.assertEqual(A.data['status'], 'MERGED')
-        self.assertEqual(A.reported, 2)
-
-        self.assertReportedStat('gerrit.event.comment-added', value='1|c')
-        self.assertReportedStat('zuul.pipeline.gate.current_changes',
-                                value='1|g')
-        self.assertReportedStat('zuul.pipeline.gate.job.project-merge.SUCCESS',
-                                kind='ms')
-        self.assertReportedStat('zuul.pipeline.gate.job.project-merge.SUCCESS',
-                                value='1|c')
-        self.assertReportedStat('zuul.pipeline.gate.resident_time', kind='ms')
-        self.assertReportedStat('zuul.pipeline.gate.total_changes',
-                                value='1|c')
-        self.assertReportedStat(
-            'zuul.pipeline.gate.org.project.resident_time', kind='ms')
-        self.assertReportedStat(
-            'zuul.pipeline.gate.org.project.total_changes', value='1|c')
-
-        for build in self.builds:
-            self.assertEqual(build.parameters['ZUUL_VOTING'], '1')
+        self.assertEqual(A.reported, 2,
+                         "A should report start and success")
+        self.assertIn('tenant-one-gate', A.messages[1],
+                      "A should transit tenant-one gate")
+        self.assertNotIn('tenant-two-gate', A.messages[1],
+                         "A should *not* transit tenant-two gate")
+
+        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+        B.addApproval('CRVW', 2)
+        self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
+        self.waitUntilSettled()
+        self.assertEqual(self.getJobFromHistory('project2-test1').result,
+                         'SUCCESS')
+        self.assertEqual(B.data['status'], 'MERGED')
+        self.assertEqual(B.reported, 2,
+                         "B should report start and success")
+        self.assertIn('tenant-two-gate', B.messages[1],
+                      "B should transit tenant-two gate")
+        self.assertNotIn('tenant-one-gate', B.messages[1],
+                         "B should *not* transit tenant-one gate")
+
+        self.assertEqual(A.reported, 2, "Activity in tenant two should"
+                         "not affect tenant one")

zuul/scheduler.py

@@ -59,6 +59,29 @@ def deep_format(obj, paramdict):
     return ret
 
 
+def extend_dict(a, b):
+    """Extend dictionary a (which will be modified in place) with the
+       contents of b.  This is designed for Zuul yaml files which are
+       typically dictionaries of lists of dictionaries, e.g.,
+       {'pipelines': ['name': 'gate']}.  If two such dictionaries each
+       define a pipeline, the result will be a single dictionary with
+       a pipelines entry whose value is a two-element list."""
+
+    for k, v in b.items():
+        if k not in a:
+            a[k] = v
+        elif isinstance(v, dict) and isinstance(a[k], dict):
+            extend_dict(a[k], v)
+        elif isinstance(v, list) and isinstance(a[k], list):
+            a[k] += v
+        elif isinstance(v, list):
+            a[k] = [a[k]] + v
+        elif isinstance(a[k], list):
+            a[k] += [v]
+        else:
+            raise Exception("Unhandled case in extend_dict at %s" % (k,))
+
+
 class ManagementEvent(object):
     """An event that should be processed within the main queue run loop"""
     def __init__(self):
@@ -331,6 +354,7 @@ class Scheduler(threading.Thread):
                                 config_path)
         with open(config_path) as config_file:
             data = yaml.load(config_file)
+        base = os.path.dirname(os.path.realpath(config_path))
 
         validator = layoutvalidator.ConfigValidator()
         validator.validate(data, connections)
@@ -338,26 +362,21 @@ class Scheduler(threading.Thread):
         for conf_tenant in data['tenants']:
             tenant = model.Tenant(conf_tenant['name'])
             abide.tenants[tenant.name] = tenant
+            tenant_config = {}
             for fn in conf_tenant.get('include', []):
                 if not os.path.isabs(fn):
-                    base = os.path.dirname(os.path.realpath(config_path))
                     fn = os.path.join(base, fn)
                 fn = os.path.expanduser(fn)
-                tenant.layout = self._parseLayout(fn, connections)
+                with open(fn) as config_file:
+                    incdata = yaml.load(config_file)
+                    extend_dict(tenant_config, incdata)
+            tenant.layout = self._parseLayout(base, tenant_config, connections)
         return abide
 
-    def _parseLayout(self, config_path, connections):
+    def _parseLayout(self, base, data, connections):
         layout = model.Layout()
         project_templates = {}
 
-        if config_path:
-            config_path = os.path.expanduser(config_path)
-            if not os.path.exists(config_path):
-                raise Exception("Unable to read layout config file at %s" %
-                                config_path)
-        with open(config_path) as config_file:
-            data = yaml.load(config_file)
-
         validator = layoutvalidator.LayoutValidator()
         validator.validate(data, connections)
 
@@ -366,7 +385,6 @@ class Scheduler(threading.Thread):
             if 'python-file' in include:
                 fn = include['python-file']
                 if not os.path.isabs(fn):
-                    base = os.path.dirname(os.path.realpath(config_path))
                     fn = os.path.join(base, fn)
                 fn = os.path.expanduser(fn)
                 execfile(fn, config_env)