Add job tags

This allows us to add arbitrary string 'tags' to jobs.  These may
then be inspected inside of parameter functions for any purpose.
In OpenStack, we will likely pass these through untouched to the
the build so that they can be picked up by the logstash worker
and we can record extra metadata about jobs.

Change-Id: Ibc00c6d30cdfe4678864adb13421a4d9f71f5128

doc/source/zuul.rst

@@ -765,6 +765,12 @@ each job as it builds a list from the project specification.
   Boolean value (``true`` or ``false``) that indicates whatever
   a job is voting or not.  Default: ``true``.
 
+**tags (optional)**
+  A list of arbitrary strings which will be associated with the job.
+  Can be used by the parameter-function to alter behavior based on
+  their presence on a job.  If the job name is a regular expression,
+  tags will accumulate on jobs that match.
+
 **parameter-function (optional)**
   Specifies a function that should be applied to the parameters before
   the job is launched.  The function should be defined in a python file

tests/base.py

@@ -620,6 +620,7 @@ class FakeBuild(threading.Thread):
             BuildHistory(name=self.name, number=self.number,
                          result=result, changes=changes, node=self.node,
                          uuid=self.unique, description=self.description,
+                         parameters=self.parameters,
                          pipeline=self.parameters['ZUUL_PIPELINE'])
         )
 

tests/fixtures/layout-tags.yaml

@@ -0,0 +1,42 @@
+includes:
+  - python-file: tags_custom_functions.py
+
+pipelines:
+  - name: check
+    manager: IndependentPipelineManager
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        verified: 1
+    failure:
+      gerrit:
+        verified: -1
+
+jobs:
+  - name: ^.*$
+    parameter-function: apply_tags
+  - name: ^.*-merge$
+    failure-message: Unable to merge change
+    hold-following-changes: true
+    tags: merge
+  - name: project1-merge
+    tags:
+      - project1
+      - extratag
+
+projects:
+  - name: org/project1
+    check:
+      - project1-merge:
+        - project1-test1
+        - project1-test2
+        - project1-project2-integration
+
+  - name: org/project2
+    check:
+      - project2-merge:
+        - project2-test1
+        - project2-test2
+        - project1-project2-integration

tests/fixtures/layout.yaml

@@ -107,6 +107,7 @@ jobs:
   - name: ^.*-merge$
     failure-message: Unable to merge change
     hold-following-changes: true
+    tags: merge
   - name: nonvoting-project-test2
     voting: false
   - name: project-testfile
@@ -120,6 +121,10 @@ jobs:
     mutex: test-mutex
   - name: mutex-two
     mutex: test-mutex
+  - name: project1-merge
+    tags:
+      - project1
+      - extratag
 
 project-templates:
   - name: test-one-and-two

tests/fixtures/tags_custom_functions.py

@@ -0,0 +1,2 @@
+def apply_tags(item, job, params):
+    params['BUILD_TAGS'] = ' '.join(sorted(job.tags))

tests/test_scheduler.py

@@ -2747,6 +2747,25 @@ class TestScheduler(ZuulTestCase):
         self.assertEqual(B.data['status'], 'MERGED')
         self.assertEqual(B.reported, 2)
 
+    def test_tags(self):
+        "Test job tags"
+        self.config.set('zuul', 'layout_config',
+                        'tests/fixtures/layout-tags.yaml')
+        self.sched.reconfigure(self.config)
+
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        results = {'project1-merge': 'extratag merge project1',
+                   'project2-merge': 'merge'}
+
+        for build in self.history:
+            self.assertEqual(results.get(build.name, ''),
+                             build.parameters.get('BUILD_TAGS'))
+
     def test_timer(self):
         "Test that a periodic job is triggered"
         self.worker.hold_jobs_in_build = True

zuul/layoutvalidator.py

@@ -104,6 +104,7 @@ class LayoutSchema(object):
            'hold-following-changes': bool,
            'voting': bool,
            'mutex': str,
+           'tags': toList(str),
            'parameter-function': str,
            'branch': toList(str),
            'files': toList(str),

zuul/model.py

@@ -444,6 +444,7 @@ class Job(object):
         self.failure_pattern = None
         self.success_pattern = None
         self.parameter_function = None
+        self.tags = set()
         self.mutex = None
         # A metajob should only supply values for attributes that have
         # been explicitly provided, so avoid setting boolean defaults.
@@ -493,6 +494,11 @@ class Job(object):
             self.swift.update(other.swift)
         if other.mutex:
             self.mutex = other.mutex
+        # Tags are merged via a union rather than a destructive copy
+        # because they are intended to accumulate as metajobs are
+        # applied.
+        if other.tags:
+            self.tags = self.tags.union(other.tags)
         # Only non-None values should be copied for boolean attributes.
         if other.hold_following_changes is not None:
             self.hold_following_changes = other.hold_following_changes

zuul/scheduler.py

@@ -527,6 +527,13 @@ class Scheduler(threading.Thread):
             m = config_job.get('mutex', None)
             if m is not None:
                 job.mutex = m
+            tags = toList(config_job.get('tags'))
+            if tags:
+                # Tags are merged via a union rather than a
+                # destructive copy because they are intended to
+                # accumulate onto any previously applied tags from
+                # metajobs.
+                job.tags = job.tags.union(set(tags))
             fname = config_job.get('parameter-function', None)
             if fname:
                 func = config_env.get(fname, None)