From 456f2fb74885ba8b69432fccfeda2f7e6dbb4124 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Tue, 9 Feb 2016 09:29:33 -0800 Subject: [PATCH] Add job tags This allows us to add arbitrary string 'tags' to jobs. These may then be inspected inside of parameter functions for any purpose. In OpenStack, we will likely pass these through untouched to the the build so that they can be picked up by the logstash worker and we can record extra metadata about jobs. Change-Id: Ibc00c6d30cdfe4678864adb13421a4d9f71f5128 --- doc/source/zuul.rst | 6 ++++ tests/base.py | 1 + tests/fixtures/layout-tags.yaml | 42 +++++++++++++++++++++++++ tests/fixtures/layout.yaml | 5 +++ tests/fixtures/tags_custom_functions.py | 2 ++ tests/test_scheduler.py | 19 +++++++++++ zuul/layoutvalidator.py | 1 + zuul/model.py | 6 ++++ zuul/scheduler.py | 7 +++++ 9 files changed, 89 insertions(+) create mode 100644 tests/fixtures/layout-tags.yaml create mode 100644 tests/fixtures/tags_custom_functions.py diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst index b5b8d7bf19..d8d72e69ce 100644 --- a/doc/source/zuul.rst +++ b/doc/source/zuul.rst @@ -765,6 +765,12 @@ each job as it builds a list from the project specification. Boolean value (``true`` or ``false``) that indicates whatever a job is voting or not. Default: ``true``. +**tags (optional)** + A list of arbitrary strings which will be associated with the job. + Can be used by the parameter-function to alter behavior based on + their presence on a job. If the job name is a regular expression, + tags will accumulate on jobs that match. + **parameter-function (optional)** Specifies a function that should be applied to the parameters before the job is launched. The function should be defined in a python file diff --git a/tests/base.py b/tests/base.py index f3bfa4ea8a..405caa0ded 100755 --- a/tests/base.py +++ b/tests/base.py @@ -620,6 +620,7 @@ class FakeBuild(threading.Thread): BuildHistory(name=self.name, number=self.number, result=result, changes=changes, node=self.node, uuid=self.unique, description=self.description, + parameters=self.parameters, pipeline=self.parameters['ZUUL_PIPELINE']) ) diff --git a/tests/fixtures/layout-tags.yaml b/tests/fixtures/layout-tags.yaml new file mode 100644 index 0000000000..d5b8bf94f4 --- /dev/null +++ b/tests/fixtures/layout-tags.yaml @@ -0,0 +1,42 @@ +includes: + - python-file: tags_custom_functions.py + +pipelines: + - name: check + manager: IndependentPipelineManager + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + verified: 1 + failure: + gerrit: + verified: -1 + +jobs: + - name: ^.*$ + parameter-function: apply_tags + - name: ^.*-merge$ + failure-message: Unable to merge change + hold-following-changes: true + tags: merge + - name: project1-merge + tags: + - project1 + - extratag + +projects: + - name: org/project1 + check: + - project1-merge: + - project1-test1 + - project1-test2 + - project1-project2-integration + + - name: org/project2 + check: + - project2-merge: + - project2-test1 + - project2-test2 + - project1-project2-integration diff --git a/tests/fixtures/layout.yaml b/tests/fixtures/layout.yaml index e8f035e5f4..2e48ff1d0c 100644 --- a/tests/fixtures/layout.yaml +++ b/tests/fixtures/layout.yaml @@ -107,6 +107,7 @@ jobs: - name: ^.*-merge$ failure-message: Unable to merge change hold-following-changes: true + tags: merge - name: nonvoting-project-test2 voting: false - name: project-testfile @@ -120,6 +121,10 @@ jobs: mutex: test-mutex - name: mutex-two mutex: test-mutex + - name: project1-merge + tags: + - project1 + - extratag project-templates: - name: test-one-and-two diff --git a/tests/fixtures/tags_custom_functions.py b/tests/fixtures/tags_custom_functions.py new file mode 100644 index 0000000000..67e7ef1449 --- /dev/null +++ b/tests/fixtures/tags_custom_functions.py @@ -0,0 +1,2 @@ +def apply_tags(item, job, params): + params['BUILD_TAGS'] = ' '.join(sorted(job.tags)) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 8960e3af75..744a7f9d29 100755 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -2747,6 +2747,25 @@ class TestScheduler(ZuulTestCase): self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(B.reported, 2) + def test_tags(self): + "Test job tags" + self.config.set('zuul', 'layout_config', + 'tests/fixtures/layout-tags.yaml') + self.sched.reconfigure(self.config) + + A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') + B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + results = {'project1-merge': 'extratag merge project1', + 'project2-merge': 'merge'} + + for build in self.history: + self.assertEqual(results.get(build.name, ''), + build.parameters.get('BUILD_TAGS')) + def test_timer(self): "Test that a periodic job is triggered" self.worker.hold_jobs_in_build = True diff --git a/zuul/layoutvalidator.py b/zuul/layoutvalidator.py index a01eed3e76..e1e8ac6048 100644 --- a/zuul/layoutvalidator.py +++ b/zuul/layoutvalidator.py @@ -104,6 +104,7 @@ class LayoutSchema(object): 'hold-following-changes': bool, 'voting': bool, 'mutex': str, + 'tags': toList(str), 'parameter-function': str, 'branch': toList(str), 'files': toList(str), diff --git a/zuul/model.py b/zuul/model.py index 75f727dfc3..d2cf13bc7f 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -444,6 +444,7 @@ class Job(object): self.failure_pattern = None self.success_pattern = None self.parameter_function = None + self.tags = set() self.mutex = None # A metajob should only supply values for attributes that have # been explicitly provided, so avoid setting boolean defaults. @@ -493,6 +494,11 @@ class Job(object): self.swift.update(other.swift) if other.mutex: self.mutex = other.mutex + # Tags are merged via a union rather than a destructive copy + # because they are intended to accumulate as metajobs are + # applied. + if other.tags: + self.tags = self.tags.union(other.tags) # Only non-None values should be copied for boolean attributes. if other.hold_following_changes is not None: self.hold_following_changes = other.hold_following_changes diff --git a/zuul/scheduler.py b/zuul/scheduler.py index e1aa0c213b..d44006b729 100644 --- a/zuul/scheduler.py +++ b/zuul/scheduler.py @@ -527,6 +527,13 @@ class Scheduler(threading.Thread): m = config_job.get('mutex', None) if m is not None: job.mutex = m + tags = toList(config_job.get('tags')) + if tags: + # Tags are merged via a union rather than a + # destructive copy because they are intended to + # accumulate onto any previously applied tags from + # metajobs. + job.tags = job.tags.union(set(tags)) fname = config_job.get('parameter-function', None) if fname: func = config_env.get(fname, None)