From 37c3d8c86d3beb678bfd16adc3ce690fc3fcf78c Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Wed, 13 Dec 2017 15:06:11 -0800 Subject: [PATCH] Add implied-branches pragma directive When two projects have dissimilar stable branch names, for example, stable/pike and stable/jewel, but should generally be used together and therefore share job variants, it can be difficult to make that happen. Currently, one must add explicit multi-branch matchers to every such job and project-template. This allows a user to add a pragma directive to indicate all the jobs in a file should apply to multiple branches. Change-Id: I57cf159992d8f501cbaf41aef19562951ef6b7ea --- doc/source/user/config.rst | 42 ++++++- .../git/common-config/zuul.yaml | 61 ++++++++++ .../git/org_project1/README | 1 + .../git/org_project1/playbooks/test-job1.yaml | 2 + .../git/org_project1/playbooks/test-job2.yaml | 2 + .../git/org_project1/zuul.yaml | 13 +++ .../git/org_project2/README | 1 + .../git/org_project2/zuul.yaml | 7 ++ .../config/pragma-multibranch/main.yaml | 9 ++ tests/unit/test_v3.py | 109 ++++++++++++++++++ zuul/configloader.py | 14 ++- zuul/model.py | 1 + 12 files changed, 258 insertions(+), 4 deletions(-) create mode 100644 tests/fixtures/config/pragma-multibranch/git/common-config/zuul.yaml create mode 100644 tests/fixtures/config/pragma-multibranch/git/org_project1/README create mode 100644 tests/fixtures/config/pragma-multibranch/git/org_project1/playbooks/test-job1.yaml create mode 100644 tests/fixtures/config/pragma-multibranch/git/org_project1/playbooks/test-job2.yaml create mode 100644 tests/fixtures/config/pragma-multibranch/git/org_project1/zuul.yaml create mode 100644 tests/fixtures/config/pragma-multibranch/git/org_project2/README create mode 100644 tests/fixtures/config/pragma-multibranch/git/org_project2/zuul.yaml create mode 100644 tests/fixtures/config/pragma-multibranch/main.yaml diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst index 4151eda113..94f7a142c1 100644 --- a/doc/source/user/config.rst +++ b/doc/source/user/config.rst @@ -1311,7 +1311,7 @@ pragma directives may not be set and then unset within the same file. .. attr:: pragma - The pragma item currently only supports one attribute: + The pragma item currently supports the following attributes: .. attr:: implied-branch-matchers @@ -1326,3 +1326,43 @@ pragma directives may not be set and then unset within the same file. Note that if a job contains an explicit branch matcher, it will be used regardless of the value supplied here. + + .. attr:: implied-branches + + This is a list of regular expressions, just as + :attr:`job.branches`, which may be used to supply the value of + the implied branch matcher for all jobs in a file. + + This may be useful if two projects share jobs but have + dissimilar branch names. If, for example, two projects have + stable maintenance branches with dissimilar names, but both + should use the same job variants, this directive may be used to + indicate that all of the jobs defined in the stable branch of + the first project may also be used for the stable branch of teh + other. For example: + + .. code-block:: yaml + + - pragma: + implied-branches: + - stable/foo + - stable/bar + + The above code, when added to the ``stable/foo`` branch of a + project would indicate that the job variants described in that + file should not only be used for changes to ``stable/foo``, but + also on changes to ``stable/bar``, which may be in another + project. + + Note that if a job contains an explicit branch matcher, it will + be used regardless of the value supplied here. + + Note also that the presence of `implied-branches` does not + automatically set `implied-branch-matchers`. Zuul will still + decide if implied branch matchers are warranted at all, using + the heuristics described in :attr:`job.branches`, and only use + the value supplied here if that is the case. If you want to + declare specific implied branches on, for example, a + :term:`config-project` project (which normally would not use + implied branches), you must set `implied-branch-matchers` as + well. diff --git a/tests/fixtures/config/pragma-multibranch/git/common-config/zuul.yaml b/tests/fixtures/config/pragma-multibranch/git/common-config/zuul.yaml new file mode 100644 index 0000000000..dc83f9ddfb --- /dev/null +++ b/tests/fixtures/config/pragma-multibranch/git/common-config/zuul.yaml @@ -0,0 +1,61 @@ +- pipeline: + name: check + manager: independent + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + Verified: 1 + failure: + gerrit: + Verified: -1 + +- pipeline: + name: gate + manager: dependent + post-review: True + trigger: + gerrit: + - event: comment-added + approval: + - Approved: 1 + success: + gerrit: + Verified: 2 + submit: true + failure: + gerrit: + Verified: -2 + start: + gerrit: + Verified: 0 + precedence: high + +- job: + name: base + parent: null + +- project: + name: common-config + check: + jobs: [] + gate: + jobs: + - noop + +- project: + name: org/project1 + check: + jobs: [] + gate: + jobs: + - noop + +- project: + name: org/project2 + check: + jobs: [] + gate: + jobs: + - noop diff --git a/tests/fixtures/config/pragma-multibranch/git/org_project1/README b/tests/fixtures/config/pragma-multibranch/git/org_project1/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/pragma-multibranch/git/org_project1/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/pragma-multibranch/git/org_project1/playbooks/test-job1.yaml b/tests/fixtures/config/pragma-multibranch/git/org_project1/playbooks/test-job1.yaml new file mode 100644 index 0000000000..f679dceaef --- /dev/null +++ b/tests/fixtures/config/pragma-multibranch/git/org_project1/playbooks/test-job1.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/pragma-multibranch/git/org_project1/playbooks/test-job2.yaml b/tests/fixtures/config/pragma-multibranch/git/org_project1/playbooks/test-job2.yaml new file mode 100644 index 0000000000..f679dceaef --- /dev/null +++ b/tests/fixtures/config/pragma-multibranch/git/org_project1/playbooks/test-job2.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/pragma-multibranch/git/org_project1/zuul.yaml b/tests/fixtures/config/pragma-multibranch/git/org_project1/zuul.yaml new file mode 100644 index 0000000000..6c8352a237 --- /dev/null +++ b/tests/fixtures/config/pragma-multibranch/git/org_project1/zuul.yaml @@ -0,0 +1,13 @@ +- job: + name: test-job1 + run: playbooks/test-job1.yaml + +- job: + name: test-job2 + run: playbooks/test-job2.yaml + +- project-template: + name: test-template + check: + jobs: + - test-job1 diff --git a/tests/fixtures/config/pragma-multibranch/git/org_project2/README b/tests/fixtures/config/pragma-multibranch/git/org_project2/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/pragma-multibranch/git/org_project2/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/pragma-multibranch/git/org_project2/zuul.yaml b/tests/fixtures/config/pragma-multibranch/git/org_project2/zuul.yaml new file mode 100644 index 0000000000..748cab2642 --- /dev/null +++ b/tests/fixtures/config/pragma-multibranch/git/org_project2/zuul.yaml @@ -0,0 +1,7 @@ +- project: + name: org/project2 + templates: + - test-template + check: + jobs: + - test-job2 diff --git a/tests/fixtures/config/pragma-multibranch/main.yaml b/tests/fixtures/config/pragma-multibranch/main.yaml new file mode 100644 index 0000000000..950b1172c5 --- /dev/null +++ b/tests/fixtures/config/pragma-multibranch/main.yaml @@ -0,0 +1,9 @@ +- tenant: + name: tenant-one + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - org/project1 + - org/project2 diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py index 1f401d0c40..e4ef619d32 100755 --- a/tests/unit/test_v3.py +++ b/tests/unit/test_v3.py @@ -2232,6 +2232,115 @@ class TestPragma(ZuulTestCase): self.assertIsNone(job.branch_matcher) +class TestPragmaMultibranch(ZuulTestCase): + tenant_config_file = 'config/pragma-multibranch/main.yaml' + + def test_no_branch_matchers(self): + self.create_branch('org/project1', 'stable/pike') + self.create_branch('org/project2', 'stable/jewel') + self.fake_gerrit.addEvent( + self.fake_gerrit.getFakeBranchCreatedEvent( + 'org/project1', 'stable/pike')) + self.fake_gerrit.addEvent( + self.fake_gerrit.getFakeBranchCreatedEvent( + 'org/project2', 'stable/jewel')) + self.waitUntilSettled() + # We want the jobs defined on the stable/pike branch of + # project1 to apply to the stable/jewel branch of project2. + + # First, without the pragma line, the jobs should not run + # because in project1 they have branch matchers for pike, so + # they will not match a jewel change. + B = self.fake_gerrit.addFakeChange('org/project2', 'stable/jewel', 'B') + self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + self.assertHistory([]) + + # Add a pragma line to disable implied branch matchers in + # project1, so that the jobs and templates apply to both + # branches. + with open(os.path.join(FIXTURE_DIR, + 'config/pragma-multibranch/git/', + 'org_project1/zuul.yaml')) as f: + config = f.read() + extra_conf = textwrap.dedent( + """ + - pragma: + implied-branch-matchers: False + """) + config = extra_conf + config + file_dict = {'zuul.yaml': config} + A = self.fake_gerrit.addFakeChange('org/project1', 'stable/pike', 'A', + files=file_dict) + A.addApproval('Code-Review', 2) + self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) + self.waitUntilSettled() + self.fake_gerrit.addEvent(A.getChangeMergedEvent()) + self.waitUntilSettled() + + # Now verify that when we propose a change to jewel, we get + # the pike/jewel jobs. + self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + self.assertHistory([ + dict(name='test-job1', result='SUCCESS', changes='1,1'), + dict(name='test-job2', result='SUCCESS', changes='1,1'), + ], ordered=False) + + def test_supplied_branch_matchers(self): + self.create_branch('org/project1', 'stable/pike') + self.create_branch('org/project2', 'stable/jewel') + self.fake_gerrit.addEvent( + self.fake_gerrit.getFakeBranchCreatedEvent( + 'org/project1', 'stable/pike')) + self.fake_gerrit.addEvent( + self.fake_gerrit.getFakeBranchCreatedEvent( + 'org/project2', 'stable/jewel')) + self.waitUntilSettled() + # We want the jobs defined on the stable/pike branch of + # project1 to apply to the stable/jewel branch of project2. + + # First, without the pragma line, the jobs should not run + # because in project1 they have branch matchers for pike, so + # they will not match a jewel change. + B = self.fake_gerrit.addFakeChange('org/project2', 'stable/jewel', 'B') + self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + self.assertHistory([]) + + # Add a pragma line to disable implied branch matchers in + # project1, so that the jobs and templates apply to both + # branches. + with open(os.path.join(FIXTURE_DIR, + 'config/pragma-multibranch/git/', + 'org_project1/zuul.yaml')) as f: + config = f.read() + extra_conf = textwrap.dedent( + """ + - pragma: + implied-branches: + - stable/pike + - stable/jewel + """) + config = extra_conf + config + file_dict = {'zuul.yaml': config} + A = self.fake_gerrit.addFakeChange('org/project1', 'stable/pike', 'A', + files=file_dict) + A.addApproval('Code-Review', 2) + self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) + self.waitUntilSettled() + self.fake_gerrit.addEvent(A.getChangeMergedEvent()) + self.waitUntilSettled() + # Now verify that when we propose a change to jewel, we get + # the pike/jewel jobs. + self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + self.assertHistory([ + dict(name='test-job1', result='SUCCESS', changes='1,1'), + dict(name='test-job2', result='SUCCESS', changes='1,1'), + ], ordered=False) + + class TestBaseJobs(ZuulTestCase): tenant_config_file = 'config/base-jobs/main.yaml' diff --git a/zuul/configloader.py b/zuul/configloader.py index 227e352116..10e023fd68 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -358,6 +358,7 @@ class EncryptedPKCS1_OAEP(yaml.YAMLObject): class PragmaParser(object): pragma = { 'implied-branch-matchers': bool, + 'implied-branches': to_list(str), '_source_context': model.SourceContext, '_start_mark': ZuulMark, } @@ -372,11 +373,14 @@ class PragmaParser(object): self.schema(conf) bm = conf.get('implied-branch-matchers') - if bm is None: - return source_context = conf['_source_context'] - source_context.implied_branch_matchers = bm + if bm is not None: + source_context.implied_branch_matchers = bm + + branches = conf.get('implied-branches') + if branches is not None: + source_context.implied_branches = as_list(branches) class NodeSetParser(object): @@ -528,6 +532,8 @@ class JobParser(object): # If the user has set a pragma directive for this, use the # value (if unset, the value is None). if job.source_context.implied_branch_matchers is True: + if job.source_context.implied_branches is not None: + return job.source_context.implied_branches return [job.source_context.branch] elif job.source_context.implied_branch_matchers is False: return None @@ -543,6 +549,8 @@ class JobParser(object): if len(branches) == 1: return None + if job.source_context.implied_branches is not None: + return job.source_context.implied_branches return [job.source_context.branch] @staticmethod diff --git a/zuul/model.py b/zuul/model.py index e53a357d28..12b85c9d9c 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -641,6 +641,7 @@ class SourceContext(object): self.path = path self.trusted = trusted self.implied_branch_matchers = None + self.implied_branches = None def __str__(self): return '%s/%s@%s' % (self.project, self.path, self.branch)