From 7fb04515997dd99fad8586529c86bbc1a382ad8c Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Tue, 23 Jan 2018 13:23:13 -0800 Subject: [PATCH] Use override-checkout to select jobs If a job is defined in a project with "master" and "stable" branches, and a user wants to inherit from that job in a project with only a "devel" branch, Zuul will not run the job because there are no matching variants. To correct this, have Zuul use the value of override-checkout to mask the branch of the item under test so that it appears to match the 'override-checkout' branch. Change-Id: I4619a6f9fe9c3203a7285e5eb4f74c52b650c737 --- doc/source/user/config.rst | 17 +++ .../git/common-config/playbooks/base.yaml | 2 + .../git/common-config/zuul.yaml | 22 ++++ .../branch-mismatch/git/org_project1/README | 1 + .../git/org_project1/zuul.yaml | 7 ++ .../branch-mismatch/git/org_project2/README | 1 + .../git/org_project2/zuul.yaml | 13 +++ .../fixtures/config/branch-mismatch/main.yaml | 9 ++ tests/unit/test_v3.py | 36 ++++++ zuul/configloader.py | 4 +- zuul/model.py | 108 ++++++++++++++---- 11 files changed, 196 insertions(+), 24 deletions(-) create mode 100644 tests/fixtures/config/branch-mismatch/git/common-config/playbooks/base.yaml create mode 100644 tests/fixtures/config/branch-mismatch/git/common-config/zuul.yaml create mode 100644 tests/fixtures/config/branch-mismatch/git/org_project1/README create mode 100644 tests/fixtures/config/branch-mismatch/git/org_project1/zuul.yaml create mode 100644 tests/fixtures/config/branch-mismatch/git/org_project2/README create mode 100644 tests/fixtures/config/branch-mismatch/git/org_project2/zuul.yaml create mode 100644 tests/fixtures/config/branch-mismatch/main.yaml diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst index 525cb3892a..f28601bfea 100644 --- a/doc/source/user/config.rst +++ b/doc/source/user/config.rst @@ -692,6 +692,11 @@ Here is an example of two job definitions: attribute to apply this behavior to a subset of a job's projects. + This value is also used to help select which variants of a job + to run. If ``override-checkout`` is set, then Zuul will use + this value instead of the branch of the item being tested when + collecting jobs to run. + .. attr:: timeout The time in seconds that the job should be allowed to run before @@ -837,6 +842,12 @@ Here is an example of two job definitions: :attr:`job.override-checkout` attribute to apply the same behavior to all projects in a job. + This value is also used to help select which variants of a + job to run. If ``override-checkout`` is set, then Zuul will + use this value instead of the branch of the item being tested + when collecting any jobs to run which are defined in this + project. + .. attr:: vars A dictionary of variables to supply to Ansible. When inheriting @@ -895,6 +906,12 @@ Here is an example of two job definitions: branch of an item, then that job is not run for the item. Otherwise, all of the job variants which match that branch (and any other selection criteria) are used when freezing the job. + However, if :attr:`job.override-checkout` or + :attr:`job.required-projects.override-checkout` are set for a + project, Zuul will attempt to use the job variants which match + the values supplied in ``override-checkout`` for jobs defined in + those projects. This can be used to run a job defined in one + project on another project without a matching branch. This example illustrates a job called *run-tests* which uses a nodeset based on the current release of an operating system to diff --git a/tests/fixtures/config/branch-mismatch/git/common-config/playbooks/base.yaml b/tests/fixtures/config/branch-mismatch/git/common-config/playbooks/base.yaml new file mode 100644 index 0000000000..f679dceaef --- /dev/null +++ b/tests/fixtures/config/branch-mismatch/git/common-config/playbooks/base.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/branch-mismatch/git/common-config/zuul.yaml b/tests/fixtures/config/branch-mismatch/git/common-config/zuul.yaml new file mode 100644 index 0000000000..9954846b89 --- /dev/null +++ b/tests/fixtures/config/branch-mismatch/git/common-config/zuul.yaml @@ -0,0 +1,22 @@ +- pipeline: + name: check + manager: independent + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + Verified: 1 + failure: + gerrit: + Verified: -1 + +- job: + name: base + parent: null + run: playbooks/base.yaml + +- project: + name: common-config + check: + jobs: [] diff --git a/tests/fixtures/config/branch-mismatch/git/org_project1/README b/tests/fixtures/config/branch-mismatch/git/org_project1/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/branch-mismatch/git/org_project1/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/branch-mismatch/git/org_project1/zuul.yaml b/tests/fixtures/config/branch-mismatch/git/org_project1/zuul.yaml new file mode 100644 index 0000000000..809f830e39 --- /dev/null +++ b/tests/fixtures/config/branch-mismatch/git/org_project1/zuul.yaml @@ -0,0 +1,7 @@ +- job: + name: project-test1 + +- project: + check: + jobs: + - project-test1 diff --git a/tests/fixtures/config/branch-mismatch/git/org_project2/README b/tests/fixtures/config/branch-mismatch/git/org_project2/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/branch-mismatch/git/org_project2/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/branch-mismatch/git/org_project2/zuul.yaml b/tests/fixtures/config/branch-mismatch/git/org_project2/zuul.yaml new file mode 100644 index 0000000000..3a8e9dfbc7 --- /dev/null +++ b/tests/fixtures/config/branch-mismatch/git/org_project2/zuul.yaml @@ -0,0 +1,13 @@ +- job: + name: project-test2 + parent: project-test1 + override-checkout: stable + +- project: + check: + jobs: + - project-test1: + required-projects: + - name: org/project1 + override-checkout: stable + - project-test2 diff --git a/tests/fixtures/config/branch-mismatch/main.yaml b/tests/fixtures/config/branch-mismatch/main.yaml new file mode 100644 index 0000000000..950b1172c5 --- /dev/null +++ b/tests/fixtures/config/branch-mismatch/main.yaml @@ -0,0 +1,9 @@ +- tenant: + name: tenant-one + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - org/project1 + - org/project2 diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py index 163a58b904..164d5b4ee7 100755 --- a/tests/unit/test_v3.py +++ b/tests/unit/test_v3.py @@ -497,6 +497,42 @@ class TestBranchVariants(ZuulTestCase): self.waitUntilSettled() +class TestBranchMismatch(ZuulTestCase): + tenant_config_file = 'config/branch-mismatch/main.yaml' + + def test_job_override_branch(self): + "Test that override-checkout overrides branch matchers as well" + + # Make sure the parent job repo is branched, so it gets + # implied branch matchers. + self.create_branch('org/project1', 'stable') + self.fake_gerrit.addEvent( + self.fake_gerrit.getFakeBranchCreatedEvent( + 'org/project1', 'stable')) + + # The child job repo should have a branch which does not exist + # in the parent job repo. + self.create_branch('org/project2', 'devel') + self.fake_gerrit.addEvent( + self.fake_gerrit.getFakeBranchCreatedEvent( + 'org/project2', 'devel')) + + # A job in a repo with a weird branch name should use the + # parent job from the parent job's master (default) branch. + A = self.fake_gerrit.addFakeChange('org/project2', 'devel', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + # project-test2 should run because it inherits from + # project-test1 and we will use the fallback branch to find + # project-test1 variants, but project-test1 itself, even + # though it is in the project-pipeline config, should not run + # because it doesn't directly match. + self.assertHistory([ + dict(name='project-test1', result='SUCCESS', changes='1,1'), + dict(name='project-test2', result='SUCCESS', changes='1,1'), + ], ordered=False) + + class TestCentralJobs(ZuulTestCase): tenant_config_file = 'config/central-jobs/main.yaml' diff --git a/zuul/configloader.py b/zuul/configloader.py index d622370434..43bfbc51f5 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -700,10 +700,10 @@ class JobParser(object): (trusted, project) = tenant.getProject(project_name) if project is None: raise Exception("Unknown project %s" % (project_name,)) - job_project = model.JobProject(project_name, + job_project = model.JobProject(project.canonical_name, project_override_branch, project_override_checkout) - new_projects[project_name] = job_project + new_projects[project.canonical_name] = job_project job.required_projects = new_projects tags = conf.get('tags') diff --git a/zuul/model.py b/zuul/model.py index 29c5a9d7e0..a6d2bccbcc 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -1105,8 +1105,18 @@ class Job(object): self.inheritance_path = self.inheritance_path + (repr(other),) - def changeMatches(self, change): - if self.branch_matcher and not self.branch_matcher.matches(change): + def changeMatches(self, change, override_branch=None): + if override_branch is None: + branch_change = change + else: + # If an override branch is supplied, create a very basic + # change (a Ref) and set its branch to the override + # branch. + branch_change = Ref(change.project) + branch_change.ref = override_branch + + if self.branch_matcher and not self.branch_matcher.matches( + branch_change): return False if self.file_matcher and not self.file_matcher.matches(change): @@ -2068,9 +2078,6 @@ class Ref(object): def isUpdateOf(self, other): return False - def filterJobs(self, jobs): - return filter(lambda job: job.changeMatches(self), jobs) - def getRelatedChanges(self): return set() @@ -2618,21 +2625,33 @@ class Layout(object): def addProjectConfig(self, project_config): self.project_configs[project_config.name] = project_config - def collectJobs(self, item, jobname, change, path=None, jobs=None, - stack=None): - if stack is None: - stack = [] - if jobs is None: - jobs = [] - if path is None: - path = [] - path.append(jobname) + def _updateOverrideCheckouts(self, override_checkouts, job): + # Update the values in an override_checkouts dict with those + # in a job. Used in collectJobVariants. + if job.override_checkout: + override_checkouts[None] = job.override_checkout + for req in job.required_projects.values(): + if req.override_checkout: + override_checkouts[req.project_name] = req.override_checkout + + def _collectJobVariants(self, item, jobname, change, path, jobs, stack, + override_checkouts, indent): matched = False - indent = len(path) + 1 - item.debug("Collecting job variants for {jobname}".format( - jobname=jobname), indent=indent) + local_override_checkouts = override_checkouts.copy() + override_branch = None + project = None for variant in self.getJobs(jobname): - if not variant.changeMatches(change): + if project is None and variant.source_context: + project = variant.source_context.project + if override_checkouts.get(None) is not None: + override_branch = override_checkouts.get(None) + override_branch = override_checkouts.get( + project.canonical_name, override_branch) + branches = self.tenant.getProjectBranches(project) + if override_branch not in branches: + override_branch = None + if not variant.changeMatches(change, + override_branch=override_branch): self.log.debug("Variant %s did not match %s", repr(variant), change) item.debug("Variant {variant} did not match".format( @@ -2648,17 +2667,53 @@ class Layout(object): parent = self.tenant.default_base_job else: parent = None + self._updateOverrideCheckouts(local_override_checkouts, variant) if parent and parent not in path: if parent in stack: raise Exception("Dependency cycle in jobs: %s" % stack) self.collectJobs(item, parent, change, path, jobs, - stack + [jobname]) + stack + [jobname], local_override_checkouts) matched = True - jobs.append(variant) + if variant not in jobs: + jobs.append(variant) + return matched + + def collectJobs(self, item, jobname, change, path=None, jobs=None, + stack=None, override_checkouts=None): + # Stack is the recursion stack of job parent names. Each time + # we go up a level, we add to stack, and it's popped as we + # descend. + if stack is None: + stack = [] + # Jobs is the list of jobs we've accumulated. + if jobs is None: + jobs = [] + # Path is the list of job names we've examined. It + # accumulates and never reduces. If more than one job has the + # same parent, this will prevent us from adding it a second + # time. + if path is None: + path = [] + # Override_checkouts is a dictionary of canonical project + # names -> branch names. It is not mutated, but instead new + # copies are made and updated as we ascend the hierarchy, so + # higher levels don't affect lower levels after we descend. + # It's used to override the branch matchers for jobs. + if override_checkouts is None: + override_checkouts = {} + path.append(jobname) + matched = False + indent = len(path) + 1 + msg = "Collecting job variants for {jobname}".format(jobname=jobname) + self.log.debug(msg) + item.debug(msg, indent=indent) + matched = self._collectJobVariants( + item, jobname, change, path, jobs, stack, override_checkouts, + indent) if not matched: self.log.debug("No matching parents for job %s and change %s", jobname, change) - item.debug("No matching parent for {jobname}".format( + item.debug("No matching parents for {jobname}".format( jobname=repr(jobname)), indent=indent) raise NoMatchingParentError() return jobs @@ -2673,8 +2728,17 @@ class Layout(object): self.log.debug("Collecting jobs %s for %s", jobname, change) item.debug("Freezing job {jobname}".format( jobname=jobname), indent=1) + # Create the initial list of override_checkouts, which are + # used as we walk up the hierarchy to expand the set of + # jobs which match. + override_checkouts = {} + for variant in job_list.jobs[jobname]: + if variant.changeMatches(change): + self._updateOverrideCheckouts(override_checkouts, variant) try: - variants = self.collectJobs(item, jobname, change) + variants = self.collectJobs( + item, jobname, change, + override_checkouts=override_checkouts) except NoMatchingParentError: variants = None if not variants: