diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst index e0526f5e60..597062eecc 100644 --- a/doc/source/user/config.rst +++ b/doc/source/user/config.rst @@ -692,6 +692,11 @@ Here is an example of two job definitions: attribute to apply this behavior to a subset of a job's projects. + This value is also used to help select which variants of a job + to run. If ``override-checkout`` is set, then Zuul will use + this value instead of the branch of the item being tested when + collecting jobs to run. + .. attr:: timeout The time in seconds that the job should be allowed to run before @@ -837,6 +842,12 @@ Here is an example of two job definitions: :attr:`job.override-checkout` attribute to apply the same behavior to all projects in a job. + This value is also used to help select which variants of a + job to run. If ``override-checkout`` is set, then Zuul will + use this value instead of the branch of the item being tested + when collecting any jobs to run which are defined in this + project. + .. attr:: vars A dictionary of variables to supply to Ansible. When inheriting @@ -895,6 +906,12 @@ Here is an example of two job definitions: branch of an item, then that job is not run for the item. Otherwise, all of the job variants which match that branch (and any other selection criteria) are used when freezing the job. + However, if :attr:`job.override-checkout` or + :attr:`job.required-projects.override-checkout` are set for a + project, Zuul will attempt to use the job variants which match + the values supplied in ``override-checkout`` for jobs defined in + those projects. This can be used to run a job defined in one + project on another project without a matching branch. This example illustrates a job called *run-tests* which uses a nodeset based on the current release of an operating system to diff --git a/tests/fixtures/config/branch-mismatch/git/common-config/playbooks/base.yaml b/tests/fixtures/config/branch-mismatch/git/common-config/playbooks/base.yaml new file mode 100644 index 0000000000..f679dceaef --- /dev/null +++ b/tests/fixtures/config/branch-mismatch/git/common-config/playbooks/base.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/branch-mismatch/git/common-config/zuul.yaml b/tests/fixtures/config/branch-mismatch/git/common-config/zuul.yaml new file mode 100644 index 0000000000..9954846b89 --- /dev/null +++ b/tests/fixtures/config/branch-mismatch/git/common-config/zuul.yaml @@ -0,0 +1,22 @@ +- pipeline: + name: check + manager: independent + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + Verified: 1 + failure: + gerrit: + Verified: -1 + +- job: + name: base + parent: null + run: playbooks/base.yaml + +- project: + name: common-config + check: + jobs: [] diff --git a/tests/fixtures/config/branch-mismatch/git/org_project1/README b/tests/fixtures/config/branch-mismatch/git/org_project1/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/branch-mismatch/git/org_project1/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/branch-mismatch/git/org_project1/zuul.yaml b/tests/fixtures/config/branch-mismatch/git/org_project1/zuul.yaml new file mode 100644 index 0000000000..809f830e39 --- /dev/null +++ b/tests/fixtures/config/branch-mismatch/git/org_project1/zuul.yaml @@ -0,0 +1,7 @@ +- job: + name: project-test1 + +- project: + check: + jobs: + - project-test1 diff --git a/tests/fixtures/config/branch-mismatch/git/org_project2/README b/tests/fixtures/config/branch-mismatch/git/org_project2/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/branch-mismatch/git/org_project2/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/branch-mismatch/git/org_project2/zuul.yaml b/tests/fixtures/config/branch-mismatch/git/org_project2/zuul.yaml new file mode 100644 index 0000000000..3a8e9dfbc7 --- /dev/null +++ b/tests/fixtures/config/branch-mismatch/git/org_project2/zuul.yaml @@ -0,0 +1,13 @@ +- job: + name: project-test2 + parent: project-test1 + override-checkout: stable + +- project: + check: + jobs: + - project-test1: + required-projects: + - name: org/project1 + override-checkout: stable + - project-test2 diff --git a/tests/fixtures/config/branch-mismatch/main.yaml b/tests/fixtures/config/branch-mismatch/main.yaml new file mode 100644 index 0000000000..950b1172c5 --- /dev/null +++ b/tests/fixtures/config/branch-mismatch/main.yaml @@ -0,0 +1,9 @@ +- tenant: + name: tenant-one + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - org/project1 + - org/project2 diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py index 4af5b47654..4cb4a41342 100755 --- a/tests/unit/test_v3.py +++ b/tests/unit/test_v3.py @@ -497,6 +497,42 @@ class TestBranchVariants(ZuulTestCase): self.waitUntilSettled() +class TestBranchMismatch(ZuulTestCase): + tenant_config_file = 'config/branch-mismatch/main.yaml' + + def test_job_override_branch(self): + "Test that override-checkout overrides branch matchers as well" + + # Make sure the parent job repo is branched, so it gets + # implied branch matchers. + self.create_branch('org/project1', 'stable') + self.fake_gerrit.addEvent( + self.fake_gerrit.getFakeBranchCreatedEvent( + 'org/project1', 'stable')) + + # The child job repo should have a branch which does not exist + # in the parent job repo. + self.create_branch('org/project2', 'devel') + self.fake_gerrit.addEvent( + self.fake_gerrit.getFakeBranchCreatedEvent( + 'org/project2', 'devel')) + + # A job in a repo with a weird branch name should use the + # parent job from the parent job's master (default) branch. + A = self.fake_gerrit.addFakeChange('org/project2', 'devel', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + # project-test2 should run because it inherits from + # project-test1 and we will use the fallback branch to find + # project-test1 variants, but project-test1 itself, even + # though it is in the project-pipeline config, should not run + # because it doesn't directly match. + self.assertHistory([ + dict(name='project-test1', result='SUCCESS', changes='1,1'), + dict(name='project-test2', result='SUCCESS', changes='1,1'), + ], ordered=False) + + class TestCentralJobs(ZuulTestCase): tenant_config_file = 'config/central-jobs/main.yaml' diff --git a/zuul/configloader.py b/zuul/configloader.py index 4f9390772b..be1bd63912 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -700,10 +700,10 @@ class JobParser(object): (trusted, project) = tenant.getProject(project_name) if project is None: raise Exception("Unknown project %s" % (project_name,)) - job_project = model.JobProject(project_name, + job_project = model.JobProject(project.canonical_name, project_override_branch, project_override_checkout) - new_projects[project_name] = job_project + new_projects[project.canonical_name] = job_project job.required_projects = new_projects tags = conf.get('tags') diff --git a/zuul/model.py b/zuul/model.py index 96ec85b805..0685f8288c 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -1108,8 +1108,18 @@ class Job(object): self.inheritance_path = self.inheritance_path + (repr(other),) - def changeMatches(self, change): - if self.branch_matcher and not self.branch_matcher.matches(change): + def changeMatches(self, change, override_branch=None): + if override_branch is None: + branch_change = change + else: + # If an override branch is supplied, create a very basic + # change (a Ref) and set its branch to the override + # branch. + branch_change = Ref(change.project) + branch_change.ref = override_branch + + if self.branch_matcher and not self.branch_matcher.matches( + branch_change): return False if self.file_matcher and not self.file_matcher.matches(change): @@ -2071,9 +2081,6 @@ class Ref(object): def isUpdateOf(self, other): return False - def filterJobs(self, jobs): - return filter(lambda job: job.changeMatches(self), jobs) - def getRelatedChanges(self): return set() @@ -2668,21 +2675,33 @@ class Layout(object): def addProjectConfig(self, project_config): self.project_configs[project_config.name] = project_config - def collectJobs(self, item, jobname, change, path=None, jobs=None, - stack=None): - if stack is None: - stack = [] - if jobs is None: - jobs = [] - if path is None: - path = [] - path.append(jobname) + def _updateOverrideCheckouts(self, override_checkouts, job): + # Update the values in an override_checkouts dict with those + # in a job. Used in collectJobVariants. + if job.override_checkout: + override_checkouts[None] = job.override_checkout + for req in job.required_projects.values(): + if req.override_checkout: + override_checkouts[req.project_name] = req.override_checkout + + def _collectJobVariants(self, item, jobname, change, path, jobs, stack, + override_checkouts, indent): matched = False - indent = len(path) + 1 - item.debug("Collecting job variants for {jobname}".format( - jobname=jobname), indent=indent) + local_override_checkouts = override_checkouts.copy() + override_branch = None + project = None for variant in self.getJobs(jobname): - if not variant.changeMatches(change): + if project is None and variant.source_context: + project = variant.source_context.project + if override_checkouts.get(None) is not None: + override_branch = override_checkouts.get(None) + override_branch = override_checkouts.get( + project.canonical_name, override_branch) + branches = self.tenant.getProjectBranches(project) + if override_branch not in branches: + override_branch = None + if not variant.changeMatches(change, + override_branch=override_branch): self.log.debug("Variant %s did not match %s", repr(variant), change) item.debug("Variant {variant} did not match".format( @@ -2698,17 +2717,53 @@ class Layout(object): parent = self.tenant.default_base_job else: parent = None + self._updateOverrideCheckouts(local_override_checkouts, variant) if parent and parent not in path: if parent in stack: raise Exception("Dependency cycle in jobs: %s" % stack) self.collectJobs(item, parent, change, path, jobs, - stack + [jobname]) + stack + [jobname], local_override_checkouts) matched = True - jobs.append(variant) + if variant not in jobs: + jobs.append(variant) + return matched + + def collectJobs(self, item, jobname, change, path=None, jobs=None, + stack=None, override_checkouts=None): + # Stack is the recursion stack of job parent names. Each time + # we go up a level, we add to stack, and it's popped as we + # descend. + if stack is None: + stack = [] + # Jobs is the list of jobs we've accumulated. + if jobs is None: + jobs = [] + # Path is the list of job names we've examined. It + # accumulates and never reduces. If more than one job has the + # same parent, this will prevent us from adding it a second + # time. + if path is None: + path = [] + # Override_checkouts is a dictionary of canonical project + # names -> branch names. It is not mutated, but instead new + # copies are made and updated as we ascend the hierarchy, so + # higher levels don't affect lower levels after we descend. + # It's used to override the branch matchers for jobs. + if override_checkouts is None: + override_checkouts = {} + path.append(jobname) + matched = False + indent = len(path) + 1 + msg = "Collecting job variants for {jobname}".format(jobname=jobname) + self.log.debug(msg) + item.debug(msg, indent=indent) + matched = self._collectJobVariants( + item, jobname, change, path, jobs, stack, override_checkouts, + indent) if not matched: self.log.debug("No matching parents for job %s and change %s", jobname, change) - item.debug("No matching parent for {jobname}".format( + item.debug("No matching parents for {jobname}".format( jobname=repr(jobname)), indent=indent) raise NoMatchingParentError() return jobs @@ -2723,8 +2778,17 @@ class Layout(object): self.log.debug("Collecting jobs %s for %s", jobname, change) item.debug("Freezing job {jobname}".format( jobname=jobname), indent=1) + # Create the initial list of override_checkouts, which are + # used as we walk up the hierarchy to expand the set of + # jobs which match. + override_checkouts = {} + for variant in job_list.jobs[jobname]: + if variant.changeMatches(change): + self._updateOverrideCheckouts(override_checkouts, variant) try: - variants = self.collectJobs(item, jobname, change) + variants = self.collectJobs( + item, jobname, change, + override_checkouts=override_checkouts) except NoMatchingParentError: variants = None if not variants: