From 81515adbd634b10118b5eaa8cd97dad784357ece Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Mon, 1 Oct 2012 18:29:08 -0700 Subject: [PATCH] Add ZUUL_COMMIT. To facilitate using zuul with just the jenkins git plugin, add the ZUUL_COMMIT parameter. Also, always include ZUUL_REF, whether the job is pre- or post-commit. If it's pre, it will be a refs/zuul ref, if it's post, it will be refs/tags or a branch name. Change-Id: I88c38a28dcd552b2540095d36caacd10acf167b8 Reviewed-on: https://review.openstack.org/13934 Reviewed-by: Clark Boylan Approved: James E. Blair Tested-by: Jenkins --- doc/source/launchers.rst | 89 ++++++++++++++++++++++++++-------------- tests/test_scheduler.py | 4 ++ zuul/launcher/jenkins.py | 43 +++++++++++++++---- zuul/merger.py | 10 ++++- zuul/model.py | 4 +- zuul/scheduler.py | 19 +++++---- 6 files changed, 117 insertions(+), 52 deletions(-) diff --git a/doc/source/launchers.rst b/doc/source/launchers.rst index 34aa4eac74..eb8cd10efc 100644 --- a/doc/source/launchers.rst +++ b/doc/source/launchers.rst @@ -63,44 +63,71 @@ Zuul will pass some parameters to Jenkins for every job it launches. Check **This build is parameterized**, and add the following fields with the type **String Parameter**: -**UUID** +**ZUUL_UUID** Zuul provided key to link builds with Gerrit events -**GERRIT_PROJECT** - Zuul provided project name -**GERRIT_BRANCH** - Zuul provided branch name -**GERRIT_CHANGES** - Zuul provided list of dependent changes to merge +**ZUUL_REF** + Zuul provided ref that includes commit(s) to build +**ZUUL_COMMIT** + The commit SHA1 at the head of ZUUL_REF -You may find it useful to use the ``GERRIT_*`` variables in your job. -In particular, ``GERRIT_CHANGES`` indicates the change or changes that -should be tested. If Zuul has decided that more than one change -should be merged and tested together, they will all be listed in -``GERRIT_CHANGES``. The format for the description of one change is:: +Those are the only required parameters. The UUID is needed for Zuul +to keep track of the build, and the REF and COMMIT parameters are for +use in preparing the git repo for the build. The following parameters +will be sent for all builds, but are not required so you do not need +to configure Jenkins to accept them if you do not plan on using them: - project:branch:refspec +**ZUUL_PROJECT** + The project that triggered this build +**ZUUL_PIPELINE** + The Zuul pipeline that is building this job -And multiple changes are separated by a carat ("^"). E.g.:: +The following parameters are optional and will only be provided for +builds associated with changes (i.e., in response to patchset-created +or comment-added events): - testproject:master:refs/changes/20/420/1^testproject:master:refs/changes/21/421/1" +**ZUUL_BRANCH** + The target branch for the change that triggered this build +**ZUUL_CHANGE** + The Gerrit change ID for the change that triggered this build +**ZUUL_CHANGE_IDS** + All of the Gerrit change IDs that are included in this build (useful + when the DependentPipelineManager combines changes for testing) +**ZUUL_PATCHSET** + The Gerrit patchset number for the change that triggered this build -The OpenStack project uses the following script to update the -repository in a workspace and merge appropriate changes: +The following parameters are optional and will only be provided for +post-merge (ref-updated) builds: - https://github.com/openstack/openstack-ci-puppet/blob/master/modules/jenkins/files/slave_scripts/gerrit-git-prep.sh +**ZUUL_OLDREV** + The SHA1 of the old revision at this ref (recall the ref name is + in ZUUL_REF) +**ZUUL_NEWREV** + The SHA1 of the new revision at this ref (recall the ref name is + in ZUUL_REF) +**ZUUL_SHORT_OLDREV** + The shortened (7 character) SHA1 of the old revision +**ZUUL_SHORT_NEWREV** + The shortened (7 character) SHA1 of the new revision -Gerrit events that do not include a change (e.g., ref-updated events -which are emitted after a git ref is updated (i.e., a commit is merged -to master)) require a slightly different set of parameters: +In order to test the correct build, configure the Jenkins Git SCM +plugin as follows:: -**UUID** - Zuul provided key to link builds with Gerrit events -**GERRIT_PROJECT** - Zuul provided project name -**GERRIT_REFNAME** - Zuul provided ref name -**GERRIT_OLDREV** - Zuul provided old reference for ref-updated -**GERRIT_NEWREV** - Zuul provided new reference for ref-updated + Source Code Management: + Git + Repositories: + Repository URL: + Advanced: + Refspec: ${ZUUL_REF} + Branches to build: + Branch Specifier: ${ZUUL_COMMIT} + Advanced: + Clean after checkout: True +That should be sufficient for a job that only builds a single project. +If you have multiple interrelated projects (i.e., they share a Zuul +Change Queue) that are built together, you may be able to configure +the Git plugin to prepare them, or you may chose to use a shell script +instead. The OpenStack project uses the following script to prepare +the workspace for its integration testing: + + https://github.com/openstack-ci/devstack-gate/blob/master/devstack-vm-gate-wrap.sh diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index d59a66abd9..9b621c4b38 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -128,11 +128,15 @@ def job_has_changes(*args): path = os.path.join(GIT_ROOT, project) repo = git.Repo(path) ref = job.parameters['ZUUL_REF'] + sha = job.parameters['ZUUL_COMMIT'] repo_messages = [c.message.strip() for c in repo.iter_commits(ref)] + repo_shas = [c.hexsha for c in repo.iter_commits(ref)] commit_messages = ['%s-1' % commit.subject for commit in commits] for msg in commit_messages: if msg not in repo_messages: return False + if repo_shas[0] != sha: + return False return True diff --git a/zuul/launcher/jenkins.py b/zuul/launcher/jenkins.py index 6a73cbcfbc..31a548e5fd 100644 --- a/zuul/launcher/jenkins.py +++ b/zuul/launcher/jenkins.py @@ -219,21 +219,23 @@ class Jenkins(object): dependent_changes = dependent_changes[:] dependent_changes.reverse() uuid = str(uuid4().hex) - params = dict(UUID=uuid, - GERRIT_PROJECT=change.project.name, + params = dict(UUID=uuid, # deprecated + ZUUL_UUID=uuid, + GERRIT_PROJECT=change.project.name, # deprecated ZUUL_PROJECT=change.project.name) params['ZUUL_PIPELINE'] = pipeline.name if hasattr(change, 'refspec'): changes_str = '^'.join( ['%s:%s:%s' % (c.project.name, c.branch, c.refspec) for c in dependent_changes + [change]]) - params['GERRIT_BRANCH'] = change.branch + params['GERRIT_BRANCH'] = change.branch # deprecated params['ZUUL_BRANCH'] = change.branch - params['GERRIT_CHANGES'] = changes_str + params['GERRIT_CHANGES'] = changes_str # deprecated params['ZUUL_CHANGES'] = changes_str params['ZUUL_REF'] = ('refs/zuul/%s/%s' % (change.branch, change.current_build_set.ref)) + params['ZUUL_COMMIT'] = change.current_build_set.commit zuul_changes = ' '.join(['%s,%s' % (c.number, c.patchset) for c in dependent_changes + [change]]) @@ -241,14 +243,41 @@ class Jenkins(object): params['ZUUL_CHANGE'] = str(change.number) params['ZUUL_PATCHSET'] = str(change.patchset) if hasattr(change, 'ref'): - params['GERRIT_REFNAME'] = change.ref + params['GERRIT_REFNAME'] = change.ref # deprecated params['ZUUL_REFNAME'] = change.ref - params['GERRIT_OLDREV'] = change.oldrev + params['GERRIT_OLDREV'] = change.oldrev # deprecated params['ZUUL_OLDREV'] = change.oldrev - params['GERRIT_NEWREV'] = change.newrev + params['GERRIT_NEWREV'] = change.newrev # deprecated params['ZUUL_NEWREV'] = change.newrev + params['ZUUL_SHORT_OLDREV'] = change.oldrev[:7] params['ZUUL_SHORT_NEWREV'] = change.newrev[:7] + params['ZUUL_REF'] = change.ref + params['ZUUL_COMMIT'] = change.newrev + + # This is what we should be heading toward for parameters: + + # required: + # ZUUL_UUID + # ZUUL_REF (/refs/zuul/..., /refs/tags/foo, master) + # ZUUL_COMMIT + + # optional: + # ZUUL_PROJECT + # ZUUL_PIPELINE + + # optional (changes only): + # ZUUL_BRANCH + # ZUUL_CHANGE + # ZUUL_CHANGE_IDS + # ZUUL_PATCHSET + + # optional (ref updated only): + # ZUUL_OLDREV + # ZUUL_NEWREV + # ZUUL_SHORT_NEWREV + # ZUUL_SHORT_OLDREV + if callable(job.parameter_function): job.parameter_function(change, params) self.log.debug("Custom parameter function used for job %s, " diff --git a/zuul/merger.py b/zuul/merger.py index 3e72ee8045..7cd3315ff3 100644 --- a/zuul/merger.py +++ b/zuul/merger.py @@ -81,6 +81,7 @@ class Repo(object): def setZuulRef(self, ref, commit): self.repo.refs[ref].commit = commit + return self.repo.refs[ref].commit def push(self, local, remote): self.log.debug("Pushing %s:%s to %s " % (local, remote, @@ -112,6 +113,7 @@ class Merger(object): def mergeChanges(self, changes, target_ref=None, mode=None): projects = {} + commit = None # Reset all repos involved in the change set for change in changes: branches = projects.get(change.project, []) @@ -148,7 +150,11 @@ class Merger(object): repo.merge(change.refspec) elif mode == model.CHERRY_PICK: repo.cherryPick(change.refspec) - repo.setZuulRef(change.branch + '/' + target_ref, 'HEAD') + # Keep track of the last commit, it's the commit that + # will be passed to jenkins because it's the commit + # for the triggering change + commit = repo.setZuulRef(change.branch + '/' + target_ref, + 'HEAD').hexsha except: self.log.info("Unable to merge %s" % change) return False @@ -169,4 +175,4 @@ class Merger(object): self.log.error("Ref %s did not show up in repo" % ref) return False - return True + return commit diff --git a/zuul/model.py b/zuul/model.py index 79fdbfb128..13981319ea 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -402,6 +402,7 @@ class BuildSet(object): self.next_build_set = None self.previous_build_set = None self.ref = None + self.commit = None self.unable_to_merge = False def setConfiguration(self): @@ -416,9 +417,6 @@ class BuildSet(object): if not self.ref: self.ref = 'Z' + uuid4().hex - def getRef(self): - return self.ref - def addBuild(self, build): self.builds[build.job.name] = build build.build_set = self diff --git a/zuul/scheduler.py b/zuul/scheduler.py index 33bac35c6e..880cfc3543 100644 --- a/zuul/scheduler.py +++ b/zuul/scheduler.py @@ -531,18 +531,18 @@ class BasePipelineManager(object): def _launchJobs(self, change, jobs): self.log.debug("Launching jobs for change %s" % change) - ref = change.current_build_set.getRef() + ref = change.current_build_set.ref if hasattr(change, 'refspec') and not ref: change.current_build_set.setConfiguration() - ref = change.current_build_set.getRef() + ref = change.current_build_set.ref mode = model.MERGE_IF_NECESSARY - merged = self.sched.merger.mergeChanges([change], ref, mode=mode) - if not merged: + commit = self.sched.merger.mergeChanges([change], ref, mode=mode) + if not commit: self.log.info("Unable to merge change %s" % change) self.pipeline.setUnableToMerge(change) self.possiblyReportChange(change) return - + change.current_build_set.commit = commit for job in self.pipeline.findJobsToRun(change): self.log.debug("Found job %s for change %s" % (job, change)) try: @@ -956,19 +956,20 @@ class DependentPipelineManager(BasePipelineManager): def _launchJobs(self, change, jobs): self.log.debug("Launching jobs for change %s" % change) - ref = change.current_build_set.getRef() + ref = change.current_build_set.ref if hasattr(change, 'refspec') and not ref: change.current_build_set.setConfiguration() - ref = change.current_build_set.getRef() + ref = change.current_build_set.ref dependent_changes = self._getDependentChanges(change) dependent_changes.reverse() all_changes = dependent_changes + [change] - merged = self.sched.merger.mergeChanges(all_changes, ref) - if not merged: + commit = self.sched.merger.mergeChanges(all_changes, ref) + if not commit: self.log.info("Unable to merge changes %s" % all_changes) self.pipeline.setUnableToMerge(change) self.possiblyReportChange(change) return + change.current_build_set.commit = commit #TODO: remove this line after GERRIT_CHANGES is gone dependent_changes = self._getDependentChanges(change) for job in jobs: