Add merging capability.
Add git repo management and merging. When collecting changes to be tested together, merge or cherry-pick those changes into the zuul-managed repos, and create a unique ref for that configuration. Pass the ref to Jenkins instead of the string description of the changes, so that Jenkins only needs to checkout that one ref. This moves the complexity of merging and managing multiple commits out of Jenkins and into Zuul. The GERRIT_CHANGES variable is deprecated (along with the rest of the GERRIT_* variables) and will be removed in a future patch (which will contain a documentation update). Change-Id: I126c9030223c07a30f7092e2273ebd7605d9f3df Reviewed-on: https://review.openstack.org/11349 Reviewed-by: Monty Taylor <mordred@inaugust.com> Reviewed-by: Clark Boylan <clark.boylan@gmail.com> Approved: James E. Blair <corvus@inaugust.com> Tested-by: Jenkins
This commit is contained in:
parent
597e7c4c85
commit
4886cc186f
|
@ -42,6 +42,7 @@ jobs:
|
|||
|
||||
projects:
|
||||
- name: org/project
|
||||
merge-mode: cherry-pick
|
||||
check:
|
||||
- project-merge:
|
||||
- project-test1
|
||||
|
|
|
@ -10,3 +10,4 @@ sshkey=none
|
|||
|
||||
[zuul]
|
||||
layout_config=layout.yaml
|
||||
git_dir=/tmp/zuul-test/git
|
||||
|
|
|
@ -28,6 +28,8 @@ import pprint
|
|||
import re
|
||||
import urllib2
|
||||
import urlparse
|
||||
import shutil
|
||||
import git
|
||||
|
||||
import zuul
|
||||
import zuul.scheduler
|
||||
|
@ -49,6 +51,75 @@ def random_sha1():
|
|||
return hashlib.sha1(str(random.random())).hexdigest()
|
||||
|
||||
|
||||
class ChangeReference(git.Reference):
|
||||
_common_path_default = "refs/changes"
|
||||
_points_to_commits_only = True
|
||||
|
||||
|
||||
def init_repo(project):
|
||||
parts = project.split('/')
|
||||
path = os.path.join("/tmp/zuul-test/upstream", *parts[:-1])
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
path = os.path.join("/tmp/zuul-test/upstream", project)
|
||||
repo = git.Repo.init(path)
|
||||
|
||||
fn = os.path.join(path, 'README')
|
||||
f = open(fn, 'w')
|
||||
f.write("test\n")
|
||||
f.close()
|
||||
repo.index.add([fn])
|
||||
repo.index.commit('initial commit')
|
||||
repo.create_head('master')
|
||||
repo.create_tag('init')
|
||||
|
||||
|
||||
def add_fake_change_to_repo(project, branch, change_num, patchset, msg):
|
||||
path = os.path.join("/tmp/zuul-test/upstream", project)
|
||||
repo = git.Repo(path)
|
||||
ref = ChangeReference.create(repo, '1/%s/%s' % (change_num,
|
||||
patchset),
|
||||
'refs/tags/init')
|
||||
repo.head.reference = ref
|
||||
repo.head.reset(index=True, working_tree=True)
|
||||
repo.git.clean('-x', '-f', '-d')
|
||||
|
||||
path = os.path.join("/tmp/zuul-test/upstream", project)
|
||||
fn = os.path.join(path, '%s-%s' % (branch, change_num))
|
||||
f = open(fn, 'w')
|
||||
f.write("test\n")
|
||||
f.close()
|
||||
repo.index.add([fn])
|
||||
repo.index.commit(msg)
|
||||
|
||||
|
||||
def ref_has_change(ref, change):
|
||||
path = os.path.join("/tmp/zuul-test/git", change.project)
|
||||
repo = git.Repo(path)
|
||||
for commit in repo.iter_commits(ref):
|
||||
if commit.message.strip() == ('%s-1' % change.subject):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def job_has_changes(*args):
|
||||
job = args[0]
|
||||
commits = args[1:]
|
||||
project = job.parameters['ZUUL_PROJECT']
|
||||
path = os.path.join("/tmp/zuul-test/git", project)
|
||||
repo = git.Repo(path)
|
||||
ref = job.parameters['ZUUL_REF']
|
||||
repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
|
||||
commit_messages = ['%s-1' % commit.subject for commit in commits]
|
||||
print 'checking that job %s has changes:' % ref
|
||||
print ' commit messages:', commit_messages
|
||||
print ' repo messages :', repo_messages
|
||||
for msg in commit_messages:
|
||||
if msg not in repo_messages:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class FakeChange(object):
|
||||
categories = {'APRV': ('Approved', -1, 1),
|
||||
'CRVW': ('Code-Review', -2, 2),
|
||||
|
@ -106,6 +177,9 @@ class FakeChange(object):
|
|||
self.data['currentPatchSet'] = d
|
||||
self.patchsets.append(d)
|
||||
self.data['submitRecords'] = self.getSubmitRecords()
|
||||
add_fake_change_to_repo(self.project, self.branch,
|
||||
self.number, self.latest_patchset,
|
||||
self.subject + '-' + str(self.latest_patchset))
|
||||
|
||||
def addApproval(self, category, value):
|
||||
approval = {'description': self.categories[category][0],
|
||||
|
@ -321,7 +395,7 @@ class FakeJenkinsJob(threading.Thread):
|
|||
result = 'SUCCESS'
|
||||
if self.jenkins.fakeShouldFailTest(
|
||||
self.name,
|
||||
self.parameters['GERRIT_CHANGES']):
|
||||
self.parameters['ZUUL_REF']):
|
||||
result = 'FAILURE'
|
||||
if self.aborted:
|
||||
result = 'ABORTED'
|
||||
|
@ -386,10 +460,10 @@ class FakeJenkins(object):
|
|||
l.append(change)
|
||||
self.fail_tests[name] = l
|
||||
|
||||
def fakeShouldFailTest(self, name, changes):
|
||||
def fakeShouldFailTest(self, name, ref):
|
||||
l = self.fail_tests.get(name, [])
|
||||
for change in l:
|
||||
if change in changes:
|
||||
if ref_has_change(ref, change):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
@ -476,10 +550,25 @@ class FakeURLOpener(object):
|
|||
return ret
|
||||
|
||||
|
||||
class FakeGerritTrigger(zuul.trigger.gerrit.Gerrit):
|
||||
def getGitUrl(self, project):
|
||||
return "/tmp/zuul-test/upstream/%s" % project
|
||||
|
||||
|
||||
class testScheduler(unittest.TestCase):
|
||||
log = logging.getLogger("zuul.test")
|
||||
|
||||
def setUp(self):
|
||||
if os.path.exists("/tmp/zuul-test"):
|
||||
shutil.rmtree("/tmp/zuul-test")
|
||||
os.makedirs("/tmp/zuul-test")
|
||||
os.makedirs("/tmp/zuul-test/upstream")
|
||||
os.makedirs("/tmp/zuul-test/git")
|
||||
|
||||
# For each project in config:
|
||||
init_repo("org/project")
|
||||
init_repo("org/project1")
|
||||
init_repo("org/project2")
|
||||
self.config = CONFIG
|
||||
self.sched = zuul.scheduler.Scheduler()
|
||||
|
||||
|
@ -503,7 +592,7 @@ class testScheduler(unittest.TestCase):
|
|||
|
||||
zuul.lib.gerrit.Gerrit = FakeGerrit
|
||||
|
||||
self.gerrit = zuul.trigger.gerrit.Gerrit(self.config, self.sched)
|
||||
self.gerrit = FakeGerritTrigger(self.config, self.sched)
|
||||
self.gerrit.replication_timeout = 1.5
|
||||
self.gerrit.replication_retry_interval = 0.5
|
||||
self.fake_gerrit = self.gerrit.gerrit
|
||||
|
@ -520,6 +609,7 @@ class testScheduler(unittest.TestCase):
|
|||
self.gerrit.stop()
|
||||
self.sched.stop()
|
||||
self.sched.join()
|
||||
#shutil.rmtree("/tmp/zuul-test")
|
||||
|
||||
def waitUntilSettled(self):
|
||||
self.log.debug("Waiting until settled...")
|
||||
|
@ -582,77 +672,51 @@ class testScheduler(unittest.TestCase):
|
|||
jobs = self.fake_jenkins.all_jobs
|
||||
assert len(jobs) == 1
|
||||
assert jobs[0].name == 'project-merge'
|
||||
assert (jobs[0].parameters['GERRIT_CHANGES'] ==
|
||||
'org/project:master:refs/changes/1/1/1')
|
||||
assert job_has_changes(jobs[0], A)
|
||||
|
||||
self.fake_jenkins.fakeRelease('.*-merge')
|
||||
self.waitUntilSettled()
|
||||
assert len(jobs) == 3
|
||||
assert jobs[0].name == 'project-test1'
|
||||
assert (jobs[0].parameters['GERRIT_CHANGES'] ==
|
||||
'org/project:master:refs/changes/1/1/1')
|
||||
assert job_has_changes(jobs[0], A)
|
||||
assert jobs[1].name == 'project-test2'
|
||||
assert (jobs[1].parameters['GERRIT_CHANGES'] ==
|
||||
'org/project:master:refs/changes/1/1/1')
|
||||
assert job_has_changes(jobs[1], A)
|
||||
assert jobs[2].name == 'project-merge'
|
||||
assert (jobs[2].parameters['GERRIT_CHANGES'] ==
|
||||
'org/project:master:refs/changes/1/1/1^'
|
||||
'org/project:master:refs/changes/1/2/1')
|
||||
assert job_has_changes(jobs[2], A, B)
|
||||
|
||||
self.fake_jenkins.fakeRelease('.*-merge')
|
||||
self.waitUntilSettled()
|
||||
assert len(jobs) == 5
|
||||
assert jobs[0].name == 'project-test1'
|
||||
assert (jobs[0].parameters['GERRIT_CHANGES'] ==
|
||||
'org/project:master:refs/changes/1/1/1')
|
||||
assert job_has_changes(jobs[0], A)
|
||||
assert jobs[1].name == 'project-test2'
|
||||
assert (jobs[1].parameters['GERRIT_CHANGES'] ==
|
||||
'org/project:master:refs/changes/1/1/1')
|
||||
assert job_has_changes(jobs[1], A)
|
||||
|
||||
assert jobs[2].name == 'project-test1'
|
||||
assert (jobs[2].parameters['GERRIT_CHANGES'] ==
|
||||
'org/project:master:refs/changes/1/1/1^'
|
||||
'org/project:master:refs/changes/1/2/1')
|
||||
assert job_has_changes(jobs[2], A, B)
|
||||
assert jobs[3].name == 'project-test2'
|
||||
assert (jobs[3].parameters['GERRIT_CHANGES'] ==
|
||||
'org/project:master:refs/changes/1/1/1^'
|
||||
'org/project:master:refs/changes/1/2/1')
|
||||
assert job_has_changes(jobs[3], A, B)
|
||||
|
||||
assert jobs[4].name == 'project-merge'
|
||||
assert (jobs[4].parameters['GERRIT_CHANGES'] ==
|
||||
'org/project:master:refs/changes/1/1/1^'
|
||||
'org/project:master:refs/changes/1/2/1^'
|
||||
'org/project:master:refs/changes/1/3/1')
|
||||
assert job_has_changes(jobs[4], A, B, C)
|
||||
|
||||
self.fake_jenkins.fakeRelease('.*-merge')
|
||||
self.waitUntilSettled()
|
||||
assert len(jobs) == 6
|
||||
assert jobs[0].name == 'project-test1'
|
||||
assert (jobs[0].parameters['GERRIT_CHANGES'] ==
|
||||
'org/project:master:refs/changes/1/1/1')
|
||||
assert job_has_changes(jobs[0], A)
|
||||
assert jobs[1].name == 'project-test2'
|
||||
assert (jobs[1].parameters['GERRIT_CHANGES'] ==
|
||||
'org/project:master:refs/changes/1/1/1')
|
||||
assert job_has_changes(jobs[1], A)
|
||||
|
||||
assert jobs[2].name == 'project-test1'
|
||||
assert (jobs[2].parameters['GERRIT_CHANGES'] ==
|
||||
'org/project:master:refs/changes/1/1/1^'
|
||||
'org/project:master:refs/changes/1/2/1')
|
||||
assert job_has_changes(jobs[2], A, B)
|
||||
assert jobs[3].name == 'project-test2'
|
||||
assert (jobs[3].parameters['GERRIT_CHANGES'] ==
|
||||
'org/project:master:refs/changes/1/1/1^'
|
||||
'org/project:master:refs/changes/1/2/1')
|
||||
assert job_has_changes(jobs[3], A, B)
|
||||
|
||||
assert jobs[4].name == 'project-test1'
|
||||
assert (jobs[4].parameters['GERRIT_CHANGES'] ==
|
||||
'org/project:master:refs/changes/1/1/1^'
|
||||
'org/project:master:refs/changes/1/2/1^'
|
||||
'org/project:master:refs/changes/1/3/1')
|
||||
assert job_has_changes(jobs[4], A, B, C)
|
||||
assert jobs[5].name == 'project-test2'
|
||||
assert (jobs[5].parameters['GERRIT_CHANGES'] ==
|
||||
'org/project:master:refs/changes/1/1/1^'
|
||||
'org/project:master:refs/changes/1/2/1^'
|
||||
'org/project:master:refs/changes/1/3/1')
|
||||
assert job_has_changes(jobs[5], A, B, C)
|
||||
|
||||
self.fake_jenkins.hold_jobs_in_build = False
|
||||
self.fake_jenkins.fakeRelease()
|
||||
|
@ -678,9 +742,7 @@ class testScheduler(unittest.TestCase):
|
|||
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
|
||||
self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
|
||||
|
||||
self.fake_jenkins.fakeAddFailTest(
|
||||
'project-test1',
|
||||
'org/project:master:refs/changes/1/1/1')
|
||||
self.fake_jenkins.fakeAddFailTest('project-test1', A)
|
||||
|
||||
self.waitUntilSettled()
|
||||
jobs = self.fake_jenkins.job_history
|
||||
|
@ -710,11 +772,9 @@ class testScheduler(unittest.TestCase):
|
|||
# There should be one merge job at the head of each queue running
|
||||
assert len(jobs) == 2
|
||||
assert jobs[0].name == 'project-merge'
|
||||
assert (jobs[0].parameters['GERRIT_CHANGES'] ==
|
||||
'org/project:master:refs/changes/1/1/1')
|
||||
assert job_has_changes(jobs[0], A)
|
||||
assert jobs[1].name == 'project1-merge'
|
||||
assert (jobs[1].parameters['GERRIT_CHANGES'] ==
|
||||
'org/project1:master:refs/changes/1/2/1')
|
||||
assert job_has_changes(jobs[1], B)
|
||||
|
||||
# Release the current merge jobs
|
||||
self.fake_jenkins.fakeRelease('.*-merge')
|
||||
|
@ -751,9 +811,7 @@ class testScheduler(unittest.TestCase):
|
|||
B.addApproval('CRVW', 2)
|
||||
C.addApproval('CRVW', 2)
|
||||
|
||||
self.fake_jenkins.fakeAddFailTest(
|
||||
'project-test1',
|
||||
'org/project:master:refs/changes/1/1/1')
|
||||
self.fake_jenkins.fakeAddFailTest('project-test1', A)
|
||||
|
||||
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
|
||||
self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
|
||||
|
@ -765,8 +823,7 @@ class testScheduler(unittest.TestCase):
|
|||
|
||||
assert len(jobs) == 1
|
||||
assert jobs[0].name == 'project-merge'
|
||||
assert (jobs[0].parameters['GERRIT_CHANGES'] ==
|
||||
'org/project:master:refs/changes/1/1/1')
|
||||
assert job_has_changes(jobs[0], A)
|
||||
|
||||
self.fake_jenkins.fakeRelease('.*-merge')
|
||||
self.waitUntilSettled()
|
||||
|
@ -813,9 +870,7 @@ class testScheduler(unittest.TestCase):
|
|||
B.addApproval('CRVW', 2)
|
||||
C.addApproval('CRVW', 2)
|
||||
|
||||
self.fake_jenkins.fakeAddFailTest(
|
||||
'project-test1',
|
||||
'org/project:master:refs/changes/1/1/1')
|
||||
self.fake_jenkins.fakeAddFailTest('project-test1', A)
|
||||
|
||||
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
|
||||
self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
|
||||
|
@ -829,8 +884,7 @@ class testScheduler(unittest.TestCase):
|
|||
assert len(jobs) == 1
|
||||
assert len(queue) == 1
|
||||
assert jobs[0].name == 'project-merge'
|
||||
assert (jobs[0].parameters['GERRIT_CHANGES'] ==
|
||||
'org/project:master:refs/changes/1/1/1')
|
||||
assert job_has_changes(jobs[0], A)
|
||||
|
||||
self.fake_jenkins.fakeRelease('.*-merge')
|
||||
self.waitUntilSettled()
|
||||
|
@ -913,7 +967,7 @@ class testScheduler(unittest.TestCase):
|
|||
assert C.reported == 2
|
||||
|
||||
def test_can_merge(self):
|
||||
"Test that whether a change is ready to merge"
|
||||
"Test whether a change is ready to merge"
|
||||
# TODO: move to test_gerrit (this is a unit test!)
|
||||
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||
a = self.sched.trigger.getChange(1, 2)
|
||||
|
@ -929,3 +983,37 @@ class testScheduler(unittest.TestCase):
|
|||
assert self.sched.trigger.canMerge(a, mgr.getSubmitAllowNeeds())
|
||||
|
||||
return True
|
||||
|
||||
def test_build_configuration(self):
|
||||
"Test that zuul merges the right commits for testing"
|
||||
self.fake_jenkins.hold_jobs_in_queue = True
|
||||
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
|
||||
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
|
||||
A.addApproval('CRVW', 2)
|
||||
B.addApproval('CRVW', 2)
|
||||
C.addApproval('CRVW', 2)
|
||||
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
|
||||
self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
|
||||
self.fake_gerrit.addEvent(C.addApproval('APRV', 1))
|
||||
self.waitUntilSettled()
|
||||
|
||||
jobs = self.fake_jenkins.all_jobs
|
||||
|
||||
self.fake_jenkins.fakeRelease('.*-merge')
|
||||
self.waitUntilSettled()
|
||||
self.fake_jenkins.fakeRelease('.*-merge')
|
||||
self.waitUntilSettled()
|
||||
self.fake_jenkins.fakeRelease('.*-merge')
|
||||
self.waitUntilSettled()
|
||||
ref = jobs[-1].parameters['ZUUL_REF']
|
||||
self.fake_jenkins.hold_jobs_in_queue = False
|
||||
self.fake_jenkins.fakeRelease()
|
||||
|
||||
path = os.path.join("/tmp/zuul-test/git/org/project")
|
||||
repo = git.Repo(path)
|
||||
repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
|
||||
repo_messages.reverse()
|
||||
print ' repo messages :', repo_messages
|
||||
correct_messages = ['initial commit', 'A-1', 'B-1', 'C-1']
|
||||
assert repo_messages == correct_messages
|
||||
|
|
|
@ -3,3 +3,4 @@ python-jenkins
|
|||
Paste
|
||||
webob
|
||||
paramiko
|
||||
GitPython>=0.3.2.RC1
|
||||
|
|
|
@ -24,7 +24,7 @@ import time
|
|||
import urllib # for extending jenkins lib
|
||||
import urllib2 # for extending jenkins lib
|
||||
import urlparse
|
||||
from uuid import uuid1
|
||||
from uuid import uuid4
|
||||
|
||||
import jenkins
|
||||
from paste import httpserver
|
||||
|
@ -207,24 +207,32 @@ class Jenkins(object):
|
|||
self.cleanup_thread.stop()
|
||||
self.cleanup_thread.join()
|
||||
|
||||
#TODO: remove dependent_changes
|
||||
def launch(self, job, change, dependent_changes=[]):
|
||||
self.log.info("Launch job %s for change %s with dependent changes %s" %
|
||||
(job, change, dependent_changes))
|
||||
dependent_changes = dependent_changes[:]
|
||||
dependent_changes.reverse()
|
||||
uuid = str(uuid1())
|
||||
uuid = str(uuid4().hex)
|
||||
params = dict(UUID=uuid,
|
||||
GERRIT_PROJECT=change.project.name)
|
||||
GERRIT_PROJECT=change.project.name,
|
||||
ZUUL_PROJECT=change.project.name)
|
||||
if hasattr(change, 'refspec'):
|
||||
changes_str = '^'.join(
|
||||
['%s:%s:%s' % (c.project.name, c.branch, c.refspec)
|
||||
for c in dependent_changes + [change]])
|
||||
params['GERRIT_BRANCH'] = change.branch
|
||||
params['ZUUL_BRANCH'] = change.branch
|
||||
params['GERRIT_CHANGES'] = changes_str
|
||||
params['ZUUL_REF'] = 'refs/zuul/%s/%s' % (change.branch,
|
||||
change.current_build_set.ref)
|
||||
if hasattr(change, 'ref'):
|
||||
params['GERRIT_REFNAME'] = change.ref
|
||||
params['ZUUL_REFNAME'] = change.ref
|
||||
params['GERRIT_OLDREV'] = change.oldrev
|
||||
params['ZUUL_OLDREV'] = change.oldrev
|
||||
params['GERRIT_NEWREV'] = change.newrev
|
||||
params['ZUUL_NEWREV'] = change.newrev
|
||||
|
||||
if callable(job.parameter_function):
|
||||
job.parameter_function(change, params)
|
||||
|
|
|
@ -0,0 +1,126 @@
|
|||
# Copyright 2012 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import git
|
||||
import os
|
||||
import logging
|
||||
import model
|
||||
|
||||
|
||||
class ZuulReference(git.Reference):
|
||||
_common_path_default = "refs/zuul"
|
||||
_points_to_commits_only = True
|
||||
|
||||
|
||||
class Repo(object):
|
||||
log = logging.getLogger("zuul.Repo")
|
||||
|
||||
def __init__(self, remote, local):
|
||||
self.remote_url = remote
|
||||
self.local_path = local
|
||||
self._ensure_cloned()
|
||||
self.repo = git.Repo(self.local_path)
|
||||
|
||||
def _ensure_cloned(self):
|
||||
if not os.path.exists(self.local_path):
|
||||
self.log.debug("Cloning from %s to %s" % (self.remote_url,
|
||||
self.local_path))
|
||||
git.Repo.clone_from(self.remote_url, self.local_path)
|
||||
|
||||
def reset(self):
|
||||
self.log.debug("Resetting repository %s" % self.local_path)
|
||||
origin = self.repo.remotes.origin
|
||||
origin.update()
|
||||
self.repo.head.reference = origin.refs.master
|
||||
self.repo.head.reset(index=True, working_tree=True)
|
||||
self.repo.git.clean('-x', '-f', '-d')
|
||||
|
||||
def cherryPick(self, ref):
|
||||
self.log.debug("Cherry-picking %s" % ref)
|
||||
origin = self.repo.remotes.origin
|
||||
origin.fetch(ref)
|
||||
self.repo.git.cherry_pick("FETCH_HEAD")
|
||||
|
||||
def merge(self, ref):
|
||||
self.log.debug("Merging %s" % ref)
|
||||
origin = self.repo.remotes.origin
|
||||
origin.fetch(ref)
|
||||
self.repo.git.merge("FETCH_HEAD")
|
||||
|
||||
def createZuulRef(self, ref):
|
||||
ref = ZuulReference.create(self.repo, ref, 'HEAD')
|
||||
return ref
|
||||
|
||||
def setZuulRef(self, ref, commit):
|
||||
self.repo.refs[ref].commit = commit
|
||||
|
||||
|
||||
class Merger(object):
|
||||
log = logging.getLogger("zuul.Merger")
|
||||
|
||||
def __init__(self, working_root):
|
||||
self.repos = {}
|
||||
self.working_root = working_root
|
||||
if not os.path.exists(working_root):
|
||||
os.makedirs(working_root)
|
||||
|
||||
def addProject(self, project, url):
|
||||
try:
|
||||
path = os.path.join(self.working_root, project.name)
|
||||
repo = Repo(url, path)
|
||||
self.repos[project] = repo
|
||||
except:
|
||||
self.log.exception("Unable to initialize repo for %s" % project)
|
||||
|
||||
def getRepo(self, project):
|
||||
return self.repos.get(project, None)
|
||||
|
||||
def mergeChanges(self, changes, target_ref=None, mode=None):
|
||||
projects = {}
|
||||
# Reset all repos involved in the change set
|
||||
for change in changes:
|
||||
branches = projects.get(change.project, [])
|
||||
if change.branch not in branches:
|
||||
repo = self.getRepo(change.project)
|
||||
if not repo:
|
||||
self.log.error("Unable to find repo for %s" %
|
||||
change.project)
|
||||
return False
|
||||
try:
|
||||
repo.reset()
|
||||
except:
|
||||
self.log.exception("Unable to reset repo %s" % repo)
|
||||
return False
|
||||
|
||||
if target_ref:
|
||||
repo.createZuulRef(change.branch + '/' + target_ref)
|
||||
branches.append(change.branch)
|
||||
projects[change.project] = branches
|
||||
|
||||
# Merge all the changes
|
||||
for change in changes:
|
||||
repo = self.getRepo(change.project)
|
||||
try:
|
||||
if not mode:
|
||||
mode = change.project.merge_mode
|
||||
if mode == model.MERGE_IF_NECESSARY:
|
||||
repo.merge(change.refspec)
|
||||
elif mode == model.CHERRY_PICK:
|
||||
repo.cherryPick(change.refspec)
|
||||
repo.setZuulRef(change.branch + '/' + target_ref, 'HEAD')
|
||||
except:
|
||||
self.log.exception("Unable to merge %s" % change)
|
||||
return False
|
||||
|
||||
return True
|
|
@ -14,6 +14,13 @@
|
|||
|
||||
import re
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
FAST_FORWARD_ONLY = 1
|
||||
MERGE_ALWAYS = 2
|
||||
MERGE_IF_NECESSARY = 3
|
||||
CHERRY_PICK = 4
|
||||
|
||||
|
||||
class Pipeline(object):
|
||||
|
@ -327,6 +334,7 @@ class ChangeQueue(object):
|
|||
class Project(object):
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.merge_mode = MERGE_IF_NECESSARY
|
||||
|
||||
def __str__(self):
|
||||
return self.name
|
||||
|
@ -424,11 +432,9 @@ class BuildSet(object):
|
|||
self.result = None
|
||||
self.next_build_set = None
|
||||
self.previous_build_set = None
|
||||
self.ref = None
|
||||
|
||||
def addBuild(self, build):
|
||||
self.builds[build.job.name] = build
|
||||
build.build_set = self
|
||||
|
||||
def setConfiguration(self):
|
||||
# The change isn't enqueued until after it's created
|
||||
# so we don't know what the other changes ahead will be
|
||||
# until jobs start.
|
||||
|
@ -437,6 +443,15 @@ class BuildSet(object):
|
|||
while next_change:
|
||||
self.other_changes.append(next_change)
|
||||
next_change = next_change.change_ahead
|
||||
if not self.ref:
|
||||
self.ref = 'Z' + uuid4().hex
|
||||
|
||||
def getRef(self):
|
||||
return self.ref
|
||||
|
||||
def addBuild(self, build):
|
||||
self.builds[build.job.name] = build
|
||||
build.build_set = self
|
||||
|
||||
def getBuild(self, job_name):
|
||||
return self.builds.get(job_name)
|
||||
|
|
|
@ -20,7 +20,9 @@ import re
|
|||
import threading
|
||||
import yaml
|
||||
|
||||
import model
|
||||
from model import Pipeline, Job, Project, ChangeQueue, EventFilter
|
||||
import merger
|
||||
|
||||
|
||||
class Scheduler(threading.Thread):
|
||||
|
@ -139,6 +141,9 @@ class Scheduler(threading.Thread):
|
|||
for config_project in data['projects']:
|
||||
project = Project(config_project['name'])
|
||||
self.projects[config_project['name']] = project
|
||||
mode = config_project.get('merge-mode')
|
||||
if mode and mode == 'cherry-pick':
|
||||
project.merge_mode = model.CHERRY_PICK
|
||||
for pipeline in self.pipelines.values():
|
||||
if pipeline.name in config_project:
|
||||
job_tree = pipeline.addProject(project)
|
||||
|
@ -154,6 +159,15 @@ class Scheduler(threading.Thread):
|
|||
for pipeline in self.pipelines.values():
|
||||
pipeline.manager._postConfig()
|
||||
|
||||
if self.config.has_option('zuul', 'git_dir'):
|
||||
merge_root = self.config.get('zuul', 'git_dir')
|
||||
else:
|
||||
merge_root = '/var/lib/zuul/git'
|
||||
self.merger = merger.Merger(merge_root)
|
||||
for project in self.projects.values():
|
||||
url = self.trigger.getGitUrl(project)
|
||||
self.merger.addProject(project, url)
|
||||
|
||||
def getJob(self, name):
|
||||
if name in self.jobs:
|
||||
return self.jobs[name]
|
||||
|
@ -464,6 +478,13 @@ class BasePipelineManager(object):
|
|||
|
||||
def launchJobs(self, change):
|
||||
self.log.debug("Launching jobs for change %s" % change)
|
||||
ref = change.current_build_set.getRef()
|
||||
if not ref:
|
||||
change.current_build_set.setConfiguration()
|
||||
ref = change.current_build_set.getRef()
|
||||
self.sched.merger.mergeChanges([change], ref,
|
||||
mode=model.MERGE_IF_NECESSARY)
|
||||
|
||||
for job in self.pipeline.findJobsToRun(change):
|
||||
self.log.debug("Found job %s for change %s" % (job, change))
|
||||
try:
|
||||
|
@ -720,12 +741,21 @@ class DependentPipelineManager(BasePipelineManager):
|
|||
|
||||
def launchJobs(self, change):
|
||||
self.log.debug("Launching jobs for change %s" % change)
|
||||
ref = change.current_build_set.getRef()
|
||||
if not ref:
|
||||
change.current_build_set.setConfiguration()
|
||||
ref = change.current_build_set.getRef()
|
||||
dependent_changes = self._getDependentChanges(change)
|
||||
dependent_changes.reverse()
|
||||
self.sched.merger.mergeChanges(dependent_changes + [change], ref)
|
||||
|
||||
#TODO: remove this line after GERRIT_CHANGES is gone
|
||||
dependent_changes = self._getDependentChanges(change)
|
||||
for job in self.pipeline.findJobsToRun(change):
|
||||
self.log.debug("Found job %s for change %s" % (job, change))
|
||||
try:
|
||||
build = self.sched.launcher.launch(job,
|
||||
change,
|
||||
#TODO: remove dependent_changes after GERRIT_CHANGES is gone
|
||||
build = self.sched.launcher.launch(job, change,
|
||||
dependent_changes)
|
||||
self.building_jobs[build] = change
|
||||
self.log.debug("Adding build %s of job %s to change %s" %
|
||||
|
|
|
@ -79,6 +79,7 @@ class Gerrit(object):
|
|||
|
||||
def __init__(self, config, sched):
|
||||
self.sched = sched
|
||||
self.config = config
|
||||
self.server = config.get('gerrit', 'server')
|
||||
user = config.get('gerrit', 'user')
|
||||
if config.has_option('gerrit', 'sshkey'):
|
||||
|
@ -270,3 +271,13 @@ class Gerrit(object):
|
|||
change.needed_by_changes.append(dep)
|
||||
|
||||
return change
|
||||
|
||||
def getGitUrl(self, project):
|
||||
server = self.config.get('gerrit', 'server')
|
||||
user = self.config.get('gerrit', 'user')
|
||||
if self.config.has_option('gerrit', 'port'):
|
||||
port = self.config.get('gerrit', 'port')
|
||||
else:
|
||||
port = 29418
|
||||
url = 'ssh://%s@%s:%s/%s' % (user, server, port, project.name)
|
||||
return url
|
||||
|
|
Loading…
Reference in New Issue