From 9ca3983a75e980ab11ae2e964855236d71a7458c Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Tue, 28 Jan 2014 12:27:13 -0800 Subject: [PATCH] Remove Zuul ref replication. This reverts commit 87650fa736fb61c6b305f9ea8c3907e4f0168cd9. The problem of distributing the load of serving Zuul Git refs will be addressed by separating the merger component so that it can be scaled out. Remove this feature which is hopefully new enough that no one is using it. Change-Id: Id2be95c85f5c3464a66537ae3095024a964ee1c0 --- NEWS.rst | 4 +--- doc/source/triggers.rst | 40 +++++----------------------------------- doc/source/zuul.rst | 10 ---------- tests/test_scheduler.py | 33 --------------------------------- zuul/merger.py | 35 ++--------------------------------- zuul/scheduler.py | 9 ++------- 6 files changed, 10 insertions(+), 121 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 52f5c4f231..ba6c9860e0 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -1,9 +1,7 @@ Since 2.0.0: * The push_change_refs option which specified that Zuul refs should be - pushed to Gerrit has been removed. Similar functionality may be - obtained using the replication feature. See the Triggers - documentation for details. + pushed to Gerrit has been removed. Since 1.3.0: diff --git a/doc/source/triggers.rst b/doc/source/triggers.rst index 21aab13c46..287246cc78 100644 --- a/doc/source/triggers.rst +++ b/doc/source/triggers.rst @@ -49,20 +49,17 @@ and checking it out. The parameters that provide this information are described in :ref:`launchers`. These references need to be made available via a Git repository that -is available to Jenkins. You may accomplish this by either serving -Zuul's git repositories directly, allowing Zuul to push the references -back to Gerrit, or pushing the references to a third location. -Instructions for each of these alternatives are in the following -sections. +is available to Jenkins. This is accomplished by serving Zuul's Git +repositories directly. Serving Zuul Git Repos """""""""""""""""""""" Zuul maintains its own copies of any needed Git repositories in the directory specified by ``git_dir`` in the ``zuul`` section of -zuul.conf (by default, /var/lib/zuul/git). If you want to serve -Zuul's Git repositories in order to provide Zuul refs for Jenkins, you -can configure Apache to do so using the following directives:: +zuul.conf (by default, /var/lib/zuul/git). To directly serve Zuul's +Git repositories in order to provide Zuul refs for Jenkins, you can +configure Apache to do so using the following directives:: SetEnv GIT_PROJECT_ROOT /var/lib/zuul/git SetEnv GIT_HTTP_EXPORT_ALL @@ -81,33 +78,6 @@ depending on what the state of Zuul's repository is when the clone happens). They are, however, suitable for automated systems that respond to Zuul triggers. -Pushing to Gerrit -""""""""""""""""" - -If you want to push Zuul refs back to Gerrit, set the following -permissions for your project (or ``All-Projects``) in Gerrit (where -``CI Tools`` is a group of which the user you created above is a -member):: - - [access "refs/zuul/*"] - create = group CI Tools - push = +force CI Tools - pushMerge = group CI Tools - forgeAuthor = group CI Tools - [access "refs/for/refs/zuul/*"] - pushMerge = group CI Tools - -And set the following in ``zuul.conf``: - - [replication] - url1=ssh://user@review.example.com:29418/ - -Pushing to Another Location -""""""""""""""""""""""""""" - -Simply set one or more destination URLs in the ``replication`` section -of zuul.conf as above. - Timer ----- diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst index d2b394ba79..ef8c6e91f2 100644 --- a/doc/source/zuul.rst +++ b/doc/source/zuul.rst @@ -154,16 +154,6 @@ smtp This can be overridden by individual pipelines. ``default_to=you@example.com`` -replication -""""""""""" - -Zuul can push the refs it creates to any number of servers. To do so, -list the git push URLs in this section, one per line as follows:: - - [replication] - url1=ssh://user@host1.example.com:port/path/to/repo - url2=ssh://user@host2.example.com:port/path/to/repo - layout.yaml ~~~~~~~~~~~ diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 9787ae1d73..82b0cbd0c4 100755 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -3025,39 +3025,6 @@ class TestScheduler(testtools.TestCase): self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(B.reported, 2) - def test_push_urls(self): - "Test that Zuul can push refs to multiple URLs" - upstream_path = os.path.join(self.upstream_root, 'org/project') - replica1 = os.path.join(self.upstream_root, 'replica1') - replica2 = os.path.join(self.upstream_root, 'replica2') - - self.config.add_section('replication') - self.config.set('replication', 'url1', 'file://%s' % replica1) - self.config.set('replication', 'url2', 'file://%s' % replica2) - self.sched.reconfigure(self.config) - - r1 = git.Repo.clone_from(upstream_path, replica1 + '/org/project.git') - r2 = git.Repo.clone_from(upstream_path, replica2 + '/org/project.git') - - A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') - A.addApproval('CRVW', 2) - self.fake_gerrit.addEvent(A.addApproval('APRV', 1)) - B = self.fake_gerrit.addFakeChange('org/project', 'mp', 'B') - B.addApproval('CRVW', 2) - self.fake_gerrit.addEvent(B.addApproval('APRV', 1)) - self.waitUntilSettled() - count = 0 - for ref in r1.refs: - if ref.path.startswith('refs/zuul'): - count += 1 - self.assertEqual(count, 3) - - count = 0 - for ref in r2.refs: - if ref.path.startswith('refs/zuul'): - count += 1 - self.assertEqual(count, 3) - def test_timer(self): "Test that a periodic job is triggered" self.worker.hold_jobs_in_build = True diff --git a/zuul/merger.py b/zuul/merger.py index a580a2121b..095da5eca5 100644 --- a/zuul/merger.py +++ b/zuul/merger.py @@ -16,7 +16,6 @@ import git import os import logging import model -import threading class ZuulReference(git.Reference): @@ -132,11 +131,6 @@ class Repo(object): self.remote_url)) repo.remotes.origin.push('%s:%s' % (local, remote)) - def push_url(self, url, refspecs): - repo = self.createRepoObject() - self.log.debug("Pushing %s to %s" % (refspecs, url)) - repo.git.push([url] + refspecs) - def update(self): repo = self.createRepoObject() self.log.debug("Updating repository %s" % self.local_path) @@ -147,7 +141,7 @@ class Repo(object): class Merger(object): log = logging.getLogger("zuul.Merger") - def __init__(self, working_root, sshkey, email, username, replicate_urls): + def __init__(self, working_root, sshkey, email, username): self.repos = {} self.working_root = working_root if not os.path.exists(working_root): @@ -156,7 +150,6 @@ class Merger(object): self._makeSSHWrapper(sshkey) self.email = email self.username = username - self.replicate_urls = replicate_urls def _makeSSHWrapper(self, key): name = os.path.join(self.working_root, '.ssh_wrapper') @@ -223,25 +216,6 @@ class Merger(object): return False return commit - def replicateRefspecs(self, refspecs): - threads = [] - for url in self.replicate_urls: - t = threading.Thread(target=self._replicate, - args=(url, refspecs)) - t.start() - threads.append(t) - for t in threads: - t.join() - - def _replicate(self, url, project_refspecs): - try: - for project, refspecs in project_refspecs.items(): - repo = self.getRepo(project) - repo.push_url(os.path.join(url, project.name + '.git'), - refspecs) - except Exception: - self.log.exception("Exception pushing to %s" % url) - def mergeChanges(self, items, target_ref=None): # Merge shortcuts: # if this is the only change just merge it against its branch. @@ -280,7 +254,6 @@ class Merger(object): return commit project_branches = [] - replicate_refspecs = {} for i in reversed(items): # Here we create all of the necessary zuul refs and potentially # push them back to Gerrit. @@ -300,10 +273,6 @@ class Merger(object): self.log.exception("Unable to set zuul ref %s for " "change %s" % (zuul_ref, i.change)) return False - ref = 'refs/zuul/' + i.change.branch + '/' + target_ref - refspecs = replicate_refspecs.get(i.change.project, []) - refspecs.append('%s:%s' % (ref, ref)) - replicate_refspecs[i.change.project] = refspecs project_branches.append((i.change.project, i.change.branch)) - self.replicateRefspecs(replicate_refspecs) + return commit diff --git a/zuul/scheduler.py b/zuul/scheduler.py index 73034c7bda..d1c03d3642 100644 --- a/zuul/scheduler.py +++ b/zuul/scheduler.py @@ -367,11 +367,6 @@ class Scheduler(threading.Thread): else: merge_name = None - replicate_urls = [] - if self.config.has_section('replication'): - for k, v in self.config.items('replication'): - replicate_urls.append(v) - if self.config.has_option('gerrit', 'sshkey'): sshkey = self.config.get('gerrit', 'sshkey') else: @@ -380,8 +375,8 @@ class Scheduler(threading.Thread): # TODO: The merger should have an upstream repo independent of # triggers, and then each trigger should provide a fetch # location. - self.merger = merger.Merger(merge_root, sshkey, merge_email, - merge_name, replicate_urls) + self.merger = merger.Merger(merge_root, sshkey, + merge_email, merge_name) for project in self.layout.projects.values(): url = self.triggers['gerrit'].getGitUrl(project) self.merger.addProject(project, url)