Browse Source

Remove Zuul ref replication.

This reverts commit 87650fa736.

The problem of distributing the load of serving Zuul Git refs will
be addressed by separating the merger component so that it can be
scaled out.  Remove this feature which is hopefully new enough that
no one is using it.

Change-Id: Id2be95c85f5c3464a66537ae3095024a964ee1c0
changes/09/70309/3
James E. Blair 8 years ago
parent
commit
9ca3983a75
  1. 4
      NEWS.rst
  2. 40
      doc/source/triggers.rst
  3. 10
      doc/source/zuul.rst
  4. 33
      tests/test_scheduler.py
  5. 35
      zuul/merger.py
  6. 9
      zuul/scheduler.py

4
NEWS.rst

@ -1,9 +1,7 @@
Since 2.0.0:
* The push_change_refs option which specified that Zuul refs should be
pushed to Gerrit has been removed. Similar functionality may be
obtained using the replication feature. See the Triggers
documentation for details.
pushed to Gerrit has been removed.
Since 1.3.0:

40
doc/source/triggers.rst

@ -49,20 +49,17 @@ and checking it out. The parameters that provide this information are
described in :ref:`launchers`.
These references need to be made available via a Git repository that
is available to Jenkins. You may accomplish this by either serving
Zuul's git repositories directly, allowing Zuul to push the references
back to Gerrit, or pushing the references to a third location.
Instructions for each of these alternatives are in the following
sections.
is available to Jenkins. This is accomplished by serving Zuul's Git
repositories directly.
Serving Zuul Git Repos
""""""""""""""""""""""
Zuul maintains its own copies of any needed Git repositories in the
directory specified by ``git_dir`` in the ``zuul`` section of
zuul.conf (by default, /var/lib/zuul/git). If you want to serve
Zuul's Git repositories in order to provide Zuul refs for Jenkins, you
can configure Apache to do so using the following directives::
zuul.conf (by default, /var/lib/zuul/git). To directly serve Zuul's
Git repositories in order to provide Zuul refs for Jenkins, you can
configure Apache to do so using the following directives::
SetEnv GIT_PROJECT_ROOT /var/lib/zuul/git
SetEnv GIT_HTTP_EXPORT_ALL
@ -81,33 +78,6 @@ depending on what the state of Zuul's repository is when the clone
happens). They are, however, suitable for automated systems that
respond to Zuul triggers.
Pushing to Gerrit
"""""""""""""""""
If you want to push Zuul refs back to Gerrit, set the following
permissions for your project (or ``All-Projects``) in Gerrit (where
``CI Tools`` is a group of which the user you created above is a
member)::
[access "refs/zuul/*"]
create = group CI Tools
push = +force CI Tools
pushMerge = group CI Tools
forgeAuthor = group CI Tools
[access "refs/for/refs/zuul/*"]
pushMerge = group CI Tools
And set the following in ``zuul.conf``:
[replication]
url1=ssh://user@review.example.com:29418/
Pushing to Another Location
"""""""""""""""""""""""""""
Simply set one or more destination URLs in the ``replication`` section
of zuul.conf as above.
Timer
-----

10
doc/source/zuul.rst

@ -154,16 +154,6 @@ smtp
This can be overridden by individual pipelines.
``default_to=you@example.com``
replication
"""""""""""
Zuul can push the refs it creates to any number of servers. To do so,
list the git push URLs in this section, one per line as follows::
[replication]
url1=ssh://user@host1.example.com:port/path/to/repo
url2=ssh://user@host2.example.com:port/path/to/repo
layout.yaml
~~~~~~~~~~~

33
tests/test_scheduler.py

@ -3025,39 +3025,6 @@ class TestScheduler(testtools.TestCase):
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(B.reported, 2)
def test_push_urls(self):
"Test that Zuul can push refs to multiple URLs"
upstream_path = os.path.join(self.upstream_root, 'org/project')
replica1 = os.path.join(self.upstream_root, 'replica1')
replica2 = os.path.join(self.upstream_root, 'replica2')
self.config.add_section('replication')
self.config.set('replication', 'url1', 'file://%s' % replica1)
self.config.set('replication', 'url2', 'file://%s' % replica2)
self.sched.reconfigure(self.config)
r1 = git.Repo.clone_from(upstream_path, replica1 + '/org/project.git')
r2 = git.Repo.clone_from(upstream_path, replica2 + '/org/project.git')
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('CRVW', 2)
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
B = self.fake_gerrit.addFakeChange('org/project', 'mp', 'B')
B.addApproval('CRVW', 2)
self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
self.waitUntilSettled()
count = 0
for ref in r1.refs:
if ref.path.startswith('refs/zuul'):
count += 1
self.assertEqual(count, 3)
count = 0
for ref in r2.refs:
if ref.path.startswith('refs/zuul'):
count += 1
self.assertEqual(count, 3)
def test_timer(self):
"Test that a periodic job is triggered"
self.worker.hold_jobs_in_build = True

35
zuul/merger.py

@ -16,7 +16,6 @@ import git
import os
import logging
import model
import threading
class ZuulReference(git.Reference):
@ -132,11 +131,6 @@ class Repo(object):
self.remote_url))
repo.remotes.origin.push('%s:%s' % (local, remote))
def push_url(self, url, refspecs):
repo = self.createRepoObject()
self.log.debug("Pushing %s to %s" % (refspecs, url))
repo.git.push([url] + refspecs)
def update(self):
repo = self.createRepoObject()
self.log.debug("Updating repository %s" % self.local_path)
@ -147,7 +141,7 @@ class Repo(object):
class Merger(object):
log = logging.getLogger("zuul.Merger")
def __init__(self, working_root, sshkey, email, username, replicate_urls):
def __init__(self, working_root, sshkey, email, username):
self.repos = {}
self.working_root = working_root
if not os.path.exists(working_root):
@ -156,7 +150,6 @@ class Merger(object):
self._makeSSHWrapper(sshkey)
self.email = email
self.username = username
self.replicate_urls = replicate_urls
def _makeSSHWrapper(self, key):
name = os.path.join(self.working_root, '.ssh_wrapper')
@ -223,25 +216,6 @@ class Merger(object):
return False
return commit
def replicateRefspecs(self, refspecs):
threads = []
for url in self.replicate_urls:
t = threading.Thread(target=self._replicate,
args=(url, refspecs))
t.start()
threads.append(t)
for t in threads:
t.join()
def _replicate(self, url, project_refspecs):
try:
for project, refspecs in project_refspecs.items():
repo = self.getRepo(project)
repo.push_url(os.path.join(url, project.name + '.git'),
refspecs)
except Exception:
self.log.exception("Exception pushing to %s" % url)
def mergeChanges(self, items, target_ref=None):
# Merge shortcuts:
# if this is the only change just merge it against its branch.
@ -280,7 +254,6 @@ class Merger(object):
return commit
project_branches = []
replicate_refspecs = {}
for i in reversed(items):
# Here we create all of the necessary zuul refs and potentially
# push them back to Gerrit.
@ -300,10 +273,6 @@ class Merger(object):
self.log.exception("Unable to set zuul ref %s for "
"change %s" % (zuul_ref, i.change))
return False
ref = 'refs/zuul/' + i.change.branch + '/' + target_ref
refspecs = replicate_refspecs.get(i.change.project, [])
refspecs.append('%s:%s' % (ref, ref))
replicate_refspecs[i.change.project] = refspecs
project_branches.append((i.change.project, i.change.branch))
self.replicateRefspecs(replicate_refspecs)
return commit

9
zuul/scheduler.py

@ -367,11 +367,6 @@ class Scheduler(threading.Thread):
else:
merge_name = None
replicate_urls = []
if self.config.has_section('replication'):
for k, v in self.config.items('replication'):
replicate_urls.append(v)
if self.config.has_option('gerrit', 'sshkey'):
sshkey = self.config.get('gerrit', 'sshkey')
else:
@ -380,8 +375,8 @@ class Scheduler(threading.Thread):
# TODO: The merger should have an upstream repo independent of
# triggers, and then each trigger should provide a fetch
# location.
self.merger = merger.Merger(merge_root, sshkey, merge_email,
merge_name, replicate_urls)
self.merger = merger.Merger(merge_root, sshkey,
merge_email, merge_name)
for project in self.layout.projects.values():
url = self.triggers['gerrit'].getGitUrl(project)
self.merger.addProject(project, url)

Loading…
Cancel
Save