diff --git a/doc/source/admin/drivers/gerrit.rst b/doc/source/admin/drivers/gerrit.rst index 935cb32901..5433df3c23 100644 --- a/doc/source/admin/drivers/gerrit.rst +++ b/doc/source/admin/drivers/gerrit.rst @@ -36,6 +36,7 @@ The supported options in ``zuul.conf`` connections are: The connection must set ``driver=gerrit`` for Gerrit connections. .. attr:: server + :required: Fully qualified domain name of Gerrit server. @@ -58,8 +59,9 @@ The supported options in ``zuul.conf`` connections are: Gerrit server port. .. attr:: baseurl + :default: https://{server} - Path to Gerrit web interface. + Path to Gerrit web interface. Omit the trailing ``/``. .. attr:: gitweb_url_template :default: {baseurl}/gitweb?p={project.name}.git;a=commitdiff;h={sha} @@ -87,6 +89,40 @@ The supported options in ``zuul.conf`` connections are: SSH connection keepalive timeout; ``0`` disables. + .. attr:: password + + The HTTP authentication password for the user. This is + optional, but if it is provided, Zuul will report to Gerrit via + HTTP rather than SSH. It is required in order for file and line + comments to reported (the Gerrit SSH API only supports review + messages). Retrieve this password from the ``HTTP Password`` + section of the ``Settings`` page in Gerrit. + + .. attr:: auth_type + :default: digest + + The HTTP authentication mechanism. + + .. value:: digest + + HTTP Digest authentication; the default for most Gerrit + installations. + + .. value:: basic + + HTTP Basic authentication. + + .. value:: form + + Zuul will submit a username and password to a form in order + to authenticate. + + .. attr:: verify_ssl + :default: true + + When using a self-signed certificate, this may be set to + ``false`` to disable SSL certificate verification. + Trigger Configuration --------------------- diff --git a/releasenotes/notes/gerrit-http-d134f509d6b49f7a.yaml b/releasenotes/notes/gerrit-http-d134f509d6b49f7a.yaml new file mode 100644 index 0000000000..31b9b7a0eb --- /dev/null +++ b/releasenotes/notes/gerrit-http-d134f509d6b49f7a.yaml @@ -0,0 +1,5 @@ +--- +features: + - The Gerrit driver can now (optionally) report via HTTP instead of + SSH. In the future, this will be used to report file and line + comments (the SSH API only supports review messages). diff --git a/tests/base.py b/tests/base.py index 391596d886..67ca61a52b 100644 --- a/tests/base.py +++ b/tests/base.py @@ -478,6 +478,82 @@ class FakeGerritChange(object): self.reported += 1 +class GerritWebServer(object): + + def __init__(self, fake_gerrit): + super(GerritWebServer, self).__init__() + self.fake_gerrit = fake_gerrit + + def start(self): + fake_gerrit = self.fake_gerrit + + class Server(http.server.SimpleHTTPRequestHandler): + log = logging.getLogger("zuul.test.FakeGerritConnection") + review_re = re.compile('/a/changes/(.*?)/revisions/(.*?)/review') + submit_re = re.compile('/a/changes/(.*?)/submit') + + def do_POST(self): + path = self.path + self.log.debug("Got POST %s", path) + + data = self.rfile.read(int(self.headers['Content-Length'])) + data = json.loads(data.decode('utf-8')) + self.log.debug("Got data %s", data) + + m = self.review_re.match(path) + if m: + return self.review(m.group(1), m.group(2), data) + m = self.submit_re.match(path) + if m: + return self.submit(m.group(1), data) + self.send_response(500) + self.end_headers() + + def _404(self): + self.send_response(404) + self.end_headers() + + def _get_change(self, change_id): + for c in fake_gerrit.changes.values(): + if c.data['id'] == change_id: + return c + + def review(self, change_id, revision, data): + change = self._get_change(change_id) + if not change: + return self._404() + + message = data['message'] + action = data['labels'] + fake_gerrit._test_handle_review( + int(change.data['number']), message, action) + self.send_response(200) + self.end_headers() + + def submit(self, change_id, data): + change = self._get_change(change_id) + if not change: + return self._404() + + message = None + action = {'submit': True} + fake_gerrit._test_handle_review( + int(change.data['number']), message, action) + self.send_response(200) + self.end_headers() + + self.httpd = socketserver.ThreadingTCPServer(('', 0), Server) + self.port = self.httpd.socket.getsockname()[1] + self.thread = threading.Thread(name='GerritWebServer', + target=self.httpd.serve_forever) + self.thread.daemon = True + self.thread.start() + + def stop(self): + self.httpd.shutdown() + self.thread.join() + + class FakeGerritConnection(gerritconnection.GerritConnection): """A Fake Gerrit connection for use in tests. @@ -490,6 +566,15 @@ class FakeGerritConnection(gerritconnection.GerritConnection): def __init__(self, driver, connection_name, connection_config, changes_db=None, upstream_root=None): + + if connection_config.get('password'): + self.web_server = GerritWebServer(self) + self.web_server.start() + url = 'http://localhost:%s' % self.web_server.port + connection_config['baseurl'] = url + else: + self.web_server = None + super(FakeGerritConnection, self).__init__(driver, connection_name, connection_config) @@ -570,9 +655,15 @@ class FakeGerritConnection(gerritconnection.GerritConnection): } return event - def review(self, project, changeid, message, action): - number, ps = changeid.split(',') - change = self.changes[int(number)] + def review(self, change, message, action): + if self.web_server: + return super(FakeGerritConnection, self).review( + change, message, action) + self._test_handle_review(int(change.number), message, action) + + def _test_handle_review(self, change_number, message, action): + # Handle a review action from a test + change = self.changes[change_number] # Add the approval back onto the change (ie simulate what gerrit would # do). @@ -588,7 +679,8 @@ class FakeGerritConnection(gerritconnection.GerritConnection): if cat != 'submit': change.addApproval(cat, action[cat], username=self.user) - change.messages.append(message) + if message: + change.messages.append(message) if 'submit' in action: change.setMerged() @@ -2340,6 +2432,9 @@ class ZuulTestCase(BaseTestCase): con = FakeGerritConnection(driver, name, config, changes_db=db, upstream_root=self.upstream_root) + if con.web_server: + self.addCleanup(con.web_server.stop) + self.event_queues.append(con.event_queue) setattr(self, 'fake_' + name, con) return con @@ -2651,6 +2746,7 @@ class ZuulTestCase(BaseTestCase): 'pydevd.Reader', 'pydevd.Writer', 'socketserver_Thread', + 'GerritWebServer', ] threads = [t for t in threading.enumerate() if t.name not in whitelist] diff --git a/tests/fixtures/zuul-gerrit-web.conf b/tests/fixtures/zuul-gerrit-web.conf new file mode 100644 index 0000000000..d79140f7fc --- /dev/null +++ b/tests/fixtures/zuul-gerrit-web.conf @@ -0,0 +1,32 @@ +[gearman] +server=127.0.0.1 + +[statsd] +# note, use 127.0.0.1 rather than localhost to avoid getting ipv6 +# see: https://github.com/jsocol/pystatsd/issues/61 +server=127.0.0.1 + +[scheduler] +tenant_config=main.yaml + +[merger] +git_dir=/tmp/zuul-test/merger-git +git_user_email=zuul@example.com +git_user_name=zuul + +[executor] +git_dir=/tmp/zuul-test/executor-git + +[connection gerrit] +driver=gerrit +server=review.example.com +user=jenkins +sshkey=fake_id_rsa_path +password=badpassword + +[connection smtp] +driver=smtp +server=localhost +port=25 +default_from=zuul@example.com +default_to=you@example.com diff --git a/tests/unit/test_gerrit.py b/tests/unit/test_gerrit.py index 1c5caff0a1..1d76f2b636 100644 --- a/tests/unit/test_gerrit.py +++ b/tests/unit/test_gerrit.py @@ -16,7 +16,7 @@ import os from unittest import mock import tests.base -from tests.base import BaseTestCase +from tests.base import BaseTestCase, ZuulTestCase from zuul.driver.gerrit import GerritDriver from zuul.driver.gerrit.gerritconnection import GerritConnection @@ -76,3 +76,27 @@ class TestGerrit(BaseTestCase): 'simple_query_pagination_old_3'] expected_patches = 5 self.run_query(files, expected_patches) + + +class TestGerritWeb(ZuulTestCase): + config_file = 'zuul-gerrit-web.conf' + tenant_config_file = 'config/single-tenant/main.yaml' + + def test_jobs_executed(self): + "Test that jobs are executed and a change is merged" + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + A.addApproval('Code-Review', 2) + self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) + self.waitUntilSettled() + self.assertEqual(self.getJobFromHistory('project-merge').result, + 'SUCCESS') + self.assertEqual(self.getJobFromHistory('project-test1').result, + 'SUCCESS') + self.assertEqual(self.getJobFromHistory('project-test2').result, + 'SUCCESS') + self.assertEqual(A.data['status'], 'MERGED') + self.assertEqual(A.reported, 2) + self.assertEqual(self.getJobFromHistory('project-test1').node, + 'label1') + self.assertEqual(self.getJobFromHistory('project-test2').node, + 'label1') diff --git a/zuul/driver/gerrit/auth.py b/zuul/driver/gerrit/auth.py new file mode 100644 index 0000000000..61836e347f --- /dev/null +++ b/zuul/driver/gerrit/auth.py @@ -0,0 +1,69 @@ +# Copyright 2015 Christoph Gysin +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import requests + +from urllib.parse import urlparse + + +class FormAuth(requests.auth.AuthBase): + log = logging.getLogger('zuul.GerritConnection') + + def __init__(self, username, password): + self.username = username + self.password = password + + def _retry_using_form_auth(self, response, args): + adapter = requests.adapters.HTTPAdapter() + request = _copy_request(response.request) + + u = urlparse.urlparse(response.url) + url = urlparse.urlunparse([u.scheme, u.netloc, '/login', + None, None, None]) + auth = {'username': self.username, + 'password': self.password} + request2 = requests.Request('POST', url, data=auth).prepare() + response2 = adapter.send(request2, **args) + + if response2.status_code == 401: + self.log.error('Login failed: Invalid username or password?') + return response + + cookie = response2.headers.get('set-cookie') + if cookie is not None: + request.headers['Cookie'] = cookie + + response3 = adapter.send(request, **args) + return response3 + + def _response_hook(self, response, **kwargs): + if response.status_code == 401: + return self._retry_using_form_auth(response, kwargs) + return response + + def __call__(self, request): + request.headers["Connection"] = "Keep-Alive" + request.register_hook('response', self._response_hook) + return request + + +def _copy_request(request): + new_request = requests.PreparedRequest() + new_request.method = request.method + new_request.url = request.url + new_request.body = request.body + new_request.hooks = request.hooks + new_request.headers = request.headers.copy() + return new_request diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py index 6117983389..c81efef43e 100644 --- a/zuul/driver/gerrit/gerritconnection.py +++ b/zuul/driver/gerrit/gerritconnection.py @@ -25,6 +25,7 @@ import pprint import shlex import queue import voluptuous as v +import requests from typing import Dict, List @@ -32,6 +33,11 @@ from zuul.connection import BaseConnection from zuul.model import Ref, Tag, Branch, Project from zuul import exceptions from zuul.driver.gerrit.gerritmodel import GerritChange, GerritTriggerEvent +from zuul.driver.gerrit.auth import FormAuth +from zuul import version as zuul_version + +# HTTP timeout in seconds +TIMEOUT = 30 class GerritEventConnector(threading.Thread): @@ -317,6 +323,53 @@ class GerritConnection(BaseConnection): self.gerrit_event_connector = None self.source = driver.getSource(self) + self.session = None + self.password = self.connection_config.get('password', None) + if self.password: + self.auth_type = self.connection_config.get('auth_type', None) + self.verify_ssl = self.connection_config.get('verify_ssl', True) + if self.verify_ssl not in ['true', 'True', '1', 1, 'TRUE']: + self.verify_ssl = False + self.user_agent = 'Zuul/%s %s' % ( + zuul_version.release_string, + requests.utils.default_user_agent()) + self.session = requests.Session() + if self.auth_type == 'basic': + authclass = requests.auth.HTTPBasicAuth + elif self.auth_type == 'form': + authclass = FormAuth + else: + authclass = requests.auth.HTTPDigestAuth + self.auth = authclass( + self.user, self.password) + + def url(self, path): + return self.baseurl + '/a/' + path + + def post(self, path, data): + url = self.url(path) + self.log.debug('POST: %s' % (url,)) + self.log.debug('data: %s' % (data,)) + r = self.session.post( + url, data=json.dumps(data).encode('utf8'), + verify=self.verify_ssl, + auth=self.auth, timeout=TIMEOUT, + headers={'Content-Type': 'application/json;charset=UTF-8', + 'User-Agent': self.user_agent}) + self.log.debug('Received: %s %s' % (r.status_code, r.text,)) + if r.status_code != 200: + raise Exception("Received response %s" % (r.status_code,)) + ret = None + if r.text and len(r.text) > 4: + try: + ret = json.loads(r.text[4:]) + except Exception: + self.log.exception( + "Unable to parse result %s from post to %s" % + (r.text, url)) + raise + return ret + def getProject(self, name: str) -> Project: return self.projects.get(name) @@ -477,6 +530,7 @@ class GerritConnection(BaseConnection): if 'project' not in data: raise exceptions.ChangeNotFound(change.number, change.patchset) change.project = self.source.getProject(data['project']) + change.id = data['id'] change.branch = data['branch'] change.url = data['url'] urlparse = urllib.parse.urlparse(self.baseurl) @@ -492,6 +546,7 @@ class GerritConnection(BaseConnection): for ps in data['patchSets']: if str(ps['number']) == change.patchset: change.ref = ps['ref'] + change.commit = ps['revision'] for f in ps.get('files', []): files.append(f['file']) if int(ps['number']) > int(max_ps): @@ -721,7 +776,15 @@ class GerritConnection(BaseConnection): def eventDone(self): self.event_queue.task_done() - def review(self, project, change, message, action={}): + def review(self, change, message, action={}): + if self.session: + meth = self.review_http + else: + meth = self.review_ssh + return meth(change, message, action) + + def review_ssh(self, change, message, action={}): + project = change.project.name cmd = 'gerrit review --project %s' % project if message: cmd += ' --message %s' % shlex.quote(message) @@ -730,10 +793,51 @@ class GerritConnection(BaseConnection): cmd += ' --%s' % key else: cmd += ' --label %s=%s' % (key, val) - cmd += ' %s' % change + changeid = '%s,%s' % (change.number, change.patchset) + cmd += ' %s' % changeid out, err = self._ssh(cmd) return err + def review_http(self, change, message, action={}, + file_comments={}): + data = dict(message=message, + strict_labels=False) + submit = False + labels = {} + for key, val in action.items(): + if val is True: + if key == 'submit': + submit = True + else: + labels[key] = val + if change.is_current_patchset: + if labels: + data['labels'] = labels + if file_comments: + data['comments'] = file_comments + # { path: [ + # {line=42, message='foobar'}, + # {line=40, message='baz'}, + # ] + # } + for x in range(1, 4): + try: + self.post('changes/%s/revisions/%s/review' % + (change.id, change.commit), + data) + break + except Exception: + self.log.exception( + "Error submitting data to gerrit, attempt %s", x) + time.sleep(x * 10) + if change.is_current_patchset and submit: + try: + self.post('changes/%s/submit' % (change.id,), {}) + except Exception: + self.log.exception( + "Error submitting data to gerrit, attempt %s", x) + time.sleep(x * 10) + def query(self, query): args = '--all-approvals --comments --commit-message' args += ' --current-patch-set --dependencies --files' diff --git a/zuul/driver/gerrit/gerritreporter.py b/zuul/driver/gerrit/gerritreporter.py index 90c95e344a..5450f277f0 100644 --- a/zuul/driver/gerrit/gerritreporter.py +++ b/zuul/driver/gerrit/gerritreporter.py @@ -42,12 +42,10 @@ class GerritReporter(BaseReporter): self.log.debug("Report change %s, params %s, message: %s" % (item.change, self.config, message)) - changeid = '%s,%s' % (item.change.number, item.change.patchset) item.change._ref_sha = item.change.project.source.getRefSha( item.change.project, 'refs/heads/' + item.change.branch) - return self.connection.review(item.change.project.name, changeid, - message, self.config) + return self.connection.review(item.change, message, self.config) def getSubmitAllowNeeds(self): """Get a list of code review labels that are allowed to be