Support HTTP-only Gerrit
This adds support for performing queries and git clones over HTTP (reporting over HTTP was already supported). This will happen automatically for any connection with a password configured. Otherwise, the SSH connection will be used as before. Change-Id: I11920a13615de103eb3d8fb305eacbbcb30e5e40
This commit is contained in:
James E. Blair
parent
4d8325a3cc
commit
947b7b1dcb
171
tests/base.py
171
tests/base.py
|
|
@ -42,6 +42,7 @@ import time
|
|||
import uuid
|
||||
import socketserver
|
||||
import http.server
|
||||
import urllib.parse
|
||||
|
||||
import git
|
||||
import gear
|
||||
|
|
@ -188,6 +189,7 @@ class FakeGerritChange(object):
|
|||
self.subject = subject
|
||||
self.latest_patchset = 0
|
||||
self.depends_on_change = None
|
||||
self.depends_on_patchset = None
|
||||
self.needed_by_changes = []
|
||||
self.fail_merge = False
|
||||
self.messages = []
|
||||
|
|
@ -490,13 +492,14 @@ class FakeGerritChange(object):
|
|||
|
||||
def setDependsOn(self, other, patchset):
|
||||
self.depends_on_change = other
|
||||
self.depends_on_patchset = patchset
|
||||
d = {'id': other.data['id'],
|
||||
'number': other.data['number'],
|
||||
'ref': other.patchsets[patchset - 1]['ref']
|
||||
}
|
||||
self.data['dependsOn'] = [d]
|
||||
|
||||
other.needed_by_changes.append(self)
|
||||
other.needed_by_changes.append((self, len(self.patchsets)))
|
||||
needed = other.data.get('neededBy', [])
|
||||
d = {'id': self.data['id'],
|
||||
'number': self.data['number'],
|
||||
|
|
@ -517,6 +520,100 @@ class FakeGerritChange(object):
|
|||
d['isCurrentPatchSet'] = False
|
||||
return json.loads(json.dumps(self.data))
|
||||
|
||||
def queryHTTP(self):
|
||||
self.queried += 1
|
||||
labels = {}
|
||||
for cat in self.categories:
|
||||
labels[cat] = {}
|
||||
for app in self.patchsets[-1]['approvals']:
|
||||
label = labels[app['type']]
|
||||
_, label_min, label_max = self.categories[app['type']]
|
||||
val = int(app['value'])
|
||||
label_all = label.setdefault('all', [])
|
||||
label_all.append({
|
||||
"value": val,
|
||||
"username": app['by']['username'],
|
||||
"email": app['by']['email'],
|
||||
"date": str(datetime.datetime.fromtimestamp(app['grantedOn'])),
|
||||
})
|
||||
if val == label_min:
|
||||
label['blocking'] = True
|
||||
if 'rejected' not in label:
|
||||
label['rejected'] = app['by']
|
||||
if val == label_max:
|
||||
if 'approved' not in label:
|
||||
label['approved'] = app['by']
|
||||
revisions = {}
|
||||
rev = self.patchsets[-1]
|
||||
num = len(self.patchsets)
|
||||
files = {}
|
||||
for f in rev['files']:
|
||||
files[f['file']] = {"status": f['type'][0]} # ADDED -> A
|
||||
parent = '0000000000000000000000000000000000000000'
|
||||
if self.depends_on_change:
|
||||
parent = self.depends_on_change.patchsets[
|
||||
self.depends_on_patchset - 1]['revision']
|
||||
revisions[rev['revision']] = {
|
||||
"kind": "REWORK",
|
||||
"_number": num,
|
||||
"created": rev['createdOn'],
|
||||
"uploader": rev['uploader'],
|
||||
"ref": rev['ref'],
|
||||
"commit": {
|
||||
"subject": self.subject,
|
||||
"message": self.data['commitMessage'],
|
||||
"parents": [{
|
||||
"commit": parent,
|
||||
}]
|
||||
},
|
||||
"files": files
|
||||
}
|
||||
data = {
|
||||
"id": self.project + '~' + self.branch + '~' + self.data['id'],
|
||||
"project": self.project,
|
||||
"branch": self.branch,
|
||||
"hashtags": [],
|
||||
"change_id": self.data['id'],
|
||||
"subject": self.subject,
|
||||
"status": self.data['status'],
|
||||
"created": self.data['createdOn'],
|
||||
"updated": self.data['lastUpdated'],
|
||||
"_number": self.number,
|
||||
"owner": self.data['owner'],
|
||||
"labels": labels,
|
||||
"current_revision": self.patchsets[-1]['revision'],
|
||||
"revisions": revisions,
|
||||
"requirements": []
|
||||
}
|
||||
return json.loads(json.dumps(data))
|
||||
|
||||
def queryRevisionHTTP(self, revision):
|
||||
for ps in self.patchsets:
|
||||
if ps['revision'] == revision:
|
||||
break
|
||||
else:
|
||||
return None
|
||||
changes = []
|
||||
if self.depends_on_change:
|
||||
changes.append({
|
||||
"commit": {
|
||||
"commit": self.depends_on_change.patchsets[
|
||||
self.depends_on_patchset - 1]['revision'],
|
||||
},
|
||||
"_change_number": self.depends_on_change.number,
|
||||
"_revision_number": self.depends_on_patchset
|
||||
})
|
||||
for (needed_by_change, needed_by_patchset) in self.needed_by_changes:
|
||||
changes.append({
|
||||
"commit": {
|
||||
"commit": needed_by_change.patchsets[
|
||||
needed_by_patchset - 1]['revision'],
|
||||
},
|
||||
"_change_number": needed_by_change.number,
|
||||
"_revision_number": needed_by_patchset,
|
||||
})
|
||||
return {"changes": changes}
|
||||
|
||||
def setMerged(self):
|
||||
if (self.depends_on_change and
|
||||
self.depends_on_change.data['status'] != 'MERGED'):
|
||||
|
|
@ -554,6 +651,9 @@ class GerritWebServer(object):
|
|||
update_checks_re = re.compile(
|
||||
r'/a/changes/(.*)/revisions/(.*?)/checks/(.*)')
|
||||
list_checkers_re = re.compile('/a/plugins/checks/checkers/')
|
||||
change_re = re.compile(r'/a/changes/(.*)\?o=.*')
|
||||
related_re = re.compile(r'/a/changes/(.*)/revisions/(.*)/related')
|
||||
change_search_re = re.compile(r'/a/changes/\?n=500.*&q=(.*)')
|
||||
|
||||
def do_POST(self):
|
||||
path = self.path
|
||||
|
|
@ -580,6 +680,15 @@ class GerritWebServer(object):
|
|||
path = self.path
|
||||
self.log.debug("Got GET %s", path)
|
||||
|
||||
m = self.change_re.match(path)
|
||||
if m:
|
||||
return self.get_change(m.group(1))
|
||||
m = self.related_re.match(path)
|
||||
if m:
|
||||
return self.get_related(m.group(1), m.group(2))
|
||||
m = self.change_search_re.match(path)
|
||||
if m:
|
||||
return self.get_changes(m.group(1))
|
||||
m = self.pending_checks_re.match(path)
|
||||
if m:
|
||||
return self.get_pending_checks(m.group(1), m.group(2))
|
||||
|
|
@ -663,6 +772,40 @@ class GerritWebServer(object):
|
|||
self.log.debug("Get checkers")
|
||||
self.send_data(fake_gerrit.fake_checkers)
|
||||
|
||||
def get_change(self, number):
|
||||
change = fake_gerrit.changes.get(int(number))
|
||||
if not change:
|
||||
return self._404()
|
||||
|
||||
self.send_data(change.queryHTTP())
|
||||
self.end_headers()
|
||||
|
||||
def get_related(self, number, revision):
|
||||
change = fake_gerrit.changes.get(int(number))
|
||||
if not change:
|
||||
return self._404()
|
||||
data = change.queryRevisionHTTP(revision)
|
||||
if data is None:
|
||||
return self._404()
|
||||
self.send_data(data)
|
||||
self.end_headers()
|
||||
|
||||
def get_changes(self, query):
|
||||
self.log.debug("simpleQueryHTTP: %s", query)
|
||||
query = urllib.parse.unquote(query)
|
||||
fake_gerrit.queries.append(query)
|
||||
results = []
|
||||
if query.startswith('(') and 'OR' in query:
|
||||
query = query[1:-1]
|
||||
for q in query.split(' OR '):
|
||||
for r in fake_gerrit._simpleQuery(q, http=True):
|
||||
if r not in results:
|
||||
results.append(r)
|
||||
else:
|
||||
results = fake_gerrit._simpleQuery(query, http=True)
|
||||
self.send_data(results)
|
||||
self.end_headers()
|
||||
|
||||
def send_data(self, data):
|
||||
data = json.dumps(data).encode('utf-8')
|
||||
data = b")]}'\n" + data
|
||||
|
|
@ -852,25 +995,27 @@ class FakeGerritConnection(gerritconnection.GerritConnection):
|
|||
if message:
|
||||
change.setReported()
|
||||
|
||||
def query(self, number, event=None):
|
||||
if type(number) == int:
|
||||
return self.queryChange(number, event=event)
|
||||
raise Exception("Could not query %s %s" % (type(number, number)))
|
||||
|
||||
def queryChange(self, number, event=None):
|
||||
def queryChangeSSH(self, number, event=None):
|
||||
self.log.debug("Query change SSH: %s", number)
|
||||
change = self.changes.get(int(number))
|
||||
if change:
|
||||
return change.query()
|
||||
return {}
|
||||
|
||||
def _simpleQuery(self, query):
|
||||
def _simpleQuery(self, query, http=False):
|
||||
if http:
|
||||
def queryMethod(change):
|
||||
return change.queryHTTP()
|
||||
else:
|
||||
def queryMethod(change):
|
||||
return change.query()
|
||||
# the query can be in parenthesis so strip them if needed
|
||||
if query.startswith('('):
|
||||
query = query[1:-1]
|
||||
if query.startswith('change:'):
|
||||
# Query a specific changeid
|
||||
changeid = query[len('change:'):]
|
||||
l = [change.query() for change in self.changes.values()
|
||||
l = [queryMethod(change) for change in self.changes.values()
|
||||
if (change.data['id'] == changeid or
|
||||
change.data['number'] == changeid)]
|
||||
elif query.startswith('message:'):
|
||||
|
|
@ -879,16 +1024,16 @@ class FakeGerritConnection(gerritconnection.GerritConnection):
|
|||
# Remove quoting if it is there
|
||||
if msg.startswith('{') and msg.endswith('}'):
|
||||
msg = msg[1:-1]
|
||||
l = [change.query() for change in self.changes.values()
|
||||
l = [queryMethod(change) for change in self.changes.values()
|
||||
if msg in change.data['commitMessage']]
|
||||
else:
|
||||
# Query all open changes
|
||||
l = [change.query() for change in self.changes.values()]
|
||||
l = [queryMethod(change) for change in self.changes.values()]
|
||||
return l
|
||||
|
||||
def simpleQuery(self, query, event=None):
|
||||
def simpleQuerySSH(self, query, event=None):
|
||||
log = get_annotated_logger(self.log, event)
|
||||
log.debug("simpleQuery: %s", query)
|
||||
log.debug("simpleQuerySSH: %s", query)
|
||||
self.queries.append(query)
|
||||
results = []
|
||||
if query.startswith('(') and 'OR' in query:
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ URL_FORMATS = [
|
|||
'{baseurl}/{change_no}',
|
||||
'{baseurl}/#/c/{change_no}',
|
||||
'{baseurl}/c/{project}/+/{change_no}/',
|
||||
'{change_id}',
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -60,7 +61,8 @@ class TestGerritCRD(ZuulTestCase):
|
|||
|
||||
url = url_fmt.format(baseurl=B.gerrit.baseurl.rstrip('/'),
|
||||
project=B.project,
|
||||
change_no=B.number)
|
||||
change_no=B.number,
|
||||
change_id=B.data['id'])
|
||||
A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
|
||||
A.subject, url)
|
||||
|
||||
|
|
@ -101,6 +103,7 @@ class TestGerritCRD(ZuulTestCase):
|
|||
# changes - repeat the simple test on each of the 3 to ensure they can be
|
||||
# parsed, the other tests just use the default URL schema provided in
|
||||
# FakeGerritChange.data['url'] .
|
||||
# This list also includes the legacy change id.
|
||||
def test_crd_gate_url_schema0(self):
|
||||
self._test_crd_gate(URL_FORMATS[0])
|
||||
|
||||
|
|
@ -110,6 +113,9 @@ class TestGerritCRD(ZuulTestCase):
|
|||
def test_crd_gate_url_schema2(self):
|
||||
self._test_crd_gate(URL_FORMATS[2])
|
||||
|
||||
def test_crd_gate_legacy_id(self):
|
||||
self._test_crd_gate(URL_FORMATS[3])
|
||||
|
||||
def test_crd_gate_triangle(self):
|
||||
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
|
||||
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ from uuid import uuid4
|
|||
from zuul.connection import BaseConnection
|
||||
from zuul.lib.logutil import get_annotated_logger
|
||||
from zuul.model import Ref, Tag, Branch, Project
|
||||
from zuul import exceptions
|
||||
from zuul.driver.gerrit.gerritmodel import GerritChange, GerritTriggerEvent
|
||||
from zuul.driver.gerrit.auth import FormAuth
|
||||
from zuul import version as zuul_version
|
||||
|
|
@ -44,6 +43,68 @@ from zuul import version as zuul_version
|
|||
TIMEOUT = 30
|
||||
|
||||
|
||||
class GerritChangeData(object):
|
||||
"""Compatability layer for SSH/HTTP
|
||||
|
||||
This class holds the raw data returned from a change query over
|
||||
SSH or HTTP. Most of the work of parsing the data and storing it
|
||||
on the change is in the gerritmodel.GerritChange class, however
|
||||
this does perform a small amount of parsing of dependencies since
|
||||
they are handled outside of that class. This provides an API to
|
||||
that data independent of the source.
|
||||
|
||||
"""
|
||||
|
||||
SSH = 1
|
||||
HTTP = 2
|
||||
|
||||
def __init__(self, fmt, data, related=None):
|
||||
self.format = fmt
|
||||
self.data = data
|
||||
|
||||
if fmt == self.SSH:
|
||||
self.parseSSH(data)
|
||||
else:
|
||||
self.parseHTTP(data)
|
||||
if related:
|
||||
self.parseRelatedHTTP(data, related)
|
||||
|
||||
def parseSSH(self, data):
|
||||
self.needed_by = []
|
||||
self.depends_on = None
|
||||
self.message = data['commitMessage']
|
||||
self.current_patchset = str(data['currentPatchSet']['number'])
|
||||
self.number = str(data['number'])
|
||||
|
||||
if 'dependsOn' in data:
|
||||
parts = data['dependsOn'][0]['ref'].split('/')
|
||||
self.depends_on = (parts[3], parts[4])
|
||||
|
||||
for needed in data.get('neededBy', []):
|
||||
parts = needed['ref'].split('/')
|
||||
self.needed_by.append((parts[3], parts[4]))
|
||||
|
||||
def parseHTTP(self, data):
|
||||
rev = data['revisions'][data['current_revision']]
|
||||
self.message = rev['commit']['message']
|
||||
self.current_patchset = str(rev['_number'])
|
||||
self.number = str(data['_number'])
|
||||
|
||||
def parseRelatedHTTP(self, data, related):
|
||||
self.needed_by = []
|
||||
self.depends_on = None
|
||||
current_rev = data['revisions'][data['current_revision']]
|
||||
for change in related['changes']:
|
||||
for parent in current_rev['commit']['parents']:
|
||||
if change['commit']['commit'] == parent['commit']:
|
||||
self.depends_on = (change['_change_number'],
|
||||
change['_revision_number'])
|
||||
break
|
||||
else:
|
||||
self.needed_by.append((change['_change_number'],
|
||||
change['_revision_number']))
|
||||
|
||||
|
||||
class GerritEventConnector(threading.Thread):
|
||||
"""Move events from Gerrit to the scheduler."""
|
||||
|
||||
|
|
@ -380,6 +441,13 @@ class GerritConnection(BaseConnection):
|
|||
self.port = int(self.connection_config.get('port', 29418))
|
||||
self.keyfile = self.connection_config.get('sshkey', None)
|
||||
self.keepalive = int(self.connection_config.get('keepalive', 60))
|
||||
# TODO(corvus): Document this when the checks api is stable;
|
||||
# it's not useful without it.
|
||||
self.enable_stream_events = self.connection_config.get(
|
||||
'stream_events', True)
|
||||
if self.enable_stream_events not in [
|
||||
'true', 'True', '1', 1, 'TRUE', True]:
|
||||
self.enable_stream_events = False
|
||||
self.watcher_thread = None
|
||||
self.poller_thread = None
|
||||
self.event_queue = queue.Queue()
|
||||
|
|
@ -627,7 +695,7 @@ class GerritConnection(BaseConnection):
|
|||
log.debug("Updating %s: Running query %s to find needed changes",
|
||||
change, query)
|
||||
records.extend(self.simpleQuery(query, event=event))
|
||||
return records
|
||||
return [(x.number, x.current_patchset) for x in records]
|
||||
|
||||
def _getNeededByFromCommit(self, change_id, change, event):
|
||||
log = get_annotated_logger(self.log, event)
|
||||
|
|
@ -639,11 +707,10 @@ class GerritConnection(BaseConnection):
|
|||
results = self.simpleQuery(query, event=event)
|
||||
for result in results:
|
||||
for match in self.depends_on_re.findall(
|
||||
result['commitMessage']):
|
||||
result.message):
|
||||
if match != change_id:
|
||||
continue
|
||||
key = (str(result['number']),
|
||||
str(result['currentPatchSet']['number']))
|
||||
key = (result.number, result.current_patchset)
|
||||
if key in seen:
|
||||
continue
|
||||
log.debug("Updating %s: Found change %s,%s "
|
||||
|
|
@ -651,7 +718,7 @@ class GerritConnection(BaseConnection):
|
|||
change, key[0], key[1], change_id)
|
||||
seen.add(key)
|
||||
records.append(result)
|
||||
return records
|
||||
return [(x.number, x.current_patchset) for x in records]
|
||||
|
||||
def _updateChange(self, change, event, history):
|
||||
log = get_annotated_logger(self.log, event)
|
||||
|
|
@ -679,54 +746,16 @@ class GerritConnection(BaseConnection):
|
|||
|
||||
log.info("Updating %s", change)
|
||||
data = self.queryChange(change.number, event=event)
|
||||
change._data = data
|
||||
change.update(data, self)
|
||||
|
||||
if change.patchset is None:
|
||||
change.patchset = str(data['currentPatchSet']['number'])
|
||||
if 'project' not in data:
|
||||
raise exceptions.ChangeNotFound(change.number, change.patchset)
|
||||
change.project = self.source.getProject(data['project'])
|
||||
change.id = data['id']
|
||||
change.branch = data['branch']
|
||||
change.url = data['url']
|
||||
urlparse = urllib.parse.urlparse(self.baseurl)
|
||||
baseurl = "%s%s" % (urlparse.netloc, urlparse.path)
|
||||
baseurl = baseurl.rstrip('/')
|
||||
change.uris = [
|
||||
'%s/%s' % (baseurl, change.number),
|
||||
'%s/#/c/%s' % (baseurl, change.number),
|
||||
'%s/c/%s/+/%s' % (baseurl, change.project.name, change.number),
|
||||
]
|
||||
if not change.is_merged:
|
||||
self._updateChangeDependencies(log, change, data, event, history)
|
||||
|
||||
max_ps = 0
|
||||
files = []
|
||||
for ps in data['patchSets']:
|
||||
if str(ps['number']) == change.patchset:
|
||||
change.ref = ps['ref']
|
||||
change.commit = ps['revision']
|
||||
for f in ps.get('files', []):
|
||||
files.append(f['file'])
|
||||
if int(ps['number']) > int(max_ps):
|
||||
max_ps = str(ps['number'])
|
||||
if max_ps == change.patchset:
|
||||
change.is_current_patchset = True
|
||||
else:
|
||||
change.is_current_patchset = False
|
||||
change.files = files
|
||||
self.sched.onChangeUpdated(change, event)
|
||||
|
||||
change.is_merged = self._isMerged(change)
|
||||
change.approvals = data['currentPatchSet'].get('approvals', [])
|
||||
change.open = data['open']
|
||||
change.status = data['status']
|
||||
change.owner = data['owner']
|
||||
change.message = data['commitMessage']
|
||||
|
||||
if change.is_merged:
|
||||
# This change is merged, so we don't need to look any further
|
||||
# for dependencies.
|
||||
log.debug("Updating %s: change is merged", change)
|
||||
return change
|
||||
return change
|
||||
|
||||
def _updateChangeDependencies(self, log, change, data, event, history):
|
||||
if history is None:
|
||||
history = []
|
||||
else:
|
||||
|
|
@ -735,9 +764,8 @@ class GerritConnection(BaseConnection):
|
|||
|
||||
needs_changes = set()
|
||||
git_needs_changes = []
|
||||
if 'dependsOn' in data:
|
||||
parts = data['dependsOn'][0]['ref'].split('/')
|
||||
dep_num, dep_ps = parts[3], parts[4]
|
||||
if data.depends_on is not None:
|
||||
dep_num, dep_ps = data.depends_on
|
||||
log.debug("Updating %s: Getting git-dependent change %s,%s",
|
||||
change, dep_num, dep_ps)
|
||||
dep = self._getChange(dep_num, dep_ps, history=history,
|
||||
|
|
@ -751,10 +779,8 @@ class GerritConnection(BaseConnection):
|
|||
change.git_needs_changes = git_needs_changes
|
||||
|
||||
compat_needs_changes = []
|
||||
for record in self._getDependsOnFromCommit(data['commitMessage'],
|
||||
change, event):
|
||||
dep_num = str(record['number'])
|
||||
dep_ps = str(record['currentPatchSet']['number'])
|
||||
for (dep_num, dep_ps) in self._getDependsOnFromCommit(
|
||||
change.message, change, event):
|
||||
log.debug("Updating %s: Getting commit-dependent "
|
||||
"change %s,%s", change, dep_num, dep_ps)
|
||||
dep = self._getChange(dep_num, dep_ps, history=history,
|
||||
|
|
@ -766,24 +792,20 @@ class GerritConnection(BaseConnection):
|
|||
|
||||
needed_by_changes = set()
|
||||
git_needed_by_changes = []
|
||||
if 'neededBy' in data:
|
||||
for needed in data['neededBy']:
|
||||
parts = needed['ref'].split('/')
|
||||
dep_num, dep_ps = parts[3], parts[4]
|
||||
log.debug("Updating %s: Getting git-needed change %s,%s",
|
||||
change, dep_num, dep_ps)
|
||||
dep = self._getChange(dep_num, dep_ps, history=history,
|
||||
event=event)
|
||||
if (dep.open and dep.is_current_patchset and
|
||||
dep not in needed_by_changes):
|
||||
git_needed_by_changes.append(dep)
|
||||
needed_by_changes.add(dep)
|
||||
for (dep_num, dep_ps) in data.needed_by:
|
||||
log.debug("Updating %s: Getting git-needed change %s,%s",
|
||||
change, dep_num, dep_ps)
|
||||
dep = self._getChange(dep_num, dep_ps, history=history,
|
||||
event=event)
|
||||
if (dep.open and dep.is_current_patchset and
|
||||
dep not in needed_by_changes):
|
||||
git_needed_by_changes.append(dep)
|
||||
needed_by_changes.add(dep)
|
||||
change.git_needed_by_changes = git_needed_by_changes
|
||||
|
||||
compat_needed_by_changes = []
|
||||
for record in self._getNeededByFromCommit(data['id'], change, event):
|
||||
dep_num = str(record['number'])
|
||||
dep_ps = str(record['currentPatchSet']['number'])
|
||||
for (dep_num, dep_ps) in self._getNeededByFromCommit(
|
||||
change.id, change, event):
|
||||
log.debug("Updating %s: Getting commit-needed change %s,%s",
|
||||
change, dep_num, dep_ps)
|
||||
# Because a commit needed-by may be a cross-repo
|
||||
|
|
@ -800,10 +822,6 @@ class GerritConnection(BaseConnection):
|
|||
needed_by_changes.add(dep)
|
||||
change.compat_needed_by_changes = compat_needed_by_changes
|
||||
|
||||
self.sched.onChangeUpdated(change, event)
|
||||
|
||||
return change
|
||||
|
||||
def isMerged(self, change, head=None):
|
||||
self.log.debug("Checking if change %s is merged" % change)
|
||||
if not change.number:
|
||||
|
|
@ -813,8 +831,7 @@ class GerritConnection(BaseConnection):
|
|||
return True
|
||||
|
||||
data = self.queryChange(change.number)
|
||||
change._data = data
|
||||
change.is_merged = self._isMerged(change)
|
||||
change.update(data, self)
|
||||
if change.is_merged:
|
||||
self.log.debug("Change %s is merged" % (change,))
|
||||
else:
|
||||
|
|
@ -834,17 +851,6 @@ class GerritConnection(BaseConnection):
|
|||
(change))
|
||||
return False
|
||||
|
||||
def _isMerged(self, change):
|
||||
data = change._data
|
||||
if not data:
|
||||
return False
|
||||
status = data.get('status')
|
||||
if not status:
|
||||
return False
|
||||
if status == 'MERGED':
|
||||
return True
|
||||
return False
|
||||
|
||||
def _waitForRefSha(self, project: Project,
|
||||
ref: str, old_sha: str='') -> bool:
|
||||
# Wait for the ref to show up in the repo
|
||||
|
|
@ -873,35 +879,9 @@ class GerritConnection(BaseConnection):
|
|||
# Good question. It's probably ref-updated, which, ah,
|
||||
# means it's merged.
|
||||
return True
|
||||
data = change._data
|
||||
if not data:
|
||||
return False
|
||||
if 'submitRecords' not in data:
|
||||
return False
|
||||
try:
|
||||
for sr in data['submitRecords']:
|
||||
if sr['status'] == 'OK':
|
||||
return True
|
||||
elif sr['status'] == 'NOT_READY':
|
||||
for label in sr['labels']:
|
||||
if label['status'] in ['OK', 'MAY']:
|
||||
continue
|
||||
elif label['status'] in ['NEED', 'REJECT']:
|
||||
# It may be our own rejection, so we ignore
|
||||
if label['label'] not in allow_needs:
|
||||
return False
|
||||
continue
|
||||
else:
|
||||
# IMPOSSIBLE
|
||||
return False
|
||||
else:
|
||||
# CLOSED, RULE_ERROR
|
||||
return False
|
||||
except Exception:
|
||||
log.exception("Exception determining whether change"
|
||||
"%s can merge:", change)
|
||||
return False
|
||||
return True
|
||||
if change.missing_labels < set(allow_needs):
|
||||
return True
|
||||
return False
|
||||
|
||||
def getProjectOpenChanges(self, project: Project) -> List[GerritChange]:
|
||||
# This is a best-effort function in case Gerrit is unable to return
|
||||
|
|
@ -914,11 +894,10 @@ class GerritConnection(BaseConnection):
|
|||
for record in data:
|
||||
try:
|
||||
changes.append(
|
||||
self._getChange(record['number'],
|
||||
record['currentPatchSet']['number']))
|
||||
self._getChange(record.number, record.current_patchset))
|
||||
except Exception:
|
||||
self.log.exception("Unable to query change %s" %
|
||||
(record.get('number'),))
|
||||
self.log.exception("Unable to query change %s",
|
||||
record.number)
|
||||
return changes
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -1065,12 +1044,11 @@ class GerritConnection(BaseConnection):
|
|||
"Error submitting data to gerrit, attempt %s", x)
|
||||
time.sleep(x * 10)
|
||||
|
||||
def query(self, query, event=None):
|
||||
def queryChangeSSH(self, number, event=None):
|
||||
args = '--all-approvals --comments --commit-message'
|
||||
args += ' --current-patch-set --dependencies --files'
|
||||
args += ' --patch-sets --submit-records'
|
||||
cmd = 'gerrit query --format json %s %s' % (
|
||||
args, query)
|
||||
cmd = 'gerrit query --format json %s %s' % (args, number)
|
||||
out, err = self._ssh(cmd)
|
||||
if not out:
|
||||
return False
|
||||
|
|
@ -1085,10 +1063,23 @@ class GerritConnection(BaseConnection):
|
|||
pprint.pformat(data))
|
||||
return data
|
||||
|
||||
def queryChange(self, number, event=None):
|
||||
return self.query('change:%s' % number, event=event)
|
||||
def queryChangeHTTP(self, number, event=None):
|
||||
data = self.get('changes/%s?o=DETAILED_ACCOUNTS&o=CURRENT_REVISION&'
|
||||
'o=CURRENT_COMMIT&o=CURRENT_FILES&o=LABELS&'
|
||||
'o=DETAILED_LABELS' % (number,))
|
||||
related = self.get('changes/%s/revisions/%s/related' % (
|
||||
number, data['current_revision']))
|
||||
return data, related
|
||||
|
||||
def simpleQuery(self, query, event=None):
|
||||
def queryChange(self, number, event=None):
|
||||
if self.session:
|
||||
data, related = self.queryChangeHTTP(number, event=event)
|
||||
return GerritChangeData(GerritChangeData.HTTP, data, related)
|
||||
else:
|
||||
data = self.queryChangeSSH(number, event=event)
|
||||
return GerritChangeData(GerritChangeData.SSH, data)
|
||||
|
||||
def simpleQuerySSH(self, query, event=None):
|
||||
def _query_chunk(query, event):
|
||||
args = '--commit-message --current-patch-set'
|
||||
|
||||
|
|
@ -1140,9 +1131,63 @@ class GerritConnection(BaseConnection):
|
|||
"%s %s" % (query, resume), event)
|
||||
return alldata
|
||||
|
||||
def simpleQueryHTTP(self, query, event=None):
|
||||
iolog = get_annotated_logger(self.iolog, event)
|
||||
changes = []
|
||||
sortkey = ''
|
||||
done = False
|
||||
offset = 0
|
||||
while not done:
|
||||
# We don't actually want to limit to 500, but that's the
|
||||
# server-side default, and if we don't specify this, we
|
||||
# won't get a _more_changes flag.
|
||||
q = ('changes/?n=500%s&o=CURRENT_REVISION&o=CURRENT_COMMIT&'
|
||||
'q=%s' % (sortkey, query))
|
||||
iolog.debug('Query: %s', q)
|
||||
batch = self.get(q)
|
||||
iolog.debug("Received data from Gerrit query: \n%s",
|
||||
pprint.pformat(batch))
|
||||
done = True
|
||||
if batch:
|
||||
changes += batch
|
||||
if '_more_changes' in batch[-1]:
|
||||
done = False
|
||||
if '_sortkey' in batch[-1]:
|
||||
sortkey = '&N=%s' % (batch[-1]['_sortkey'],)
|
||||
else:
|
||||
offset += len(batch)
|
||||
sortkey = '&start=%s' % (offset,)
|
||||
return changes
|
||||
|
||||
def simpleQuery(self, query, event=None):
|
||||
if self.session:
|
||||
# None of the users of this method require dependency
|
||||
# data, so we only perform the change query and omit the
|
||||
# related changes query.
|
||||
alldata = self.simpleQueryHTTP(query, event=event)
|
||||
return [GerritChangeData(GerritChangeData.HTTP, data)
|
||||
for data in alldata]
|
||||
else:
|
||||
alldata = self.simpleQuerySSH(query, event=event)
|
||||
return [GerritChangeData(GerritChangeData.SSH, data)
|
||||
for data in alldata]
|
||||
|
||||
def _uploadPack(self, project: Project) -> str:
|
||||
cmd = "git-upload-pack %s" % project.name
|
||||
out, err = self._ssh(cmd, "0000")
|
||||
if self.session:
|
||||
url = ('%s/%s/info/refs?service=git-upload-pack' %
|
||||
(self.baseurl, project.name))
|
||||
r = self.session.get(
|
||||
url,
|
||||
verify=self.verify_ssl,
|
||||
auth=self.auth, timeout=TIMEOUT,
|
||||
headers={'User-Agent': self.user_agent})
|
||||
self.iolog.debug('Received: %s %s' % (r.status_code, r.text,))
|
||||
if r.status_code != 200:
|
||||
raise Exception("Received response %s" % (r.status_code,))
|
||||
out = r.text[r.text.find('\n') + 5:]
|
||||
else:
|
||||
cmd = "git-upload-pack %s" % project.name
|
||||
out, err = self._ssh(cmd, "0000")
|
||||
return out
|
||||
|
||||
def _open(self):
|
||||
|
|
@ -1231,8 +1276,14 @@ class GerritConnection(BaseConnection):
|
|||
return ret
|
||||
|
||||
def getGitUrl(self, project: Project) -> str:
|
||||
url = 'ssh://%s@%s:%s/%s' % (self.user, self.server, self.port,
|
||||
project.name)
|
||||
if self.session:
|
||||
baseurl = list(urllib.parse.urlparse(self.baseurl))
|
||||
baseurl[1] = '%s:%s@%s' % (self.user, self.password, baseurl[1])
|
||||
baseurl = urllib.parse.urlunparse(baseurl)
|
||||
url = ('%s/%s' % (baseurl, project.name))
|
||||
else:
|
||||
url = 'ssh://%s@%s:%s/%s' % (self.user, self.server, self.port,
|
||||
project.name)
|
||||
return url
|
||||
|
||||
def _getWebUrl(self, project: Project, sha: str=None) -> str:
|
||||
|
|
@ -1243,7 +1294,8 @@ class GerritConnection(BaseConnection):
|
|||
|
||||
def onLoad(self):
|
||||
self.log.debug("Starting Gerrit Connection/Watchers")
|
||||
self._start_watcher_thread()
|
||||
if self.enable_stream_events:
|
||||
self._start_watcher_thread()
|
||||
self._start_poller_thread()
|
||||
self._start_event_connector()
|
||||
|
||||
|
|
|
|||
|
|
@ -15,10 +15,14 @@
|
|||
import copy
|
||||
import re
|
||||
import time
|
||||
import urllib.parse
|
||||
|
||||
import dateutil.parser
|
||||
|
||||
from zuul.model import EventFilter, RefFilter
|
||||
from zuul.model import Change, TriggerEvent
|
||||
from zuul.driver.util import time_to_seconds
|
||||
from zuul import exceptions
|
||||
|
||||
|
||||
EMPTY_GIT_REF = '0' * 40 # git sha of all zeros, used during creates/deletes
|
||||
|
|
@ -29,6 +33,121 @@ class GerritChange(Change):
|
|||
super(GerritChange, self).__init__(project)
|
||||
self.approvals = []
|
||||
|
||||
def update(self, data, connection):
|
||||
if data.format == data.SSH:
|
||||
self.updateFromSSH(data.data, connection)
|
||||
else:
|
||||
self.updateFromHTTP(data.data, connection)
|
||||
|
||||
def updateFromSSH(self, data, connection):
|
||||
if self.patchset is None:
|
||||
self.patchset = str(data['currentPatchSet']['number'])
|
||||
if 'project' not in data:
|
||||
raise exceptions.ChangeNotFound(self.number, self.patchset)
|
||||
self.project = connection.source.getProject(data['project'])
|
||||
self.id = data['id']
|
||||
self.branch = data['branch']
|
||||
self.url = data['url']
|
||||
urlparse = urllib.parse.urlparse(connection.baseurl)
|
||||
baseurl = "%s%s" % (urlparse.netloc, urlparse.path)
|
||||
baseurl = baseurl.rstrip('/')
|
||||
self.uris = [
|
||||
'%s/%s' % (baseurl, self.number),
|
||||
'%s/#/c/%s' % (baseurl, self.number),
|
||||
'%s/c/%s/+/%s' % (baseurl, self.project.name, self.number),
|
||||
]
|
||||
|
||||
max_ps = 0
|
||||
files = []
|
||||
for ps in data['patchSets']:
|
||||
if str(ps['number']) == self.patchset:
|
||||
self.ref = ps['ref']
|
||||
self.commit = ps['revision']
|
||||
for f in ps.get('files', []):
|
||||
files.append(f['file'])
|
||||
if int(ps['number']) > int(max_ps):
|
||||
max_ps = str(ps['number'])
|
||||
if max_ps == self.patchset:
|
||||
self.is_current_patchset = True
|
||||
else:
|
||||
self.is_current_patchset = False
|
||||
self.files = files
|
||||
|
||||
self.is_merged = data.get('status', '') == 'MERGED'
|
||||
self.approvals = data['currentPatchSet'].get('approvals', [])
|
||||
self.open = data['open']
|
||||
self.status = data['status']
|
||||
self.owner = data['owner']
|
||||
self.message = data['commitMessage']
|
||||
|
||||
self.missing_labels = set()
|
||||
for sr in data.get('submitRecords', []):
|
||||
if sr['status'] == 'NOT_READY':
|
||||
for label in sr['labels']:
|
||||
if label['status'] in ['OK', 'MAY']:
|
||||
continue
|
||||
elif label['status'] in ['NEED', 'REJECT']:
|
||||
self.missing_labels.add(label['label'])
|
||||
|
||||
def updateFromHTTP(self, data, connection):
|
||||
urlparse = urllib.parse.urlparse(connection.baseurl)
|
||||
baseurl = "%s%s" % (urlparse.netloc, urlparse.path)
|
||||
baseurl = baseurl.rstrip('/')
|
||||
current_revision = data['revisions'][data['current_revision']]
|
||||
if self.patchset is None:
|
||||
self.patchset = str(current_revision['_number'])
|
||||
self.project = connection.source.getProject(data['project'])
|
||||
self.id = data['change_id']
|
||||
self.branch = data['branch']
|
||||
self.url = '%s/%s' % (baseurl, self.number)
|
||||
self.uris = [
|
||||
'%s/%s' % (baseurl, self.number),
|
||||
'%s/#/c/%s' % (baseurl, self.number),
|
||||
'%s/c/%s/+/%s' % (baseurl, self.project.name, self.number),
|
||||
]
|
||||
|
||||
files = []
|
||||
if str(current_revision['_number']) == self.patchset:
|
||||
self.ref = current_revision['ref']
|
||||
self.commit = data['current_revision']
|
||||
files = current_revision.get('files', []).keys()
|
||||
self.is_current_patchset = True
|
||||
else:
|
||||
self.is_current_patchset = False
|
||||
self.files = files
|
||||
|
||||
self.is_merged = data['status'] == 'MERGED'
|
||||
self.approvals = []
|
||||
self.missing_labels = set()
|
||||
for label_name, label_data in data.get('labels', {}).items():
|
||||
for app in label_data.get('all', []):
|
||||
if app.get('value', 0) == 0:
|
||||
continue
|
||||
by = {}
|
||||
for k in ('name', 'username', 'email'):
|
||||
if k in app:
|
||||
by[k] = app[k]
|
||||
self.approvals.append({
|
||||
"type": label_name,
|
||||
"description": label_name,
|
||||
"value": app['value'],
|
||||
"grantedOn":
|
||||
dateutil.parser.parse(app['date']).timestamp(),
|
||||
"by": by,
|
||||
})
|
||||
if label_data.get('optional', False):
|
||||
continue
|
||||
if label_data.get('blocking', False):
|
||||
self.missing_labels.add(label_name)
|
||||
continue
|
||||
if 'approved' in label_data:
|
||||
continue
|
||||
self.missing_labels.add(label_name)
|
||||
self.open = data['status'] == 'NEW'
|
||||
self.status = data['status']
|
||||
self.owner = data['owner']
|
||||
self.message = current_revision['commit']['message']
|
||||
|
||||
|
||||
class GerritTriggerEvent(TriggerEvent):
|
||||
"""Incoming event from an external system."""
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ class GerritSource(BaseSource):
|
|||
if not results:
|
||||
return None
|
||||
change = self.connection._getChange(
|
||||
results[0]['number'], results[0]['currentPatchSet']['number'])
|
||||
results[0].number, results[0].current_patchset)
|
||||
return change
|
||||
|
||||
def getChangesDependingOn(self, change, projects, tenant):
|
||||
|
|
@ -89,7 +89,7 @@ class GerritSource(BaseSource):
|
|||
results = self.connection.simpleQuery(query)
|
||||
seen = set()
|
||||
for result in results:
|
||||
for match in find_dependency_headers(result['commitMessage']):
|
||||
for match in find_dependency_headers(result.message):
|
||||
found = False
|
||||
for uri in change.uris:
|
||||
if uri in match:
|
||||
|
|
@ -97,13 +97,12 @@ class GerritSource(BaseSource):
|
|||
break
|
||||
if not found:
|
||||
continue
|
||||
key = (str(result['number']),
|
||||
str(result['currentPatchSet']['number']))
|
||||
key = (result.number, result.current_patchset)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
change = self.connection._getChange(
|
||||
result['number'], result['currentPatchSet']['number'])
|
||||
result.number, result.current_patchset)
|
||||
changes.append(change)
|
||||
return changes
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue