811e2e9334
Make sure we update the referenced change object on a new gerrit event rather than waiting to remake the queue item. This was a performance regression in the connection changes. Change-Id: I2a967f0347352a7674deb550e34fb94d1d903e89
325 lines
12 KiB
Python
325 lines
12 KiB
Python
# Copyright 2012 Hewlett-Packard Development Company, L.P.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import logging
|
|
import re
|
|
import time
|
|
from zuul import exceptions
|
|
from zuul.model import Change, Ref, NullChange
|
|
from zuul.source import BaseSource
|
|
|
|
|
|
class GerritSource(BaseSource):
|
|
name = 'gerrit'
|
|
log = logging.getLogger("zuul.source.Gerrit")
|
|
replication_timeout = 300
|
|
replication_retry_interval = 5
|
|
|
|
depends_on_re = re.compile(r"^Depends-On: (I[0-9a-f]{40})\s*$",
|
|
re.MULTILINE | re.IGNORECASE)
|
|
|
|
def getRefSha(self, project, ref):
|
|
refs = {}
|
|
try:
|
|
refs = self.connection.getInfoRefs(project)
|
|
except:
|
|
self.log.exception("Exception looking for ref %s" %
|
|
ref)
|
|
sha = refs.get(ref, '')
|
|
return sha
|
|
|
|
def _waitForRefSha(self, project, ref, old_sha=''):
|
|
# Wait for the ref to show up in the repo
|
|
start = time.time()
|
|
while time.time() - start < self.replication_timeout:
|
|
sha = self.getRefSha(project.name, ref)
|
|
if old_sha != sha:
|
|
return True
|
|
time.sleep(self.replication_retry_interval)
|
|
return False
|
|
|
|
def isMerged(self, change, head=None):
|
|
self.log.debug("Checking if change %s is merged" % change)
|
|
if not change.number:
|
|
self.log.debug("Change has no number; considering it merged")
|
|
# Good question. It's probably ref-updated, which, ah,
|
|
# means it's merged.
|
|
return True
|
|
|
|
data = self.connection.query(change.number)
|
|
change._data = data
|
|
change.is_merged = self._isMerged(change)
|
|
if not head:
|
|
return change.is_merged
|
|
if not change.is_merged:
|
|
return False
|
|
|
|
ref = 'refs/heads/' + change.branch
|
|
self.log.debug("Waiting for %s to appear in git repo" % (change))
|
|
if self._waitForRefSha(change.project, ref, change._ref_sha):
|
|
self.log.debug("Change %s is in the git repo" %
|
|
(change))
|
|
return True
|
|
self.log.debug("Change %s did not appear in the git repo" %
|
|
(change))
|
|
return False
|
|
|
|
def _isMerged(self, change):
|
|
data = change._data
|
|
if not data:
|
|
return False
|
|
status = data.get('status')
|
|
if not status:
|
|
return False
|
|
self.log.debug("Change %s status: %s" % (change, status))
|
|
if status == 'MERGED':
|
|
return True
|
|
return False
|
|
|
|
def canMerge(self, change, allow_needs):
|
|
if not change.number:
|
|
self.log.debug("Change has no number; considering it merged")
|
|
# Good question. It's probably ref-updated, which, ah,
|
|
# means it's merged.
|
|
return True
|
|
data = change._data
|
|
if not data:
|
|
return False
|
|
if 'submitRecords' not in data:
|
|
return False
|
|
try:
|
|
for sr in data['submitRecords']:
|
|
if sr['status'] == 'OK':
|
|
return True
|
|
elif sr['status'] == 'NOT_READY':
|
|
for label in sr['labels']:
|
|
if label['status'] in ['OK', 'MAY']:
|
|
continue
|
|
elif label['status'] in ['NEED', 'REJECT']:
|
|
# It may be our own rejection, so we ignore
|
|
if label['label'].lower() not in allow_needs:
|
|
return False
|
|
continue
|
|
else:
|
|
# IMPOSSIBLE
|
|
return False
|
|
else:
|
|
# CLOSED, RULE_ERROR
|
|
return False
|
|
except:
|
|
self.log.exception("Exception determining whether change"
|
|
"%s can merge:" % change)
|
|
return False
|
|
return True
|
|
|
|
def postConfig(self):
|
|
pass
|
|
|
|
def getChange(self, event, project):
|
|
if event.change_number:
|
|
refresh = False
|
|
change = self._getChange(event.change_number, event.patch_number,
|
|
refresh=refresh)
|
|
elif event.ref:
|
|
change = Ref(project)
|
|
change.ref = event.ref
|
|
change.oldrev = event.oldrev
|
|
change.newrev = event.newrev
|
|
change.url = self._getGitwebUrl(project, sha=event.newrev)
|
|
else:
|
|
change = NullChange(project)
|
|
return change
|
|
|
|
def _getChange(self, number, patchset, refresh=False, history=None):
|
|
key = '%s,%s' % (number, patchset)
|
|
change = self.connection.getCachedChange(key)
|
|
if change and not refresh:
|
|
return change
|
|
if not change:
|
|
change = Change(None)
|
|
change.number = number
|
|
change.patchset = patchset
|
|
key = '%s,%s' % (change.number, change.patchset)
|
|
self.connection.updateChangeCache(key, change)
|
|
try:
|
|
self._updateChange(change, history)
|
|
except Exception:
|
|
self.connection.deleteCachedChange(key)
|
|
raise
|
|
return change
|
|
|
|
def getProjectOpenChanges(self, project):
|
|
# This is a best-effort function in case Gerrit is unable to return
|
|
# a particular change. It happens.
|
|
query = "project:%s status:open" % (project.name,)
|
|
self.log.debug("Running query %s to get project open changes" %
|
|
(query,))
|
|
data = self.connection.simpleQuery(query)
|
|
changes = []
|
|
for record in data:
|
|
try:
|
|
changes.append(
|
|
self._getChange(record['number'],
|
|
record['currentPatchSet']['number']))
|
|
except Exception:
|
|
self.log.exception("Unable to query change %s" %
|
|
(record.get('number'),))
|
|
return changes
|
|
|
|
def _getDependsOnFromCommit(self, message):
|
|
records = []
|
|
seen = set()
|
|
for match in self.depends_on_re.findall(message):
|
|
if match in seen:
|
|
self.log.debug("Ignoring duplicate Depends-On: %s" %
|
|
(match,))
|
|
continue
|
|
seen.add(match)
|
|
query = "change:%s" % (match,)
|
|
self.log.debug("Running query %s to find needed changes" %
|
|
(query,))
|
|
records.extend(self.connection.simpleQuery(query))
|
|
return records
|
|
|
|
def _getNeededByFromCommit(self, change_id):
|
|
records = []
|
|
seen = set()
|
|
query = 'message:%s' % change_id
|
|
self.log.debug("Running query %s to find changes needed-by" %
|
|
(query,))
|
|
results = self.connection.simpleQuery(query)
|
|
for result in results:
|
|
for match in self.depends_on_re.findall(
|
|
result['commitMessage']):
|
|
if match != change_id:
|
|
continue
|
|
key = (result['number'], result['currentPatchSet']['number'])
|
|
if key in seen:
|
|
continue
|
|
self.log.debug("Found change %s,%s needs %s from commit" %
|
|
(key[0], key[1], change_id))
|
|
seen.add(key)
|
|
records.append(result)
|
|
return records
|
|
|
|
def _updateChange(self, change, history=None):
|
|
self.log.info("Updating information for %s,%s" %
|
|
(change.number, change.patchset))
|
|
data = self.connection.query(change.number)
|
|
change._data = data
|
|
|
|
if change.patchset is None:
|
|
change.patchset = data['currentPatchSet']['number']
|
|
|
|
if 'project' not in data:
|
|
raise exceptions.ChangeNotFound(change.number, change.patchset)
|
|
# If updated changed came as a dependent on
|
|
# and its project is not defined,
|
|
# then create a 'foreign' project for it in layout
|
|
change.project = self.sched.getProject(data['project'],
|
|
create_foreign=bool(history))
|
|
change.branch = data['branch']
|
|
change.url = data['url']
|
|
max_ps = 0
|
|
files = []
|
|
for ps in data['patchSets']:
|
|
if ps['number'] == change.patchset:
|
|
change.refspec = ps['ref']
|
|
for f in ps.get('files', []):
|
|
files.append(f['file'])
|
|
if int(ps['number']) > int(max_ps):
|
|
max_ps = ps['number']
|
|
if max_ps == change.patchset:
|
|
change.is_current_patchset = True
|
|
else:
|
|
change.is_current_patchset = False
|
|
change.files = files
|
|
|
|
change.is_merged = self._isMerged(change)
|
|
change.approvals = data['currentPatchSet'].get('approvals', [])
|
|
change.open = data['open']
|
|
change.status = data['status']
|
|
change.owner = data['owner']
|
|
|
|
if change.is_merged:
|
|
# This change is merged, so we don't need to look any further
|
|
# for dependencies.
|
|
return change
|
|
|
|
if history is None:
|
|
history = []
|
|
else:
|
|
history = history[:]
|
|
history.append(change.number)
|
|
|
|
needs_changes = []
|
|
if 'dependsOn' in data:
|
|
parts = data['dependsOn'][0]['ref'].split('/')
|
|
dep_num, dep_ps = parts[3], parts[4]
|
|
if dep_num in history:
|
|
raise Exception("Dependency cycle detected: %s in %s" % (
|
|
dep_num, history))
|
|
self.log.debug("Getting git-dependent change %s,%s" %
|
|
(dep_num, dep_ps))
|
|
dep = self._getChange(dep_num, dep_ps, history=history)
|
|
if (not dep.is_merged) and dep not in needs_changes:
|
|
needs_changes.append(dep)
|
|
|
|
for record in self._getDependsOnFromCommit(data['commitMessage']):
|
|
dep_num = record['number']
|
|
dep_ps = record['currentPatchSet']['number']
|
|
if dep_num in history:
|
|
raise Exception("Dependency cycle detected: %s in %s" % (
|
|
dep_num, history))
|
|
self.log.debug("Getting commit-dependent change %s,%s" %
|
|
(dep_num, dep_ps))
|
|
dep = self._getChange(dep_num, dep_ps, history=history)
|
|
if (not dep.is_merged) and dep not in needs_changes:
|
|
needs_changes.append(dep)
|
|
change.needs_changes = needs_changes
|
|
|
|
needed_by_changes = []
|
|
if 'neededBy' in data:
|
|
for needed in data['neededBy']:
|
|
parts = needed['ref'].split('/')
|
|
dep_num, dep_ps = parts[3], parts[4]
|
|
dep = self._getChange(dep_num, dep_ps)
|
|
if (not dep.is_merged) and dep.is_current_patchset:
|
|
needed_by_changes.append(dep)
|
|
|
|
for record in self._getNeededByFromCommit(data['id']):
|
|
dep_num = record['number']
|
|
dep_ps = record['currentPatchSet']['number']
|
|
self.log.debug("Getting commit-needed change %s,%s" %
|
|
(dep_num, dep_ps))
|
|
# Because a commit needed-by may be a cross-repo
|
|
# dependency, cause that change to refresh so that it will
|
|
# reference the latest patchset of its Depends-On (this
|
|
# change).
|
|
dep = self._getChange(dep_num, dep_ps, refresh=True)
|
|
if (not dep.is_merged) and dep.is_current_patchset:
|
|
needed_by_changes.append(dep)
|
|
change.needed_by_changes = needed_by_changes
|
|
|
|
return change
|
|
|
|
def getGitUrl(self, project):
|
|
return self.connection.getGitUrl(project)
|
|
|
|
def _getGitwebUrl(self, project, sha=None):
|
|
return self.connection.getGitwebUrl(project, sha)
|
|
|
|
def maintainCache(self, relevant):
|
|
self.connection.maintainCache(relevant)
|