Merge "Annotate github logs with the event id"

This commit is contained in:
Zuul 2019-05-20 18:15:21 +00:00 committed by Gerrit Code Review
commit a690e9f858
1 changed files with 41 additions and 32 deletions

View File

@ -140,26 +140,26 @@ class GithubGearmanWorker(object):
body = args.get("body")
delivery = headers.get('x-github-delivery')
self.log.debug("Github Webhook Received: {delivery}".format(
delivery=delivery))
log = get_annotated_logger(self.log, delivery)
log.debug("Github Webhook Received")
# TODO(jlk): Validate project in the request is a project we know
try:
self.__dispatch_event(body, headers)
self.__dispatch_event(body, headers, log)
output = {'return_code': 200}
except Exception:
output = {'return_code': 503}
self.log.exception("Exception handling Github event:")
log.exception("Exception handling Github event:")
return output
def __dispatch_event(self, body, headers):
def __dispatch_event(self, body, headers, log):
try:
event = headers['x-github-event']
self.log.debug("X-Github-Event: " + event)
log.debug("X-Github-Event: " + event)
except KeyError:
self.log.debug("Request headers missing the X-Github-Event.")
log.debug("Request headers missing the X-Github-Event.")
raise Exception('Please specify a X-Github-Event header.')
delivery = headers.get('x-github-delivery')
@ -167,7 +167,7 @@ class GithubGearmanWorker(object):
self.connection.addEvent(body, event, delivery)
except Exception:
message = 'Exception deserializing JSON body'
self.log.exception(message)
log.exception(message)
# TODO(jlk): Raise this as something different?
raise Exception(message)
@ -251,12 +251,14 @@ class GithubEventProcessor(object):
if event:
event.delivery = self.delivery
event.zuul_event_id = self.delivery
project = self.connection.source.getProject(event.project_name)
if event.change_number:
self.connection._getChange(project,
event.change_number,
event.patch_number,
refresh=True)
refresh=True,
event=event)
self.log.debug("Refreshed change %s,%s",
event.change_number, event.patch_number)
@ -852,7 +854,8 @@ class GithubConnection(BaseConnection):
project = self.source.getProject(event.project_name)
if event.change_number:
change = self._getChange(project, event.change_number,
event.patch_number, refresh=refresh)
event.patch_number, refresh=refresh,
event=event)
if hasattr(event, 'change_url') and event.change_url:
change.url = event.change_url
else:
@ -883,7 +886,8 @@ class GithubConnection(BaseConnection):
change.files = self.getPushedFileNames(event)
return change
def _getChange(self, project, number, patchset=None, refresh=False):
def _getChange(self, project, number, patchset=None, refresh=False,
event=None):
key = (project.name, number, patchset)
change = self._change_cache.get(key)
if change and not refresh:
@ -895,7 +899,7 @@ class GithubConnection(BaseConnection):
change.patchset = patchset
self._change_cache[key] = change
try:
self._updateChange(change)
self._updateChange(change, event)
except Exception:
if key in self._change_cache:
del self._change_cache[key]
@ -971,9 +975,11 @@ class GithubConnection(BaseConnection):
return changes
def _updateChange(self, change):
self.log.info("Updating %s" % (change,))
change.pr, pr_obj = self.getPull(change.project.name, change.number)
def _updateChange(self, change, event):
log = get_annotated_logger(self.log, event)
log.info("Updating %s" % (change,))
change.pr, pr_obj = self.getPull(
change.project.name, change.number, log=log)
change.ref = "refs/pull/%s/head" % change.number
change.branch = change.pr.get('base').get('ref')
@ -986,9 +992,9 @@ class GithubConnection(BaseConnection):
# the first 300 changed files of a PR in alphabetical order.
# https://developer.github.com/v3/pulls/#list-pull-requests-files
if len(change.files) < change.pr.get('changed_files', 0):
self.log.warning("Got only %s files but PR has %s files.",
len(change.files),
change.pr.get('changed_files', 0))
log.warning("Got only %s files but PR has %s files.",
len(change.files),
change.pr.get('changed_files', 0))
# In this case explicitly set change.files to None to signalize
# that we need to ask the mergers later in pipeline processing.
# We cannot query the files here using the mergers because this
@ -1003,10 +1009,10 @@ class GithubConnection(BaseConnection):
if not change.is_merged:
change.is_merged = change.pr.get('merged')
change.status = self._get_statuses(change.project,
change.patchset)
change.reviews = self.getPullReviews(pr_obj, change.project,
change.number)
change.status = self._get_statuses(
change.project, change.patchset, event)
change.reviews = self.getPullReviews(
pr_obj, change.project, change.number, event)
change.labels = change.pr.get('labels')
# ensure message is at least an empty string
message = change.pr.get("body") or ""
@ -1241,11 +1247,12 @@ class GithubConnection(BaseConnection):
return None
return pulls.pop()
def getPullReviews(self, pr_obj, project, number):
def getPullReviews(self, pr_obj, project, number, event):
log = get_annotated_logger(self.log, event)
# make a list out of the reviews so that we complete our
# API transaction
revs = [review.as_dict() for review in pr_obj.reviews()]
self.log.debug('Got reviews for PR %s#%s', project, number)
log.debug('Got reviews for PR %s#%s', project, number)
permissions = {}
reviews = {}
@ -1291,9 +1298,9 @@ class GithubConnection(BaseConnection):
if review['grantedOn'] > reviews[user]['grantedOn']:
if (review['type'] == 'commented' and reviews[user]['type']
in ('approved', 'changes_requested')):
self.log.debug("Discarding comment review %s due to "
"an existing vote %s" % (review,
reviews[user]))
log.debug("Discarding comment review %s due to "
"an existing vote %s" % (review,
reviews[user]))
pass
else:
reviews[user] = review
@ -1418,7 +1425,8 @@ class GithubConnection(BaseConnection):
time.sleep(1)
return self._getCommit(repository, sha, retries - 1)
def getCommitStatuses(self, project, sha):
def getCommitStatuses(self, project, sha, zuul_event_id=None):
log = get_annotated_logger(self.log, zuul_event_id)
github = self.getGithubClient(project)
owner, proj = project.split('/')
repository = github.repository(owner, proj)
@ -1429,8 +1437,8 @@ class GithubConnection(BaseConnection):
# API transaction
statuses = [status.as_dict() for status in commit.statuses()]
self.log.debug("Got commit statuses for sha %s on %s", sha, project)
self.log_rate_limit(self.log, github)
log.debug("Got commit statuses for sha %s on %s", sha, project)
self.log_rate_limit(log, github)
return statuses
def setCommitStatus(self, project, sha, state, url='', description='',
@ -1477,7 +1485,7 @@ class GithubConnection(BaseConnection):
def _ghTimestampToDate(self, timestamp):
return time.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ')
def _get_statuses(self, project, sha):
def _get_statuses(self, project, sha, event):
# A ref can have more than one status from each context,
# however the API returns them in order, newest first.
# So we can keep track of which contexts we've already seen
@ -1487,7 +1495,8 @@ class GithubConnection(BaseConnection):
# by user, so that we can require/trigger by user too.
seen = []
statuses = []
for status in self.getCommitStatuses(project.name, sha):
for status in self.getCommitStatuses(
project.name, sha, event):
stuple = _status_as_tuple(status)
if "%s:%s" % (stuple[0], stuple[1]) not in seen:
statuses.append("%s:%s:%s" % stuple)