Move Item formatting into Reporters

This will allow a reporter to decide how to handle the results of
each item. It can use the common plain text formatter (as has been
the case, '_formatItemReport') or it may generate a report itself.

This will be useful for the MySQL reporter where it will want to
create an entry in a table for each build.

Action reporters are now configured with the action type, so they can
react differently for the success, failure, etc.

Co-Authored-By: Jan Hruban <jan.hruban@gooddata.com>

Change-Id: Ib270334ff694fdff69a3028db8d6d7fed1f05176
This commit is contained in:
Joshua Hesketh 2015-09-14 14:50:01 -06:00
parent de95865f48
commit 385d11e2ed
4 changed files with 153 additions and 115 deletions

View File

@ -28,12 +28,16 @@ class BaseReporter(object):
self.reporter_config = reporter_config
self.sched = sched
self.connection = connection
self._action = None
def setAction(self, action):
self._action = action
def stop(self):
"""Stop the reporter."""
@abc.abstractmethod
def report(self, source, change, message):
def report(self, source, pipeline, item):
"""Send the compiled report message."""
def getSubmitAllowNeeds(self):
@ -46,3 +50,115 @@ class BaseReporter(object):
def postConfig(self):
"""Run tasks after configuration is reloaded"""
def _getFormatter(self):
format_methods = {
'start': self._formatItemReportStart,
'success': self._formatItemReportSuccess,
'failure': self._formatItemReportFailure,
'merge-failure': self._formatItemReportMergeFailure,
'disabled': self._formatItemReportDisabled
}
return format_methods[self._action]
def _formatItemReport(self, pipeline, item):
"""Format a report from the given items. Usually to provide results to
a reporter taking free-form text."""
ret = self._getFormatter()(pipeline, item)
if pipeline.footer_message:
ret += '\n' + pipeline.footer_message
return ret
def _formatItemReportStart(self, pipeline, item):
msg = "Starting %s jobs." % pipeline.name
if self.sched.config.has_option('zuul', 'status_url'):
msg += "\n" + self.sched.config.get('zuul', 'status_url')
return msg
def _formatItemReportSuccess(self, pipeline, item):
return (pipeline.success_message + '\n\n' +
self._formatItemReportJobs(pipeline, item))
def _formatItemReportFailure(self, pipeline, item):
if item.dequeued_needing_change:
msg = 'This change depends on a change that failed to merge.\n'
else:
msg = (pipeline.failure_message + '\n\n' +
self._formatItemReportJobs(pipeline, item))
return msg
def _formatItemReportMergeFailure(self, pipeline, item):
return pipeline.merge_failure_message
def _formatItemReportDisabled(self, pipeline, item):
if item.current_build_set.result == 'SUCCESS':
return self._formatItemReportSuccess(pipeline, item)
elif item.current_build_set.result == 'FAILURE':
return self._formatItemReportFailure(pipeline, item)
else:
return self._formatItemReport(pipeline, item)
def _formatItemReportJobs(self, pipeline, item):
# Return the list of jobs portion of the report
ret = ''
if self.sched.config.has_option('zuul', 'url_pattern'):
url_pattern = self.sched.config.get('zuul', 'url_pattern')
else:
url_pattern = None
for job in pipeline.getJobs(item):
build = item.current_build_set.getBuild(job.name)
result = build.result
pattern = url_pattern
if result == 'SUCCESS':
if job.success_message:
result = job.success_message
if job.success_pattern:
pattern = job.success_pattern
elif result == 'FAILURE':
if job.failure_message:
result = job.failure_message
if job.failure_pattern:
pattern = job.failure_pattern
if pattern:
url = pattern.format(change=item.change,
pipeline=pipeline,
job=job,
build=build)
else:
url = build.url or job.name
if not job.voting:
voting = ' (non-voting)'
else:
voting = ''
if self.sched.config and self.sched.config.has_option(
'zuul', 'report_times'):
report_times = self.sched.config.getboolean(
'zuul', 'report_times')
else:
report_times = True
if report_times and build.end_time and build.start_time:
dt = int(build.end_time - build.start_time)
m, s = divmod(dt, 60)
h, m = divmod(m, 60)
if h:
elapsed = ' in %dh %02dm %02ds' % (h, m, s)
elif m:
elapsed = ' in %dm %02ds' % (m, s)
else:
elapsed = ' in %ds' % (s)
else:
elapsed = ''
name = ''
if self.sched.config.has_option('zuul', 'job_name_in_report'):
if self.sched.config.getboolean('zuul',
'job_name_in_report'):
name = job.name + ' '
ret += '- %s%s : %s%s%s\n' % (name, url, result, elapsed,
voting)
return ret

View File

@ -25,16 +25,18 @@ class GerritReporter(BaseReporter):
name = 'gerrit'
log = logging.getLogger("zuul.reporter.gerrit.Reporter")
def report(self, source, change, message):
def report(self, source, pipeline, item):
"""Send a message to gerrit."""
self.log.debug("Report change %s, params %s, message: %s" %
(change, self.reporter_config, message))
changeid = '%s,%s' % (change.number, change.patchset)
change._ref_sha = source.getRefSha(change.project.name,
'refs/heads/' + change.branch)
message = self._formatItemReport(pipeline, item)
return self.connection.review(change.project.name, changeid, message,
self.reporter_config)
self.log.debug("Report change %s, params %s, message: %s" %
(item.change, self.reporter_config, message))
changeid = '%s,%s' % (item.change.number, item.change.patchset)
item.change._ref_sha = source.getRefSha(
item.change.project.name, 'refs/heads/' + item.change.branch)
return self.connection.review(item.change.project.name, changeid,
message, self.reporter_config)
def getSubmitAllowNeeds(self):
"""Get a list of code review labels that are allowed to be

View File

@ -24,10 +24,12 @@ class SMTPReporter(BaseReporter):
name = 'smtp'
log = logging.getLogger("zuul.reporter.smtp.Reporter")
def report(self, source, change, message):
def report(self, source, pipeline, item):
"""Send the compiled report message via smtp."""
message = self._formatItemReport(pipeline, item)
self.log.debug("Report change %s, params %s, message: %s" %
(change, self.reporter_config, message))
(item.change, self.reporter_config, message))
from_email = self.reporter_config['from'] \
if 'from' in self.reporter_config else None
@ -35,9 +37,10 @@ class SMTPReporter(BaseReporter):
if 'to' in self.reporter_config else None
if 'subject' in self.reporter_config:
subject = self.reporter_config['subject'].format(change=change)
subject = self.reporter_config['subject'].format(
change=item.change)
else:
subject = "Report for change %s" % change
subject = "Report for change %s" % item.change
self.connection.sendMail(subject, message, from_email, to_email)

View File

@ -374,6 +374,7 @@ class Scheduler(threading.Thread):
in conf_pipeline.get(conf_key).items():
reporter = self._getReporterDriver(reporter_name,
params)
reporter.setAction(conf_key)
reporter_set.append(reporter)
setattr(pipeline, action, reporter_set)
@ -1054,12 +1055,6 @@ class BasePipelineManager(object):
self.pipeline = pipeline
self.event_filters = []
self.changeish_filters = []
if self.sched.config and self.sched.config.has_option(
'zuul', 'report_times'):
self.report_times = self.sched.config.getboolean(
'zuul', 'report_times')
else:
self.report_times = True
def __str__(self):
return "<%s %s>" % (self.__class__.__name__, self.pipeline.name)
@ -1153,32 +1148,30 @@ class BasePipelineManager(object):
return True
return False
def reportStart(self, change):
def reportStart(self, item):
if not self.pipeline._disabled:
try:
self.log.info("Reporting start, action %s change %s" %
(self.pipeline.start_actions, change))
msg = "Starting %s jobs." % self.pipeline.name
if self.sched.config.has_option('zuul', 'status_url'):
msg += "\n" + self.sched.config.get('zuul', 'status_url')
self.log.info("Reporting start, action %s item %s" %
(self.pipeline.start_actions, item))
ret = self.sendReport(self.pipeline.start_actions,
self.pipeline.source, change, msg)
self.pipeline.source, item)
if ret:
self.log.error("Reporting change start %s received: %s" %
(change, ret))
self.log.error("Reporting item start %s received: %s" %
(item, ret))
except:
self.log.exception("Exception while reporting start:")
def sendReport(self, action_reporters, source, change, message):
def sendReport(self, action_reporters, source, item,
message=None):
"""Sends the built message off to configured reporters.
Takes the action_reporters, change, message and extra options and
Takes the action_reporters, item, message and extra options and
sends them to the pluggable reporters.
"""
report_errors = []
if len(action_reporters) > 0:
for reporter in action_reporters:
ret = reporter.report(source, change, message)
ret = reporter.report(source, self.pipeline, item)
if ret:
report_errors.append(ret)
if len(report_errors) == 0:
@ -1314,14 +1307,14 @@ class BasePipelineManager(object):
self.log.debug("Adding change %s to queue %s" %
(change, change_queue))
if not quiet:
if len(self.pipeline.start_actions) > 0:
self.reportStart(change)
item = change_queue.enqueueChange(change)
if enqueue_time:
item.enqueue_time = enqueue_time
item.live = live
self.reportStats(item)
if not quiet:
if len(self.pipeline.start_actions) > 0:
self.reportStart(item)
self.enqueueChangesBehind(change, quiet, ignore_requirements,
change_queue)
for trigger in self.sched.triggers.values():
@ -1636,95 +1629,19 @@ class BasePipelineManager(object):
self.pipeline._consecutive_failures >= self.pipeline.disable_at):
self.pipeline._disabled = True
if actions:
report = self.formatReport(item)
try:
self.log.info("Reporting change %s, actions: %s" %
(item.change, actions))
ret = self.sendReport(actions, self.pipeline.source,
item.change, report)
self.log.info("Reporting item %s, actions: %s" %
(item, actions))
ret = self.sendReport(actions, self.pipeline.source, item)
if ret:
self.log.error("Reporting change %s received: %s" %
(item.change, ret))
self.log.error("Reporting item %s received: %s" %
(item, ret))
except:
self.log.exception("Exception while reporting:")
item.setReportedResult('ERROR')
self.updateBuildDescriptions(item.current_build_set)
return ret
def formatReport(self, item):
ret = ''
if item.dequeued_needing_change:
ret += 'This change depends on a change that failed to merge.\n'
elif not self.pipeline.didMergerSucceed(item):
ret += self.pipeline.merge_failure_message
else:
if self.pipeline.didAllJobsSucceed(item):
ret += self.pipeline.success_message + '\n\n'
else:
ret += self.pipeline.failure_message + '\n\n'
ret += self._formatReportJobs(item)
if self.pipeline.footer_message:
ret += '\n' + self.pipeline.footer_message
return ret
def _formatReportJobs(self, item):
# Return the list of jobs portion of the report
ret = ''
if self.sched.config.has_option('zuul', 'url_pattern'):
url_pattern = self.sched.config.get('zuul', 'url_pattern')
else:
url_pattern = None
for job in self.pipeline.getJobs(item):
build = item.current_build_set.getBuild(job.name)
result = build.result
pattern = url_pattern
if result == 'SUCCESS':
if job.success_message:
result = job.success_message
if job.success_pattern:
pattern = job.success_pattern
elif result == 'FAILURE':
if job.failure_message:
result = job.failure_message
if job.failure_pattern:
pattern = job.failure_pattern
if pattern:
url = pattern.format(change=item.change,
pipeline=self.pipeline,
job=job,
build=build)
else:
url = build.url or job.name
if not job.voting:
voting = ' (non-voting)'
else:
voting = ''
if self.report_times and build.end_time and build.start_time:
dt = int(build.end_time - build.start_time)
m, s = divmod(dt, 60)
h, m = divmod(m, 60)
if h:
elapsed = ' in %dh %02dm %02ds' % (h, m, s)
elif m:
elapsed = ' in %dm %02ds' % (m, s)
else:
elapsed = ' in %ds' % (s)
else:
elapsed = ''
name = ''
if self.sched.config.has_option('zuul', 'job_name_in_report'):
if self.sched.config.getboolean('zuul',
'job_name_in_report'):
name = job.name + ' '
ret += '- %s%s : %s%s%s\n' % (name, url, result, elapsed,
voting)
return ret
def formatDescription(self, build):
concurrent_changes = ''
concurrent_builds = ''