From 385d11e2eddad010dd8de2989668092249acddd3 Mon Sep 17 00:00:00 2001 From: Joshua Hesketh Date: Mon, 14 Sep 2015 14:50:01 -0600 Subject: [PATCH] Move Item formatting into Reporters This will allow a reporter to decide how to handle the results of each item. It can use the common plain text formatter (as has been the case, '_formatItemReport') or it may generate a report itself. This will be useful for the MySQL reporter where it will want to create an entry in a table for each build. Action reporters are now configured with the action type, so they can react differently for the success, failure, etc. Co-Authored-By: Jan Hruban Change-Id: Ib270334ff694fdff69a3028db8d6d7fed1f05176 --- zuul/reporter/__init__.py | 118 ++++++++++++++++++++++++++++++++++++- zuul/reporter/gerrit.py | 18 +++--- zuul/reporter/smtp.py | 11 ++-- zuul/scheduler.py | 121 ++++++-------------------------------- 4 files changed, 153 insertions(+), 115 deletions(-) diff --git a/zuul/reporter/__init__.py b/zuul/reporter/__init__.py index d9857daf1a..e29f9a776f 100644 --- a/zuul/reporter/__init__.py +++ b/zuul/reporter/__init__.py @@ -28,12 +28,16 @@ class BaseReporter(object): self.reporter_config = reporter_config self.sched = sched self.connection = connection + self._action = None + + def setAction(self, action): + self._action = action def stop(self): """Stop the reporter.""" @abc.abstractmethod - def report(self, source, change, message): + def report(self, source, pipeline, item): """Send the compiled report message.""" def getSubmitAllowNeeds(self): @@ -46,3 +50,115 @@ class BaseReporter(object): def postConfig(self): """Run tasks after configuration is reloaded""" + + def _getFormatter(self): + format_methods = { + 'start': self._formatItemReportStart, + 'success': self._formatItemReportSuccess, + 'failure': self._formatItemReportFailure, + 'merge-failure': self._formatItemReportMergeFailure, + 'disabled': self._formatItemReportDisabled + } + return format_methods[self._action] + + def _formatItemReport(self, pipeline, item): + """Format a report from the given items. Usually to provide results to + a reporter taking free-form text.""" + ret = self._getFormatter()(pipeline, item) + + if pipeline.footer_message: + ret += '\n' + pipeline.footer_message + + return ret + + def _formatItemReportStart(self, pipeline, item): + msg = "Starting %s jobs." % pipeline.name + if self.sched.config.has_option('zuul', 'status_url'): + msg += "\n" + self.sched.config.get('zuul', 'status_url') + return msg + + def _formatItemReportSuccess(self, pipeline, item): + return (pipeline.success_message + '\n\n' + + self._formatItemReportJobs(pipeline, item)) + + def _formatItemReportFailure(self, pipeline, item): + if item.dequeued_needing_change: + msg = 'This change depends on a change that failed to merge.\n' + else: + msg = (pipeline.failure_message + '\n\n' + + self._formatItemReportJobs(pipeline, item)) + return msg + + def _formatItemReportMergeFailure(self, pipeline, item): + return pipeline.merge_failure_message + + def _formatItemReportDisabled(self, pipeline, item): + if item.current_build_set.result == 'SUCCESS': + return self._formatItemReportSuccess(pipeline, item) + elif item.current_build_set.result == 'FAILURE': + return self._formatItemReportFailure(pipeline, item) + else: + return self._formatItemReport(pipeline, item) + + def _formatItemReportJobs(self, pipeline, item): + # Return the list of jobs portion of the report + ret = '' + + if self.sched.config.has_option('zuul', 'url_pattern'): + url_pattern = self.sched.config.get('zuul', 'url_pattern') + else: + url_pattern = None + + for job in pipeline.getJobs(item): + build = item.current_build_set.getBuild(job.name) + result = build.result + pattern = url_pattern + if result == 'SUCCESS': + if job.success_message: + result = job.success_message + if job.success_pattern: + pattern = job.success_pattern + elif result == 'FAILURE': + if job.failure_message: + result = job.failure_message + if job.failure_pattern: + pattern = job.failure_pattern + if pattern: + url = pattern.format(change=item.change, + pipeline=pipeline, + job=job, + build=build) + else: + url = build.url or job.name + if not job.voting: + voting = ' (non-voting)' + else: + voting = '' + + if self.sched.config and self.sched.config.has_option( + 'zuul', 'report_times'): + report_times = self.sched.config.getboolean( + 'zuul', 'report_times') + else: + report_times = True + + if report_times and build.end_time and build.start_time: + dt = int(build.end_time - build.start_time) + m, s = divmod(dt, 60) + h, m = divmod(m, 60) + if h: + elapsed = ' in %dh %02dm %02ds' % (h, m, s) + elif m: + elapsed = ' in %dm %02ds' % (m, s) + else: + elapsed = ' in %ds' % (s) + else: + elapsed = '' + name = '' + if self.sched.config.has_option('zuul', 'job_name_in_report'): + if self.sched.config.getboolean('zuul', + 'job_name_in_report'): + name = job.name + ' ' + ret += '- %s%s : %s%s%s\n' % (name, url, result, elapsed, + voting) + return ret diff --git a/zuul/reporter/gerrit.py b/zuul/reporter/gerrit.py index e1b05714a7..14274490f8 100644 --- a/zuul/reporter/gerrit.py +++ b/zuul/reporter/gerrit.py @@ -25,16 +25,18 @@ class GerritReporter(BaseReporter): name = 'gerrit' log = logging.getLogger("zuul.reporter.gerrit.Reporter") - def report(self, source, change, message): + def report(self, source, pipeline, item): """Send a message to gerrit.""" - self.log.debug("Report change %s, params %s, message: %s" % - (change, self.reporter_config, message)) - changeid = '%s,%s' % (change.number, change.patchset) - change._ref_sha = source.getRefSha(change.project.name, - 'refs/heads/' + change.branch) + message = self._formatItemReport(pipeline, item) - return self.connection.review(change.project.name, changeid, message, - self.reporter_config) + self.log.debug("Report change %s, params %s, message: %s" % + (item.change, self.reporter_config, message)) + changeid = '%s,%s' % (item.change.number, item.change.patchset) + item.change._ref_sha = source.getRefSha( + item.change.project.name, 'refs/heads/' + item.change.branch) + + return self.connection.review(item.change.project.name, changeid, + message, self.reporter_config) def getSubmitAllowNeeds(self): """Get a list of code review labels that are allowed to be diff --git a/zuul/reporter/smtp.py b/zuul/reporter/smtp.py index 4daa6df380..586b941365 100644 --- a/zuul/reporter/smtp.py +++ b/zuul/reporter/smtp.py @@ -24,10 +24,12 @@ class SMTPReporter(BaseReporter): name = 'smtp' log = logging.getLogger("zuul.reporter.smtp.Reporter") - def report(self, source, change, message): + def report(self, source, pipeline, item): """Send the compiled report message via smtp.""" + message = self._formatItemReport(pipeline, item) + self.log.debug("Report change %s, params %s, message: %s" % - (change, self.reporter_config, message)) + (item.change, self.reporter_config, message)) from_email = self.reporter_config['from'] \ if 'from' in self.reporter_config else None @@ -35,9 +37,10 @@ class SMTPReporter(BaseReporter): if 'to' in self.reporter_config else None if 'subject' in self.reporter_config: - subject = self.reporter_config['subject'].format(change=change) + subject = self.reporter_config['subject'].format( + change=item.change) else: - subject = "Report for change %s" % change + subject = "Report for change %s" % item.change self.connection.sendMail(subject, message, from_email, to_email) diff --git a/zuul/scheduler.py b/zuul/scheduler.py index 91bcf13d34..f8321d14e2 100644 --- a/zuul/scheduler.py +++ b/zuul/scheduler.py @@ -374,6 +374,7 @@ class Scheduler(threading.Thread): in conf_pipeline.get(conf_key).items(): reporter = self._getReporterDriver(reporter_name, params) + reporter.setAction(conf_key) reporter_set.append(reporter) setattr(pipeline, action, reporter_set) @@ -1054,12 +1055,6 @@ class BasePipelineManager(object): self.pipeline = pipeline self.event_filters = [] self.changeish_filters = [] - if self.sched.config and self.sched.config.has_option( - 'zuul', 'report_times'): - self.report_times = self.sched.config.getboolean( - 'zuul', 'report_times') - else: - self.report_times = True def __str__(self): return "<%s %s>" % (self.__class__.__name__, self.pipeline.name) @@ -1153,32 +1148,30 @@ class BasePipelineManager(object): return True return False - def reportStart(self, change): + def reportStart(self, item): if not self.pipeline._disabled: try: - self.log.info("Reporting start, action %s change %s" % - (self.pipeline.start_actions, change)) - msg = "Starting %s jobs." % self.pipeline.name - if self.sched.config.has_option('zuul', 'status_url'): - msg += "\n" + self.sched.config.get('zuul', 'status_url') + self.log.info("Reporting start, action %s item %s" % + (self.pipeline.start_actions, item)) ret = self.sendReport(self.pipeline.start_actions, - self.pipeline.source, change, msg) + self.pipeline.source, item) if ret: - self.log.error("Reporting change start %s received: %s" % - (change, ret)) + self.log.error("Reporting item start %s received: %s" % + (item, ret)) except: self.log.exception("Exception while reporting start:") - def sendReport(self, action_reporters, source, change, message): + def sendReport(self, action_reporters, source, item, + message=None): """Sends the built message off to configured reporters. - Takes the action_reporters, change, message and extra options and + Takes the action_reporters, item, message and extra options and sends them to the pluggable reporters. """ report_errors = [] if len(action_reporters) > 0: for reporter in action_reporters: - ret = reporter.report(source, change, message) + ret = reporter.report(source, self.pipeline, item) if ret: report_errors.append(ret) if len(report_errors) == 0: @@ -1314,14 +1307,14 @@ class BasePipelineManager(object): self.log.debug("Adding change %s to queue %s" % (change, change_queue)) - if not quiet: - if len(self.pipeline.start_actions) > 0: - self.reportStart(change) item = change_queue.enqueueChange(change) if enqueue_time: item.enqueue_time = enqueue_time item.live = live self.reportStats(item) + if not quiet: + if len(self.pipeline.start_actions) > 0: + self.reportStart(item) self.enqueueChangesBehind(change, quiet, ignore_requirements, change_queue) for trigger in self.sched.triggers.values(): @@ -1636,95 +1629,19 @@ class BasePipelineManager(object): self.pipeline._consecutive_failures >= self.pipeline.disable_at): self.pipeline._disabled = True if actions: - report = self.formatReport(item) try: - self.log.info("Reporting change %s, actions: %s" % - (item.change, actions)) - ret = self.sendReport(actions, self.pipeline.source, - item.change, report) + self.log.info("Reporting item %s, actions: %s" % + (item, actions)) + ret = self.sendReport(actions, self.pipeline.source, item) if ret: - self.log.error("Reporting change %s received: %s" % - (item.change, ret)) + self.log.error("Reporting item %s received: %s" % + (item, ret)) except: self.log.exception("Exception while reporting:") item.setReportedResult('ERROR') self.updateBuildDescriptions(item.current_build_set) return ret - def formatReport(self, item): - ret = '' - - if item.dequeued_needing_change: - ret += 'This change depends on a change that failed to merge.\n' - elif not self.pipeline.didMergerSucceed(item): - ret += self.pipeline.merge_failure_message - else: - if self.pipeline.didAllJobsSucceed(item): - ret += self.pipeline.success_message + '\n\n' - else: - ret += self.pipeline.failure_message + '\n\n' - ret += self._formatReportJobs(item) - - if self.pipeline.footer_message: - ret += '\n' + self.pipeline.footer_message - - return ret - - def _formatReportJobs(self, item): - # Return the list of jobs portion of the report - ret = '' - - if self.sched.config.has_option('zuul', 'url_pattern'): - url_pattern = self.sched.config.get('zuul', 'url_pattern') - else: - url_pattern = None - - for job in self.pipeline.getJobs(item): - build = item.current_build_set.getBuild(job.name) - result = build.result - pattern = url_pattern - if result == 'SUCCESS': - if job.success_message: - result = job.success_message - if job.success_pattern: - pattern = job.success_pattern - elif result == 'FAILURE': - if job.failure_message: - result = job.failure_message - if job.failure_pattern: - pattern = job.failure_pattern - if pattern: - url = pattern.format(change=item.change, - pipeline=self.pipeline, - job=job, - build=build) - else: - url = build.url or job.name - if not job.voting: - voting = ' (non-voting)' - else: - voting = '' - if self.report_times and build.end_time and build.start_time: - dt = int(build.end_time - build.start_time) - m, s = divmod(dt, 60) - h, m = divmod(m, 60) - if h: - elapsed = ' in %dh %02dm %02ds' % (h, m, s) - elif m: - elapsed = ' in %dm %02ds' % (m, s) - else: - elapsed = ' in %ds' % (s) - else: - elapsed = '' - name = '' - if self.sched.config.has_option('zuul', 'job_name_in_report'): - if self.sched.config.getboolean('zuul', - 'job_name_in_report'): - name = job.name + ' ' - ret += '- %s%s : %s%s%s\n' % (name, url, result, elapsed, - voting) - return ret - def formatDescription(self, build): concurrent_changes = '' concurrent_builds = ''