Add debug project-pipeline option

This may be set by a project to help debug why a job is or is not
running.  It works speculatively, and so can be used to debug
a single change.

Change-Id: I1957d21fe7775f786935e5d7d4bdf65b86eb5e4d

doc/source/user/config.rst

@@ -1090,6 +1090,14 @@ pipeline.
          changes which break the others.  This is a free-form string;
          just set the same value for each group of projects.
 
+      .. attr:: debug
+
+         If this is set to `true`, Zuul will include debugging
+         information in reports it makes about items in the pipeline.
+         This should not normally be set, but in situations were it is
+         difficult to determine why Zuul did or did not run a certain
+         job, the additional information this provides may help.
+
 .. _project-template:
 
 Project Template

tests/unit/test_v3.py

@@ -1552,6 +1552,32 @@ class TestInRepoConfig(ZuulTestCase):
                       C.messages[0],
                       "C should have an error reported")
 
+    def test_pipeline_debug(self):
+        in_repo_conf = textwrap.dedent(
+            """
+            - job:
+                name: project-test1
+                run: playbooks/project-test1.yaml
+            - project:
+                name: org/project
+                check:
+                  debug: True
+                  jobs:
+                    - project-test1
+            """)
+
+        file_dict = {'.zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertEqual(A.data['status'], 'NEW')
+        self.assertEqual(A.reported, 1,
+                         "A should report success")
+        self.assertIn('Debug information:',
+                      A.messages[0], "A should have debug info")
+
 
 class TestInRepoJoin(ZuulTestCase):
     # In this config, org/project is not a member of any pipelines, so

zuul/configloader.py

@@ -789,7 +789,11 @@ class ProjectTemplateParser(object):
 
         job = {str: vs.Any(str, JobParser.job_attributes)}
         job_list = [vs.Any(str, job)]
-        pipeline_contents = {'queue': str, 'jobs': job_list}
+        pipeline_contents = {
+            'queue': str,
+            'debug': bool,
+            'jobs': job_list,
+        }
 
         for p in self.layout.pipelines.values():
             project_template[p.name] = pipeline_contents
@@ -809,6 +813,7 @@ class ProjectTemplateParser(object):
             project_pipeline = model.ProjectPipelineConfig()
             project_template.pipelines[pipeline.name] = project_pipeline
             project_pipeline.queue_name = conf_pipeline.get('queue')
+            project_pipeline.debug = conf_pipeline.get('debug')
             self.parseJobList(
                 conf_pipeline.get('jobs', []),
                 source_context, start_mark, project_pipeline.job_list)
@@ -859,7 +864,11 @@ class ProjectParser(object):
 
         job = {str: vs.Any(str, JobParser.job_attributes)}
         job_list = [vs.Any(str, job)]
-        pipeline_contents = {'queue': str, 'jobs': job_list}
+        pipeline_contents = {
+            'queue': str,
+            'debug': bool,
+            'jobs': job_list
+        }
 
         for p in self.layout.pipelines.values():
             project[p.name] = pipeline_contents
@@ -920,6 +929,7 @@ class ProjectParser(object):
         for pipeline in self.layout.pipelines.values():
             project_pipeline = model.ProjectPipelineConfig()
             queue_name = None
+            debug = False
             # For every template, iterate over the job tree and replace or
             # create the jobs in the final definition as needed.
             pipeline_defined = False
@@ -932,8 +942,12 @@ class ProjectParser(object):
                         implied_branch)
                     if template_pipeline.queue_name:
                         queue_name = template_pipeline.queue_name
+                    if template_pipeline.debug is not None:
+                        debug = template_pipeline.debug
             if queue_name:
                 project_pipeline.queue_name = queue_name
+            if debug:
+                project_pipeline.debug = True
             if pipeline_defined:
                 project_config.pipelines[pipeline.name] = project_pipeline
         return project_config

zuul/model.py

@@ -1337,6 +1337,7 @@ class BuildSet(object):
         self.unable_to_merge = False
         self.config_error = None  # None or an error message string.
         self.failing_reasons = []
+        self.debug_messages = []
         self.merge_state = self.NEW
         self.nodesets = {}  # job -> nodeset
         self.node_requests = {}  # job -> reqs
@@ -1501,6 +1502,17 @@ class QueueItem(object):
     def setReportedResult(self, result):
         self.current_build_set.result = result
 
+    def debug(self, msg, indent=0):
+        ppc = self.layout.getProjectPipelineConfig(self.change.project,
+                                                   self.pipeline)
+        if not ppc.debug:
+            return
+        if indent:
+            indent = '  ' * indent
+        else:
+            indent = ''
+        self.current_build_set.debug_messages.append(indent + msg)
+
     def freezeJobGraph(self):
         """Find or create actual matching jobs for this item's change and
         store the resulting job tree."""
@@ -2220,6 +2232,7 @@ class ProjectPipelineConfig(object):
     def __init__(self):
         self.job_list = JobList()
         self.queue_name = None
+        self.debug = False
         self.merge_mode = None
 
 
@@ -2545,7 +2558,8 @@ class Layout(object):
     def addProjectConfig(self, project_config):
         self.project_configs[project_config.name] = project_config
 
-    def collectJobs(self, jobname, change, path=None, jobs=None, stack=None):
+    def collectJobs(self, item, jobname, change, path=None, jobs=None,
+                    stack=None):
         if stack is None:
             stack = []
         if jobs is None:
@@ -2554,13 +2568,20 @@ class Layout(object):
             path = []
         path.append(jobname)
         matched = False
+        indent = len(path) + 1
+        item.debug("Collecting job variants for {jobname}".format(
+            jobname=jobname), indent=indent)
         for variant in self.getJobs(jobname):
             if not variant.changeMatches(change):
                 self.log.debug("Variant %s did not match %s", repr(variant),
                                change)
+                item.debug("Variant {variant} did not match".format(
+                    variant=repr(variant)), indent=indent)
                 continue
             else:
                 self.log.debug("Variant %s matched %s", repr(variant), change)
+                item.debug("Variant {variant} matched".format(
+                    variant=repr(variant)), indent=indent)
             if not variant.isBase():
                 parent = variant.parent
                 if not jobs and parent is None:
@@ -2570,30 +2591,38 @@ class Layout(object):
             if parent and parent not in path:
                 if parent in stack:
                     raise Exception("Dependency cycle in jobs: %s" % stack)
-                self.collectJobs(parent, change, path, jobs, stack + [jobname])
+                self.collectJobs(item, parent, change, path, jobs,
+                                 stack + [jobname])
             matched = True
             jobs.append(variant)
         if not matched:
+            self.log.debug("No matching parents for job %s and change %s",
+                           jobname, change)
+            item.debug("No matching parent for {jobname}".format(
+                jobname=repr(jobname)), indent=indent)
             raise NoMatchingParentError()
         return jobs
 
     def _createJobGraph(self, item, job_list, job_graph):
         change = item.change
         pipeline = item.pipeline
+        item.debug("Freezing job graph")
         for jobname in job_list.jobs:
             # This is the final job we are constructing
             frozen_job = None
             self.log.debug("Collecting jobs %s for %s", jobname, change)
+            item.debug("Freezing job {jobname}".format(
+                jobname=jobname), indent=1)
             try:
-                variants = self.collectJobs(jobname, change)
+                variants = self.collectJobs(item, jobname, change)
             except NoMatchingParentError:
-                self.log.debug("No matching parents for job %s and change %s",
-                               jobname, change)
                 variants = None
             if not variants:
                 # A change must match at least one defined job variant
                 # (that is to say that it must match more than just
                 # the job that is defined in the tree).
+                item.debug("No matching variants for {jobname}".format(
+                    jobname=jobname), indent=2)
                 continue
             for variant in variants:
                 if frozen_job is None:
@@ -2612,12 +2641,18 @@ class Layout(object):
                     matched = True
                     self.log.debug("Pipeline variant %s matched %s",
                                    repr(variant), change)
+                    item.debug("Pipeline variant {variant} matched".format(
+                        variant=repr(variant)), indent=2)
             else:
                 self.log.debug("Pipeline variant %s did not match %s",
                                repr(variant), change)
+                item.debug("Pipeline variant {variant} did not match".format(
+                    variant=repr(variant)), indent=2)
             if not matched:
                 # A change must match at least one project pipeline
                 # job variant.
+                item.debug("No matching pipeline variants for {jobname}".
+                           format(jobname=jobname), indent=2)
                 continue
             if (frozen_job.allowed_projects and
                 change.project.name not in frozen_job.allowed_projects):

zuul/reporter/__init__.py

@@ -64,6 +64,10 @@ class BaseReporter(object, metaclass=abc.ABCMeta):
         a reporter taking free-form text."""
         ret = self._getFormatter()(item, with_jobs)
 
+        if item.current_build_set.debug_messages:
+            debug = '\n  '.join(item.current_build_set.debug_messages)
+            ret += '\nDebug information:\n  ' + debug + '\n'
+
         if item.pipeline.footer_message:
             ret += '\n' + item.pipeline.footer_message