Add debug project-pipeline option

This may be set by a project to help debug why a job is or is not
running.  It works speculatively, and so can be used to debug
a single change.

Change-Id: I1957d21fe7775f786935e5d7d4bdf65b86eb5e4d
This commit is contained in:
James E. Blair 2017-12-13 17:18:34 -08:00
parent 37c3d8c86d
commit 1ef8f7c65f
5 changed files with 94 additions and 7 deletions

View File

@ -1090,6 +1090,14 @@ pipeline.
changes which break the others. This is a free-form string;
just set the same value for each group of projects.
.. attr:: debug
If this is set to `true`, Zuul will include debugging
information in reports it makes about items in the pipeline.
This should not normally be set, but in situations were it is
difficult to determine why Zuul did or did not run a certain
job, the additional information this provides may help.
.. _project-template:
Project Template

View File

@ -1552,6 +1552,32 @@ class TestInRepoConfig(ZuulTestCase):
C.messages[0],
"C should have an error reported")
def test_pipeline_debug(self):
in_repo_conf = textwrap.dedent(
"""
- job:
name: project-test1
run: playbooks/project-test1.yaml
- project:
name: org/project
check:
debug: True
jobs:
- project-test1
""")
file_dict = {'.zuul.yaml': in_repo_conf}
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
files=file_dict)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(A.reported, 1,
"A should report success")
self.assertIn('Debug information:',
A.messages[0], "A should have debug info")
class TestInRepoJoin(ZuulTestCase):
# In this config, org/project is not a member of any pipelines, so

View File

@ -789,7 +789,11 @@ class ProjectTemplateParser(object):
job = {str: vs.Any(str, JobParser.job_attributes)}
job_list = [vs.Any(str, job)]
pipeline_contents = {'queue': str, 'jobs': job_list}
pipeline_contents = {
'queue': str,
'debug': bool,
'jobs': job_list,
}
for p in self.layout.pipelines.values():
project_template[p.name] = pipeline_contents
@ -809,6 +813,7 @@ class ProjectTemplateParser(object):
project_pipeline = model.ProjectPipelineConfig()
project_template.pipelines[pipeline.name] = project_pipeline
project_pipeline.queue_name = conf_pipeline.get('queue')
project_pipeline.debug = conf_pipeline.get('debug')
self.parseJobList(
conf_pipeline.get('jobs', []),
source_context, start_mark, project_pipeline.job_list)
@ -859,7 +864,11 @@ class ProjectParser(object):
job = {str: vs.Any(str, JobParser.job_attributes)}
job_list = [vs.Any(str, job)]
pipeline_contents = {'queue': str, 'jobs': job_list}
pipeline_contents = {
'queue': str,
'debug': bool,
'jobs': job_list
}
for p in self.layout.pipelines.values():
project[p.name] = pipeline_contents
@ -920,6 +929,7 @@ class ProjectParser(object):
for pipeline in self.layout.pipelines.values():
project_pipeline = model.ProjectPipelineConfig()
queue_name = None
debug = False
# For every template, iterate over the job tree and replace or
# create the jobs in the final definition as needed.
pipeline_defined = False
@ -932,8 +942,12 @@ class ProjectParser(object):
implied_branch)
if template_pipeline.queue_name:
queue_name = template_pipeline.queue_name
if template_pipeline.debug is not None:
debug = template_pipeline.debug
if queue_name:
project_pipeline.queue_name = queue_name
if debug:
project_pipeline.debug = True
if pipeline_defined:
project_config.pipelines[pipeline.name] = project_pipeline
return project_config

View File

@ -1337,6 +1337,7 @@ class BuildSet(object):
self.unable_to_merge = False
self.config_error = None # None or an error message string.
self.failing_reasons = []
self.debug_messages = []
self.merge_state = self.NEW
self.nodesets = {} # job -> nodeset
self.node_requests = {} # job -> reqs
@ -1501,6 +1502,17 @@ class QueueItem(object):
def setReportedResult(self, result):
self.current_build_set.result = result
def debug(self, msg, indent=0):
ppc = self.layout.getProjectPipelineConfig(self.change.project,
self.pipeline)
if not ppc.debug:
return
if indent:
indent = ' ' * indent
else:
indent = ''
self.current_build_set.debug_messages.append(indent + msg)
def freezeJobGraph(self):
"""Find or create actual matching jobs for this item's change and
store the resulting job tree."""
@ -2220,6 +2232,7 @@ class ProjectPipelineConfig(object):
def __init__(self):
self.job_list = JobList()
self.queue_name = None
self.debug = False
self.merge_mode = None
@ -2545,7 +2558,8 @@ class Layout(object):
def addProjectConfig(self, project_config):
self.project_configs[project_config.name] = project_config
def collectJobs(self, jobname, change, path=None, jobs=None, stack=None):
def collectJobs(self, item, jobname, change, path=None, jobs=None,
stack=None):
if stack is None:
stack = []
if jobs is None:
@ -2554,13 +2568,20 @@ class Layout(object):
path = []
path.append(jobname)
matched = False
indent = len(path) + 1
item.debug("Collecting job variants for {jobname}".format(
jobname=jobname), indent=indent)
for variant in self.getJobs(jobname):
if not variant.changeMatches(change):
self.log.debug("Variant %s did not match %s", repr(variant),
change)
item.debug("Variant {variant} did not match".format(
variant=repr(variant)), indent=indent)
continue
else:
self.log.debug("Variant %s matched %s", repr(variant), change)
item.debug("Variant {variant} matched".format(
variant=repr(variant)), indent=indent)
if not variant.isBase():
parent = variant.parent
if not jobs and parent is None:
@ -2570,30 +2591,38 @@ class Layout(object):
if parent and parent not in path:
if parent in stack:
raise Exception("Dependency cycle in jobs: %s" % stack)
self.collectJobs(parent, change, path, jobs, stack + [jobname])
self.collectJobs(item, parent, change, path, jobs,
stack + [jobname])
matched = True
jobs.append(variant)
if not matched:
self.log.debug("No matching parents for job %s and change %s",
jobname, change)
item.debug("No matching parent for {jobname}".format(
jobname=repr(jobname)), indent=indent)
raise NoMatchingParentError()
return jobs
def _createJobGraph(self, item, job_list, job_graph):
change = item.change
pipeline = item.pipeline
item.debug("Freezing job graph")
for jobname in job_list.jobs:
# This is the final job we are constructing
frozen_job = None
self.log.debug("Collecting jobs %s for %s", jobname, change)
item.debug("Freezing job {jobname}".format(
jobname=jobname), indent=1)
try:
variants = self.collectJobs(jobname, change)
variants = self.collectJobs(item, jobname, change)
except NoMatchingParentError:
self.log.debug("No matching parents for job %s and change %s",
jobname, change)
variants = None
if not variants:
# A change must match at least one defined job variant
# (that is to say that it must match more than just
# the job that is defined in the tree).
item.debug("No matching variants for {jobname}".format(
jobname=jobname), indent=2)
continue
for variant in variants:
if frozen_job is None:
@ -2612,12 +2641,18 @@ class Layout(object):
matched = True
self.log.debug("Pipeline variant %s matched %s",
repr(variant), change)
item.debug("Pipeline variant {variant} matched".format(
variant=repr(variant)), indent=2)
else:
self.log.debug("Pipeline variant %s did not match %s",
repr(variant), change)
item.debug("Pipeline variant {variant} did not match".format(
variant=repr(variant)), indent=2)
if not matched:
# A change must match at least one project pipeline
# job variant.
item.debug("No matching pipeline variants for {jobname}".
format(jobname=jobname), indent=2)
continue
if (frozen_job.allowed_projects and
change.project.name not in frozen_job.allowed_projects):

View File

@ -64,6 +64,10 @@ class BaseReporter(object, metaclass=abc.ABCMeta):
a reporter taking free-form text."""
ret = self._getFormatter()(item, with_jobs)
if item.current_build_set.debug_messages:
debug = '\n '.join(item.current_build_set.debug_messages)
ret += '\nDebug information:\n ' + debug + '\n'
if item.pipeline.footer_message:
ret += '\n' + item.pipeline.footer_message