Browse Source

Grab json log contents for final post playbook failures

If the final post playbook fails, something has gone wrong with log
uploading, which means it's very hard to debug. Grab the contents of the
json log file, extract the log for the last playbook and add it to the
executor log.

Change-Id: Ia930311e121c350e73e41b20e9b742b2eac9c9f6
changes/52/510952/7
Monty Taylor 5 years ago
parent
commit
0e2489a4cd
No known key found for this signature in database
GPG Key ID: 7BAE94BC7141A594
  1. 4
      tests/fixtures/config/job-output/git/common-config/playbooks/job-output-failure-post.yaml
  2. 11
      tests/fixtures/config/job-output/git/common-config/zuul.yaml
  3. 1
      tests/fixtures/config/job-output/git/org_project2/README
  4. 1
      tests/fixtures/config/job-output/main.yaml
  5. 38
      tests/unit/test_v3.py
  6. 28
      zuul/executor/server.py

4
tests/fixtures/config/job-output/git/common-config/playbooks/job-output-failure-post.yaml vendored

@ -0,0 +1,4 @@
- hosts: all
tasks:
- shell: echo "Failure test {{ zuul.executor.src_root }}"
- shell: exit 1

11
tests/fixtures/config/job-output/git/common-config/zuul.yaml vendored

@ -20,8 +20,19 @@
parent: base
name: job-output
- job:
name: job-output-failure
run: playbooks/job-output
post-run: playbooks/job-output-failure-post
- project:
name: org/project
check:
jobs:
- job-output
- project:
name: org/project2
check:
jobs:
- job-output-failure

1
tests/fixtures/config/job-output/git/org_project2/README vendored

@ -0,0 +1 @@
test

1
tests/fixtures/config/job-output/main.yaml vendored

@ -6,3 +6,4 @@
- common-config
untrusted-projects:
- org/project
- org/project2

38
tests/unit/test_v3.py

@ -14,7 +14,9 @@
# License for the specific language governing permissions and limitations
# under the License.
import io
import json
import logging
import os
import textwrap
import gc
@ -1844,7 +1846,8 @@ class TestJobOutput(AnsibleZuulTestCase):
return f.read()
def test_job_output(self):
# Verify that command standard output appears in the job output
# Verify that command standard output appears in the job output,
# and that failures in the final playbook get logged.
# This currently only verifies we receive output from
# localhost. Notably, it does not verify we receive output
@ -1869,3 +1872,36 @@ class TestJobOutput(AnsibleZuulTestCase):
self.assertIn(token,
self._get_file(self.history[0],
'work/logs/job-output.txt'))
def test_job_output_failure_log(self):
logger = logging.getLogger('zuul.AnsibleJob')
output = io.StringIO()
logger.addHandler(logging.StreamHandler(output))
# Verify that a failure in the last post playbook emits the contents
# of the json output to the log
self.executor_server.keep_jobdir = True
A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertHistory([
dict(name='job-output-failure',
result='POST_FAILURE', changes='1,1'),
], ordered=False)
token = 'Standard output test %s' % (self.history[0].jobdir.src_root)
j = json.loads(self._get_file(self.history[0],
'work/logs/job-output.json'))
self.assertEqual(token,
j[0]['plays'][0]['tasks'][0]
['hosts']['localhost']['stdout'])
print(self._get_file(self.history[0],
'work/logs/job-output.json'))
self.assertIn(token,
self._get_file(self.history[0],
'work/logs/job-output.txt'))
log_output = output.getvalue()
self.assertIn('Final playbook failed', log_output)
self.assertIn('Failure test', log_output)

28
zuul/executor/server.py

@ -1229,8 +1229,36 @@ class AnsibleJob(object):
# precedence over the post result.
if not pre_failed:
result = 'POST_FAILURE'
if (index + 1) == len(self.jobdir.post_playbooks):
self._logFinalPlaybookError()
return result
def _logFinalPlaybookError(self):
# Failures in the final post playbook can include failures
# uploading logs, which makes diagnosing issues difficult.
# Grab the output from the last playbook from the json
# file and log it.
json_output = self.jobdir.job_output_file.replace('txt', 'json')
self.log.debug("Final playbook failed")
if not os.path.exists(json_output):
self.log.debug("JSON logfile {logfile} is missing".format(
logfile=json_output))
return
try:
output = json.load(open(json_output, 'r'))
last_playbook = output[-1]
# Transform json to yaml - because it's easier to read and given
# the size of the data it'll be extra-hard to read this as an
# all on one line stringified nested dict.
yaml_out = yaml.safe_dump(last_playbook, default_flow_style=False)
for line in yaml_out.split('\n'):
self.log.debug(line)
except Exception:
self.log.exception(
"Could not decode json from {logfile}".format(
logfile=json_output))
def getHostList(self, args):
hosts = []
for node in args['nodes']:

Loading…
Cancel
Save