Add callback plugin to emit json

Tried first with the upstream callback plugin, but it is a stdout
plugin, so needs to take over stdout to work. We need stdout for
executor communication. Then tried subclassing- but the magical ansible
module plugin loading fun happened again. Just copy it in and modify it
slightly for now.

We add playbook, phase and index information. We also read the previous
file back in and append to it on subsequent runs. This may be a memory
issue. HOWEVER - the current construction will hold all of an individual
play in memory anyway. Most of our content size concerns are around
devstack jobs where the bulk of the content will be in a single playbook
anyway - so although ram pressure may be a real thing - we may need to
solve it on the single playbook level anyway. But for now, this should
get us the data.

Change-Id: Ic1becaf2f3ab345da22fa62314f1296d76777fec
changes/15/484515/1
Monty Taylor 5 years ago
parent c3dbefdf4f
commit 83509426c8
No known key found for this signature in database
GPG Key ID: 7BAE94BC7141A594
  1. 138
      zuul/ansible/callback/zuul_json.py
  2. 3
      zuul/executor/server.py

@ -0,0 +1,138 @@
# (c) 2016, Matt Martz <matt@sivel.net>
# (c) 2017, Red Hat, Inc.
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# Copy of github.com/ansible/ansible/lib/ansible/plugins/callback/json.py
# We need to run as a secondary callback not a stdout and we need to control
# the output file location via a zuul environment variable similar to how we
# do in zuul_stream.
# Subclassing wreaks havoc on the module loader and namepsaces
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import json
import os
from ansible.plugins.callback import CallbackBase
try:
# It's here in 2.4
from ansible.vars import strip_internal_keys
except ImportError:
# It's here in 2.3
from ansible.vars.manager import strip_internal_keys
class CallbackModule(CallbackBase):
CALLBACK_VERSION = 2.0
# aggregate means we can be loaded and not be the stdout plugin
CALLBACK_TYPE = 'aggregate'
CALLBACK_NAME = 'zuul_json'
def __init__(self, display=None):
super(CallbackModule, self).__init__(display)
self.results = []
self.output_path = os.path.splitext(
os.environ['ZUUL_JOB_OUTPUT_FILE'])[0] + '.json'
# For now, just read in the old file and write it all out again
# This may well not scale from a memory perspective- but let's see how
# it goes.
if os.path.exists(self.output_path):
self.results = json.load(open(self.output_path, 'r'))
def _get_playbook_name(self, work_dir):
playbook = self._playbook_name
if work_dir and playbook.startswith(work_dir):
playbook = playbook.replace(work_dir.rstrip('/') + '/', '')
# Lop off the first two path elements - ansible/pre_playbook_0
for prefix in ('pre', 'playbook', 'post'):
full_prefix = 'ansible/{prefix}_'.format(prefix=prefix)
if playbook.startswith(full_prefix):
playbook = playbook.split(os.path.sep, 2)[2]
return playbook
def _new_play(self, play, phase, index, work_dir):
return {
'play': {
'name': play.name,
'id': str(play._uuid),
'phase': phase,
'index': index,
'playbook': self._get_playbook_name(work_dir),
},
'tasks': []
}
def _new_task(self, task):
return {
'task': {
'name': task.name,
'id': str(task._uuid)
},
'hosts': {}
}
def v2_playbook_on_start(self, playbook):
self._playbook_name = os.path.splitext(playbook._file_name)[0]
def v2_playbook_on_play_start(self, play):
# Get the hostvars from just one host - the vars we're looking for will
# be identical on all of them
hostvars = next(iter(play._variable_manager._hostvars.values()))
phase = hostvars.get('zuul_execution_phase')
index = hostvars.get('zuul_execution_phase_index')
# TODO(mordred) For now, protect this to make it not absurdly strange
# to run local tests with the callback plugin enabled. Remove once we
# have a "run playbook like zuul runs playbook" tool.
work_dir = None
if 'zuul' in hostvars and 'executor' in hostvars['zuul']:
# imply work_dir from src_root
work_dir = os.path.dirname(
hostvars['zuul']['executor']['src_root'])
self.results.append(self._new_play(play, phase, index, work_dir))
def v2_playbook_on_task_start(self, task, is_conditional):
self.results[-1]['tasks'].append(self._new_task(task))
def v2_runner_on_ok(self, result, **kwargs):
host = result._host
if result._result.get('_ansible_no_log', False):
self.results[-1]['tasks'][-1]['hosts'][host.name] = dict(
censored="the output has been hidden due to the fact that"
" 'no_log: true' was specified for this result")
else:
clean_result = strip_internal_keys(result._result)
self.results[-1]['tasks'][-1]['hosts'][host.name] = clean_result
def v2_playbook_on_stats(self, stats):
"""Display info about playbook statistics"""
hosts = sorted(stats.processed.keys())
summary = {}
for h in hosts:
s = stats.summarize(h)
summary[h] = s
output = {
'plays': self.results,
'stats': summary
}
json.dump(output, open(self.output_path, 'w'),
indent=4, sort_keys=True, separators=(',', ': '))
v2_runner_on_failed = v2_runner_on_ok
v2_runner_on_unreachable = v2_runner_on_ok
v2_runner_on_skipped = v2_runner_on_ok

@ -1211,6 +1211,7 @@ class AnsibleJob(object):
config.write('callback_plugins = %s\n'
% self.executor_server.callback_dir)
config.write('stdout_callback = zuul_stream\n')
config.write('callback_whitelist = zuul_json\n')
# bump the timeout because busy nodes may take more than
# 10s to respond
config.write('timeout = 30\n')
@ -1353,6 +1354,8 @@ class AnsibleJob(object):
# TODO(mordred) If/when we rework use of logger in ansible-playbook
# we'll want to change how this works to use that as well. For now,
# this is what we need to do.
# TODO(mordred) We probably want to put this into the json output
# as well.
with open(self.jobdir.job_output_file, 'a') as job_output:
job_output.write("{now} | ANSIBLE PARSE ERROR\n".format(
now=datetime.datetime.now()))

Loading…
Cancel
Save