Add callback plugin to emit json
Tried first with the upstream callback plugin, but it is a stdout
plugin, so needs to take over stdout to work. We need stdout for
executor communication. Then tried subclassing- but the magical ansible
module plugin loading fun happened again. Just copy it in and modify it
slightly for now.
We add playbook, phase and index information. We also read the previous
file back in and append to it on subsequent runs. This may be a memory
issue. HOWEVER - the current construction will hold all of an individual
play in memory anyway. Most of our content size concerns are around
devstack jobs where the bulk of the content will be in a single playbook
anyway - so although ram pressure may be a real thing - we may need to
solve it on the single playbook level anyway. But for now, this should
get us the data.
Change-Id: Ic1becaf2f3ab345da22fa62314f1296d76777fec
diff --git a/zuul/ansible/callback/zuul_json.py b/zuul/ansible/callback/zuul_json.py
new file mode 100644
index 0000000..017c27e
--- /dev/null
+++ b/zuul/ansible/callback/zuul_json.py
@@ -0,0 +1,138 @@
+# (c) 2016, Matt Martz <matt@sivel.net>
+# (c) 2017, Red Hat, Inc.
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
+
+# Copy of github.com/ansible/ansible/lib/ansible/plugins/callback/json.py
+# We need to run as a secondary callback not a stdout and we need to control
+# the output file location via a zuul environment variable similar to how we
+# do in zuul_stream.
+# Subclassing wreaks havoc on the module loader and namepsaces
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+import os
+
+from ansible.plugins.callback import CallbackBase
+try:
+ # It's here in 2.4
+ from ansible.vars import strip_internal_keys
+except ImportError:
+ # It's here in 2.3
+ from ansible.vars.manager import strip_internal_keys
+
+
+class CallbackModule(CallbackBase):
+ CALLBACK_VERSION = 2.0
+ # aggregate means we can be loaded and not be the stdout plugin
+ CALLBACK_TYPE = 'aggregate'
+ CALLBACK_NAME = 'zuul_json'
+
+ def __init__(self, display=None):
+ super(CallbackModule, self).__init__(display)
+ self.results = []
+ self.output_path = os.path.splitext(
+ os.environ['ZUUL_JOB_OUTPUT_FILE'])[0] + '.json'
+ # For now, just read in the old file and write it all out again
+ # This may well not scale from a memory perspective- but let's see how
+ # it goes.
+ if os.path.exists(self.output_path):
+ self.results = json.load(open(self.output_path, 'r'))
+
+ def _get_playbook_name(self, work_dir):
+
+ playbook = self._playbook_name
+ if work_dir and playbook.startswith(work_dir):
+ playbook = playbook.replace(work_dir.rstrip('/') + '/', '')
+ # Lop off the first two path elements - ansible/pre_playbook_0
+ for prefix in ('pre', 'playbook', 'post'):
+ full_prefix = 'ansible/{prefix}_'.format(prefix=prefix)
+ if playbook.startswith(full_prefix):
+ playbook = playbook.split(os.path.sep, 2)[2]
+ return playbook
+
+ def _new_play(self, play, phase, index, work_dir):
+ return {
+ 'play': {
+ 'name': play.name,
+ 'id': str(play._uuid),
+ 'phase': phase,
+ 'index': index,
+ 'playbook': self._get_playbook_name(work_dir),
+ },
+ 'tasks': []
+ }
+
+ def _new_task(self, task):
+ return {
+ 'task': {
+ 'name': task.name,
+ 'id': str(task._uuid)
+ },
+ 'hosts': {}
+ }
+
+ def v2_playbook_on_start(self, playbook):
+ self._playbook_name = os.path.splitext(playbook._file_name)[0]
+
+ def v2_playbook_on_play_start(self, play):
+ # Get the hostvars from just one host - the vars we're looking for will
+ # be identical on all of them
+ hostvars = next(iter(play._variable_manager._hostvars.values()))
+ phase = hostvars.get('zuul_execution_phase')
+ index = hostvars.get('zuul_execution_phase_index')
+ # TODO(mordred) For now, protect this to make it not absurdly strange
+ # to run local tests with the callback plugin enabled. Remove once we
+ # have a "run playbook like zuul runs playbook" tool.
+ work_dir = None
+ if 'zuul' in hostvars and 'executor' in hostvars['zuul']:
+ # imply work_dir from src_root
+ work_dir = os.path.dirname(
+ hostvars['zuul']['executor']['src_root'])
+ self.results.append(self._new_play(play, phase, index, work_dir))
+
+ def v2_playbook_on_task_start(self, task, is_conditional):
+ self.results[-1]['tasks'].append(self._new_task(task))
+
+ def v2_runner_on_ok(self, result, **kwargs):
+ host = result._host
+ if result._result.get('_ansible_no_log', False):
+ self.results[-1]['tasks'][-1]['hosts'][host.name] = dict(
+ censored="the output has been hidden due to the fact that"
+ " 'no_log: true' was specified for this result")
+ else:
+ clean_result = strip_internal_keys(result._result)
+ self.results[-1]['tasks'][-1]['hosts'][host.name] = clean_result
+
+ def v2_playbook_on_stats(self, stats):
+ """Display info about playbook statistics"""
+ hosts = sorted(stats.processed.keys())
+
+ summary = {}
+ for h in hosts:
+ s = stats.summarize(h)
+ summary[h] = s
+
+ output = {
+ 'plays': self.results,
+ 'stats': summary
+ }
+
+ json.dump(output, open(self.output_path, 'w'),
+ indent=4, sort_keys=True, separators=(',', ': '))
+
+ v2_runner_on_failed = v2_runner_on_ok
+ v2_runner_on_unreachable = v2_runner_on_ok
+ v2_runner_on_skipped = v2_runner_on_ok
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 824a47a..8ab369c 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -1211,6 +1211,7 @@
config.write('callback_plugins = %s\n'
% self.executor_server.callback_dir)
config.write('stdout_callback = zuul_stream\n')
+ config.write('callback_whitelist = zuul_json\n')
# bump the timeout because busy nodes may take more than
# 10s to respond
config.write('timeout = 30\n')
@@ -1353,6 +1354,8 @@
# TODO(mordred) If/when we rework use of logger in ansible-playbook
# we'll want to change how this works to use that as well. For now,
# this is what we need to do.
+ # TODO(mordred) We probably want to put this into the json output
+ # as well.
with open(self.jobdir.job_output_file, 'a') as job_output:
job_output.write("{now} | ANSIBLE PARSE ERROR\n".format(
now=datetime.datetime.now()))