Merge "Return executor errors to user" into feature/zuulv3
diff --git a/tests/base.py b/tests/base.py
index 1cc9999..fb94638 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -1308,7 +1308,7 @@
self.running_builds.append(build)
self.job_builds[job.unique] = build
args = json.loads(job.arguments)
- args['vars']['zuul']['_test'] = dict(test_root=self._test_root)
+ args['zuul']['_test'] = dict(test_root=self._test_root)
job.arguments = json.dumps(args)
self.job_workers[job.unique] = RecordingAnsibleJob(self, job)
self.job_workers[job.unique].run()
diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py
index f360866..0cfe3da 100644
--- a/tests/unit/test_github_driver.py
+++ b/tests/unit/test_github_driver.py
@@ -46,7 +46,7 @@
self.getJobFromHistory('project-test2').result)
job = self.getJobFromHistory('project-test2')
- zuulvars = job.parameters['vars']['zuul']
+ zuulvars = job.parameters['zuul']
self.assertEqual(A.number, zuulvars['change'])
self.assertEqual(A.head_sha, zuulvars['patchset'])
self.assertEqual(1, len(A.comments))
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 61bf9f8..ac2a779 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -2757,7 +2757,7 @@
for build in self.history:
self.assertEqual(results.get(build.uuid, ''),
- build.parameters['vars']['zuul'].get('tags'))
+ build.parameters['zuul'].get('tags'))
def test_timer(self):
"Test that a periodic job is triggered"
diff --git a/zuul/ansible/callback/zuul_json.py b/zuul/ansible/callback/zuul_json.py
new file mode 100644
index 0000000..017c27e
--- /dev/null
+++ b/zuul/ansible/callback/zuul_json.py
@@ -0,0 +1,138 @@
+# (c) 2016, Matt Martz <matt@sivel.net>
+# (c) 2017, Red Hat, Inc.
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
+
+# Copy of github.com/ansible/ansible/lib/ansible/plugins/callback/json.py
+# We need to run as a secondary callback not a stdout and we need to control
+# the output file location via a zuul environment variable similar to how we
+# do in zuul_stream.
+# Subclassing wreaks havoc on the module loader and namepsaces
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+import os
+
+from ansible.plugins.callback import CallbackBase
+try:
+ # It's here in 2.4
+ from ansible.vars import strip_internal_keys
+except ImportError:
+ # It's here in 2.3
+ from ansible.vars.manager import strip_internal_keys
+
+
+class CallbackModule(CallbackBase):
+ CALLBACK_VERSION = 2.0
+ # aggregate means we can be loaded and not be the stdout plugin
+ CALLBACK_TYPE = 'aggregate'
+ CALLBACK_NAME = 'zuul_json'
+
+ def __init__(self, display=None):
+ super(CallbackModule, self).__init__(display)
+ self.results = []
+ self.output_path = os.path.splitext(
+ os.environ['ZUUL_JOB_OUTPUT_FILE'])[0] + '.json'
+ # For now, just read in the old file and write it all out again
+ # This may well not scale from a memory perspective- but let's see how
+ # it goes.
+ if os.path.exists(self.output_path):
+ self.results = json.load(open(self.output_path, 'r'))
+
+ def _get_playbook_name(self, work_dir):
+
+ playbook = self._playbook_name
+ if work_dir and playbook.startswith(work_dir):
+ playbook = playbook.replace(work_dir.rstrip('/') + '/', '')
+ # Lop off the first two path elements - ansible/pre_playbook_0
+ for prefix in ('pre', 'playbook', 'post'):
+ full_prefix = 'ansible/{prefix}_'.format(prefix=prefix)
+ if playbook.startswith(full_prefix):
+ playbook = playbook.split(os.path.sep, 2)[2]
+ return playbook
+
+ def _new_play(self, play, phase, index, work_dir):
+ return {
+ 'play': {
+ 'name': play.name,
+ 'id': str(play._uuid),
+ 'phase': phase,
+ 'index': index,
+ 'playbook': self._get_playbook_name(work_dir),
+ },
+ 'tasks': []
+ }
+
+ def _new_task(self, task):
+ return {
+ 'task': {
+ 'name': task.name,
+ 'id': str(task._uuid)
+ },
+ 'hosts': {}
+ }
+
+ def v2_playbook_on_start(self, playbook):
+ self._playbook_name = os.path.splitext(playbook._file_name)[0]
+
+ def v2_playbook_on_play_start(self, play):
+ # Get the hostvars from just one host - the vars we're looking for will
+ # be identical on all of them
+ hostvars = next(iter(play._variable_manager._hostvars.values()))
+ phase = hostvars.get('zuul_execution_phase')
+ index = hostvars.get('zuul_execution_phase_index')
+ # TODO(mordred) For now, protect this to make it not absurdly strange
+ # to run local tests with the callback plugin enabled. Remove once we
+ # have a "run playbook like zuul runs playbook" tool.
+ work_dir = None
+ if 'zuul' in hostvars and 'executor' in hostvars['zuul']:
+ # imply work_dir from src_root
+ work_dir = os.path.dirname(
+ hostvars['zuul']['executor']['src_root'])
+ self.results.append(self._new_play(play, phase, index, work_dir))
+
+ def v2_playbook_on_task_start(self, task, is_conditional):
+ self.results[-1]['tasks'].append(self._new_task(task))
+
+ def v2_runner_on_ok(self, result, **kwargs):
+ host = result._host
+ if result._result.get('_ansible_no_log', False):
+ self.results[-1]['tasks'][-1]['hosts'][host.name] = dict(
+ censored="the output has been hidden due to the fact that"
+ " 'no_log: true' was specified for this result")
+ else:
+ clean_result = strip_internal_keys(result._result)
+ self.results[-1]['tasks'][-1]['hosts'][host.name] = clean_result
+
+ def v2_playbook_on_stats(self, stats):
+ """Display info about playbook statistics"""
+ hosts = sorted(stats.processed.keys())
+
+ summary = {}
+ for h in hosts:
+ s = stats.summarize(h)
+ summary[h] = s
+
+ output = {
+ 'plays': self.results,
+ 'stats': summary
+ }
+
+ json.dump(output, open(self.output_path, 'w'),
+ indent=4, sort_keys=True, separators=(',', ': '))
+
+ v2_runner_on_failed = v2_runner_on_ok
+ v2_runner_on_unreachable = v2_runner_on_ok
+ v2_runner_on_skipped = v2_runner_on_ok
diff --git a/zuul/ansible/callback/zuul_stream.py b/zuul/ansible/callback/zuul_stream.py
index cc979f2..e9f969a 100644
--- a/zuul/ansible/callback/zuul_stream.py
+++ b/zuul/ansible/callback/zuul_stream.py
@@ -312,8 +312,7 @@
else:
self._log_message(
result=result,
- status=status,
- result_dict=result_dict)
+ status=status)
elif 'results' in result_dict:
for res in result_dict['results']:
self._log_message(
@@ -342,7 +341,6 @@
self._log_message(
result=result,
msg="Item: {item}".format(item=result_dict['item']),
- result_dict=result_dict,
status=status)
else:
self._log_message(
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index 9768bc1..f764778 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -253,10 +253,12 @@
params['nodes'] = nodes
params['groups'] = [group.toDict() for group in nodeset.getGroups()]
params['vars'] = copy.deepcopy(job.variables)
+ params['secrets'] = {}
if job.auth:
for secret in job.auth.secrets:
- params['vars'][secret.name] = copy.deepcopy(secret.secret_data)
- params['vars']['zuul'] = zuul_params
+ secret_data = copy.deepcopy(secret.secret_data)
+ params['secrets'][secret.name] = secret_data
+ params['zuul'] = zuul_params
projects = set()
def make_project_dict(project, override_branch=None):
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 0246223..f291dce 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -225,6 +225,8 @@
pass
self.known_hosts = os.path.join(ssh_dir, 'known_hosts')
self.inventory = os.path.join(self.ansible_root, 'inventory.yaml')
+ self.secrets = os.path.join(self.ansible_root, 'secrets.yaml')
+ self.has_secrets = False
self.playbooks = [] # The list of candidate playbooks
self.playbook = None # A pointer to the candidate we have chosen
self.pre_playbooks = []
@@ -782,7 +784,7 @@
def _execute(self):
args = json.loads(self.job.arguments)
self.log.debug("Beginning job %s for ref %s" %
- (self.job.name, args['vars']['zuul']['ref']))
+ (self.job.name, args['zuul']['ref']))
self.log.debug("Args: %s" % (self.job.arguments,))
self.log.debug("Job root: %s" % (self.jobdir.root,))
tasks = []
@@ -1188,9 +1190,12 @@
jobdir_playbook.roles_path.append(role_path)
def prepareAnsibleFiles(self, args):
- all_vars = dict(args['vars'])
+ all_vars = args['vars'].copy()
# TODO(mordred) Hack to work around running things with python3
all_vars['ansible_python_interpreter'] = '/usr/bin/python2'
+ if 'zuul' in all_vars:
+ raise Exception("Defining vars named 'zuul' is not allowed")
+ all_vars['zuul'] = args['zuul'].copy()
all_vars['zuul']['executor'] = dict(
hostname=self.executor_server.hostname,
src_root=self.jobdir.src_root,
@@ -1209,6 +1214,15 @@
for key in node['host_keys']:
known_hosts.write('%s\n' % key)
+ secrets = args['secrets'].copy()
+ if secrets:
+ if 'zuul' in secrets:
+ raise Exception("Defining secrets named 'zuul' is not allowed")
+ with open(self.jobdir.secrets, 'w') as secrets_yaml:
+ secrets_yaml.write(
+ yaml.safe_dump(secrets, default_flow_style=False))
+ self.jobdir.has_secrets = True
+
def writeAnsibleConfig(self, jobdir_playbook):
trusted = jobdir_playbook.trusted
@@ -1227,6 +1241,7 @@
config.write('callback_plugins = %s\n'
% self.executor_server.callback_dir)
config.write('stdout_callback = zuul_stream\n')
+ config.write('callback_whitelist = zuul_json\n')
# bump the timeout because busy nodes may take more than
# 10s to respond
config.write('timeout = 30\n')
@@ -1369,6 +1384,8 @@
# TODO(mordred) If/when we rework use of logger in ansible-playbook
# we'll want to change how this works to use that as well. For now,
# this is what we need to do.
+ # TODO(mordred) We probably want to put this into the json output
+ # as well.
with open(self.jobdir.job_output_file, 'a') as job_output:
job_output.write("{now} | ANSIBLE PARSE ERROR\n".format(
now=datetime.datetime.now()))
@@ -1389,6 +1406,8 @@
verbose = '-v'
cmd = ['ansible-playbook', verbose, playbook.path]
+ if self.jobdir.has_secrets:
+ cmd.extend(['-e', '@' + self.jobdir.secrets])
if success is not None:
cmd.extend(['-e', 'success=%s' % str(bool(success))])