Merge "Add zuul.d configuration split documentation" into feature/zuulv3
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index bfb8136..4a9a99e 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -718,8 +718,23 @@
roles: a Galaxy role, which is simply a role that is installed from
Ansible Galaxy, or a Zuul role, which is a role provided by a
project managed by Zuul. Zuul roles are able to benefit from
- speculative merging and cross-project dependencies when used by jobs
- in untrusted projects.
+ speculative merging and cross-project dependencies when used by
+ playbooks in untrusted projects. Roles are added to the Ansible
+ role path in the order they appear on the job -- roles earlier in
+ the list will take precedence over those which follow.
+
+ In the case of job inheritance or variance, the roles used for each
+ of the playbooks run by the job will be only those which were
+ defined along with that playbook. If a child job inherits from a
+ parent which defines a pre and post playbook, then the pre and post
+ playbooks it inherits from the parent job will run only with the
+ roles that were defined on the parent. If the child adds its own
+ pre and post playbooks, then any roles added by the child will be
+ available to the child's playbooks. This is so that a job which
+ inherits from a parent does not inadvertantly alter the behavior of
+ the parent's playbooks by the addition of conflicting roles. Roles
+ added by a child will appear before those it inherits from its
+ parent.
A project which supplies a role may be structured in one of two
configurations: a bare role (in which the role exists at the root of
diff --git a/doc/source/user/jobs.rst b/doc/source/user/jobs.rst
index 78121bc..c2c376e 100644
--- a/doc/source/user/jobs.rst
+++ b/doc/source/user/jobs.rst
@@ -70,8 +70,41 @@
.. TODO: link to base job documentation and/or document src (and logs?) directory
+Variables
+---------
+
+Any variables specified in the job definition are available as Ansible
+host variables. They are added to the `vars` section of the inventory
+file under the `all` hosts group, so they are available to all hosts.
+Simply refer to them by the name specified in the job's `vars`
+section.
+
+Secrets
+~~~~~~~
+
+Secrets also appear as variables available to Ansible. Unlike job
+variables, these are not added to the inventory file (so that the
+inventory file may be kept for debugging purposes without revealing
+secrets). But they are still available to Ansible as normal
+variables. Because secrets are groups of variables, they will appear
+as a dictionary structure in templates, with the dictionary itself
+being the name of the secret, and its members the individual items in
+the secret. For example, a secret defined as::
+
+ - secret:
+ name: credentials
+ data:
+ username: foo
+ password: bar
+
+Might be used in a template as::
+
+ {{ credentials.username }} {{ credentials.password }}
+
+.. TODO: xref job vars
+
Zuul Variables
---------------
+~~~~~~~~~~~~~~
Zuul supplies not only the variables specified by the job definition
to Ansible, but also some variables from the executor itself. They
diff --git a/tests/base.py b/tests/base.py
index 9709bf7..fb94638 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -1308,7 +1308,7 @@
self.running_builds.append(build)
self.job_builds[job.unique] = build
args = json.loads(job.arguments)
- args['vars']['zuul']['_test'] = dict(test_root=self._test_root)
+ args['zuul']['_test'] = dict(test_root=self._test_root)
job.arguments = json.dumps(args)
self.job_workers[job.unique] = RecordingAnsibleJob(self, job)
self.job_workers[job.unique].run()
@@ -1359,12 +1359,12 @@
self.recordResult(result)
return result
- def runAnsible(self, cmd, timeout, trusted=False):
+ def runAnsible(self, cmd, timeout, config_file, trusted):
build = self.executor_server.job_builds[self.job.unique]
if self.executor_server._run_ansible:
result = super(RecordingAnsibleJob, self).runAnsible(
- cmd, timeout, trusted=trusted)
+ cmd, timeout, config_file, trusted)
else:
result = build.run()
return result
diff --git a/tests/fixtures/config/in-repo/git/common-config/zuul.yaml b/tests/fixtures/config/in-repo/git/common-config/zuul.yaml
index 1fdaf2e..fce086e 100644
--- a/tests/fixtures/config/in-repo/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/in-repo/git/common-config/zuul.yaml
@@ -32,6 +32,26 @@
verified: 0
precedence: high
+- pipeline:
+ name: gate
+ manager: dependent
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - code-review: 2
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
+
- job:
name: common-config-test
diff --git a/tests/fixtures/config/roles/git/org_project/playbooks/parent-post.yaml b/tests/fixtures/config/roles/git/org_project/playbooks/parent-post.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/roles/git/org_project/playbooks/parent-post.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/roles/git/org_project/playbooks/parent-pre.yaml b/tests/fixtures/config/roles/git/org_project/playbooks/parent-pre.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/roles/git/org_project/playbooks/parent-pre.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/roles/git/org_project/roles/README b/tests/fixtures/config/roles/git/org_project/roles/README
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/fixtures/config/roles/git/org_project/roles/README
diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py
index 8d9d127..c9e2e91 100644
--- a/tests/unit/test_connection.py
+++ b/tests/unit/test_connection.py
@@ -69,7 +69,7 @@
insp = sa.engine.reflection.Inspector(
self.connections.connections['resultsdb'].engine)
- self.assertEqual(9, len(insp.get_columns(buildset_table)))
+ self.assertEqual(10, len(insp.get_columns(buildset_table)))
self.assertEqual(10, len(insp.get_columns(build_table)))
def test_sql_results(self):
@@ -108,6 +108,7 @@
self.assertEqual(1, buildset0['patchset'])
self.assertEqual(1, buildset0['score'])
self.assertEqual('Build succeeded.', buildset0['message'])
+ self.assertEqual('tenant-one', buildset0['tenant'])
buildset0_builds = conn.execute(
sa.sql.select([reporter.connection.zuul_build_table]).
diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py
index f360866..0cfe3da 100644
--- a/tests/unit/test_github_driver.py
+++ b/tests/unit/test_github_driver.py
@@ -46,7 +46,7 @@
self.getJobFromHistory('project-test2').result)
job = self.getJobFromHistory('project-test2')
- zuulvars = job.parameters['vars']['zuul']
+ zuulvars = job.parameters['zuul']
self.assertEqual(A.number, zuulvars['change'])
self.assertEqual(A.head_sha, zuulvars['patchset'])
self.assertEqual(1, len(A.comments))
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index 7fe101e..a52a2ee 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -54,7 +54,8 @@
encryption.deserialize_rsa_keypair(f.read())
self.context = model.SourceContext(self.project, 'master',
'test', True)
- self.start_mark = yaml.Mark('name', 0, 0, 0, '', 0)
+ m = yaml.Mark('name', 0, 0, 0, '', 0)
+ self.start_mark = configloader.ZuulMark(m, m, '')
@property
def job(self):
@@ -95,9 +96,9 @@
def test_job_inheritance(self):
# This is standard job inheritance.
- base_pre = model.PlaybookContext(self.context, 'base-pre')
- base_run = model.PlaybookContext(self.context, 'base-run')
- base_post = model.PlaybookContext(self.context, 'base-post')
+ base_pre = model.PlaybookContext(self.context, 'base-pre', [])
+ base_run = model.PlaybookContext(self.context, 'base-run', [])
+ base_post = model.PlaybookContext(self.context, 'base-post', [])
base = model.Job('base')
base.timeout = 30
@@ -121,9 +122,9 @@
def test_job_variants(self):
# This simulates freezing a job.
- py27_pre = model.PlaybookContext(self.context, 'py27-pre')
- py27_run = model.PlaybookContext(self.context, 'py27-run')
- py27_post = model.PlaybookContext(self.context, 'py27-post')
+ py27_pre = model.PlaybookContext(self.context, 'py27-pre', [])
+ py27_run = model.PlaybookContext(self.context, 'py27-run', [])
+ py27_post = model.PlaybookContext(self.context, 'py27-post', [])
py27 = model.Job('py27')
py27.timeout = 30
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 61bf9f8..ac2a779 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -2757,7 +2757,7 @@
for build in self.history:
self.assertEqual(results.get(build.uuid, ''),
- build.parameters['vars']['zuul'].get('tags'))
+ build.parameters['zuul'].get('tags'))
def test_timer(self):
"Test that a periodic job is triggered"
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 87eddc6..734c45c 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -199,6 +199,52 @@
self.executor_server.release()
self.waitUntilSettled()
+ def test_dynamic_dependent_pipeline(self):
+ # Test dynamically adding a project to a
+ # dependent pipeline for the first time
+ self.executor_server.hold_jobs_in_build = True
+
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ gate_pipeline = tenant.layout.pipelines['gate']
+
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ name: project-test2
+
+ - project:
+ name: org/project
+ gate:
+ jobs:
+ - project-test2
+ """)
+
+ in_repo_playbook = textwrap.dedent(
+ """
+ - hosts: all
+ tasks: []
+ """)
+
+ file_dict = {'.zuul.yaml': in_repo_conf,
+ 'playbooks/project-test2.yaml': in_repo_playbook}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+ A.addApproval('approved', 1)
+ self.fake_gerrit.addEvent(A.addApproval('code-review', 2))
+ self.waitUntilSettled()
+
+ items = gate_pipeline.getAllItems()
+ self.assertEqual(items[0].change.number, '1')
+ self.assertEqual(items[0].change.patchset, '1')
+ self.assertTrue(items[0].live)
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ # Make sure the dynamic queue got cleaned up
+ self.assertEqual(gate_pipeline.queues, [])
+
def test_in_repo_branch(self):
in_repo_conf = textwrap.dedent(
"""
@@ -673,6 +719,25 @@
class TestRoles(ZuulTestCase):
tenant_config_file = 'config/roles/main.yaml'
+ def _assertRolePath(self, build, playbook, content):
+ path = os.path.join(self.test_root, build.uuid,
+ 'ansible', playbook, 'ansible.cfg')
+ roles_paths = []
+ with open(path) as f:
+ for line in f:
+ if line.startswith('roles_path'):
+ roles_paths.append(line)
+ print(roles_paths)
+ if content:
+ self.assertEqual(len(roles_paths), 1,
+ "Should have one roles_path line in %s" %
+ (playbook,))
+ self.assertIn(content, roles_paths[0])
+ else:
+ self.assertEqual(len(roles_paths), 0,
+ "Should have no roles_path line in %s" %
+ (playbook,))
+
def test_role(self):
# This exercises a proposed change to a role being checked out
# and used.
@@ -687,6 +752,75 @@
dict(name='project-test', result='SUCCESS', changes='1,1 2,1'),
])
+ def test_role_inheritance(self):
+ self.executor_server.hold_jobs_in_build = True
+ conf = textwrap.dedent(
+ """
+ - job:
+ name: parent
+ roles:
+ - zuul: bare-role
+ pre-run: playbooks/parent-pre
+ post-run: playbooks/parent-post
+
+ - job:
+ name: project-test
+ parent: parent
+ roles:
+ - zuul: org/project
+
+ - project:
+ name: org/project
+ check:
+ jobs:
+ - project-test
+ """)
+
+ file_dict = {'.zuul.yaml': conf}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertEqual(len(self.builds), 1)
+ build = self.getBuildByName('project-test')
+ self._assertRolePath(build, 'pre_playbook_0', 'role_0')
+ self._assertRolePath(build, 'playbook_0', 'role_0')
+ self._assertRolePath(build, 'playbook_0', 'role_1')
+ self._assertRolePath(build, 'post_playbook_0', 'role_0')
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertHistory([
+ dict(name='project-test', result='SUCCESS', changes='1,1'),
+ ])
+
+ def test_role_error(self):
+ conf = textwrap.dedent(
+ """
+ - job:
+ name: project-test
+ roles:
+ - zuul: common-config
+
+ - project:
+ name: org/project
+ check:
+ jobs:
+ - project-test
+ """)
+
+ file_dict = {'.zuul.yaml': conf}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertIn(
+ '- project-test project-test : ERROR Unable to find role',
+ A.messages[-1])
+
class TestShadow(ZuulTestCase):
tenant_config_file = 'config/shadow/main.yaml'
diff --git a/zuul/ansible/callback/zuul_json.py b/zuul/ansible/callback/zuul_json.py
new file mode 100644
index 0000000..017c27e
--- /dev/null
+++ b/zuul/ansible/callback/zuul_json.py
@@ -0,0 +1,138 @@
+# (c) 2016, Matt Martz <matt@sivel.net>
+# (c) 2017, Red Hat, Inc.
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
+
+# Copy of github.com/ansible/ansible/lib/ansible/plugins/callback/json.py
+# We need to run as a secondary callback not a stdout and we need to control
+# the output file location via a zuul environment variable similar to how we
+# do in zuul_stream.
+# Subclassing wreaks havoc on the module loader and namepsaces
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+import os
+
+from ansible.plugins.callback import CallbackBase
+try:
+ # It's here in 2.4
+ from ansible.vars import strip_internal_keys
+except ImportError:
+ # It's here in 2.3
+ from ansible.vars.manager import strip_internal_keys
+
+
+class CallbackModule(CallbackBase):
+ CALLBACK_VERSION = 2.0
+ # aggregate means we can be loaded and not be the stdout plugin
+ CALLBACK_TYPE = 'aggregate'
+ CALLBACK_NAME = 'zuul_json'
+
+ def __init__(self, display=None):
+ super(CallbackModule, self).__init__(display)
+ self.results = []
+ self.output_path = os.path.splitext(
+ os.environ['ZUUL_JOB_OUTPUT_FILE'])[0] + '.json'
+ # For now, just read in the old file and write it all out again
+ # This may well not scale from a memory perspective- but let's see how
+ # it goes.
+ if os.path.exists(self.output_path):
+ self.results = json.load(open(self.output_path, 'r'))
+
+ def _get_playbook_name(self, work_dir):
+
+ playbook = self._playbook_name
+ if work_dir and playbook.startswith(work_dir):
+ playbook = playbook.replace(work_dir.rstrip('/') + '/', '')
+ # Lop off the first two path elements - ansible/pre_playbook_0
+ for prefix in ('pre', 'playbook', 'post'):
+ full_prefix = 'ansible/{prefix}_'.format(prefix=prefix)
+ if playbook.startswith(full_prefix):
+ playbook = playbook.split(os.path.sep, 2)[2]
+ return playbook
+
+ def _new_play(self, play, phase, index, work_dir):
+ return {
+ 'play': {
+ 'name': play.name,
+ 'id': str(play._uuid),
+ 'phase': phase,
+ 'index': index,
+ 'playbook': self._get_playbook_name(work_dir),
+ },
+ 'tasks': []
+ }
+
+ def _new_task(self, task):
+ return {
+ 'task': {
+ 'name': task.name,
+ 'id': str(task._uuid)
+ },
+ 'hosts': {}
+ }
+
+ def v2_playbook_on_start(self, playbook):
+ self._playbook_name = os.path.splitext(playbook._file_name)[0]
+
+ def v2_playbook_on_play_start(self, play):
+ # Get the hostvars from just one host - the vars we're looking for will
+ # be identical on all of them
+ hostvars = next(iter(play._variable_manager._hostvars.values()))
+ phase = hostvars.get('zuul_execution_phase')
+ index = hostvars.get('zuul_execution_phase_index')
+ # TODO(mordred) For now, protect this to make it not absurdly strange
+ # to run local tests with the callback plugin enabled. Remove once we
+ # have a "run playbook like zuul runs playbook" tool.
+ work_dir = None
+ if 'zuul' in hostvars and 'executor' in hostvars['zuul']:
+ # imply work_dir from src_root
+ work_dir = os.path.dirname(
+ hostvars['zuul']['executor']['src_root'])
+ self.results.append(self._new_play(play, phase, index, work_dir))
+
+ def v2_playbook_on_task_start(self, task, is_conditional):
+ self.results[-1]['tasks'].append(self._new_task(task))
+
+ def v2_runner_on_ok(self, result, **kwargs):
+ host = result._host
+ if result._result.get('_ansible_no_log', False):
+ self.results[-1]['tasks'][-1]['hosts'][host.name] = dict(
+ censored="the output has been hidden due to the fact that"
+ " 'no_log: true' was specified for this result")
+ else:
+ clean_result = strip_internal_keys(result._result)
+ self.results[-1]['tasks'][-1]['hosts'][host.name] = clean_result
+
+ def v2_playbook_on_stats(self, stats):
+ """Display info about playbook statistics"""
+ hosts = sorted(stats.processed.keys())
+
+ summary = {}
+ for h in hosts:
+ s = stats.summarize(h)
+ summary[h] = s
+
+ output = {
+ 'plays': self.results,
+ 'stats': summary
+ }
+
+ json.dump(output, open(self.output_path, 'w'),
+ indent=4, sort_keys=True, separators=(',', ': '))
+
+ v2_runner_on_failed = v2_runner_on_ok
+ v2_runner_on_unreachable = v2_runner_on_ok
+ v2_runner_on_skipped = v2_runner_on_ok
diff --git a/zuul/ansible/callback/zuul_stream.py b/zuul/ansible/callback/zuul_stream.py
index cc979f2..e9f969a 100644
--- a/zuul/ansible/callback/zuul_stream.py
+++ b/zuul/ansible/callback/zuul_stream.py
@@ -312,8 +312,7 @@
else:
self._log_message(
result=result,
- status=status,
- result_dict=result_dict)
+ status=status)
elif 'results' in result_dict:
for res in result_dict['results']:
self._log_message(
@@ -342,7 +341,6 @@
self._log_message(
result=result,
msg="Item: {item}".format(item=result_dict['item']),
- result_dict=result_dict,
status=status)
else:
self._log_message(
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 254527a..f8e2d15 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -15,8 +15,8 @@
import copy
import os
import logging
-import pprint
import textwrap
+import io
import voluptuous as vs
@@ -131,7 +131,7 @@
{error}
- The error appears in a {stanza} stanza with the content:
+ The error appears in the following {stanza} stanza:
{content}
@@ -140,20 +140,42 @@
m = m.format(intro=intro,
error=indent(str(e)),
stanza=stanza,
- content=indent(pprint.pformat(conf)),
+ content=indent(start_mark.snippet.rstrip()),
start_mark=str(start_mark))
raise ConfigurationSyntaxError(m)
+class ZuulMark(object):
+ # The yaml mark class differs between the C and python versions.
+ # The C version does not provide a snippet, and also appears to
+ # lose data under some circumstances.
+ def __init__(self, start_mark, end_mark, stream):
+ self.name = start_mark.name
+ self.index = start_mark.index
+ self.line = start_mark.line
+ self.column = start_mark.column
+ self.snippet = stream[start_mark.index:end_mark.index]
+
+ def __str__(self):
+ return ' in "{name}", line {line}, column {column}'.format(
+ name=self.name,
+ line=self.line + 1,
+ column=self.column + 1,
+ )
+
+
class ZuulSafeLoader(yaml.SafeLoader):
zuul_node_types = frozenset(('job', 'nodeset', 'secret', 'pipeline',
'project', 'project-template',
'semaphore'))
def __init__(self, stream, context):
- super(ZuulSafeLoader, self).__init__(stream)
+ wrapped_stream = io.StringIO(stream)
+ wrapped_stream.name = str(context)
+ super(ZuulSafeLoader, self).__init__(wrapped_stream)
self.name = str(context)
self.zuul_context = context
+ self.zuul_stream = stream
def construct_mapping(self, node, deep=False):
r = super(ZuulSafeLoader, self).construct_mapping(node, deep)
@@ -161,7 +183,8 @@
if len(keys) == 1 and keys.intersection(self.zuul_node_types):
d = list(r.values())[0]
if isinstance(d, dict):
- d['_start_mark'] = node.start_mark
+ d['_start_mark'] = ZuulMark(node.start_mark, node.end_mark,
+ self.zuul_stream)
d['_source_context'] = self.zuul_context
return r
@@ -224,7 +247,7 @@
vs.Required('nodes'): to_list(node),
'groups': to_list(group),
'_source_context': model.SourceContext,
- '_start_mark': yaml.Mark,
+ '_start_mark': ZuulMark,
}
return vs.Schema(nodeset)
@@ -262,7 +285,7 @@
secret = {vs.Required('name'): str,
vs.Required('data'): data,
'_source_context': model.SourceContext,
- '_start_mark': yaml.Mark,
+ '_start_mark': ZuulMark,
}
return vs.Schema(secret)
@@ -319,7 +342,7 @@
'post-run': to_list(str),
'run': str,
'_source_context': model.SourceContext,
- '_start_mark': yaml.Mark,
+ '_start_mark': ZuulMark,
'roles': to_list(role),
'required-projects': to_list(vs.Any(job_project, str)),
'vars': dict,
@@ -400,21 +423,34 @@
parent = layout.getJob(conf['parent'])
job.inheritFrom(parent)
+ # Roles are part of the playbook context so we must establish
+ # them earlier than playbooks.
+ if 'roles' in conf:
+ roles = []
+ for role in conf.get('roles', []):
+ if 'zuul' in role:
+ r = JobParser._makeZuulRole(tenant, job, role)
+ if r:
+ roles.append(r)
+ job.addRoles(roles)
+
for pre_run_name in as_list(conf.get('pre-run')):
pre_run = model.PlaybookContext(job.source_context,
- pre_run_name)
+ pre_run_name, job.roles)
job.pre_run = job.pre_run + (pre_run,)
for post_run_name in as_list(conf.get('post-run')):
post_run = model.PlaybookContext(job.source_context,
- post_run_name)
+ post_run_name, job.roles)
job.post_run = (post_run,) + job.post_run
if 'run' in conf:
- run = model.PlaybookContext(job.source_context, conf['run'])
+ run = model.PlaybookContext(job.source_context, conf['run'],
+ job.roles)
job.run = (run,)
else:
if not project_pipeline:
run_name = os.path.join('playbooks', job.name)
- run = model.PlaybookContext(job.source_context, run_name)
+ run = model.PlaybookContext(job.source_context, run_name,
+ job.roles)
job.implied_run = (run,) + job.implied_run
for k in JobParser.simple_attributes:
@@ -460,15 +496,6 @@
job.dependencies = frozenset(as_list(conf.get('dependencies')))
- if 'roles' in conf:
- roles = []
- for role in conf.get('roles', []):
- if 'zuul' in role:
- r = JobParser._makeZuulRole(tenant, job, role)
- if r:
- roles.append(r)
- job.addRoles(roles)
-
variables = conf.get('vars', None)
if variables:
job.updateVariables(variables)
@@ -539,7 +566,7 @@
'merge', 'merge-resolve',
'cherry-pick'),
'_source_context': model.SourceContext,
- '_start_mark': yaml.Mark,
+ '_start_mark': ZuulMark,
}
for p in layout.pipelines.values():
@@ -603,7 +630,7 @@
'cherry-pick'),
'default-branch': str,
'_source_context': model.SourceContext,
- '_start_mark': yaml.Mark,
+ '_start_mark': ZuulMark,
}
for p in layout.pipelines.values():
@@ -758,7 +785,7 @@
'window-decrease-type': window_type,
'window-decrease-factor': window_factor,
'_source_context': model.SourceContext,
- '_start_mark': yaml.Mark,
+ '_start_mark': ZuulMark,
}
pipeline['require'] = PipelineParser.getDriverSchema('require',
connections)
@@ -865,7 +892,7 @@
semaphore = {vs.Required('name'): str,
'max': int,
'_source_context': model.SourceContext,
- '_start_mark': yaml.Mark,
+ '_start_mark': ZuulMark,
}
return vs.Schema(semaphore)
diff --git a/zuul/driver/bubblewrap/__init__.py b/zuul/driver/bubblewrap/__init__.py
index 5ec2448..3609a71 100644
--- a/zuul/driver/bubblewrap/__init__.py
+++ b/zuul/driver/bubblewrap/__init__.py
@@ -150,6 +150,7 @@
'--ro-bind', '/bin', '/bin',
'--ro-bind', '/sbin', '/sbin',
'--ro-bind', '/etc/resolv.conf', '/etc/resolv.conf',
+ '--ro-bind', '/etc/hosts', '/etc/hosts',
'--ro-bind', '{ssh_auth_sock}', '{ssh_auth_sock}',
'--dir', '{work_dir}',
'--bind', '{work_dir}', '{work_dir}',
@@ -166,6 +167,9 @@
if os.path.isdir('/lib64'):
bwrap_command.extend(['--ro-bind', '/lib64', '/lib64'])
+ if os.path.isfile('/etc/nsswitch.conf'):
+ bwrap_command.extend(['--ro-bind', '/etc/nsswitch.conf',
+ '/etc/nsswitch.conf'])
return bwrap_command
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index 1a9e37b..a4a4c12 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -824,6 +824,9 @@
# Create a repo object
repository = github.repository(owner, proj)
+ if not repository:
+ return 'none'
+
# Build up a URL
url = repository._build_url('collaborators', login, 'permission',
base_url=repository._api)
diff --git a/zuul/driver/sql/alembic_reporter/versions/f86c9871ee67_add_tenant_column.py b/zuul/driver/sql/alembic_reporter/versions/f86c9871ee67_add_tenant_column.py
new file mode 100644
index 0000000..7728bd4
--- /dev/null
+++ b/zuul/driver/sql/alembic_reporter/versions/f86c9871ee67_add_tenant_column.py
@@ -0,0 +1,38 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Add tenant column
+
+Revision ID: f86c9871ee67
+Revises: 20126015a87d
+Create Date: 2017-07-17 05:47:48.189767
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'f86c9871ee67'
+down_revision = '20126015a87d'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+ op.add_column('zuul_buildset', sa.Column('tenant', sa.String(255)))
+
+
+def downgrade():
+ op.drop_column('zuul_buildset', 'tenant')
diff --git a/zuul/driver/sql/sqlconnection.py b/zuul/driver/sql/sqlconnection.py
index e478d33..0e3f0dd 100644
--- a/zuul/driver/sql/sqlconnection.py
+++ b/zuul/driver/sql/sqlconnection.py
@@ -85,6 +85,7 @@
sa.Column('ref', sa.String(255)),
sa.Column('score', sa.Integer, nullable=True),
sa.Column('message', sa.TEXT()),
+ sa.Column('tenant', sa.String(255)),
)
zuul_build_table = sa.Table(
diff --git a/zuul/driver/sql/sqlreporter.py b/zuul/driver/sql/sqlreporter.py
index 5f93ce8..214b667 100644
--- a/zuul/driver/sql/sqlreporter.py
+++ b/zuul/driver/sql/sqlreporter.py
@@ -52,6 +52,7 @@
score=self.result_score,
message=self._formatItemReport(
item, with_jobs=False),
+ tenant=item.pipeline.layout.tenant.name,
)
buildset_ins_result = conn.execute(buildset_ins)
build_inserts = []
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index c36d569..f764778 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -237,7 +237,6 @@
params['playbooks'] = [x.toDict() for x in job.run]
params['pre_playbooks'] = [x.toDict() for x in job.pre_run]
params['post_playbooks'] = [x.toDict() for x in job.post_run]
- params['roles'] = [x.toDict() for x in job.roles]
nodeset = item.current_build_set.getJobNodeSet(job.name)
nodes = []
@@ -254,10 +253,12 @@
params['nodes'] = nodes
params['groups'] = [group.toDict() for group in nodeset.getGroups()]
params['vars'] = copy.deepcopy(job.variables)
+ params['secrets'] = {}
if job.auth:
for secret in job.auth.secrets:
- params['vars'][secret.name] = copy.deepcopy(secret.secret_data)
- params['vars']['zuul'] = zuul_params
+ secret_data = copy.deepcopy(secret.secret_data)
+ params['secrets'][secret.name] = secret_data
+ params['zuul'] = zuul_params
projects = set()
def make_project_dict(project, override_branch=None):
@@ -377,6 +378,7 @@
build.node_name = data.get('node_name')
if result is None:
result = data.get('result')
+ build.error_detail = data.get('error_detail')
if result is None:
if (build.build_set.getTries(build.job.name) >=
build.job.attempts):
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index cdd082e..f291dce 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -41,6 +41,16 @@
DEFAULT_FINGER_PORT = 79
+class ExecutorError(Exception):
+ """A non-transient run-time executor error
+
+ This class represents error conditions detected by the executor
+ when preparing to run a job which we know are consistently fatal.
+ Zuul should not reschedule the build in these cases.
+ """
+ pass
+
+
class Watchdog(object):
def __init__(self, timeout, function, args):
self.timeout = timeout
@@ -72,13 +82,6 @@
self._running = False
-class JobDirPlaybook(object):
- def __init__(self, root):
- self.root = root
- self.trusted = None
- self.path = None
-
-
class SshAgent(object):
log = logging.getLogger("zuul.ExecutorServer")
@@ -122,8 +125,8 @@
subprocess.check_output(['ssh-add', key_path], env=env,
stderr=subprocess.PIPE)
except subprocess.CalledProcessError as e:
- self.log.error('ssh-add failed. stdout: %s, stderr: %s',
- e.output, e.stderr)
+ self.log.exception('ssh-add failed. stdout: %s, stderr: %s',
+ e.output, e.stderr)
raise
self.log.info('Added SSH Key {}'.format(key_path))
@@ -151,6 +154,24 @@
return result
+class JobDirPlaybook(object):
+ def __init__(self, root):
+ self.root = root
+ self.trusted = None
+ self.path = None
+ self.roles = []
+ self.roles_path = []
+ self.ansible_config = os.path.join(self.root, 'ansible.cfg')
+ self.project_link = os.path.join(self.root, 'project')
+
+ def addRole(self):
+ count = len(self.roles)
+ root = os.path.join(self.root, 'role_%i' % (count,))
+ os.makedirs(root)
+ self.roles.append(root)
+ return root
+
+
class JobDir(object):
def __init__(self, root, keep, build_uuid):
'''
@@ -163,11 +184,23 @@
'''
# root
# ansible
- # trusted.cfg
- # untrusted.cfg
+ # inventory.yaml
+ # playbook_0
+ # project -> ../trusted/project_0/...
+ # role_0 -> ../trusted/project_0/...
+ # trusted
+ # project_0
+ # <git.example.com>
+ # <project>
# work
+ # .ssh
+ # known_hosts
# src
+ # <git.example.com>
+ # <project>
# logs
+ # job-output.txt
+ # results.json
self.keep = keep
if root:
tmpdir = root
@@ -175,16 +208,16 @@
tmpdir = tempfile.gettempdir()
self.root = os.path.join(tmpdir, build_uuid)
os.mkdir(self.root, 0o700)
- # Work
self.work_root = os.path.join(self.root, 'work')
os.makedirs(self.work_root)
self.src_root = os.path.join(self.work_root, 'src')
os.makedirs(self.src_root)
self.log_root = os.path.join(self.work_root, 'logs')
os.makedirs(self.log_root)
- # Ansible
self.ansible_root = os.path.join(self.root, 'ansible')
os.makedirs(self.ansible_root)
+ self.trusted_root = os.path.join(self.root, 'trusted')
+ os.makedirs(self.trusted_root)
ssh_dir = os.path.join(self.work_root, '.ssh')
os.mkdir(ssh_dir, 0o700)
self.result_data_file = os.path.join(self.work_root, 'results.json')
@@ -192,17 +225,29 @@
pass
self.known_hosts = os.path.join(ssh_dir, 'known_hosts')
self.inventory = os.path.join(self.ansible_root, 'inventory.yaml')
+ self.secrets = os.path.join(self.ansible_root, 'secrets.yaml')
+ self.has_secrets = False
self.playbooks = [] # The list of candidate playbooks
self.playbook = None # A pointer to the candidate we have chosen
self.pre_playbooks = []
self.post_playbooks = []
- self.roles = []
- self.trusted_roles_path = []
- self.untrusted_roles_path = []
- self.untrusted_config = os.path.join(
- self.ansible_root, 'untrusted.cfg')
- self.trusted_config = os.path.join(self.ansible_root, 'trusted.cfg')
self.job_output_file = os.path.join(self.log_root, 'job-output.txt')
+ self.trusted_projects = []
+ self.trusted_project_index = {}
+
+ def addTrustedProject(self, canonical_name, branch):
+ # Trusted projects are placed in their own directories so that
+ # we can support using different branches of the same project
+ # in different playbooks.
+ count = len(self.trusted_projects)
+ root = os.path.join(self.trusted_root, 'project_%i' % (count,))
+ os.makedirs(root)
+ self.trusted_projects.append(root)
+ self.trusted_project_index[(canonical_name, branch)] = root
+ return root
+
+ def getTrustedProject(self, canonical_name, branch):
+ return self.trusted_project_index.get((canonical_name, branch))
def addPrePlaybook(self):
count = len(self.pre_playbooks)
@@ -228,17 +273,6 @@
self.playbooks.append(playbook)
return playbook
- def addRole(self):
- count = len(self.roles)
- root = os.path.join(self.ansible_root, 'role_%i' % (count,))
- os.makedirs(root)
- trusted = os.path.join(root, 'trusted')
- os.makedirs(trusted)
- untrusted = os.path.join(root, 'untrusted')
- os.makedirs(untrusted)
- self.roles.append(root)
- return root
-
def cleanup(self):
if not self.keep:
shutil.rmtree(self.root)
@@ -722,6 +756,11 @@
self.executor_server.keep_jobdir,
str(self.job.unique))
self._execute()
+ except ExecutorError as e:
+ result_data = json.dumps(dict(result='ERROR',
+ error_detail=e.args[0]))
+ self.log.debug("Sending result: %s" % (result_data,))
+ self.job.sendWorkComplete(result_data)
except Exception:
self.log.exception("Exception while executing job")
self.job.sendWorkException(traceback.format_exc())
@@ -745,7 +784,7 @@
def _execute(self):
args = json.loads(self.job.arguments)
self.log.debug("Beginning job %s for ref %s" %
- (self.job.name, args['vars']['zuul']['ref']))
+ (self.job.name, args['zuul']['ref']))
self.log.debug("Args: %s" % (self.job.arguments,))
self.log.debug("Job root: %s" % (self.jobdir.root,))
tasks = []
@@ -760,8 +799,13 @@
projects.add((project['connection'], project['name']))
# ...as well as all playbook and role projects.
- repos = (args['pre_playbooks'] + args['playbooks'] +
- args['post_playbooks'] + args['roles'])
+ repos = []
+ playbooks = (args['pre_playbooks'] + args['playbooks'] +
+ args['post_playbooks'])
+ for playbook in playbooks:
+ repos.append(playbook)
+ repos += playbook['roles']
+
for repo in repos:
self.log.debug("Job %s: updating playbook or role %s" %
(self.job.unique, repo))
@@ -806,12 +850,9 @@
for repo in repos.values():
repo.deleteRemote('origin')
- # is the playbook in a repo that we have already prepared?
- trusted, untrusted = self.preparePlaybookRepos(args)
+ # This prepares each playbook and the roles needed for each.
+ self.preparePlaybooks(args)
- self.prepareRoles(args, trusted, untrusted)
-
- # TODOv3: Ansible the ansible thing here.
self.prepareAnsibleFiles(args)
data = {
@@ -889,8 +930,9 @@
project_name, project_default_branch)
repo.checkoutLocalBranch(project_default_branch)
else:
- raise Exception("Project %s does not have the default branch %s" %
- (project_name, project_default_branch))
+ raise ExecutorError("Project %s does not have the "
+ "default branch %s" %
+ (project_name, project_default_branch))
def runPlaybooks(self, args):
result = None
@@ -981,9 +1023,9 @@
'''
for entry in os.listdir(path):
if os.path.isdir(entry) and entry.endswith('_plugins'):
- raise Exception(
- "Ansible plugin dir %s found adjacent to playbook %s in"
- " non-trusted repo." % (entry, path))
+ raise ExecutorError(
+ "Ansible plugin dir %s found adjacent to playbook %s in "
+ "non-trusted repo." % (entry, path))
def findPlaybook(self, path, required=False, trusted=False):
for ext in ['.yaml', '.yml']:
@@ -994,45 +1036,32 @@
self._blockPluginDirs(playbook_dir)
return fn
if required:
- raise Exception("Unable to find playbook %s" % path)
+ raise ExecutorError("Unable to find playbook %s" % path)
return None
- def preparePlaybookRepos(self, args):
- trusted = untrusted = False
+ def preparePlaybooks(self, args):
for playbook in args['pre_playbooks']:
jobdir_playbook = self.jobdir.addPrePlaybook()
- self.preparePlaybookRepo(jobdir_playbook, playbook,
- args, required=True)
- if playbook['trusted']:
- trusted = True
- else:
- untrusted = True
+ self.preparePlaybook(jobdir_playbook, playbook,
+ args, required=True)
for playbook in args['playbooks']:
jobdir_playbook = self.jobdir.addPlaybook()
- self.preparePlaybookRepo(jobdir_playbook, playbook,
- args, required=False)
- if playbook['trusted']:
- trusted = True
- else:
- untrusted = True
+ self.preparePlaybook(jobdir_playbook, playbook,
+ args, required=False)
if jobdir_playbook.path is not None:
self.jobdir.playbook = jobdir_playbook
break
+
if self.jobdir.playbook is None:
- raise Exception("No valid playbook found")
+ raise ExecutorError("No valid playbook found")
for playbook in args['post_playbooks']:
jobdir_playbook = self.jobdir.addPostPlaybook()
- self.preparePlaybookRepo(jobdir_playbook, playbook,
- args, required=True)
- if playbook['trusted']:
- trusted = True
- else:
- untrusted = True
- return (trusted, untrusted)
+ self.preparePlaybook(jobdir_playbook, playbook,
+ args, required=True)
- def preparePlaybookRepo(self, jobdir_playbook, playbook, args, required):
+ def preparePlaybook(self, jobdir_playbook, playbook, args, required):
self.log.debug("Prepare playbook repo for %s" % (playbook,))
# Check out the playbook repo if needed and set the path to
# the playbook that should be run.
@@ -1040,6 +1069,7 @@
source = self.executor_server.connections.getSource(
playbook['connection'])
project = source.getProject(playbook['project'])
+ path = None
if not playbook['trusted']:
# This is a project repo, so it is safe to use the already
# checked out version (from speculative merging) of the
@@ -1052,34 +1082,48 @@
project.canonical_hostname,
project.name,
playbook['path'])
- jobdir_playbook.path = self.findPlaybook(
- path,
- required=required,
- trusted=playbook['trusted'])
- return
- # The playbook repo is either a config repo, or it isn't in
- # the stack of changes we are testing, so check out the branch
- # tip into a dedicated space.
+ break
+ if not path:
+ # The playbook repo is either a config repo, or it isn't in
+ # the stack of changes we are testing, so check out the branch
+ # tip into a dedicated space.
+ path = self.checkoutTrustedProject(project, playbook['branch'])
+ path = os.path.join(path, playbook['path'])
- merger = self.executor_server._getMerger(jobdir_playbook.root,
- self.log)
- merger.checkoutBranch(playbook['connection'], project.name,
- playbook['branch'])
-
- path = os.path.join(jobdir_playbook.root,
- project.canonical_hostname,
- project.name,
- playbook['path'])
jobdir_playbook.path = self.findPlaybook(
path,
required=required,
trusted=playbook['trusted'])
- def prepareRoles(self, args, trusted, untrusted):
- for role in args['roles']:
- if role['type'] == 'zuul':
- root = self.jobdir.addRole()
- self.prepareZuulRole(args, role, root, trusted, untrusted)
+ # If this playbook doesn't exist, don't bother preparing
+ # roles.
+ if not jobdir_playbook.path:
+ return
+
+ for role in playbook['roles']:
+ self.prepareRole(jobdir_playbook, role, args)
+
+ self.writeAnsibleConfig(jobdir_playbook)
+
+ def checkoutTrustedProject(self, project, branch):
+ root = self.jobdir.getTrustedProject(project.canonical_name,
+ branch)
+ if not root:
+ root = self.jobdir.addTrustedProject(project.canonical_name,
+ branch)
+ merger = self.executor_server._getMerger(root, self.log)
+ merger.checkoutBranch(project.connection_name, project.name,
+ branch)
+
+ path = os.path.join(root,
+ project.canonical_hostname,
+ project.name)
+ return path
+
+ def prepareRole(self, jobdir_playbook, role, args):
+ if role['type'] == 'zuul':
+ root = jobdir_playbook.addRole()
+ self.prepareZuulRole(jobdir_playbook, role, args, root)
def findRole(self, path, trusted=False):
d = os.path.join(path, 'tasks')
@@ -1094,107 +1138,64 @@
# This repo has a collection of roles
if not trusted:
for entry in os.listdir(d):
- self._blockPluginDirs(os.path.join(d, entry))
+ if os.path.isdir(os.path.join(d, entry)):
+ self._blockPluginDirs(os.path.join(d, entry))
return d
# It is neither a bare role, nor a collection of roles
- raise Exception("Unable to find role in %s" % (path,))
+ raise ExecutorError("Unable to find role in %s" % (path,))
- def prepareZuulRole(self, args, role, root, trusted, untrusted):
+ def prepareZuulRole(self, jobdir_playbook, role, args, root):
self.log.debug("Prepare zuul role for %s" % (role,))
# Check out the role repo if needed
source = self.executor_server.connections.getSource(
role['connection'])
project = source.getProject(role['project'])
- untrusted_role_repo = None
- trusted_role_repo = None
- trusted_root = os.path.join(root, 'trusted')
- untrusted_root = os.path.join(root, 'untrusted')
name = role['target_name']
+ path = None
- if untrusted:
- # There is at least one untrusted playbook. For that
- # case, use the already checked out version (from
- # speculative merging) of the role.
+ if not jobdir_playbook.trusted:
+ # This playbook is untrested. Use the already checked out
+ # version (from speculative merging) of the role if it
+ # exists.
for i in args['items']:
if (i['connection'] == role['connection'] and
i['project'] == role['project']):
- # We already have this repo prepared;
- # copy it into location.
-
+ # We already have this repo prepared; use it.
path = os.path.join(self.jobdir.src_root,
project.canonical_hostname,
project.name)
- # The name of the symlink is the requested name of
- # the role (which may be the repo name or may be
- # something else; this can come into play if this
- # is a bare role).
- link = os.path.join(untrusted_root, name)
- link = os.path.realpath(link)
- if not link.startswith(os.path.realpath(untrusted_root)):
- raise Exception("Invalid role name %s", name)
- os.symlink(path, link)
- untrusted_role_repo = link
break
- if trusted or not untrusted_role_repo:
- # There is at least one trusted playbook which will need a
- # trusted checkout of the role, or the role did not appear
+ if not path:
+ # This is a trusted playbook or the role did not appear
# in the dependency chain for the change (in which case,
# there is no existing untrusted checkout of it). Check
# out the branch tip into a dedicated space.
- merger = self.executor_server._getMerger(trusted_root,
- self.log)
- merger.checkoutBranch(role['connection'], project.name,
- 'master')
- orig_repo_path = os.path.join(trusted_root,
- project.canonical_hostname,
- project.name)
- if name != project.name:
- # The requested name of the role is not the same as
- # the project name, so rename the git repo as the
- # requested name. It is the only item in this
- # directory, so we don't need to worry about
- # collisions.
- target = os.path.join(trusted_root,
- project.canonical_hostname,
- name)
- target = os.path.realpath(target)
- if not target.startswith(os.path.realpath(trusted_root)):
- raise Exception("Invalid role name %s", name)
- os.rename(orig_repo_path, target)
- trusted_role_repo = target
- else:
- trusted_role_repo = orig_repo_path
+ path = self.checkoutTrustedProject(project, 'master')
- if not untrusted_role_repo:
- # In the case that there was no untrusted checkout,
- # use the trusted checkout.
- untrusted_role_repo = trusted_role_repo
- untrusted_root = trusted_root
+ # The name of the symlink is the requested name of the role
+ # (which may be the repo name or may be something else; this
+ # can come into play if this is a bare role).
+ link = os.path.join(root, name)
+ link = os.path.realpath(link)
+ if not link.startswith(os.path.realpath(root)):
+ raise ExecutorError("Invalid role name %s", name)
+ os.symlink(path, link)
- if untrusted:
- untrusted_role_path = self.findRole(untrusted_role_repo,
- trusted=False)
- if untrusted_role_path is None:
- # In the case of a bare role, add the containing directory
- untrusted_role_path = os.path.join(untrusted_root,
- project.canonical_hostname)
- self.jobdir.untrusted_roles_path.append(untrusted_role_path)
-
- if trusted:
- trusted_role_path = self.findRole(trusted_role_repo,
- trusted=True)
- if trusted_role_path is None:
- # In the case of a bare role, add the containing directory
- trusted_role_path = os.path.join(trusted_root,
- project.canonical_hostname)
- self.jobdir.trusted_roles_path.append(trusted_role_path)
+ role_path = self.findRole(link, trusted=jobdir_playbook.trusted)
+ if role_path is None:
+ # In the case of a bare role, add the containing directory
+ role_path = root
+ jobdir_playbook.roles_path.append(role_path)
def prepareAnsibleFiles(self, args):
- all_vars = dict(args['vars'])
+ all_vars = args['vars'].copy()
# TODO(mordred) Hack to work around running things with python3
all_vars['ansible_python_interpreter'] = '/usr/bin/python2'
+ if 'zuul' in all_vars:
+ raise Exception("Defining vars named 'zuul' is not allowed")
+ all_vars['zuul'] = args['zuul'].copy()
all_vars['zuul']['executor'] = dict(
hostname=self.executor_server.hostname,
src_root=self.jobdir.src_root,
@@ -1213,11 +1214,19 @@
for key in node['host_keys']:
known_hosts.write('%s\n' % key)
- self.writeAnsibleConfig(self.jobdir.untrusted_config)
- self.writeAnsibleConfig(self.jobdir.trusted_config, trusted=True)
+ secrets = args['secrets'].copy()
+ if secrets:
+ if 'zuul' in secrets:
+ raise Exception("Defining secrets named 'zuul' is not allowed")
+ with open(self.jobdir.secrets, 'w') as secrets_yaml:
+ secrets_yaml.write(
+ yaml.safe_dump(secrets, default_flow_style=False))
+ self.jobdir.has_secrets = True
- def writeAnsibleConfig(self, config_path, trusted=False):
- with open(config_path, 'w') as config:
+ def writeAnsibleConfig(self, jobdir_playbook):
+ trusted = jobdir_playbook.trusted
+
+ with open(jobdir_playbook.ansible_config, 'w') as config:
config.write('[defaults]\n')
config.write('hostfile = %s\n' % self.jobdir.inventory)
config.write('local_tmp = %s/.ansible/local_tmp\n' %
@@ -1232,6 +1241,7 @@
config.write('callback_plugins = %s\n'
% self.executor_server.callback_dir)
config.write('stdout_callback = zuul_stream\n')
+ config.write('callback_whitelist = zuul_json\n')
# bump the timeout because busy nodes may take more than
# 10s to respond
config.write('timeout = 30\n')
@@ -1240,12 +1250,10 @@
% self.executor_server.action_dir)
config.write('lookup_plugins = %s\n'
% self.executor_server.lookup_dir)
- roles_path = self.jobdir.untrusted_roles_path
- else:
- roles_path = self.jobdir.trusted_roles_path
- if roles_path:
- config.write('roles_path = %s\n' % ':'.join(roles_path))
+ if jobdir_playbook.roles_path:
+ config.write('roles_path = %s\n' % ':'.join(
+ jobdir_playbook.roles_path))
# On trusted jobs, we want to prevent the printing of args,
# since trusted jobs might have access to secrets that they may
@@ -1286,7 +1294,7 @@
except Exception:
self.log.exception("Exception while killing ansible process:")
- def runAnsible(self, cmd, timeout, trusted=False):
+ def runAnsible(self, cmd, timeout, config_file, trusted):
env_copy = os.environ.copy()
env_copy.update(self.ssh_agent.env)
env_copy['LOGNAME'] = 'zuul'
@@ -1301,10 +1309,8 @@
env_copy['PYTHONPATH'] = os.path.pathsep.join(pythonpath)
if trusted:
- config_file = self.jobdir.trusted_config
opt_prefix = 'trusted'
else:
- config_file = self.jobdir.untrusted_config
opt_prefix = 'untrusted'
ro_dirs = get_default(self.executor_server.config, 'executor',
'%s_ro_dirs' % opt_prefix)
@@ -1378,6 +1384,8 @@
# TODO(mordred) If/when we rework use of logger in ansible-playbook
# we'll want to change how this works to use that as well. For now,
# this is what we need to do.
+ # TODO(mordred) We probably want to put this into the json output
+ # as well.
with open(self.jobdir.job_output_file, 'a') as job_output:
job_output.write("{now} | ANSIBLE PARSE ERROR\n".format(
now=datetime.datetime.now()))
@@ -1398,6 +1406,8 @@
verbose = '-v'
cmd = ['ansible-playbook', verbose, playbook.path]
+ if self.jobdir.has_secrets:
+ cmd.extend(['-e', '@' + self.jobdir.secrets])
if success is not None:
cmd.extend(['-e', 'success=%s' % str(bool(success))])
@@ -1409,7 +1419,9 @@
cmd.extend(['-e', 'zuul_execution_phase_index=%s' % index])
result, code = self.runAnsible(
- cmd=cmd, timeout=timeout, trusted=playbook.trusted)
+ cmd=cmd, timeout=timeout,
+ config_file=playbook.ansible_config,
+ trusted=playbook.trusted)
self.log.debug("Ansible complete, result %s code %s" % (
self.RESULT_MAP[result], code))
return result, code
diff --git a/zuul/manager/dependent.py b/zuul/manager/dependent.py
index ada3491..411894e 100644
--- a/zuul/manager/dependent.py
+++ b/zuul/manager/dependent.py
@@ -14,6 +14,7 @@
from zuul import model
from zuul.manager import PipelineManager, StaticChangeQueueContextManager
+from zuul.manager import DynamicChangeQueueContextManager
class DependentPipelineManager(PipelineManager):
@@ -75,8 +76,17 @@
def getChangeQueue(self, change, existing=None):
if existing:
return StaticChangeQueueContextManager(existing)
- return StaticChangeQueueContextManager(
- self.pipeline.getQueue(change.project))
+ queue = self.pipeline.getQueue(change.project)
+ if queue:
+ return StaticChangeQueueContextManager(queue)
+ else:
+ # There is no existing queue for this change. Create a
+ # dynamic one for this one change's use
+ change_queue = model.ChangeQueue(self.pipeline, dynamic=True)
+ change_queue.addProject(change.project)
+ self.pipeline.addQueue(change_queue)
+ self.log.debug("Dynamically created queue %s", change_queue)
+ return DynamicChangeQueueContextManager(change_queue)
def isChangeReadyToBeEnqueued(self, change):
source = change.project.source
@@ -201,3 +211,11 @@
if failing_items:
return failing_items
return None
+
+ def dequeueItem(self, item):
+ super(DependentPipelineManager, self).dequeueItem(item)
+ # If this was a dynamic queue from a speculative change,
+ # remove the queue (if empty)
+ if item.queue.dynamic:
+ if not item.queue.queue:
+ self.pipeline.removeQueue(item.queue)
diff --git a/zuul/model.py b/zuul/model.py
index 80ef285..1df70db 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -209,11 +209,14 @@
be processed. If a Change succeeds, the Window is increased by
`window_increase_factor`. If a Change fails, the Window is decreased by
`window_decrease_factor`.
+
+ A ChangeQueue may be a dynamically created queue, which may be removed
+ from a DependentPipelineManager once empty.
"""
def __init__(self, pipeline, window=0, window_floor=1,
window_increase_type='linear', window_increase_factor=1,
window_decrease_type='exponential', window_decrease_factor=2,
- name=None):
+ name=None, dynamic=False):
self.pipeline = pipeline
if name:
self.name = name
@@ -228,6 +231,7 @@
self.window_increase_factor = window_increase_factor
self.window_decrease_type = window_decrease_type
self.window_decrease_factor = window_decrease_factor
+ self.dynamic = dynamic
def __repr__(self):
return '<ChangeQueue %s: %s>' % (self.pipeline.name, self.name)
@@ -633,11 +637,17 @@
Jobs refer to objects of this class for their main, pre, and post
playbooks so that we can keep track of which repos and security
- contexts are needed in order to run them."""
+ contexts are needed in order to run them.
- def __init__(self, source_context, path):
+ We also keep a list of roles so that playbooks only run with the
+ roles which were defined at the point the playbook was defined.
+
+ """
+
+ def __init__(self, source_context, path, roles):
self.source_context = source_context
self.path = path
+ self.roles = roles
def __repr__(self):
return '<PlaybookContext %s %s>' % (self.source_context,
@@ -650,7 +660,8 @@
if not isinstance(other, PlaybookContext):
return False
return (self.source_context == other.source_context and
- self.path == other.path)
+ self.path == other.path and
+ self.roles == other.roles)
def toDict(self):
# Render to a dict to use in passing json to the executor
@@ -659,6 +670,7 @@
project=self.source_context.project.name,
branch=self.source_context.branch,
trusted=self.source_context.trusted,
+ roles=[r.toDict() for r in self.roles],
path=self.path)
@@ -704,7 +716,7 @@
if not isinstance(other, ZuulRole):
return False
return (super(ZuulRole, self).__eq__(other) and
- self.connection_name == other.connection_name,
+ self.connection_name == other.connection_name and
self.project_name == other.project_name)
def toDict(self):
@@ -856,9 +868,9 @@
def addRoles(self, roles):
newroles = list(self.roles)
- for role in roles:
+ for role in reversed(roles):
if role not in newroles:
- newroles.append(role)
+ newroles.insert(0, role)
self.roles = tuple(newroles)
def updateVariables(self, other_vars):
@@ -1086,6 +1098,7 @@
self.url = None
self.result = None
self.result_data = {}
+ self.error_detail = None
self.build_set = None
self.execute_time = time.time()
self.start_time = None
@@ -1106,6 +1119,7 @@
def getSafeAttributes(self):
return Attributes(uuid=self.uuid,
result=self.result,
+ error_detail=self.error_detail,
result_data=self.result_data)
diff --git a/zuul/reporter/__init__.py b/zuul/reporter/__init__.py
index 95b9208..49181a7 100644
--- a/zuul/reporter/__init__.py
+++ b/zuul/reporter/__init__.py
@@ -138,7 +138,11 @@
elapsed = ' in %ds' % (s)
else:
elapsed = ''
+ if build.error_detail:
+ error = ' ' + build.error_detail
+ else:
+ error = ''
name = job.name + ' '
- ret += '- %s%s : %s%s%s\n' % (name, url, result, elapsed,
- voting)
+ ret += '- %s%s : %s%s%s%s\n' % (name, url, result, error,
+ elapsed, voting)
return ret