Merge "Run playbooks with only those roles defined thus far" into feature/zuulv3
diff --git a/doc/source/admin/drivers/timer.rst b/doc/source/admin/drivers/timer.rst
index c70df5c..c8afdd7 100644
--- a/doc/source/admin/drivers/timer.rst
+++ b/doc/source/admin/drivers/timer.rst
@@ -21,4 +21,4 @@
**time**
The time specification in cron syntax. Only the 5 part syntax is
supported, not the symbolic names. Example: ``0 0 * * *`` runs at
- midnight.
+ midnight. The first weekday is Monday.
diff --git a/doc/source/admin/tenants.rst b/doc/source/admin/tenants.rst
index 8872397..60873a9 100644
--- a/doc/source/admin/tenants.rst
+++ b/doc/source/admin/tenants.rst
@@ -35,6 +35,8 @@
- shared-jobs:
include: jobs
untrusted-projects:
+ - zuul-jobs:
+ shadow: common-config
- project1
- project2
@@ -83,6 +85,20 @@
**exclude**
A list of configuration classes that should not be loaded.
+ **shadow**
+ A list of projects which this project is permitted to shadow.
+ Normally, only one project in Zuul may contain definitions for a
+ given job. If a project earlier in the configuration defines a
+ job which a later project redefines, the later definition is
+ considered an error and is not permitted. The "shadow" attribute
+ of a project indicates that job definitions in this project which
+ conflict with the named projects should be ignored, and those in
+ the named project should be used instead. The named projects must
+ still appear earlier in the configuration. In the example above,
+ if a job definition appears in both the "common-config" and
+ "zuul-jobs" projects, the definition in "common-config" will be
+ used.
+
The order of the projects listed in a tenant is important. A job
which is defined in one project may not be redefined in another
project; therefore, once a job appears in one project, a project
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index 64dc393..744414a 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -496,12 +496,21 @@
different string. Default: "FAILURE".
**success-url**
- When a job succeeds, this URL is reported along with the result.
+ When a job succeeds, this URL is reported along with the result. If
+ this value is not supplied, Zuul uses the content of the job
+ :ref:`return value <return_values>` **zuul.log_url**. This is
+ recommended as it allows the code which stores the URL to the job
+ artifacts to report exactly where they were stored. To override
+ this value, or if it is not set, supply an absolute URL in this
+ field. If a relative URL is supplied in this field, and
+ **zuul.log_url** is set, then the two will be combined to produce
+ the URL used for the report. This can be used to specify that
+ certain jobs should "deep link" into the stored job artifacts.
Default: none.
**failure-url**
When a job fails, this URL is reported along with the result.
- Default: none.
+ Otherwise behaves the same as **success-url**.
**hold-following-changes**
In a dependent pipeline, this option may be used to indicate that no
@@ -662,32 +671,35 @@
of attempts to make before an error is reported. Default: 3.
**pre-run**
- The name of a playbook or list of playbooks to run before the main
- body of a job. The playbook is expected to reside in the
- `playbooks/` directory of the project where the job is defined.
+ The name of a playbook or list of playbooks without file extension
+ to run before the main body of a job. The full path to the playbook
+ in the repo where the job is defined is expected.
When a job inherits from a parent, the child's pre-run playbooks are
run after the parent's. See :ref:`job` for more information.
**post-run**
- The name of a playbook or list of playbooks to run after the main
- body of a job. The playbook is expected to reside in the
- `playbooks/` directory of the project where the job is defined.
+ The name of a playbook or list of playbooks without file extension
+ to run after the main body of a job. The full path to the playbook
+ in the repo where the job is defined is expected.
When a job inherits from a parent, the child's post-run playbooks
are run before the parent's. See :ref:`job` for more information.
**run**
- The name of the main playbook for this job. This parameter is not
- normally necessary, as it defaults to the name of the job. However,
- if a playbook with a different name is needed, it can be specified
- here. The playbook is expected to reside in the `playbooks/`
- directory of the project where the job is defined. When a child
- inherits from a parent, a playbook with the name of the child job is
- implicitly searched first, before falling back on the playbook used
- by the parent job (unless the child job specifies a ``run``
- attribute, in which case that value is used). Default: the name of
- the job.
+ The name of the main playbook for this job. This parameter is
+ not normally necessary, as it defaults to a playbook with the
+ same name as the job inside of the `playbooks/` directory (e.g.,
+ the `foo` job would default to `playbooks/foo`. However, if a
+ playbook with a different name is needed, it can be specified
+ here. The file extension is not required, but the full path
+ within the repo is. When a child inherits from a parent, a
+ playbook with the name of the child job is implicitly searched
+ first, before falling back on the playbook used by the parent
+ job (unless the child job specifies a ``run`` attribute, in which
+ case that value is used). Example::
+
+ run: playbooks/<name of the job>
**roles**
A list of Ansible roles to prepare for the job. Because a job runs
diff --git a/doc/source/user/jobs.rst b/doc/source/user/jobs.rst
index 58f3371..78121bc 100644
--- a/doc/source/user/jobs.rst
+++ b/doc/source/user/jobs.rst
@@ -101,6 +101,8 @@
.. TODO: describe standard lib and link to published docs for it.
+.. _return_values:
+
Return Values
-------------
diff --git a/etc/zuul.conf-sample b/etc/zuul.conf-sample
index 9b8406c..e6375a5 100644
--- a/etc/zuul.conf-sample
+++ b/etc/zuul.conf-sample
@@ -13,9 +13,6 @@
;ssl_cert=/path/to/server.pem
;ssl_key=/path/to/server.key
-[zuul]
-status_url=https://zuul.example.com/status
-
[scheduler]
tenant_config=/etc/zuul/main.yaml
log_config=/etc/zuul/logging.conf
@@ -40,6 +37,7 @@
[webapp]
listen_address=0.0.0.0
port=8001
+status_url=https://zuul.example.com/status
[connection gerrit]
driver=gerrit
diff --git a/tests/fixtures/config/data-return/git/common-config/playbooks/data-return.yaml b/tests/fixtures/config/data-return/git/common-config/playbooks/data-return.yaml
index b92ff5c..5e412c3 100644
--- a/tests/fixtures/config/data-return/git/common-config/playbooks/data-return.yaml
+++ b/tests/fixtures/config/data-return/git/common-config/playbooks/data-return.yaml
@@ -3,4 +3,4 @@
- zuul_return:
data:
zuul:
- log_url: test/log/url
+ log_url: http://example.com/test/log/url/
diff --git a/tests/fixtures/config/data-return/git/common-config/zuul.yaml b/tests/fixtures/config/data-return/git/common-config/zuul.yaml
index 8aea931..8d602f1 100644
--- a/tests/fixtures/config/data-return/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/data-return/git/common-config/zuul.yaml
@@ -15,8 +15,14 @@
- job:
name: data-return
+- job:
+ name: data-return-relative
+ run: playbooks/data-return
+ success-url: docs/index.html
+
- project:
name: org/project
check:
jobs:
- data-return
+ - data-return-relative
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 5d49d11..87eddc6 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -713,6 +713,10 @@
self.waitUntilSettled()
self.assertHistory([
dict(name='data-return', result='SUCCESS', changes='1,1'),
- ])
- self.assertIn('- data-return test/log/url',
+ dict(name='data-return-relative', result='SUCCESS', changes='1,1'),
+ ], ordered=False)
+ self.assertIn('- data-return http://example.com/test/log/url/',
+ A.messages[-1])
+ self.assertIn('- data-return-relative '
+ 'http://example.com/test/log/url/docs/index.html',
A.messages[-1])
diff --git a/zuul/ansible/callback/zuul_stream.py b/zuul/ansible/callback/zuul_stream.py
index 5c2ce8a..cc979f2 100644
--- a/zuul/ansible/callback/zuul_stream.py
+++ b/zuul/ansible/callback/zuul_stream.py
@@ -13,8 +13,15 @@
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
+# This is not needed in python3 - but it is needed in python2 because there
+# is a json module in ansible.plugins.callback and python2 gets confused.
+# Easy local testing with ansible-playbook is handy when hacking on zuul_stream
+# so just put in the __future__ statement.
+from __future__ import absolute_import
+
import datetime
import logging
+import json
import os
import socket
import threading
@@ -67,7 +74,7 @@
if not stdout_lines and stdout:
stdout_lines = stdout.split('\n')
- for key in ('changed', 'cmd', 'zuul_log_id',
+ for key in ('changed', 'cmd', 'zuul_log_id', 'invocation',
'stderr', 'stderr_lines'):
result.pop(key, None)
return stdout_lines
@@ -143,16 +150,33 @@
def v2_playbook_on_start(self, playbook):
self._playbook_name = os.path.splitext(playbook._file_name)[0]
+ def v2_playbook_on_include(self, included_file):
+ for host in included_file._hosts:
+ self._log("{host} | included: {filename}".format(
+ host=host.name,
+ filename=included_file._filename))
+
def v2_playbook_on_play_start(self, play):
self._play = play
+ # Get the hostvars from just one host - the vars we're looking for will
+ # be identical on all of them
+ hostvars = self._play._variable_manager._hostvars
+ a_host = next(iter(hostvars.keys()))
+ self.phase = hostvars[a_host]['zuul_execution_phase']
+ if self.phase != 'run':
+ self.phase = '{phase}-{index}'.format(
+ phase=self.phase,
+ index=hostvars[a_host]['zuul_execution_phase_index'])
+
+ # the name of a play defaults to the hosts string
name = play.get_name().strip()
- if not name:
- msg = u"PLAY"
- else:
- msg = u"PLAY [{playbook} : {name}]".format(
- playbook=self._playbook_name, name=name)
+ msg = u"PLAY [{phase} : {playbook} : {name}]".format(
+ phase=self.phase,
+ playbook=self._playbook_name, name=name)
self._log(msg)
+ # Log an extra blank line to get space after each play
+ self._log("")
def v2_playbook_on_task_start(self, task, is_conditional):
self._task = task
@@ -234,12 +258,11 @@
pass
else:
self._log_message(
- result=result,
- msg="Results: => {results}".format(
- results=self._dump_results(result_dict)),
- status='ERROR')
+ result=result, status='ERROR', result_dict=result_dict)
if ignore_errors:
self._log_message(result, "Ignoring Errors", status="ERROR")
+ # Log an extra blank line to get space after each task
+ self._log("")
def v2_runner_on_ok(self, result):
if (self._play.strategy == 'free'
@@ -283,11 +306,14 @@
pass
elif result._task.action not in ('command', 'shell'):
- self._log_message(
- result=result,
- msg="Results: => {results}".format(
- results=self._dump_results(result_dict)),
- status=status)
+ if 'msg' in result_dict:
+ self._log_message(msg=result_dict['msg'],
+ result=result, status=status)
+ else:
+ self._log_message(
+ result=result,
+ status=status,
+ result_dict=result_dict)
elif 'results' in result_dict:
for res in result_dict['results']:
self._log_message(
@@ -300,6 +326,8 @@
result,
"Runtime: {delta} Start: {start} End: {end}".format(
**result_dict))
+ # Log an extra blank line to get space after each task
+ self._log("")
def v2_runner_item_on_ok(self, result):
result_dict = dict(result._result)
@@ -313,9 +341,8 @@
if result._task.action not in ('command', 'shell'):
self._log_message(
result=result,
- msg="Item: {item} => {results}".format(
- item=result_dict['item'],
- results=self._dump_results(result_dict)),
+ msg="Item: {item}".format(item=result_dict['item']),
+ result_dict=result_dict,
status=status)
else:
self._log_message(
@@ -325,6 +352,8 @@
if self._deferred_result:
self._process_deferred(result)
+ # Log an extra blank line to get space after each task
+ self._log("")
def v2_runner_item_on_failed(self, result):
result_dict = dict(result._result)
@@ -333,10 +362,9 @@
if result._task.action not in ('command', 'shell'):
self._log_message(
result=result,
- msg="Item: {item} => {results}".format(
- item=result_dict['item'],
- results=self._dump_results(result_dict)),
- status='ERROR')
+ msg="Item: {item}".format(item=result_dict['item']),
+ status='ERROR',
+ result_dict=result_dict)
else:
self._log_message(
result,
@@ -345,6 +373,24 @@
if self._deferred_result:
self._process_deferred(result)
+ # Log an extra blank line to get space after each task
+ self._log("")
+
+ def v2_playbook_on_stats(self, stats):
+
+ # Log an extra blank line to get space before the stats
+ self._log("")
+ self._log("PLAY RECAP")
+
+ hosts = sorted(stats.processed.keys())
+ for host in hosts:
+ t = stats.summarize(host)
+ self._log(
+ "{host} |"
+ " ok: {ok}"
+ " changed: {changed}"
+ " unreachable: {unreachable}"
+ " failed: {failures}".format(host=host, **t))
def _process_deferred(self, result):
self._items_done = True
@@ -383,6 +429,8 @@
task=task_name,
args=args)
self._log(msg)
+ # Log an extra blank line to get space after each task
+ self._log("")
return task
def _get_task_hosts(self, task):
@@ -401,10 +449,40 @@
hosts = play_vars.keys()
return hosts
- def _log_message(self, result, msg, status="ok"):
+ def _dump_result_dict(self, result_dict):
+ result_dict = result_dict.copy()
+ for key in list(result_dict.keys()):
+ if key.startswith('_ansible'):
+ del result_dict[key]
+ zuul_filter_result(result_dict)
+ return result_dict
+
+ def _log_message(self, result, msg=None, status="ok", result_dict=None):
hostname = self._get_hostname(result)
- self._log("{host} | {status}: {msg}".format(
- host=hostname, status=status, msg=msg))
+ if result._task.no_log:
+ self._log("{host} | {msg}".format(
+ host=hostname,
+ msg="Output suppressed because no_log was given"))
+ return
+ if msg:
+ msg_lines = msg.strip().split('\n')
+ if len(msg_lines) > 1:
+ self._log("{host} | {status}:".format(
+ host=hostname, status=status))
+ for msg_line in msg_lines:
+ self._log("{host} | {msg_line}".format(
+ host=hostname, msg_line=msg_line))
+ else:
+ self._log("{host} | {status}: {msg}".format(
+ host=hostname, status=status, msg=msg))
+ else:
+ self._log("{host} | {status}".format(
+ host=hostname, status=status, msg=msg))
+ if result_dict:
+ result_string = json.dumps(self._dump_result_dict(result_dict),
+ indent=2, sort_keys=True)
+ for line in result_string.split('\n'):
+ self._log("{host} | {line}".format(host=hostname, line=line))
def _get_hostname(self, result):
delegated_vars = result._result.get('_ansible_delegated_vars', None)
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index c6b819e..25f2f86 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -920,10 +920,10 @@
result = None
pre_failed = False
- for count, playbook in enumerate(self.jobdir.pre_playbooks):
+ for index, playbook in enumerate(self.jobdir.pre_playbooks):
# TODOv3(pabelanger): Implement pre-run timeout setting.
pre_status, pre_code = self.runAnsiblePlaybook(
- playbook, args['timeout'], phase='pre', count=count)
+ playbook, args['timeout'], phase='pre', index=index)
if pre_status != self.RESULT_NORMAL or pre_code != 0:
# These should really never fail, so return None and have
# zuul try again
@@ -949,10 +949,10 @@
else:
result = 'FAILURE'
- for count, playbook in enumerate(self.jobdir.post_playbooks):
+ for index, playbook in enumerate(self.jobdir.post_playbooks):
# TODOv3(pabelanger): Implement post-run timeout setting.
post_status, post_code = self.runAnsiblePlaybook(
- playbook, args['timeout'], success, phase='post', count=count)
+ playbook, args['timeout'], success, phase='post', index=index)
if post_status != self.RESULT_NORMAL or post_code != 0:
# If we encountered a pre-failure, that takes
# precedence over the post result.
@@ -1362,7 +1362,7 @@
return (self.RESULT_NORMAL, ret)
def runAnsiblePlaybook(self, playbook, timeout, success=None,
- phase=None, count=None):
+ phase=None, index=None):
env_copy = os.environ.copy()
env_copy['LOGNAME'] = 'zuul'
@@ -1379,8 +1379,8 @@
if phase:
cmd.extend(['-e', 'zuul_execution_phase=%s' % phase])
- if count is not None:
- cmd.extend(['-e', 'zuul_execution_phase_count=%s' % count])
+ if index is not None:
+ cmd.extend(['-e', 'zuul_execution_phase_index=%s' % index])
result, code = self.runAnsible(
cmd=cmd, timeout=timeout,
diff --git a/zuul/model.py b/zuul/model.py
index 66c043d..a17a9cb 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -20,6 +20,7 @@
import struct
import time
from uuid import uuid4
+import urllib.parse
MERGER_MERGE = 1 # "git merge"
MERGER_MERGE_RESOLVE = 2 # "git merge -s resolve"
@@ -1641,13 +1642,29 @@
result = job.failure_message
if job.failure_url:
pattern = job.failure_url
- url = None
+ url = None # The final URL
+ default_url = build.result_data.get('zuul', {}).get('log_url')
if pattern:
- url = self.formatUrlPattern(pattern, job, build)
+ job_url = self.formatUrlPattern(pattern, job, build)
+ else:
+ job_url = None
+ try:
+ if job_url:
+ u = urllib.parse.urlparse(job_url)
+ if u.scheme:
+ # The job success or failure url is absolute, so it's
+ # our final url.
+ url = job_url
+ else:
+ # We have a relative job url. Combine it with our
+ # default url.
+ if default_url:
+ url = urllib.parse.urljoin(default_url, job_url)
+ except Exception:
+ self.log.exception("Error while parsing url for job %s:"
+ % (job,))
if not url:
- url = build.result_data.get('zuul', {}).get('log_url')
- if not url:
- url = build.url or job.name
+ url = default_url or build.url or job.name
return (result, url)
def formatJSON(self):