Merge "Add zuul.items to job vars" into feature/zuulv3
diff --git a/.zuul.yaml b/.zuul.yaml
index e8b070f..8095733 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -4,6 +4,6 @@
jobs:
- tox-docs
- tox-cover
- - tox-linters
+ - tox-pep8
- tox-py35
- tox-tarball
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index 4a9a99e..c137918 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -747,6 +747,12 @@
be installed (and therefore referenced from Ansible), the `name`
attribute may be used to specify an alternate.
+ A job automatically has the project in which it is defined added to
+ the roles path if that project appears to contain a role or `roles/`
+ directory. By default, the project is added to the path under its
+ own name, however, that may be changed by explicitly listing the
+ project in the roles list in the usual way.
+
.. note:: galaxy roles are not yet implemented
**galaxy**
diff --git a/etc/status/public_html/jquery.zuul.js b/etc/status/public_html/jquery.zuul.js
index aec7a46..1937cd5 100644
--- a/etc/status/public_html/jquery.zuul.js
+++ b/etc/status/public_html/jquery.zuul.js
@@ -96,7 +96,15 @@
job: function(job) {
var $job_line = $('<span />');
- if (job.url !== null) {
+ if (job.result !== null) {
+ $job_line.append(
+ $('<a />')
+ .addClass('zuul-job-name')
+ .attr('href', job.report_url)
+ .text(job.name)
+ );
+ }
+ else if (job.url !== null) {
$job_line.append(
$('<a />')
.addClass('zuul-job-name')
diff --git a/tests/base.py b/tests/base.py
index 484b9e5..2c478ad 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -410,8 +410,8 @@
needed = other.data.get('neededBy', [])
d = {'id': self.data['id'],
'number': self.data['number'],
- 'ref': self.patchsets[patchset - 1]['ref'],
- 'revision': self.patchsets[patchset - 1]['revision']
+ 'ref': self.patchsets[-1]['ref'],
+ 'revision': self.patchsets[-1]['revision']
}
needed.append(d)
other.data['neededBy'] = needed
diff --git a/tests/fixtures/config/implicit-roles/git/common-config/zuul.yaml b/tests/fixtures/config/implicit-roles/git/common-config/zuul.yaml
new file mode 100644
index 0000000..ba91fb5
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/common-config/zuul.yaml
@@ -0,0 +1,12 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
diff --git a/tests/fixtures/config/implicit-roles/git/org_norole-project/.zuul.yaml b/tests/fixtures/config/implicit-roles/git/org_norole-project/.zuul.yaml
new file mode 100644
index 0000000..74c8e8e
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/org_norole-project/.zuul.yaml
@@ -0,0 +1,15 @@
+- job:
+ name: implicit-role-fail
+
+- job:
+ name: explicit-role-fail
+ attempts: 1
+ roles:
+ - zuul: org/norole-project
+
+- project:
+ name: org/norole-project
+ check:
+ jobs:
+ - implicit-role-fail
+ - explicit-role-fail
diff --git a/tests/fixtures/config/implicit-roles/git/org_norole-project/README b/tests/fixtures/config/implicit-roles/git/org_norole-project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/org_norole-project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/implicit-roles/git/org_norole-project/playbooks/explicit-role-fail.yaml b/tests/fixtures/config/implicit-roles/git/org_norole-project/playbooks/explicit-role-fail.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/org_norole-project/playbooks/explicit-role-fail.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/implicit-roles/git/org_norole-project/playbooks/implicit-role-fail.yaml b/tests/fixtures/config/implicit-roles/git/org_norole-project/playbooks/implicit-role-fail.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/org_norole-project/playbooks/implicit-role-fail.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/implicit-roles/git/org_role-project/.zuul.yaml b/tests/fixtures/config/implicit-roles/git/org_role-project/.zuul.yaml
new file mode 100644
index 0000000..42cae95
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/org_role-project/.zuul.yaml
@@ -0,0 +1,15 @@
+- job:
+ name: implicit-role-ok
+
+- job:
+ name: explicit-role-ok
+ roles:
+ - zuul: org/role-project
+ name: role-name
+
+- project:
+ name: org/role-project
+ check:
+ jobs:
+ - implicit-role-ok
+ - explicit-role-ok
diff --git a/tests/fixtures/config/implicit-roles/git/org_role-project/README b/tests/fixtures/config/implicit-roles/git/org_role-project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/org_role-project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/implicit-roles/git/org_role-project/playbooks/explicit-role-ok.yaml b/tests/fixtures/config/implicit-roles/git/org_role-project/playbooks/explicit-role-ok.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/org_role-project/playbooks/explicit-role-ok.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/implicit-roles/git/org_role-project/playbooks/implicit-role-ok.yaml b/tests/fixtures/config/implicit-roles/git/org_role-project/playbooks/implicit-role-ok.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/org_role-project/playbooks/implicit-role-ok.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/implicit-roles/git/org_role-project/roles/README b/tests/fixtures/config/implicit-roles/git/org_role-project/roles/README
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/git/org_role-project/roles/README
diff --git a/tests/fixtures/config/implicit-roles/main.yaml b/tests/fixtures/config/implicit-roles/main.yaml
new file mode 100644
index 0000000..d5e481a
--- /dev/null
+++ b/tests/fixtures/config/implicit-roles/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - org/norole-project
+ - org/role-project
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 98ede65..d4290a9 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -1178,6 +1178,34 @@
repo.heads.master.commit = repo.commit('init')
self.test_build_configuration()
+ def test_dependent_changes_rebase(self):
+ # Test that no errors occur when we walk a dependency tree
+ # with an unused leaf node due to a rebase.
+ # Start by constructing: C -> B -> A
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
+ B.setDependsOn(A, 1)
+
+ C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
+ C.setDependsOn(B, 1)
+
+ # Then rebase to form: D -> C -> A
+ C.addPatchset() # C,2
+ C.setDependsOn(A, 1)
+
+ D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
+ D.setDependsOn(C, 2)
+
+ # Walk the entire tree
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 3)
+
+ # Verify that walking just part of the tree still works
+ self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 6)
+
def test_dependent_changes_dequeue(self):
"Test that dependent patches are not needlessly tested"
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 9c7ffea..fb80660 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -717,9 +717,7 @@
self.assertEqual(4096, private_key.key_size)
-class TestRoles(ZuulTestCase):
- tenant_config_file = 'config/roles/main.yaml'
-
+class RoleTestCase(ZuulTestCase):
def _assertRolePath(self, build, playbook, content):
path = os.path.join(self.test_root, build.uuid,
'ansible', playbook, 'ansible.cfg')
@@ -739,6 +737,10 @@
"Should have no roles_path line in %s" %
(playbook,))
+
+class TestRoles(RoleTestCase):
+ tenant_config_file = 'config/roles/main.yaml'
+
def test_role(self):
# This exercises a proposed change to a role being checked out
# and used.
@@ -823,6 +825,57 @@
A.messages[-1])
+class TestImplicitRoles(RoleTestCase):
+ tenant_config_file = 'config/implicit-roles/main.yaml'
+
+ def test_missing_roles(self):
+ # Test implicit and explicit roles for a project which does
+ # not have roles. The implicit role should be silently
+ # ignored since the project doesn't supply roles, but if a
+ # user declares an explicit role, it should error.
+ self.executor_server.hold_jobs_in_build = True
+ A = self.fake_gerrit.addFakeChange('org/norole-project', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertEqual(len(self.builds), 2)
+ build = self.getBuildByName('implicit-role-fail')
+ self._assertRolePath(build, 'playbook_0', None)
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+ # The retry_limit doesn't get recorded
+ self.assertHistory([
+ dict(name='implicit-role-fail', result='SUCCESS', changes='1,1'),
+ ])
+
+ def test_roles(self):
+ # Test implicit and explicit roles for a project which does
+ # have roles. In both cases, we should end up with the role
+ # in the path. In the explicit case, ensure we end up with
+ # the name we specified.
+ self.executor_server.hold_jobs_in_build = True
+ A = self.fake_gerrit.addFakeChange('org/role-project', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertEqual(len(self.builds), 2)
+ build = self.getBuildByName('implicit-role-ok')
+ self._assertRolePath(build, 'playbook_0', 'role_0')
+
+ build = self.getBuildByName('explicit-role-ok')
+ self._assertRolePath(build, 'playbook_0', 'role_0')
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+ self.assertHistory([
+ dict(name='implicit-role-ok', result='SUCCESS', changes='1,1'),
+ dict(name='explicit-role-ok', result='SUCCESS', changes='1,1'),
+ ], ordered=False)
+
+
class TestShadow(ZuulTestCase):
tenant_config_file = 'config/shadow/main.yaml'
diff --git a/zuul/ansible/callback/zuul_json.py b/zuul/ansible/callback/zuul_json.py
index 017c27e..612a720 100644
--- a/zuul/ansible/callback/zuul_json.py
+++ b/zuul/ansible/callback/zuul_json.py
@@ -43,34 +43,45 @@
def __init__(self, display=None):
super(CallbackModule, self).__init__(display)
self.results = []
+ self.output = []
+ self.playbook = {}
self.output_path = os.path.splitext(
os.environ['ZUUL_JOB_OUTPUT_FILE'])[0] + '.json'
# For now, just read in the old file and write it all out again
# This may well not scale from a memory perspective- but let's see how
# it goes.
if os.path.exists(self.output_path):
- self.results = json.load(open(self.output_path, 'r'))
+ self.output = json.load(open(self.output_path, 'r'))
+ self._playbook_name = None
- def _get_playbook_name(self, work_dir):
+ def _new_playbook(self, play):
+ # Get the hostvars from just one host - the vars we're looking for will
+ # be identical on all of them
+ hostvars = next(iter(play._variable_manager._hostvars.values()))
+ self._playbook_name = None
- playbook = self._playbook_name
- if work_dir and playbook.startswith(work_dir):
- playbook = playbook.replace(work_dir.rstrip('/') + '/', '')
- # Lop off the first two path elements - ansible/pre_playbook_0
- for prefix in ('pre', 'playbook', 'post'):
- full_prefix = 'ansible/{prefix}_'.format(prefix=prefix)
- if playbook.startswith(full_prefix):
- playbook = playbook.split(os.path.sep, 2)[2]
- return playbook
+ # TODO(mordred) For now, protect specific variable lookups to make it
+ # not absurdly strange to run local tests with the callback plugin
+ # enabled. Remove once we have a "run playbook like zuul runs playbook"
+ # tool.
+ phase = hostvars.get('zuul_execution_phase')
+ index = hostvars.get('zuul_execution_phase_index')
+ playbook = hostvars.get('zuul_execution_canonical_name_and_path')
+ trusted = hostvars.get('zuul_execution_trusted')
+ trusted = True if trusted == "True" else False
+ branch = hostvars.get('zuul_execution_branch')
- def _new_play(self, play, phase, index, work_dir):
+ self.playbook['playbook'] = playbook
+ self.playbook['phase'] = phase
+ self.playbook['index'] = index
+ self.playbook['trusted'] = trusted
+ self.playbook['branch'] = branch
+
+ def _new_play(self, play):
return {
'play': {
'name': play.name,
'id': str(play._uuid),
- 'phase': phase,
- 'index': index,
- 'playbook': self._get_playbook_name(work_dir),
},
'tasks': []
}
@@ -88,20 +99,10 @@
self._playbook_name = os.path.splitext(playbook._file_name)[0]
def v2_playbook_on_play_start(self, play):
- # Get the hostvars from just one host - the vars we're looking for will
- # be identical on all of them
- hostvars = next(iter(play._variable_manager._hostvars.values()))
- phase = hostvars.get('zuul_execution_phase')
- index = hostvars.get('zuul_execution_phase_index')
- # TODO(mordred) For now, protect this to make it not absurdly strange
- # to run local tests with the callback plugin enabled. Remove once we
- # have a "run playbook like zuul runs playbook" tool.
- work_dir = None
- if 'zuul' in hostvars and 'executor' in hostvars['zuul']:
- # imply work_dir from src_root
- work_dir = os.path.dirname(
- hostvars['zuul']['executor']['src_root'])
- self.results.append(self._new_play(play, phase, index, work_dir))
+ if self._playbook_name:
+ self._new_playbook(play)
+
+ self.results.append(self._new_play(play))
def v2_playbook_on_task_start(self, task, is_conditional):
self.results[-1]['tasks'].append(self._new_task(task))
@@ -125,12 +126,11 @@
s = stats.summarize(h)
summary[h] = s
- output = {
- 'plays': self.results,
- 'stats': summary
- }
+ self.playbook['plays'] = self.results
+ self.playbook['stats'] = summary
+ self.output.append(self.playbook)
- json.dump(output, open(self.output_path, 'w'),
+ json.dump(self.output, open(self.output_path, 'w'),
indent=4, sort_keys=True, separators=(',', ': '))
v2_runner_on_failed = v2_runner_on_ok
diff --git a/zuul/ansible/callback/zuul_stream.py b/zuul/ansible/callback/zuul_stream.py
index e9f969a..0461d0c 100644
--- a/zuul/ansible/callback/zuul_stream.py
+++ b/zuul/ansible/callback/zuul_stream.py
@@ -101,6 +101,7 @@
self.configure_logger()
self._items_done = False
self._deferred_result = None
+ self._playbook_name = None
def configure_logger(self):
# ansible appends timestamp, user and pid to the log lines emitted
@@ -156,23 +157,38 @@
host=host.name,
filename=included_file._filename))
- def v2_playbook_on_play_start(self, play):
- self._play = play
+ def _emit_playbook_banner(self):
# Get the hostvars from just one host - the vars we're looking for will
# be identical on all of them
- hostvars = self._play._variable_manager._hostvars
- a_host = next(iter(hostvars.keys()))
- self.phase = hostvars[a_host]['zuul_execution_phase']
- if self.phase != 'run':
- self.phase = '{phase}-{index}'.format(
- phase=self.phase,
- index=hostvars[a_host]['zuul_execution_phase_index'])
+ hostvars = next(iter(self._play._variable_manager._hostvars.values()))
+ self._playbook_name = None
+
+ phase = hostvars.get('zuul_execution_phase', '')
+ playbook = hostvars.get('zuul_execution_canonical_name_and_path')
+ trusted = hostvars.get('zuul_execution_trusted')
+ trusted = 'trusted' if trusted == "True" else 'untrusted'
+ branch = hostvars.get('zuul_execution_branch')
+
+ if phase and phase != 'run':
+ phase = '{phase}-run'.format(phase=phase)
+ phase = phase.upper()
+
+ self._log("{phase} [{trusted} : {playbook}@{branch}]".format(
+ trusted=trusted, phase=phase, playbook=playbook, branch=branch))
+
+ def v2_playbook_on_play_start(self, play):
+ self._play = play
+
+ # We can't fill in this information until the first play
+ if self._playbook_name:
+ self._emit_playbook_banner()
+
+ # Log an extra blank line to get space before each play
+ self._log("")
# the name of a play defaults to the hosts string
name = play.get_name().strip()
- msg = u"PLAY [{phase} : {playbook} : {name}]".format(
- phase=self.phase,
- playbook=self._playbook_name, name=name)
+ msg = u"PLAY [{name}]".format(name=name)
self._log(msg)
# Log an extra blank line to get space after each play
@@ -220,13 +236,17 @@
result_dict = dict(result._result)
localhost_names = ('localhost', '127.0.0.1')
is_localhost = False
+ task_host = result._host.get_name()
delegated_vars = result_dict.get('_ansible_delegated_vars', None)
if delegated_vars:
delegated_host = delegated_vars['ansible_host']
if delegated_host in localhost_names:
is_localhost = True
+ elif result._task._variable_manager is None:
+ # Handle fact gathering which doens't have a variable manager
+ if task_host == 'localhost':
+ is_localhost = True
else:
- task_host = result._host.get_name()
task_hostvars = result._task._variable_manager._hostvars[task_host]
if task_hostvars.get('ansible_host', task_hostvars.get(
'ansible_inventory_host')) in localhost_names:
@@ -376,8 +396,6 @@
def v2_playbook_on_stats(self, stats):
- # Log an extra blank line to get space before the stats
- self._log("")
self._log("PLAY RECAP")
hosts = sorted(stats.processed.keys())
@@ -390,6 +408,10 @@
" unreachable: {unreachable}"
" failed: {failures}".format(host=host, **t))
+ # Add a spacer line after the stats so that there will be a line
+ # between each playbook
+ self._log("")
+
def _process_deferred(self, result):
self._items_done = True
result_dict = self._deferred_result
@@ -403,12 +425,14 @@
task_name = task.get_name().strip()
- args = ''
- task_args = task.args.copy()
if task.loop:
task_type = 'LOOP'
else:
task_type = 'TASK'
+
+ # TODO(mordred) With the removal of printing task args, do we really
+ # want to keep doing this section?
+ task_args = task.args.copy()
is_shell = task_args.pop('_uses_shell', False)
if is_shell and task_name == 'command':
task_name = 'shell'
@@ -418,17 +442,10 @@
task_name = '{name}: {command}'.format(
name=task_name, command=raw_params[0])
- if not task.no_log and task_args:
- args = u', '.join(u'%s=%s' % a for a in task_args.items())
- args = u' %s' % args
-
- msg = "{task_type} [{task}{args}]".format(
+ msg = "{task_type} [{task}]".format(
task_type=task_type,
- task=task_name,
- args=args)
+ task=task_name)
self._log(msg)
- # Log an extra blank line to get space after each task
- self._log("")
return task
def _get_task_hosts(self, task):
@@ -457,11 +474,16 @@
def _log_message(self, result, msg=None, status="ok", result_dict=None):
hostname = self._get_hostname(result)
+ if result_dict:
+ result_dict = self._dump_result_dict(result_dict)
if result._task.no_log:
self._log("{host} | {msg}".format(
host=hostname,
msg="Output suppressed because no_log was given"))
return
+ if not msg and set(result_dict.keys()) == set(['msg', 'failed']):
+ msg = result_dict['msg']
+ result_dict = None
if msg:
msg_lines = msg.strip().split('\n')
if len(msg_lines) > 1:
@@ -477,8 +499,7 @@
self._log("{host} | {status}".format(
host=hostname, status=status, msg=msg))
if result_dict:
- result_string = json.dumps(self._dump_result_dict(result_dict),
- indent=2, sort_keys=True)
+ result_string = json.dumps(result_dict, indent=2, sort_keys=True)
for line in result_string.split('\n'):
self._log("{host} | {line}".format(host=hostname, line=line))
diff --git a/zuul/cmd/client.py b/zuul/cmd/client.py
index dec15e7..b55aed8 100755
--- a/zuul/cmd/client.py
+++ b/zuul/cmd/client.py
@@ -91,6 +91,7 @@
cmd_show = subparsers.add_parser('show',
help='valid show subcommands')
+ cmd_show.set_defaults(func=self.show_running_jobs)
show_subparsers = cmd_show.add_subparsers(title='show')
show_running_jobs = show_subparsers.add_parser(
'running-jobs',
@@ -108,6 +109,9 @@
show_running_jobs.set_defaults(func=self.show_running_jobs)
self.args = parser.parse_args()
+ if not getattr(self.args, 'func', None):
+ parser.print_help()
+ sys.exit(1)
if self.args.func == self.enqueue_ref:
if self.args.oldrev == self.args.newrev:
parser.error("The old and new revisions must not be the same.")
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 6dc3274..7640dfc 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -17,6 +17,7 @@
import logging
import textwrap
import io
+import re
import voluptuous as vs
@@ -300,6 +301,8 @@
class JobParser(object):
+ ANSIBLE_ROLE_RE = re.compile(r'^(ansible[-_.+]*)*(role[-_.+]*)*')
+
@staticmethod
def getSchema():
auth = {'secrets': to_list(str),
@@ -425,14 +428,19 @@
# Roles are part of the playbook context so we must establish
# them earlier than playbooks.
+ roles = []
if 'roles' in conf:
- roles = []
for role in conf.get('roles', []):
if 'zuul' in role:
r = JobParser._makeZuulRole(tenant, job, role)
if r:
roles.append(r)
- job.addRoles(roles)
+ # A job's repo should be an implicit role source for that job,
+ # but not in a project-pipeline variant.
+ if not project_pipeline:
+ r = JobParser._makeImplicitRole(job)
+ roles.insert(0, r)
+ job.addRoles(roles)
for pre_run_name in as_list(conf.get('pre-run')):
pre_run = model.PlaybookContext(job.source_context,
@@ -554,6 +562,16 @@
project.connection_name,
project.name)
+ @staticmethod
+ def _makeImplicitRole(job):
+ project = job.source_context.project
+ name = project.name.split('/')[-1]
+ name = JobParser.ANSIBLE_ROLE_RE.sub('', name)
+ return model.ZuulRole(name,
+ project.connection_name,
+ project.name,
+ implicit=True)
+
class ProjectTemplateParser(object):
log = logging.getLogger("zuul.ProjectTemplateParser")
diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py
index 5ad4e7a..8f8465a 100644
--- a/zuul/driver/gerrit/gerritconnection.py
+++ b/zuul/driver/gerrit/gerritconnection.py
@@ -392,10 +392,14 @@
def _updateChange(self, change, history=None):
- # In case this change is already in the history we have a cyclic
- # dependency and don't need to update ourselves again as this gets
- # done in a previous frame of the call stack.
- if history and change.number in history:
+ # In case this change is already in the history we have a
+ # cyclic dependency and don't need to update ourselves again
+ # as this gets done in a previous frame of the call stack.
+ # NOTE(jeblair): I don't think it's possible to hit this case
+ # anymore as all paths hit the change cache first.
+ if (history and change.number and change.patchset and
+ (change.number, change.patchset) in history):
+ self.log.debug("Change %s is in history" % (change,))
return change
self.log.info("Updating %s" % (change,))
@@ -441,7 +445,7 @@
history = []
else:
history = history[:]
- history.append(change.number)
+ history.append((change.number, change.patchset))
needs_changes = []
if 'dependsOn' in data:
@@ -486,7 +490,7 @@
# reference the latest patchset of its Depends-On (this
# change). In case the dep is already in history we already
# refreshed this change so refresh is not needed in this case.
- refresh = dep_num not in history
+ refresh = (dep_num, dep_ps) not in history
dep = self._getChange(
dep_num, dep_ps, refresh=refresh, history=history)
if (not dep.is_merged) and dep.is_current_patchset:
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 6c7ceb4..ef5363c 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -51,6 +51,10 @@
pass
+class RoleNotFoundError(ExecutorError):
+ pass
+
+
class Watchdog(object):
def __init__(self, timeout, function, args):
self.timeout = timeout
@@ -158,6 +162,8 @@
def __init__(self, root):
self.root = root
self.trusted = None
+ self.branch = None
+ self.canonical_name_and_path = None
self.path = None
self.roles = []
self.roles_path = []
@@ -183,6 +189,8 @@
log streaming daemon find job logs.
'''
# root
+ # .ansible
+ # fact-cache/localhost
# ansible
# inventory.yaml
# playbook_0
@@ -220,6 +228,18 @@
os.makedirs(self.trusted_root)
ssh_dir = os.path.join(self.work_root, '.ssh')
os.mkdir(ssh_dir, 0o700)
+ # Create ansible cache directory
+ ansible_cache = os.path.join(self.root, '.ansible')
+ self.fact_cache = os.path.join(ansible_cache, 'fact-cache')
+ os.makedirs(self.fact_cache)
+ localhost_facts = os.path.join(self.fact_cache, 'localhost')
+ # NOTE(pabelanger): We do not want to leak zuul-executor facts to other
+ # playbooks now that smart fact gathering is enabled by default. We
+ # can have ansible skip populating the cache with information by the
+ # doing the following.
+ with open(localhost_facts, 'w') as f:
+ f.write('{"module_setup": true}')
+
self.result_data_file = os.path.join(self.work_root, 'results.json')
with open(self.result_data_file, 'w'):
pass
@@ -1085,10 +1105,13 @@
self.log.debug("Prepare playbook repo for %s" % (playbook,))
# Check out the playbook repo if needed and set the path to
# the playbook that should be run.
- jobdir_playbook.trusted = playbook['trusted']
source = self.executor_server.connections.getSource(
playbook['connection'])
project = source.getProject(playbook['project'])
+ jobdir_playbook.trusted = playbook['trusted']
+ jobdir_playbook.branch = playbook['branch']
+ jobdir_playbook.canonical_name_and_path = os.path.join(
+ project.canonical_name, playbook['path'])
path = None
if not playbook['trusted']:
# This is a project repo, so it is safe to use the already
@@ -1157,12 +1180,14 @@
if os.path.isdir(d):
# This repo has a collection of roles
if not trusted:
+ self._blockPluginDirs(d)
for entry in os.listdir(d):
- if os.path.isdir(os.path.join(d, entry)):
- self._blockPluginDirs(os.path.join(d, entry))
+ entry_path = os.path.join(d, entry)
+ if os.path.isdir(entry_path):
+ self._blockPluginDirs(entry_path)
return d
# It is neither a bare role, nor a collection of roles
- raise ExecutorError("Unable to find role in %s" % (path,))
+ raise RoleNotFoundError("Unable to find role in %s" % (path,))
def prepareZuulRole(self, jobdir_playbook, role, args, root):
self.log.debug("Prepare zuul role for %s" % (role,))
@@ -1203,10 +1228,17 @@
raise ExecutorError("Invalid role name %s", name)
os.symlink(path, link)
- role_path = self.findRole(link, trusted=jobdir_playbook.trusted)
+ try:
+ role_path = self.findRole(link, trusted=jobdir_playbook.trusted)
+ except RoleNotFoundError:
+ if role['implicit']:
+ self.log.info("Implicit role not found in %s", link)
+ return
+ raise
if role_path is None:
# In the case of a bare role, add the containing directory
role_path = root
+ self.log.debug("Adding role path %s", role_path)
jobdir_playbook.roles_path.append(role_path)
def prepareAnsibleFiles(self, args):
@@ -1254,7 +1286,10 @@
config.write('remote_tmp = %s/.ansible/remote_tmp\n' %
self.jobdir.root)
config.write('retry_files_enabled = False\n')
- config.write('gathering = explicit\n')
+ config.write('gathering = smart\n')
+ config.write('fact_caching = jsonfile\n')
+ config.write('fact_caching_connection = %s\n' %
+ self.jobdir.fact_cache)
config.write('library = %s\n'
% self.executor_server.library_dir)
config.write('command_warnings = False\n')
@@ -1317,7 +1352,6 @@
def runAnsible(self, cmd, timeout, config_file, trusted):
env_copy = os.environ.copy()
env_copy.update(self.ssh_agent.env)
- env_copy['LOGNAME'] = 'zuul'
env_copy['ZUUL_JOB_OUTPUT_FILE'] = self.jobdir.job_output_file
env_copy['ZUUL_JOBDIR'] = self.jobdir.root
pythonpath = env_copy.get('PYTHONPATH')
@@ -1417,9 +1451,6 @@
def runAnsiblePlaybook(self, playbook, timeout, success=None,
phase=None, index=None):
- env_copy = os.environ.copy()
- env_copy['LOGNAME'] = 'zuul'
-
if self.executor_server.verbose:
verbose = '-vvv'
else:
@@ -1438,6 +1469,13 @@
if index is not None:
cmd.extend(['-e', 'zuul_execution_phase_index=%s' % index])
+ cmd.extend(['-e', 'zuul_execution_trusted=%s' % str(playbook.trusted)])
+ cmd.extend([
+ '-e',
+ 'zuul_execution_canonical_name_and_path=%s'
+ % playbook.canonical_name_and_path])
+ cmd.extend(['-e', 'zuul_execution_branch=%s' % str(playbook.branch)])
+
result, code = self.runAnsible(
cmd=cmd, timeout=timeout,
config_file=playbook.ansible_config,
diff --git a/zuul/model.py b/zuul/model.py
index b266c02..ed77864 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -702,10 +702,12 @@
class ZuulRole(Role):
"""A reference to an ansible role in a Zuul project."""
- def __init__(self, target_name, connection_name, project_name):
+ def __init__(self, target_name, connection_name, project_name,
+ implicit=False):
super(ZuulRole, self).__init__(target_name)
self.connection_name = connection_name
self.project_name = project_name
+ self.implicit = implicit
def __repr__(self):
return '<ZuulRole %s %s>' % (self.project_name, self.target_name)
@@ -715,6 +717,8 @@
def __eq__(self, other):
if not isinstance(other, ZuulRole):
return False
+ # Implicit is not consulted for equality so that we can handle
+ # implicit to explicit conversions.
return (super(ZuulRole, self).__eq__(other) and
self.connection_name == other.connection_name and
self.project_name == other.project_name)
@@ -725,6 +729,7 @@
d['type'] = 'zuul'
d['connection'] = self.connection_name
d['project'] = self.project_name
+ d['implicit'] = self.implicit
return d
@@ -867,11 +872,31 @@
self.run = self.implied_run
def addRoles(self, roles):
- newroles = list(self.roles)
+ newroles = []
+ # Start with a copy of the existing roles, but if any of them
+ # are implicit roles which are identified as explicit in the
+ # new roles list, replace them with the explicit version.
+ changed = False
+ for existing_role in self.roles:
+ if existing_role in roles:
+ new_role = roles[roles.index(existing_role)]
+ else:
+ new_role = None
+ if (new_role and
+ isinstance(new_role, ZuulRole) and
+ isinstance(existing_role, ZuulRole) and
+ existing_role.implicit and not new_role.implicit):
+ newroles.append(new_role)
+ changed = True
+ else:
+ newroles.append(existing_role)
+ # Now add the new roles.
for role in reversed(roles):
if role not in newroles:
newroles.insert(0, role)
- self.roles = tuple(newroles)
+ changed = True
+ if changed:
+ self.roles = tuple(newroles)
def updateVariables(self, other_vars):
v = self.variables