Merge "Move playbook out of zuul" into feature/zuulv3
diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml
index 92c66d1..1f8fdf3 100644
--- a/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml
+++ b/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml
@@ -13,3 +13,10 @@
- zuul.executor.hostname is defined
- zuul.executor.src_root is defined
- zuul.executor.log_root is defined
+
+ - name: Assert zuul.project variables are valid.
+ assert:
+ that:
+ - zuul.project.name == 'org/project'
+ - zuul.project.canonical_hostname == 'review.example.com'
+ - zuul.project.canonical_name == 'review.example.com/org/project'
diff --git a/tests/fixtures/config/in-repo/git/org_project1/README b/tests/fixtures/config/in-repo/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/in-repo/git/org_project1/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/in-repo/main.yaml b/tests/fixtures/config/in-repo/main.yaml
index 208e274..5f57245 100644
--- a/tests/fixtures/config/in-repo/main.yaml
+++ b/tests/fixtures/config/in-repo/main.yaml
@@ -6,3 +6,4 @@
- common-config
untrusted-projects:
- org/project
+ - org/project1
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 3919418..2168a7f 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -191,6 +191,61 @@
dict(name='project-test1', result='SUCCESS', changes='2,1'),
dict(name='project-test2', result='SUCCESS', changes='3,1')])
+ def test_crd_dynamic_config_branch(self):
+ # Test that we can create a job in one repo and be able to use
+ # it from a different branch on a different repo.
+
+ self.create_branch('org/project1', 'stable')
+
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ name: project-test2
+
+ - project:
+ name: org/project
+ check:
+ jobs:
+ - project-test2
+ """)
+
+ in_repo_playbook = textwrap.dedent(
+ """
+ - hosts: all
+ tasks: []
+ """)
+
+ file_dict = {'.zuul.yaml': in_repo_conf,
+ 'playbooks/project-test2.yaml': in_repo_playbook}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+
+ second_repo_conf = textwrap.dedent(
+ """
+ - project:
+ name: org/project1
+ check:
+ jobs:
+ - project-test2
+ """)
+
+ second_file_dict = {'.zuul.yaml': second_repo_conf}
+ B = self.fake_gerrit.addFakeChange('org/project1', 'stable', 'B',
+ files=second_file_dict)
+ B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ B.subject, A.data['id'])
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.reported, 1, "A should report")
+ self.assertHistory([
+ dict(name='project-test2', result='SUCCESS', changes='1,1'),
+ dict(name='project-test2', result='SUCCESS', changes='1,1 2,1'),
+ ])
+
def test_untrusted_syntax_error(self):
in_repo_conf = textwrap.dedent(
"""
@@ -253,6 +308,26 @@
self.assertIn('syntax error', A.messages[1],
"A should have a syntax error reported")
+ def test_untrusted_shadow_error(self):
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ name: common-config-test
+ """)
+
+ file_dict = {'.zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+ A.addApproval('code-review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(A.reported, 2,
+ "A should report start and failure")
+ self.assertIn('not permitted to shadow', A.messages[1],
+ "A should have a syntax error reported")
+
class TestAnsible(AnsibleZuulTestCase):
# A temporary class to hold new tests while others are disabled
diff --git a/zuul/configloader.py b/zuul/configloader.py
index d7cef94..070e731 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -65,6 +65,8 @@
def configuration_exceptions(stanza, conf):
try:
yield
+ except ConfigurationSyntaxError:
+ raise
except Exception as e:
conf = copy.deepcopy(conf)
context = conf.pop('_source_context')
@@ -271,7 +273,29 @@
]
@staticmethod
- def fromYaml(tenant, layout, conf):
+ def _getImpliedBranches(reference, job, project_pipeline):
+ # If the current job definition is not in the same branch as
+ # the reference definition of this job, and this is a project
+ # repo, add an implicit branch matcher for this branch
+ # (assuming there are no explicit branch matchers). But only
+ # for top-level job definitions and variants.
+ # Project-pipeline job variants should more closely attach to
+ # their branch if they appear in a project-repo.
+ if (reference and
+ reference.source_context and
+ reference.source_context.branch != job.source_context.branch):
+ same_context = False
+ else:
+ same_context = True
+
+ if (job.source_context and
+ (not job.source_context.trusted) and
+ ((not same_context) or project_pipeline)):
+ return [job.source_context.branch]
+ return None
+
+ @staticmethod
+ def fromYaml(tenant, layout, conf, project_pipeline=False):
with configuration_exceptions('job', conf):
JobParser.getSchema()(conf)
@@ -280,6 +304,8 @@
# them (e.g., "job.run = ..." rather than
# "job.run.append(...)").
+ reference = layout.jobs.get(conf['name'], [None])[0]
+
job = model.Job(conf['name'])
job.source_context = conf.get('_source_context')
if 'auth' in conf:
@@ -316,9 +342,10 @@
run = model.PlaybookContext(job.source_context, run_name)
job.run = (run,)
else:
- run_name = os.path.join('playbooks', job.name)
- run = model.PlaybookContext(job.source_context, run_name)
- job.implied_run = (run,) + job.implied_run
+ if not project_pipeline:
+ run_name = os.path.join('playbooks', job.name)
+ run = model.PlaybookContext(job.source_context, run_name)
+ job.implied_run = (run,) + job.implied_run
for k in JobParser.simple_attributes:
a = k.replace('-', '_')
@@ -350,13 +377,14 @@
job.dependencies = frozenset(as_list(conf.get('dependencies')))
- roles = []
- for role in conf.get('roles', []):
- if 'zuul' in role:
- r = JobParser._makeZuulRole(tenant, job, role)
- if r:
- roles.append(r)
- job.roles = job.roles.union(set(roles))
+ if 'roles' in conf:
+ roles = []
+ for role in conf.get('roles', []):
+ if 'zuul' in role:
+ r = JobParser._makeZuulRole(tenant, job, role)
+ if r:
+ roles.append(r)
+ job.roles = job.roles.union(set(roles))
variables = conf.get('vars', None)
if variables:
@@ -372,14 +400,20 @@
allowed.append(project.name)
job.allowed_projects = frozenset(allowed)
- # If the definition for this job came from a project repo,
- # implicitly apply a branch matcher for the branch it was on.
- if (not job.source_context.trusted):
- branches = [job.source_context.branch]
- elif 'branches' in conf:
+ # If the current job definition is not in the same branch as
+ # the reference definition of this job, and this is a project
+ # repo, add an implicit branch matcher for this branch
+ # (assuming there are no explicit branch matchers). But only
+ # for top-level job definitions and variants.
+ # Project-pipeline job variants should more closely attach to
+ # their branch if they appear in a project-repo.
+
+ branches = None
+ if (project_pipeline or 'branches' not in conf):
+ branches = JobParser._getImpliedBranches(
+ reference, job, project_pipeline)
+ if (not branches) and ('branches' in conf):
branches = as_list(conf['branches'])
- else:
- branches = None
if branches:
matchers = []
for branch in branches:
@@ -408,7 +442,7 @@
return model.ZuulRole(role.get('name', name),
project.connection_name,
- project.name, trusted)
+ project.name)
class ProjectTemplateParser(object):
@@ -456,23 +490,22 @@
start_mark, job_list):
for conf_job in conf:
if isinstance(conf_job, six.string_types):
- job = model.Job(conf_job)
- job_list.addJob(job)
+ attrs = dict(name=conf_job)
elif isinstance(conf_job, dict):
# A dictionary in a job tree may override params
jobname, attrs = conf_job.items()[0]
if attrs:
# We are overriding params, so make a new job def
attrs['name'] = jobname
- attrs['_source_context'] = source_context
- attrs['_start_mark'] = start_mark
- job_list.addJob(JobParser.fromYaml(tenant, layout, attrs))
else:
# Not overriding, so add a blank job
- job = model.Job(jobname)
- job_list.addJob(job)
+ attrs = dict(name=jobname)
else:
raise Exception("Job must be a string or dictionary")
+ attrs['_source_context'] = source_context
+ attrs['_start_mark'] = start_mark
+ job_list.addJob(JobParser.fromYaml(tenant, layout, attrs,
+ project_pipeline=True))
class ProjectParser(object):
@@ -993,7 +1026,8 @@
layout.addSecret(SecretParser.fromYaml(layout, config_secret))
for config_job in data.jobs:
- layout.addJob(JobParser.fromYaml(tenant, layout, config_job))
+ with configuration_exceptions('job', config_job):
+ layout.addJob(JobParser.fromYaml(tenant, layout, config_job))
for config_semaphore in data.semaphores:
layout.addSemaphore(SemaphoreParser.fromYaml(config_semaphore))
@@ -1122,7 +1156,8 @@
layout.addSecret(SecretParser.fromYaml(layout, config_secret))
for config_job in config.jobs:
- layout.addJob(JobParser.fromYaml(tenant, layout, config_job))
+ with configuration_exceptions('job', config_job):
+ layout.addJob(JobParser.fromYaml(tenant, layout, config_job))
for config_template in config.project_templates:
layout.addProjectTemplate(ProjectTemplateParser.fromYaml(
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index 7e2d296..9f234e9 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -183,10 +183,15 @@
dependent_items.reverse()
# TODOv3(jeblair): This ansible vars data structure will
# replace the environment variables below.
+ project = dict(
+ name=item.change.project.name,
+ canonical_hostname=item.change.project.canonical_hostname,
+ canonical_name=item.change.project.canonical_name)
+
zuul_params = dict(uuid=uuid,
pipeline=pipeline.name,
job=job.name,
- project=item.change.project.name,
+ project=project,
tags=' '.join(sorted(job.tags)))
if hasattr(item.change, 'branch'):
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index fa0f4d5..4801de2 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -108,7 +108,8 @@
self.pre_playbooks = []
self.post_playbooks = []
self.roles = []
- self.roles_path = []
+ self.trusted_roles_path = []
+ self.untrusted_roles_path = []
self.untrusted_config = os.path.join(
self.ansible_root, 'untrusted.cfg')
self.trusted_config = os.path.join(self.ansible_root, 'trusted.cfg')
@@ -142,6 +143,10 @@
count = len(self.roles)
root = os.path.join(self.ansible_root, 'role_%i' % (count,))
os.makedirs(root)
+ trusted = os.path.join(root, 'trusted')
+ os.makedirs(trusted)
+ untrusted = os.path.join(root, 'untrusted')
+ os.makedirs(untrusted)
self.roles.append(root)
return root
@@ -601,9 +606,9 @@
repo.delete_remote(repo.remotes.origin)
# is the playbook in a repo that we have already prepared?
- self.preparePlaybookRepos(args)
+ trusted, untrusted = self.preparePlaybookRepos(args)
- self.prepareRoles(args)
+ self.prepareRoles(args, trusted, untrusted)
# TODOv3: Ansible the ansible thing here.
self.prepareAnsibleFiles(args)
@@ -737,15 +742,24 @@
return None
def preparePlaybookRepos(self, args):
+ trusted = untrusted = False
for playbook in args['pre_playbooks']:
jobdir_playbook = self.jobdir.addPrePlaybook()
self.preparePlaybookRepo(jobdir_playbook, playbook,
args, required=True)
+ if playbook['trusted']:
+ trusted = True
+ else:
+ untrusted = True
for playbook in args['playbooks']:
jobdir_playbook = self.jobdir.addPlaybook()
self.preparePlaybookRepo(jobdir_playbook, playbook,
args, required=False)
+ if playbook['trusted']:
+ trusted = True
+ else:
+ untrusted = True
if jobdir_playbook.path is not None:
self.jobdir.playbook = jobdir_playbook
break
@@ -756,6 +770,11 @@
jobdir_playbook = self.jobdir.addPostPlaybook()
self.preparePlaybookRepo(jobdir_playbook, playbook,
args, required=True)
+ if playbook['trusted']:
+ trusted = True
+ else:
+ untrusted = True
+ return (trusted, untrusted)
def preparePlaybookRepo(self, jobdir_playbook, playbook, args, required):
self.log.debug("Prepare playbook repo for %s" % (playbook,))
@@ -799,11 +818,11 @@
required=required,
trusted=playbook['trusted'])
- def prepareRoles(self, args):
+ def prepareRoles(self, args, trusted, untrusted):
for role in args['roles']:
if role['type'] == 'zuul':
root = self.jobdir.addRole()
- self.prepareZuulRole(args, role, root)
+ self.prepareZuulRole(args, role, root, trusted, untrusted)
def findRole(self, path, trusted=False):
d = os.path.join(path, 'tasks')
@@ -826,17 +845,22 @@
self._blockPluginDirs(os.path.join(path, entry))
return path
- def prepareZuulRole(self, args, role, root):
+ def prepareZuulRole(self, args, role, root, trusted, untrusted):
self.log.debug("Prepare zuul role for %s" % (role,))
# Check out the role repo if needed
source = self.executor_server.connections.getSource(
role['connection'])
project = source.getProject(role['project'])
- role_repo = None
- if not role['trusted']:
- # This is a project repo, so it is safe to use the already
- # checked out version (from speculative merging) of the
- # role
+ untrusted_role_repo = None
+ trusted_role_repo = None
+ trusted_root = os.path.join(root, 'trusted')
+ untrusted_root = os.path.join(root, 'untrusted')
+ name = role['target_name']
+
+ if untrusted:
+ # There is at least one untrusted playbook. For that
+ # case, use the already checked out version (from
+ # speculative merging) of the role.
for i in args['items']:
if (i['connection'] == role['connection'] and
@@ -847,27 +871,70 @@
path = os.path.join(self.jobdir.src_root,
project.canonical_hostname,
project.name)
- link = os.path.join(root, role['name'])
+ # The name of the symlink is the requested name of
+ # the role (which may be the repo name or may be
+ # something else; this can come into play if this
+ # is a bare role).
+ link = os.path.join(untrusted_root, name)
+ link = os.path.realpath(link)
+ if not link.startswith(os.path.realpath(untrusted_root)):
+ raise Exception("Invalid role name %s", name)
os.symlink(path, link)
- role_repo = link
+ untrusted_role_repo = link
break
- # The role repo is either a config repo, or it isn't in
- # the stack of changes we are testing, so check out the branch
- # tip into a dedicated space.
-
- if not role_repo:
- merger = self.executor_server._getMerger(root)
+ if trusted or not untrusted_role_repo:
+ # There is at least one trusted playbook which will need a
+ # trusted checkout of the role, or the role did not appear
+ # in the dependency chain for the change (in which case,
+ # there is no existing untrusted checkout of it). Check
+ # out the branch tip into a dedicated space.
+ merger = self.executor_server._getMerger(trusted_root)
merger.checkoutBranch(role['connection'], project.name,
'master')
- role_repo = os.path.join(root, project.canonical_hostname,
- project.name)
+ orig_repo_path = os.path.join(trusted_root,
+ project.canonical_hostname,
+ project.name)
+ if name != project.name:
+ # The requested name of the role is not the same as
+ # the project name, so rename the git repo as the
+ # requested name. It is the only item in this
+ # directory, so we don't need to worry about
+ # collisions.
+ target = os.path.join(trusted_root,
+ project.canonical_hostname,
+ name)
+ target = os.path.realpath(target)
+ if not target.startswith(os.path.realpath(trusted_root)):
+ raise Exception("Invalid role name %s", name)
+ os.rename(orig_repo_path, target)
+ trusted_role_repo = target
+ else:
+ trusted_role_repo = orig_repo_path
- role_path = self.findRole(role_repo, trusted=role['trusted'])
- if role_path is None:
- # In the case of a bare role, add the containing directory
- role_path = os.path.join(root, project.canonical_hostname)
- self.jobdir.roles_path.append(role_path)
+ if not untrusted_role_repo:
+ # In the case that there was no untrusted checkout,
+ # use the trusted checkout.
+ untrusted_role_repo = trusted_role_repo
+ untrusted_root = trusted_root
+
+ if untrusted:
+ untrusted_role_path = self.findRole(untrusted_role_repo,
+ trusted=False)
+ if untrusted_role_path is None:
+ # In the case of a bare role, add the containing directory
+ untrusted_role_path = os.path.join(untrusted_root,
+ project.canonical_hostname)
+ self.jobdir.untrusted_roles_path.append(untrusted_role_path)
+
+ if trusted:
+ trusted_role_path = self.findRole(trusted_role_repo,
+ trusted=True)
+ if trusted_role_path is None:
+ # In the case of a bare role, add the containing directory
+ trusted_role_path = os.path.join(trusted_root,
+ project.canonical_hostname)
+ self.jobdir.trusted_roles_path.append(trusted_role_path)
def prepareAnsibleFiles(self, args):
keys = []
@@ -909,9 +976,6 @@
config.write('gathering = explicit\n')
config.write('library = %s\n'
% self.executor_server.library_dir)
- if self.jobdir.roles_path:
- config.write('roles_path = %s\n' %
- ':'.join(self.jobdir.roles_path))
config.write('command_warnings = False\n')
config.write('callback_plugins = %s\n'
% self.executor_server.callback_dir)
@@ -924,6 +988,12 @@
% self.executor_server.action_dir)
config.write('lookup_plugins = %s\n'
% self.executor_server.lookup_dir)
+ roles_path = self.jobdir.untrusted_roles_path
+ else:
+ roles_path = self.jobdir.trusted_roles_path
+
+ if roles_path:
+ config.write('roles_path = %s\n' % ':'.join(roles_path))
# On trusted jobs, we want to prevent the printing of args,
# since trusted jobs might have access to secrets that they may
diff --git a/zuul/model.py b/zuul/model.py
index af80028..73dec39 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -673,11 +673,10 @@
class ZuulRole(Role):
"""A reference to an ansible role in a Zuul project."""
- def __init__(self, target_name, connection_name, project_name, trusted):
+ def __init__(self, target_name, connection_name, project_name):
super(ZuulRole, self).__init__(target_name)
self.connection_name = connection_name
self.project_name = project_name
- self.trusted = trusted
def __repr__(self):
return '<ZuulRole %s %s>' % (self.project_name, self.target_name)
@@ -687,8 +686,7 @@
return False
return (super(ZuulRole, self).__eq__(other) and
self.connection_name == other.connection_name,
- self.project_name == other.project_name,
- self.trusted == other.trusted)
+ self.project_name == other.project_name)
def toDict(self):
# Render to a dict to use in passing json to the executor
@@ -696,7 +694,6 @@
d['type'] = 'zuul'
d['connection'] = self.connection_name
d['project'] = self.project_name
- d['trusted'] = self.trusted
return d