Merge "Use playbooks defined in repos" into feature/zuulv3
diff --git a/tests/base.py b/tests/base.py
index db73a8d..83354c9 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -1752,6 +1752,10 @@
zuul.merger.merger.reset_repo_to_head(repo)
for fn, content in files.items():
fn = os.path.join(path, fn)
+ try:
+ os.makedirs(os.path.dirname(fn))
+ except OSError:
+ pass
with open(fn, 'w') as f:
f.write(content)
repo.index.add([fn])
diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/python27.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/python27.yaml
new file mode 100644
index 0000000..227cc12
--- /dev/null
+++ b/tests/fixtures/config/ansible/git/common-config/playbooks/python27.yaml
@@ -0,0 +1,6 @@
+# TODO(jeblair): Perform an action inside of a test chroot
+- hosts: all
+ tasks:
+ - file:
+ path: /tmp/playbook.test
+ state: touch
diff --git a/tests/fixtures/config/ansible/git/common-config/zuul.yaml b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
new file mode 100644
index 0000000..f00eab7
--- /dev/null
+++ b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
@@ -0,0 +1,40 @@
+- pipeline:
+ name: check
+ manager: independent
+ source:
+ gerrit
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- pipeline:
+ name: gate
+ manager: dependent
+ success-message: Build succeeded (gate).
+ source:
+ gerrit
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
+
+- job:
+ name: python27
diff --git a/tests/fixtures/config/ansible/git/org_project/.zuul.yaml b/tests/fixtures/config/ansible/git/org_project/.zuul.yaml
new file mode 100644
index 0000000..6bedb07
--- /dev/null
+++ b/tests/fixtures/config/ansible/git/org_project/.zuul.yaml
@@ -0,0 +1,6 @@
+- project:
+ name: org/project
+
+ check:
+ jobs:
+ - python27
diff --git a/tests/fixtures/config/ansible/git/org_project/README b/tests/fixtures/config/ansible/git/org_project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/ansible/git/org_project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/ansible/main.yaml b/tests/fixtures/config/ansible/main.yaml
new file mode 100644
index 0000000..d9868fa
--- /dev/null
+++ b/tests/fixtures/config/ansible/main.yaml
@@ -0,0 +1,8 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-repos:
+ - common-config
+ project-repos:
+ - org/project
diff --git a/tests/fixtures/config/in-repo/git/org_project/playbooks/project-test1.yaml b/tests/fixtures/config/in-repo/git/org_project/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/in-repo/git/org_project/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/multi-tenant/git/common-config/playbooks/python27.yaml b/tests/fixtures/config/multi-tenant/git/common-config/playbooks/python27.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/multi-tenant/git/common-config/playbooks/python27.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/multi-tenant/git/tenant-one-config/playbooks/project1-test1.yaml b/tests/fixtures/config/multi-tenant/git/tenant-one-config/playbooks/project1-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/multi-tenant/git/tenant-one-config/playbooks/project1-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/multi-tenant/git/tenant-two-config/playbooks/project2-test1.yaml b/tests/fixtures/config/multi-tenant/git/tenant-two-config/playbooks/project2-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/multi-tenant/git/tenant-two-config/playbooks/project2-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/openstack/git/project-config/playbooks/python27.yaml b/tests/fixtures/config/openstack/git/project-config/playbooks/python27.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/openstack/git/project-config/playbooks/python27.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/openstack/git/project-config/playbooks/python35.yaml b/tests/fixtures/config/openstack/git/project-config/playbooks/python35.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/openstack/git/project-config/playbooks/python35.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index 0189340..db98d14 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -56,7 +56,7 @@
pipeline = model.Pipeline('gate', layout)
layout.addPipeline(pipeline)
queue = model.ChangeQueue(pipeline)
- project = model.Project('project')
+ project = model.Project('project', None)
base = configloader.JobParser.fromYaml(layout, {
'_source_project': project,
@@ -122,7 +122,7 @@
def test_job_auth_inheritance(self):
layout = model.Layout()
- project = model.Project('project')
+ project = model.Project('project', None)
base = configloader.JobParser.fromYaml(layout, {
'_source_project': project,
@@ -201,7 +201,7 @@
pipeline = model.Pipeline('gate', layout)
layout.addPipeline(pipeline)
queue = model.ChangeQueue(pipeline)
- project = model.Project('project')
+ project = model.Project('project', None)
base = configloader.JobParser.fromYaml(layout, {
'_source_project': project,
@@ -271,7 +271,7 @@
pipeline = model.Pipeline('gate', layout)
layout.addPipeline(pipeline)
queue = model.ChangeQueue(pipeline)
- project = model.Project('project')
+ project = model.Project('project', None)
base = configloader.JobParser.fromYaml(layout, {
'_source_project': project,
@@ -312,14 +312,14 @@
def test_job_source_project(self):
layout = model.Layout()
- base_project = model.Project('base_project')
+ base_project = model.Project('base_project', None)
base = configloader.JobParser.fromYaml(layout, {
'_source_project': base_project,
'name': 'base',
})
layout.addJob(base)
- other_project = model.Project('other_project')
+ other_project = model.Project('other_project', None)
base2 = configloader.JobParser.fromYaml(layout, {
'_source_project': other_project,
'name': 'base',
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 8853302..96f24a1 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -98,8 +98,16 @@
- project-test2
""")
+ in_repo_playbook = textwrap.dedent(
+ """
+ - hosts: all
+ tasks: []
+ """)
+
+ file_dict = {'.zuul.yaml': in_repo_conf,
+ 'playbooks/project-test2.yaml': in_repo_playbook}
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
- files={'.zuul.yaml': in_repo_conf})
+ files=file_dict)
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
@@ -110,3 +118,16 @@
"A should report start and success")
self.assertIn('tenant-one-gate', A.messages[1],
"A should transit tenant-one gate")
+
+
+class TestAnsible(AnsibleZuulTestCase):
+ # A temporary class to hold new tests while others are disabled
+
+ tenant_config_file = 'config/ansible/main.yaml'
+
+ def test_playbook(self):
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(self.getJobFromHistory('python27').result,
+ 'SUCCESS')
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 7b35a86..c864777 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -103,6 +103,7 @@
'attempts': int,
'_source_project': model.Project,
'_source_branch': vs.Any(str, None),
+ '_source_configrepo': bool,
}
return vs.Schema(job)
@@ -143,12 +144,16 @@
# accumulate onto any previously applied tags from
# metajobs.
job.tags = job.tags.union(set(tags))
- # The source attributes may not be overridden -- they are
- # always supplied by the config loader. They correspond to
- # the Project instance of the repo where it originated, and
- # the branch name.
+ # The source attributes and playbook may not be overridden --
+ # they are always supplied by the config loader. They
+ # correspond to the Project instance of the repo where it
+ # originated, and the branch name.
job.source_project = conf.get('_source_project')
job.source_branch = conf.get('_source_branch')
+ job.source_configrepo = conf.get('_source_configrepo')
+ # TODOv3(jeblair): verify the playbook exists
+ # TODOv3(jeblair): remove hardcoded extension
+ job.playbook = os.path.join('playbooks', job.name + '.yaml')
job.failure_message = conf.get('failure-message', job.failure_message)
job.success_message = conf.get('success-message', job.success_message)
job.failure_url = conf.get('failure-url', job.failure_url)
@@ -156,7 +161,7 @@
# If the definition for this job came from a project repo,
# implicitly apply a branch matcher for the branch it was on.
- if job.source_branch:
+ if (not job.source_configrepo) and job.source_branch:
branches = [job.source_branch]
elif 'branches' in conf:
branches = as_list(conf['branches'])
@@ -546,6 +551,7 @@
job = merger.getFiles(project.name, url, 'master',
files=['zuul.yaml', '.zuul.yaml'])
job.project = project
+ job.branch = 'master'
job.config_repo = True
jobs.append(job)
@@ -579,7 +585,7 @@
(job.project, fn))
if job.config_repo:
incdata = TenantParser._parseConfigRepoLayout(
- job.files[fn], job.project)
+ job.files[fn], job.project, job.branch)
config_repos_config.extend(incdata)
else:
incdata = TenantParser._parseProjectRepoLayout(
@@ -589,11 +595,10 @@
return config_repos_config, project_repos_config
@staticmethod
- def _parseConfigRepoLayout(data, project):
+ def _parseConfigRepoLayout(data, project, branch):
# This is the top-level configuration for a tenant.
config = model.UnparsedTenantConfig()
- config.extend(yaml.load(data), project)
-
+ config.extend(yaml.load(data), project, branch, True)
return config
@staticmethod
@@ -601,7 +606,7 @@
# TODOv3(jeblair): this should implement some rules to protect
# aspects of the config that should not be changed in-repo
config = model.UnparsedTenantConfig()
- config.extend(yaml.load(data), project, branch)
+ config.extend(yaml.load(data), project, branch, False)
return config
diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py
index ac644eb..627c716 100644
--- a/zuul/driver/gerrit/gerritconnection.py
+++ b/zuul/driver/gerrit/gerritconnection.py
@@ -255,7 +255,7 @@
def getProject(self, name):
if name not in self.projects:
- self.projects[name] = Project(name)
+ self.projects[name] = Project(name, self.connection_name)
return self.projects[name]
def maintainCache(self, relevant):
diff --git a/zuul/launcher/client.py b/zuul/launcher/client.py
index 9e895ef..716e59b 100644
--- a/zuul/launcher/client.py
+++ b/zuul/launcher/client.py
@@ -368,6 +368,16 @@
params['job'] = job.name
params['items'] = merger_items
params['projects'] = []
+
+ config_repos = set([x[1] for x in
+ item.pipeline.layout.tenant.config_repos])
+ if job.name != 'noop':
+ params['playbook'] = dict(
+ connection=job.source_project.connection_name,
+ config_repo=job.source_project in config_repos,
+ project=job.source_project.name,
+ branch=job.source_branch,
+ path=job.playbook)
nodes = []
for node in item.current_build_set.getJobNodeSet(job.name).getNodes():
nodes.append(dict(name=node.name, image=node.image))
diff --git a/zuul/launcher/server.py b/zuul/launcher/server.py
index bfcc8a4..58bfb07 100644
--- a/zuul/launcher/server.py
+++ b/zuul/launcher/server.py
@@ -26,7 +26,6 @@
import traceback
import gear
-import yaml
import zuul.merger
import zuul.ansible.library
@@ -76,7 +75,9 @@
os.makedirs(self.ansible_root)
self.known_hosts = os.path.join(self.ansible_root, 'known_hosts')
self.inventory = os.path.join(self.ansible_root, 'inventory')
- self.playbook = os.path.join(self.ansible_root, 'playbook')
+ self.playbook = None
+ self.playbook_root = os.path.join(self.ansible_root, 'playbook')
+ os.makedirs(self.playbook_root)
self.post_playbook = os.path.join(self.ansible_root, 'post_playbook')
self.config = os.path.join(self.ansible_root, 'ansible.cfg')
self.ansible_log = os.path.join(self.ansible_root, 'ansible_log.txt')
@@ -358,6 +359,9 @@
else:
commit = args['items'][-1]['newrev'] # noqa
+ # is the playbook in a repo that we have already prepared?
+ jobdir.playbook = self.preparePlaybookRepo(jobdir, args)
+
# TODOv3: Ansible the ansible thing here.
self.prepareAnsibleFiles(jobdir, args)
@@ -402,6 +406,36 @@
hosts.append((node['name'], dict(ansible_connection='local')))
return hosts
+ def preparePlaybookRepo(self, jobdir, args):
+ # Check out the playbook repo if needed and return the path to
+ # the playbook that should be run.
+ playbook = args['playbook']
+ source = self.connections.getSource(playbook['connection'])
+ project = source.getProject(playbook['project'])
+ # TODO(jeblair): construct the url in the merger itself
+ url = source.getGitUrl(project)
+ if not playbook['config_repo']:
+ # This is a project repo, so it is safe to use the already
+ # checked out version (from speculative merging) of the
+ # playbook
+ for i in args['items']:
+ if (i['connection_name'] == playbook['connection'] and
+ i['project'] == playbook['project']):
+ # We already have this repo prepared
+ return os.path.join(jobdir.git_root,
+ project.name,
+ playbook['path'])
+ # The playbook repo is either a config repo, or it isn't in
+ # the stack of changes we are testing, so check out the branch
+ # tip into a dedicated space.
+
+ merger = self._getMerger(jobdir.playbook_root)
+ merger.checkoutBranch(project.name, url, playbook['branch'])
+
+ return os.path.join(jobdir.playbook_root,
+ project.name,
+ playbook['path'])
+
def prepareAnsibleFiles(self, jobdir, args):
with open(jobdir.inventory, 'w') as inventory:
for host_name, host_vars in self.getHostList(args):
@@ -410,11 +444,6 @@
for k, v in host_vars.items():
inventory.write('%s=%s' % (k, v))
inventory.write('\n')
- with open(jobdir.playbook, 'w') as playbook:
- play = dict(hosts='localhost',
- tasks=[dict(name='test',
- shell='echo Hello world')])
- playbook.write(yaml.dump([play]))
with open(jobdir.config, 'w') as config:
config.write('[defaults]\n')
config.write('hostfile = %s\n' % jobdir.inventory)
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index 692dd83..3ab7b5f 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -260,6 +260,16 @@
except Exception:
self.log.exception("Unable to update %s", project)
+ def checkoutBranch(self, project, url, branch):
+ repo = self.getRepo(project, url)
+ if repo.hasBranch(branch):
+ self.log.info("Checking out branch %s of %s" % (branch, project))
+ head = repo.getBranchHead(branch)
+ repo.checkout(head)
+ else:
+ raise Exception("Project %s does not have branch %s" %
+ (project, branch))
+
def _mergeChange(self, item, ref):
repo = self.getRepo(item['project'], item['url'])
try:
diff --git a/zuul/model.py b/zuul/model.py
index 00de007..53b98a0 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -353,8 +353,9 @@
# This makes a Project instance a unique identifier for a given
# project from a given source.
- def __init__(self, name, foreign=False):
+ def __init__(self, name, connection_name, foreign=False):
self.name = name
+ self.connection_name = connection_name
# foreign projects are those referenced in dependencies
# of layout projects, this should matter
# when deciding whether to enqueue their changes
@@ -530,11 +531,14 @@
tags=set(),
mutex=None,
attempts=3,
+ source_project=None,
+ source_branch=None,
+ source_configrepo=None,
+ playbook=None,
)
def __init__(self, name):
self.name = name
- self.project_source = None
for k, v in self.attributes.items():
setattr(self, k, v)
@@ -1779,7 +1783,8 @@
r.nodesets = copy.deepcopy(self.nodesets)
return r
- def extend(self, conf, source_project=None, source_branch=None):
+ def extend(self, conf, source_project=None, source_branch=None,
+ source_configrepo=None):
if isinstance(conf, UnparsedTenantConfig):
self.pipelines.extend(conf.pipelines)
self.jobs.extend(conf.jobs)
@@ -1809,6 +1814,8 @@
value['_source_project'] = source_project
if source_branch is not None:
value['_source_branch'] = source_branch
+ if source_configrepo is not None:
+ value['_source_configrepo'] = source_configrepo
self.jobs.append(value)
elif key == 'project-template':
self.project_templates.append(value)