Support project templates
Change-Id: I37378893e1761fe33c19d936076b9ec3d4c81627
diff --git a/tests/fixtures/config/multi-tenant/tenant-one.yaml b/tests/fixtures/config/multi-tenant/tenant-one.yaml
index c9096ef..874e932 100644
--- a/tests/fixtures/config/multi-tenant/tenant-one.yaml
+++ b/tests/fixtures/config/multi-tenant/tenant-one.yaml
@@ -28,6 +28,8 @@
projects:
- name: org/project1
check:
- - project1-test1
+ jobs:
+ - project1-test1
tenant-one-gate:
- - project1-test1
+ jobs:
+ - project1-test1
diff --git a/tests/fixtures/config/multi-tenant/tenant-two.yaml b/tests/fixtures/config/multi-tenant/tenant-two.yaml
index 6cb2d9a..254d9cd 100644
--- a/tests/fixtures/config/multi-tenant/tenant-two.yaml
+++ b/tests/fixtures/config/multi-tenant/tenant-two.yaml
@@ -28,6 +28,8 @@
projects:
- name: org/project2
check:
- - project2-test1
+ jobs:
+ - project2-test1
tenant-two-gate:
- - project2-test1
+ jobs:
+ - project2-test1
diff --git a/tests/fixtures/config/project-template/common.yaml b/tests/fixtures/config/project-template/common.yaml
new file mode 100644
index 0000000..9e76bde
--- /dev/null
+++ b/tests/fixtures/config/project-template/common.yaml
@@ -0,0 +1,56 @@
+pipelines:
+ - name: check
+ manager: independent
+ source:
+ gerrit
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+ - name: gate
+ manager: dependent
+ success-message: Build succeeded (gate).
+ source:
+ gerrit
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
+
+jobs:
+ - name:
+ project-test1
+ - name:
+ project-test2
+
+project-templates:
+ - name: test-template
+ gate:
+ jobs:
+ - project-test2
+
+projects:
+ - name: org/project
+ templates:
+ - test-template
+ gate:
+ jobs:
+ - project-test1
diff --git a/tests/fixtures/config/project-template/main.yaml b/tests/fixtures/config/project-template/main.yaml
new file mode 100644
index 0000000..c89fdfa
--- /dev/null
+++ b/tests/fixtures/config/project-template/main.yaml
@@ -0,0 +1,4 @@
+tenants:
+ - name: tenant-one
+ include:
+ - common.yaml
diff --git a/tests/fixtures/config/project-template/tenant-one.yaml b/tests/fixtures/config/project-template/tenant-one.yaml
new file mode 100644
index 0000000..c9096ef
--- /dev/null
+++ b/tests/fixtures/config/project-template/tenant-one.yaml
@@ -0,0 +1,33 @@
+pipelines:
+ - name: tenant-one-gate
+ manager: dependent
+ success-message: Build succeeded (tenant-one-gate).
+ source:
+ gerrit
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
+
+jobs:
+ - name:
+ project1-test1
+
+projects:
+ - name: org/project1
+ check:
+ - project1-test1
+ tenant-one-gate:
+ - project1-test1
diff --git a/tests/fixtures/config/project-template/zuul.conf b/tests/fixtures/config/project-template/zuul.conf
new file mode 100644
index 0000000..67f3d2c
--- /dev/null
+++ b/tests/fixtures/config/project-template/zuul.conf
@@ -0,0 +1,36 @@
+[gearman]
+server=127.0.0.1
+
+[zuul]
+tenant_config=config/project-template/main.yaml
+url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}
+job_name_in_report=true
+
+[merger]
+git_dir=/tmp/zuul-test/git
+git_user_email=zuul@example.com
+git_user_name=zuul
+zuul_url=http://zuul.example.com/p
+
+[swift]
+authurl=https://identity.api.example.org/v2.0/
+user=username
+key=password
+tenant_name=" "
+
+default_container=logs
+region_name=EXP
+logserver_prefix=http://logs.example.org/server.app/
+
+[connection gerrit]
+driver=gerrit
+server=review.example.com
+user=jenkins
+sshkey=none
+
+[connection smtp]
+driver=smtp
+server=localhost
+port=25
+default_from=zuul@example.com
+default_to=you@example.com
diff --git a/tests/test_v3.py b/tests/test_v3.py
index 73efcc9..b746eae 100644
--- a/tests/test_v3.py
+++ b/tests/test_v3.py
@@ -78,7 +78,8 @@
projects:
- name: org/project
tenant-one-gate:
- - project-test1
+ jobs:
+ - project-test1
""")
self.addCommitToRepo('org/project', 'add zuul conf',
@@ -96,3 +97,22 @@
"A should report start and success")
self.assertIn('tenant-one-gate', A.messages[1],
"A should transit tenant-one gate")
+
+
+class TestProjectTemplate(ZuulTestCase):
+ config_file = 'config/project-template/zuul.conf'
+
+ def test(self):
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ A.addApproval('CRVW', 2)
+ self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+ self.waitUntilSettled()
+ self.assertEqual(self.getJobFromHistory('project-test1').result,
+ 'SUCCESS')
+ self.assertEqual(self.getJobFromHistory('project-test2').result,
+ 'SUCCESS')
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2,
+ "A should report start and success")
+ self.assertIn('gate', A.messages[1],
+ "A should transit gate")
diff --git a/zuul/configloader.py b/zuul/configloader.py
index d22106d..d09ba85 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -150,6 +150,107 @@
return job
+class ProjectTemplateParser(object):
+ log = logging.getLogger("zuul.ProjectTemplateParser")
+
+ @staticmethod
+ def getSchema(layout):
+ project_template = {vs.Required('name'): str}
+ for p in layout.pipelines.values():
+ project_template[p.name] = {'queue': str,
+ 'jobs': [vs.Any(str, dict)]}
+ return vs.Schema(project_template)
+
+ @staticmethod
+ def fromYaml(layout, conf):
+ ProjectTemplateParser.getSchema(layout)(conf)
+ project_template = model.ProjectConfig(conf['name'])
+ for pipeline in layout.pipelines.values():
+ conf_pipeline = conf.get(pipeline.name)
+ if not conf_pipeline:
+ continue
+ project_pipeline = model.ProjectPipelineConfig()
+ project_template.pipelines[pipeline.name] = project_pipeline
+ project_pipeline.queue_name = conf.get('queue')
+ project_pipeline.job_tree = ProjectTemplateParser._parseJobTree(
+ layout, conf_pipeline.get('jobs'))
+ return project_template
+
+ @staticmethod
+ def _parseJobTree(layout, conf, tree=None):
+ if not tree:
+ tree = model.JobTree(None)
+ for conf_job in conf:
+ if isinstance(conf_job, basestring):
+ tree.addJob(layout.getJob(conf_job))
+ elif isinstance(conf_job, dict):
+ # A dictionary in a job tree may override params, or
+ # be the root of a sub job tree, or both.
+ jobname, attrs = dict.items()[0]
+ jobs = attrs.pop('jobs')
+ if attrs:
+ # We are overriding params, so make a new job def
+ attrs['name'] = jobname
+ subtree = tree.addJob(JobParser.fromYaml(layout, attrs))
+ else:
+ # Not overriding, so get existing job
+ subtree = tree.addJob(layout.getJob(jobname))
+
+ if jobs:
+ # This is the root of a sub tree
+ ProjectTemplateParser._parseJobTree(layout, jobs, subtree)
+ else:
+ raise Exception("Job must be a string or dictionary")
+ return tree
+
+
+class ProjectParser(object):
+ log = logging.getLogger("zuul.ProjectParser")
+
+ @staticmethod
+ def getSchema(layout):
+ project = {vs.Required('name'): str,
+ 'templates': [str]}
+ for p in layout.pipelines.values():
+ project[p.name] = {'queue': str,
+ 'jobs': [vs.Any(str, dict)]}
+ return vs.Schema(project)
+
+ @staticmethod
+ def fromYaml(layout, conf):
+ ProjectParser.getSchema(layout)(conf)
+ conf_templates = conf.pop('templates', [])
+ # The way we construct a project definition is by parsing the
+ # definition as a template, then applying all of the
+ # templates, including the newly parsed one, in order.
+ project_template = ProjectTemplateParser.fromYaml(layout, conf)
+ configs = [layout.project_templates[name] for name in conf_templates]
+ configs.append(project_template)
+ project = model.ProjectConfig(conf['name'])
+ for pipeline in layout.pipelines.values():
+ project_pipeline = model.ProjectPipelineConfig()
+ project_pipeline.job_tree = model.JobTree(None)
+ queue_name = None
+ # For every template, iterate over the job tree and replace or
+ # create the jobs in the final definition as needed.
+ pipeline_defined = False
+ for template in configs:
+ ProjectParser.log.debug("Applying template %s to pipeline %s" %
+ (template.name, pipeline.name))
+ if pipeline.name in template.pipelines:
+ pipeline_defined = True
+ template_pipeline = template.pipelines[pipeline.name]
+ project_pipeline.job_tree.inheritFrom(
+ template_pipeline.job_tree)
+ if template_pipeline.queue_name:
+ queue_name = template_pipeline.queue_name
+ if queue_name:
+ project_pipeline.queue_name = queue_name
+ if pipeline_defined:
+ project.pipelines[pipeline.name] = project_pipeline
+ return project
+
+
class AbideValidator(object):
tenant_source = vs.Schema({'repos': [str]})
@@ -229,7 +330,6 @@
def _parseLayout(self, base, data, scheduler, connections):
layout = model.Layout()
- project_templates = {}
# TODOv3(jeblair): add validation
# validator = layoutvalidator.LayoutValidator()
@@ -329,63 +429,16 @@
manager.event_filters += trigger.getEventFilters(
conf_pipeline['trigger'][trigger_name])
- for project_template in data.get('project-templates', []):
- # Make sure the template only contains valid pipelines
- tpl = dict(
- (pipe_name, project_template.get(pipe_name))
- for pipe_name in layout.pipelines.keys()
- if pipe_name in project_template
- )
- project_templates[project_template.get('name')] = tpl
-
for config_job in data.get('jobs', []):
layout.addJob(JobParser.fromYaml(layout, config_job))
- def add_jobs(job_tree, config_jobs):
- for job in config_jobs:
- if isinstance(job, list):
- for x in job:
- add_jobs(job_tree, x)
- if isinstance(job, dict):
- for parent, children in job.items():
- parent_tree = job_tree.addJob(layout.getJob(parent))
- add_jobs(parent_tree, children)
- if isinstance(job, str):
- job_tree.addJob(layout.getJob(job))
+ for config_template in data.get('project-templates', []):
+ layout.addProjectTemplate(ProjectTemplateParser.fromYaml(
+ layout, config_template))
for config_project in data.get('projects', []):
- shortname = config_project['name'].split('/')[-1]
-
- # This is reversed due to the prepend operation below, so
- # the ultimate order is templates (in order) followed by
- # statically defined jobs.
- for requested_template in reversed(
- config_project.get('template', [])):
- # Fetch the template from 'project-templates'
- tpl = project_templates.get(
- requested_template.get('name'))
- # Expand it with the project context
- requested_template['name'] = shortname
- expanded = deep_format(tpl, requested_template)
- # Finally merge the expansion with whatever has been
- # already defined for this project. Prepend our new
- # jobs to existing ones (which may have been
- # statically defined or defined by other templates).
- for pipeline in layout.pipelines.values():
- if pipeline.name in expanded:
- config_project.update(
- {pipeline.name: expanded[pipeline.name] +
- config_project.get(pipeline.name, [])})
-
- mode = config_project.get('merge-mode', 'merge-resolve')
- for pipeline in layout.pipelines.values():
- if pipeline.name in config_project:
- project = pipeline.source.getProject(
- config_project['name'])
- project.merge_mode = model.MERGER_MAP[mode]
- job_tree = pipeline.addProject(project)
- config_jobs = config_project[pipeline.name]
- add_jobs(job_tree, config_jobs)
+ layout.addProjectConfig(ProjectParser.fromYaml(
+ layout, config_project))
for pipeline in layout.pipelines.values():
pipeline.manager._postConfig(layout)
@@ -419,31 +472,3 @@
# TODOv3(jeblair): this should implement some rules to protect
# aspects of the config that should not be changed in-repo
return yaml.load(data)
-
- def _parseSkipIf(self, config_job):
- cm = change_matcher
- skip_matchers = []
-
- for config_skip in config_job.get('skip-if', []):
- nested_matchers = []
-
- project_regex = config_skip.get('project')
- if project_regex:
- nested_matchers.append(cm.ProjectMatcher(project_regex))
-
- branch_regex = config_skip.get('branch')
- if branch_regex:
- nested_matchers.append(cm.BranchMatcher(branch_regex))
-
- file_regexes = to_list(config_skip.get('all-files-match-any'))
- if file_regexes:
- file_matchers = [cm.FileMatcher(x) for x in file_regexes]
- all_files_matcher = cm.MatchAllFiles(file_matchers)
- nested_matchers.append(all_files_matcher)
-
- # All patterns need to match a given skip-if predicate
- skip_matchers.append(cm.MatchAll(nested_matchers))
-
- if skip_matchers:
- # Any skip-if predicate can be matched to trigger a skip
- return cm.MatchAny(skip_matchers)
diff --git a/zuul/model.py b/zuul/model.py
index aa21f85..a0bacfd 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -550,6 +550,17 @@
return ret
return None
+ def inheritFrom(self, other):
+ if other.job:
+ self.job = Job(other.job.name)
+ self.job.inheritFrom(other.job)
+ for other_tree in other.job_trees:
+ this_tree = self.getJobTreeForJob(other_tree.job)
+ if not this_tree:
+ this_tree = JobTree(None)
+ self.job_trees.append(this_tree)
+ this_tree.inheritFrom(other_tree)
+
class Build(object):
def __init__(self, job, uuid):
@@ -1356,9 +1367,26 @@
return True
+class ProjectPipelineConfig(object):
+ # Represents a project cofiguration in the context of a pipeline
+ def __init__(self):
+ self.job_tree = None
+ self.queue_name = None
+ # TODOv3(jeblair): add merge mode
+
+
+class ProjectConfig(object):
+ # Represents a project cofiguration
+ def __init__(self, name):
+ self.name = name
+ self.pipelines = {}
+
+
class Layout(object):
def __init__(self):
self.projects = {}
+ self.project_configs = {}
+ self.project_templates = {}
self.pipelines = OrderedDict()
# This is a dictionary of name -> [jobs]. The first element
# of the list is the first job added with that name. It is
@@ -1371,7 +1399,7 @@
def getJob(self, name):
if name in self.jobs:
return self.jobs[name][0]
- return None
+ raise Exception("Job %s not defined" % (name,))
def getJobs(self, name):
return self.jobs.get(name, [])
@@ -1385,6 +1413,18 @@
def addPipeline(self, pipeline):
self.pipelines[pipeline.name] = pipeline
+ def addProjectTemplate(self, project_template):
+ self.project_templates[project_template.name] = project_template
+
+ def addProjectConfig(self, project_config):
+ self.project_configs[project_config.name] = project_config
+ # TODOv3(jeblair): tidy up the relationship between pipelines
+ # and projects and projectconfigs
+ for pipeline_name, pipeline_config in project_config.pipelines.items():
+ pipeline = self.pipelines[pipeline_name]
+ project = pipeline.source.getProject(project_config.name)
+ pipeline.job_trees[project] = pipeline_config.job_tree
+
class Tenant(object):
def __init__(self, name):