Merge "Allow per-repo selection of configuration classes to load" into feature/zuulv3
diff --git a/tests/fixtures/config/tenant-parser/git/common-config/zuul.yaml b/tests/fixtures/config/tenant-parser/git/common-config/zuul.yaml
new file mode 100644
index 0000000..9e52187
--- /dev/null
+++ b/tests/fixtures/config/tenant-parser/git/common-config/zuul.yaml
@@ -0,0 +1,27 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- job:
+ name: common-config-job
+
+- project:
+ name: org/project1
+ check:
+ jobs:
+ - common-config-job
+
+- project:
+ name: org/project2
+ check:
+ jobs:
+ - common-config-job
diff --git a/tests/fixtures/config/tenant-parser/git/org_project1/.zuul.yaml b/tests/fixtures/config/tenant-parser/git/org_project1/.zuul.yaml
new file mode 100644
index 0000000..cd5dba7
--- /dev/null
+++ b/tests/fixtures/config/tenant-parser/git/org_project1/.zuul.yaml
@@ -0,0 +1,8 @@
+- job:
+ name: project1-job
+
+- project:
+ name: org/project1
+ check:
+ jobs:
+ - project1-job
diff --git a/tests/fixtures/config/tenant-parser/git/org_project2/.zuul.yaml b/tests/fixtures/config/tenant-parser/git/org_project2/.zuul.yaml
new file mode 100644
index 0000000..4292c89
--- /dev/null
+++ b/tests/fixtures/config/tenant-parser/git/org_project2/.zuul.yaml
@@ -0,0 +1,8 @@
+- job:
+ name: project2-job
+
+- project:
+ name: org/project2
+ check:
+ jobs:
+ - project2-job
diff --git a/tests/fixtures/config/tenant-parser/groups.yaml b/tests/fixtures/config/tenant-parser/groups.yaml
new file mode 100644
index 0000000..f2a0d99
--- /dev/null
+++ b/tests/fixtures/config/tenant-parser/groups.yaml
@@ -0,0 +1,11 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - exclude: project
+ projects:
+ - org/project1
+ - org/project2
diff --git a/tests/fixtures/config/tenant-parser/groups2.yaml b/tests/fixtures/config/tenant-parser/groups2.yaml
new file mode 100644
index 0000000..dc8d339
--- /dev/null
+++ b/tests/fixtures/config/tenant-parser/groups2.yaml
@@ -0,0 +1,12 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - exclude: project
+ projects:
+ - org/project1
+ - org/project2:
+ exclude: job
diff --git a/tests/fixtures/config/tenant-parser/groups3.yaml b/tests/fixtures/config/tenant-parser/groups3.yaml
new file mode 100644
index 0000000..196f03a
--- /dev/null
+++ b/tests/fixtures/config/tenant-parser/groups3.yaml
@@ -0,0 +1,14 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - include: job
+ projects:
+ - org/project1
+ - org/project2:
+ include:
+ - project
+ - job
diff --git a/tests/fixtures/config/tenant-parser/override.yaml b/tests/fixtures/config/tenant-parser/override.yaml
new file mode 100644
index 0000000..87674f1
--- /dev/null
+++ b/tests/fixtures/config/tenant-parser/override.yaml
@@ -0,0 +1,11 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - org/project1:
+ exclude: project
+ - org/project2:
+ include: job
diff --git a/tests/fixtures/config/tenant-parser/simple.yaml b/tests/fixtures/config/tenant-parser/simple.yaml
new file mode 100644
index 0000000..950b117
--- /dev/null
+++ b/tests/fixtures/config/tenant-parser/simple.yaml
@@ -0,0 +1,9 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - org/project1
+ - org/project2
diff --git a/tests/unit/test_configloader.py b/tests/unit/test_configloader.py
new file mode 100644
index 0000000..faa2f61
--- /dev/null
+++ b/tests/unit/test_configloader.py
@@ -0,0 +1,188 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+from tests.base import ZuulTestCase
+
+
+class TenantParserTestCase(ZuulTestCase):
+ create_project_keys = True
+
+ CONFIG_SET = set(['pipeline', 'job', 'semaphore', 'project',
+ 'project-template', 'nodeset', 'secret'])
+ UNTRUSTED_SET = CONFIG_SET - set(['pipeline'])
+
+ def setupAllProjectKeys(self):
+ for project in ['common-config', 'org/project1', 'org/project2']:
+ self.setupProjectKeys('gerrit', project)
+
+
+class TestTenantSimple(TenantParserTestCase):
+ tenant_config_file = 'config/tenant-parser/simple.yaml'
+
+ def test_tenant_simple(self):
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ self.assertEqual(['common-config'],
+ [x.name for x in tenant.config_projects])
+ self.assertEqual(['org/project1', 'org/project2'],
+ [x.name for x in tenant.untrusted_projects])
+ self.assertEqual(self.CONFIG_SET,
+ tenant.config_projects[0].load_classes)
+ self.assertEqual(self.UNTRUSTED_SET,
+ tenant.untrusted_projects[0].load_classes)
+ self.assertEqual(self.UNTRUSTED_SET,
+ tenant.untrusted_projects[1].load_classes)
+ self.assertTrue('common-config-job' in tenant.layout.jobs)
+ self.assertTrue('project1-job' in tenant.layout.jobs)
+ self.assertTrue('project2-job' in tenant.layout.jobs)
+ project1_config = tenant.layout.project_configs.get(
+ 'review.example.com/org/project1')
+ self.assertTrue('common-config-job' in
+ project1_config.pipelines['check'].job_list.jobs)
+ self.assertTrue('project1-job' in
+ project1_config.pipelines['check'].job_list.jobs)
+ project2_config = tenant.layout.project_configs.get(
+ 'review.example.com/org/project2')
+ self.assertTrue('common-config-job' in
+ project2_config.pipelines['check'].job_list.jobs)
+ self.assertTrue('project2-job' in
+ project2_config.pipelines['check'].job_list.jobs)
+
+
+class TestTenantOverride(TenantParserTestCase):
+ tenant_config_file = 'config/tenant-parser/override.yaml'
+
+ def test_tenant_override(self):
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ self.assertEqual(['common-config'],
+ [x.name for x in tenant.config_projects])
+ self.assertEqual(['org/project1', 'org/project2'],
+ [x.name for x in tenant.untrusted_projects])
+ self.assertEqual(self.CONFIG_SET,
+ tenant.config_projects[0].load_classes)
+ self.assertEqual(self.UNTRUSTED_SET - set(['project']),
+ tenant.untrusted_projects[0].load_classes)
+ self.assertEqual(set(['job']),
+ tenant.untrusted_projects[1].load_classes)
+ self.assertTrue('common-config-job' in tenant.layout.jobs)
+ self.assertTrue('project1-job' in tenant.layout.jobs)
+ self.assertTrue('project2-job' in tenant.layout.jobs)
+ project1_config = tenant.layout.project_configs.get(
+ 'review.example.com/org/project1')
+ self.assertTrue('common-config-job' in
+ project1_config.pipelines['check'].job_list.jobs)
+ self.assertFalse('project1-job' in
+ project1_config.pipelines['check'].job_list.jobs)
+ project2_config = tenant.layout.project_configs.get(
+ 'review.example.com/org/project2')
+ self.assertTrue('common-config-job' in
+ project2_config.pipelines['check'].job_list.jobs)
+ self.assertFalse('project2-job' in
+ project2_config.pipelines['check'].job_list.jobs)
+
+
+class TestTenantGroups(TenantParserTestCase):
+ tenant_config_file = 'config/tenant-parser/groups.yaml'
+
+ def test_tenant_groups(self):
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ self.assertEqual(['common-config'],
+ [x.name for x in tenant.config_projects])
+ self.assertEqual(['org/project1', 'org/project2'],
+ [x.name for x in tenant.untrusted_projects])
+ self.assertEqual(self.CONFIG_SET,
+ tenant.config_projects[0].load_classes)
+ self.assertEqual(self.UNTRUSTED_SET - set(['project']),
+ tenant.untrusted_projects[0].load_classes)
+ self.assertEqual(self.UNTRUSTED_SET - set(['project']),
+ tenant.untrusted_projects[1].load_classes)
+ self.assertTrue('common-config-job' in tenant.layout.jobs)
+ self.assertTrue('project1-job' in tenant.layout.jobs)
+ self.assertTrue('project2-job' in tenant.layout.jobs)
+ project1_config = tenant.layout.project_configs.get(
+ 'review.example.com/org/project1')
+ self.assertTrue('common-config-job' in
+ project1_config.pipelines['check'].job_list.jobs)
+ self.assertFalse('project1-job' in
+ project1_config.pipelines['check'].job_list.jobs)
+ project2_config = tenant.layout.project_configs.get(
+ 'review.example.com/org/project2')
+ self.assertTrue('common-config-job' in
+ project2_config.pipelines['check'].job_list.jobs)
+ self.assertFalse('project2-job' in
+ project2_config.pipelines['check'].job_list.jobs)
+
+
+class TestTenantGroups2(TenantParserTestCase):
+ tenant_config_file = 'config/tenant-parser/groups2.yaml'
+
+ def test_tenant_groups2(self):
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ self.assertEqual(['common-config'],
+ [x.name for x in tenant.config_projects])
+ self.assertEqual(['org/project1', 'org/project2'],
+ [x.name for x in tenant.untrusted_projects])
+ self.assertEqual(self.CONFIG_SET,
+ tenant.config_projects[0].load_classes)
+ self.assertEqual(self.UNTRUSTED_SET - set(['project']),
+ tenant.untrusted_projects[0].load_classes)
+ self.assertEqual(self.UNTRUSTED_SET - set(['project', 'job']),
+ tenant.untrusted_projects[1].load_classes)
+ self.assertTrue('common-config-job' in tenant.layout.jobs)
+ self.assertTrue('project1-job' in tenant.layout.jobs)
+ self.assertFalse('project2-job' in tenant.layout.jobs)
+ project1_config = tenant.layout.project_configs.get(
+ 'review.example.com/org/project1')
+ self.assertTrue('common-config-job' in
+ project1_config.pipelines['check'].job_list.jobs)
+ self.assertFalse('project1-job' in
+ project1_config.pipelines['check'].job_list.jobs)
+ project2_config = tenant.layout.project_configs.get(
+ 'review.example.com/org/project2')
+ self.assertTrue('common-config-job' in
+ project2_config.pipelines['check'].job_list.jobs)
+ self.assertFalse('project2-job' in
+ project2_config.pipelines['check'].job_list.jobs)
+
+
+class TestTenantGroups3(TenantParserTestCase):
+ tenant_config_file = 'config/tenant-parser/groups3.yaml'
+
+ def test_tenant_groups3(self):
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ self.assertEqual(['common-config'],
+ [x.name for x in tenant.config_projects])
+ self.assertEqual(['org/project1', 'org/project2'],
+ [x.name for x in tenant.untrusted_projects])
+ self.assertEqual(self.CONFIG_SET,
+ tenant.config_projects[0].load_classes)
+ self.assertEqual(set(['job']),
+ tenant.untrusted_projects[0].load_classes)
+ self.assertEqual(set(['project', 'job']),
+ tenant.untrusted_projects[1].load_classes)
+ self.assertTrue('common-config-job' in tenant.layout.jobs)
+ self.assertTrue('project1-job' in tenant.layout.jobs)
+ self.assertTrue('project2-job' in tenant.layout.jobs)
+ project1_config = tenant.layout.project_configs.get(
+ 'review.example.com/org/project1')
+ self.assertTrue('common-config-job' in
+ project1_config.pipelines['check'].job_list.jobs)
+ self.assertFalse('project1-job' in
+ project1_config.pipelines['check'].job_list.jobs)
+ project2_config = tenant.layout.project_configs.get(
+ 'review.example.com/org/project2')
+ self.assertTrue('common-config-job' in
+ project2_config.pipelines['check'].job_list.jobs)
+ self.assertTrue('project2-job' in
+ project2_config.pipelines['check'].job_list.jobs)
diff --git a/zuul/configloader.py b/zuul/configloader.py
index f78e8a4..688bd2b 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -859,8 +859,28 @@
class TenantParser(object):
log = logging.getLogger("zuul.TenantParser")
- tenant_source = vs.Schema({'config-projects': [str],
- 'untrusted-projects': [str]})
+ classes = vs.Any('pipeline', 'job', 'semaphore', 'project',
+ 'project-template', 'nodeset', 'secret')
+
+ project_dict = {str: {
+ 'include': to_list(classes),
+ 'exclude': to_list(classes),
+ }}
+
+ project = vs.Any(str, project_dict)
+
+ group = {
+ 'include': to_list(classes),
+ 'exclude': to_list(classes),
+ vs.Required('projects'): to_list(project),
+ }
+
+ project_or_group = vs.Any(project, group)
+
+ tenant_source = vs.Schema({
+ 'config-projects': to_list(project_or_group),
+ 'untrusted-projects': to_list(project_or_group),
+ })
@staticmethod
def validateTenantSources(connections):
@@ -960,24 +980,84 @@
encryption.deserialize_rsa_keypair(f.read())
@staticmethod
+ def _getProject(source, conf, current_include):
+ if isinstance(conf, six.string_types):
+ # Return a project object whether conf is a dict or a str
+ project = source.getProject(conf)
+ project_include = current_include
+ else:
+ project_name = list(conf.keys())[0]
+ project = source.getProject(project_name)
+
+ project_include = frozenset(
+ as_list(conf[project_name].get('include', [])))
+ if not project_include:
+ project_include = current_include
+ project_exclude = frozenset(
+ as_list(conf[project_name].get('exclude', [])))
+ if project_exclude:
+ project_include = frozenset(project_include - project_exclude)
+
+ project.load_classes = frozenset(project_include)
+ return project
+
+ @staticmethod
+ def _getProjects(source, conf, current_include):
+ # Return a project object whether conf is a dict or a str
+ projects = []
+ if isinstance(conf, six.string_types):
+ # A simple project name string
+ projects.append(TenantParser._getProject(
+ source, conf, current_include))
+ elif len(conf.keys()) > 1 and 'projects' in conf:
+ # This is a project group
+ if 'include' in conf:
+ current_include = set(as_list(conf['include']))
+ else:
+ current_include = current_include.copy()
+ if 'exclude' in conf:
+ exclude = set(as_list(conf['exclude']))
+ current_include = current_include - exclude
+ for project in conf['projects']:
+ sub_projects = TenantParser._getProjects(source, project,
+ current_include)
+ projects.extend(sub_projects)
+ elif len(conf.keys()) == 1:
+ # A project with overrides
+ projects.append(TenantParser._getProject(
+ source, conf, current_include))
+ else:
+ raise Exception("Unable to parse project %s", conf)
+ return projects
+
+ @staticmethod
def _loadTenantProjects(project_key_dir, connections, conf_tenant):
config_projects = []
untrusted_projects = []
+ default_include = frozenset(['pipeline', 'job', 'semaphore', 'project',
+ 'secret', 'project-template', 'nodeset'])
+
for source_name, conf_source in conf_tenant.get('source', {}).items():
source = connections.getSource(source_name)
+ current_include = default_include
for conf_repo in conf_source.get('config-projects', []):
- project = source.getProject(conf_repo)
- TenantParser._loadProjectKeys(
- project_key_dir, source_name, project)
- config_projects.append(project)
+ projects = TenantParser._getProjects(source, conf_repo,
+ current_include)
+ for project in projects:
+ TenantParser._loadProjectKeys(
+ project_key_dir, source_name, project)
+ config_projects.append(project)
+ current_include = frozenset(default_include - set(['pipeline']))
for conf_repo in conf_source.get('untrusted-projects', []):
- project = source.getProject(conf_repo)
- TenantParser._loadProjectKeys(
- project_key_dir, source_name, project)
- untrusted_projects.append(project)
+ projects = TenantParser._getProjects(source, conf_repo,
+ current_include)
+ for project in projects:
+ TenantParser._loadProjectKeys(
+ project_key_dir, source_name, project)
+ untrusted_projects.append(project)
return config_projects, untrusted_projects
@@ -1090,34 +1170,78 @@
return config
@staticmethod
- def _parseLayout(base, tenant, data, scheduler, connections):
- layout = model.Layout()
-
- for config_pipeline in data.pipelines:
- layout.addPipeline(PipelineParser.fromYaml(layout, connections,
- scheduler,
- config_pipeline))
+ def _parseLayoutItems(layout, tenant, data, scheduler, connections,
+ skip_pipelines=False, skip_semaphores=False):
+ if not skip_pipelines:
+ for config_pipeline in data.pipelines:
+ classes = config_pipeline['_source_context'].\
+ project.load_classes
+ if 'pipeline' not in classes:
+ continue
+ layout.addPipeline(PipelineParser.fromYaml(
+ layout, connections,
+ scheduler, config_pipeline))
for config_nodeset in data.nodesets:
+ classes = config_nodeset['_source_context'].project.load_classes
+ if 'nodeset' not in classes:
+ continue
layout.addNodeSet(NodeSetParser.fromYaml(layout, config_nodeset))
for config_secret in data.secrets:
+ classes = config_secret['_source_context'].project.load_classes
+ if 'secret' not in classes:
+ continue
layout.addSecret(SecretParser.fromYaml(layout, config_secret))
for config_job in data.jobs:
+ classes = config_job['_source_context'].project.load_classes
+ if 'job' not in classes:
+ continue
with configuration_exceptions('job', config_job):
- layout.addJob(JobParser.fromYaml(tenant, layout, config_job))
+ job = JobParser.fromYaml(tenant, layout, config_job)
+ layout.addJob(job)
- for config_semaphore in data.semaphores:
- layout.addSemaphore(SemaphoreParser.fromYaml(config_semaphore))
+ if not skip_semaphores:
+ for config_semaphore in data.semaphores:
+ classes = config_semaphore['_source_context'].\
+ project.load_classes
+ if 'semaphore' not in classes:
+ continue
+ layout.addSemaphore(SemaphoreParser.fromYaml(config_semaphore))
for config_template in data.project_templates:
+ classes = config_template['_source_context'].project.load_classes
+ if 'project-template' not in classes:
+ continue
layout.addProjectTemplate(ProjectTemplateParser.fromYaml(
tenant, layout, config_template))
- for config_project in data.projects.values():
+ for config_projects in data.projects.values():
+ # Unlike other config classes, we expect multiple project
+ # stanzas with the same name, so that a config repo can
+ # define a project-pipeline and the project itself can
+ # augment it. To that end, config_project is a list of
+ # each of the project stanzas. Each one may be (should
+ # be!) from a different repo, so filter them according to
+ # the include/exclude rules before parsing them.
+ filtered_projects = [
+ p for p in config_projects if
+ 'project' in p['_source_context'].project.load_classes
+ ]
+
+ if not filtered_projects:
+ continue
+
layout.addProjectConfig(ProjectParser.fromYaml(
- tenant, layout, config_project))
+ tenant, layout, filtered_projects))
+
+ @staticmethod
+ def _parseLayout(base, tenant, data, scheduler, connections):
+ layout = model.Layout()
+
+ TenantParser._parseLayoutItems(layout, tenant, data,
+ scheduler, connections)
layout.tenant = tenant
@@ -1228,21 +1352,8 @@
# configuration changes.
layout.semaphores = tenant.layout.semaphores
- for config_nodeset in config.nodesets:
- layout.addNodeSet(NodeSetParser.fromYaml(layout, config_nodeset))
+ TenantParser._parseLayoutItems(layout, tenant, config, None, None,
+ skip_pipelines=True,
+ skip_semaphores=True)
- for config_secret in config.secrets:
- layout.addSecret(SecretParser.fromYaml(layout, config_secret))
-
- for config_job in config.jobs:
- with configuration_exceptions('job', config_job):
- layout.addJob(JobParser.fromYaml(tenant, layout, config_job))
-
- for config_template in config.project_templates:
- layout.addProjectTemplate(ProjectTemplateParser.fromYaml(
- tenant, layout, config_template))
-
- for config_project in config.projects.values():
- layout.addProjectConfig(ProjectParser.fromYaml(
- tenant, layout, config_project))
return layout
diff --git a/zuul/model.py b/zuul/model.py
index e504dca..25f69d7 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -336,6 +336,9 @@
self.foreign = foreign
self.unparsed_config = None
self.unparsed_branch_config = {} # branch -> UnparsedTenantConfig
+ # Configuration object classes to include or exclude when
+ # loading zuul config files.
+ self.load_classes = frozenset()
def __str__(self):
return self.name