Merge "Add TenantProjectConfig object" into feature/zuulv3
diff --git a/tests/unit/test_configloader.py b/tests/unit/test_configloader.py
index f0e606a..573ccbf 100644
--- a/tests/unit/test_configloader.py
+++ b/tests/unit/test_configloader.py
@@ -38,12 +38,16 @@
[x.name for x in tenant.config_projects])
self.assertEqual(['org/project1', 'org/project2'],
[x.name for x in tenant.untrusted_projects])
- self.assertEqual(self.CONFIG_SET,
- tenant.config_projects[0].load_classes)
- self.assertEqual(self.UNTRUSTED_SET,
- tenant.untrusted_projects[0].load_classes)
- self.assertEqual(self.UNTRUSTED_SET,
- tenant.untrusted_projects[1].load_classes)
+
+ project = tenant.config_projects[0]
+ tpc = tenant.project_configs[project.canonical_name]
+ self.assertEqual(self.CONFIG_SET, tpc.load_classes)
+ project = tenant.untrusted_projects[0]
+ tpc = tenant.project_configs[project.canonical_name]
+ self.assertEqual(self.UNTRUSTED_SET, tpc.load_classes)
+ project = tenant.untrusted_projects[1]
+ tpc = tenant.project_configs[project.canonical_name]
+ self.assertEqual(self.UNTRUSTED_SET, tpc.load_classes)
self.assertTrue('common-config-job' in tenant.layout.jobs)
self.assertTrue('project1-job' in tenant.layout.jobs)
self.assertTrue('project2-job' in tenant.layout.jobs)
@@ -70,12 +74,16 @@
[x.name for x in tenant.config_projects])
self.assertEqual(['org/project1', 'org/project2'],
[x.name for x in tenant.untrusted_projects])
- self.assertEqual(self.CONFIG_SET,
- tenant.config_projects[0].load_classes)
+ project = tenant.config_projects[0]
+ tpc = tenant.project_configs[project.canonical_name]
+ self.assertEqual(self.CONFIG_SET, tpc.load_classes)
+ project = tenant.untrusted_projects[0]
+ tpc = tenant.project_configs[project.canonical_name]
self.assertEqual(self.UNTRUSTED_SET - set(['project']),
- tenant.untrusted_projects[0].load_classes)
- self.assertEqual(set(['job']),
- tenant.untrusted_projects[1].load_classes)
+ tpc.load_classes)
+ project = tenant.untrusted_projects[1]
+ tpc = tenant.project_configs[project.canonical_name]
+ self.assertEqual(set(['job']), tpc.load_classes)
self.assertTrue('common-config-job' in tenant.layout.jobs)
self.assertTrue('project1-job' in tenant.layout.jobs)
self.assertTrue('project2-job' in tenant.layout.jobs)
@@ -102,12 +110,17 @@
[x.name for x in tenant.config_projects])
self.assertEqual(['org/project1', 'org/project2'],
[x.name for x in tenant.untrusted_projects])
- self.assertEqual(self.CONFIG_SET,
- tenant.config_projects[0].load_classes)
+ project = tenant.config_projects[0]
+ tpc = tenant.project_configs[project.canonical_name]
+ self.assertEqual(self.CONFIG_SET, tpc.load_classes)
+ project = tenant.untrusted_projects[0]
+ tpc = tenant.project_configs[project.canonical_name]
self.assertEqual(self.UNTRUSTED_SET - set(['project']),
- tenant.untrusted_projects[0].load_classes)
+ tpc.load_classes)
+ project = tenant.untrusted_projects[1]
+ tpc = tenant.project_configs[project.canonical_name]
self.assertEqual(self.UNTRUSTED_SET - set(['project']),
- tenant.untrusted_projects[1].load_classes)
+ tpc.load_classes)
self.assertTrue('common-config-job' in tenant.layout.jobs)
self.assertTrue('project1-job' in tenant.layout.jobs)
self.assertTrue('project2-job' in tenant.layout.jobs)
@@ -134,12 +147,17 @@
[x.name for x in tenant.config_projects])
self.assertEqual(['org/project1', 'org/project2'],
[x.name for x in tenant.untrusted_projects])
- self.assertEqual(self.CONFIG_SET,
- tenant.config_projects[0].load_classes)
+ project = tenant.config_projects[0]
+ tpc = tenant.project_configs[project.canonical_name]
+ self.assertEqual(self.CONFIG_SET, tpc.load_classes)
+ project = tenant.untrusted_projects[0]
+ tpc = tenant.project_configs[project.canonical_name]
self.assertEqual(self.UNTRUSTED_SET - set(['project']),
- tenant.untrusted_projects[0].load_classes)
+ tpc.load_classes)
+ project = tenant.untrusted_projects[1]
+ tpc = tenant.project_configs[project.canonical_name]
self.assertEqual(self.UNTRUSTED_SET - set(['project', 'job']),
- tenant.untrusted_projects[1].load_classes)
+ tpc.load_classes)
self.assertTrue('common-config-job' in tenant.layout.jobs)
self.assertTrue('project1-job' in tenant.layout.jobs)
self.assertFalse('project2-job' in tenant.layout.jobs)
@@ -166,12 +184,15 @@
[x.name for x in tenant.config_projects])
self.assertEqual(['org/project1', 'org/project2'],
[x.name for x in tenant.untrusted_projects])
- self.assertEqual(self.CONFIG_SET,
- tenant.config_projects[0].load_classes)
- self.assertEqual(set(['job']),
- tenant.untrusted_projects[0].load_classes)
- self.assertEqual(set(['project', 'job']),
- tenant.untrusted_projects[1].load_classes)
+ project = tenant.config_projects[0]
+ tpc = tenant.project_configs[project.canonical_name]
+ self.assertEqual(self.CONFIG_SET, tpc.load_classes)
+ project = tenant.untrusted_projects[0]
+ tpc = tenant.project_configs[project.canonical_name]
+ self.assertEqual(set(['job']), tpc.load_classes)
+ project = tenant.untrusted_projects[1]
+ tpc = tenant.project_configs[project.canonical_name]
+ self.assertEqual(set(['project', 'job']), tpc.load_classes)
self.assertTrue('common-config-job' in tenant.layout.jobs)
self.assertTrue('project1-job' in tenant.layout.jobs)
self.assertTrue('project2-job' in tenant.layout.jobs)
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index f4ca96f..3ab3305 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -42,7 +42,8 @@
self.tenant = model.Tenant('tenant')
self.layout = model.Layout()
self.project = model.Project('project', self.source)
- self.tenant.addUntrustedProject(self.project)
+ self.tpc = model.TenantProjectConfig(self.project)
+ self.tenant.addUntrustedProject(self.tpc)
self.pipeline = model.Pipeline('gate', self.layout)
self.layout.addPipeline(self.pipeline)
self.queue = model.ChangeQueue(self.pipeline)
@@ -175,7 +176,8 @@
layout.addPipeline(pipeline)
queue = model.ChangeQueue(pipeline)
project = model.Project('project', self.source)
- tenant.addUntrustedProject(project)
+ tpc = model.TenantProjectConfig(project)
+ tenant.addUntrustedProject(tpc)
base = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': self.context,
@@ -442,7 +444,8 @@
def test_job_inheritance_job_tree(self):
tenant = model.Tenant('tenant')
layout = model.Layout()
- tenant.addUntrustedProject(self.project)
+ tpc = model.TenantProjectConfig(self.project)
+ tenant.addUntrustedProject(tpc)
pipeline = model.Pipeline('gate', layout)
layout.addPipeline(pipeline)
@@ -523,7 +526,8 @@
layout.addPipeline(pipeline)
queue = model.ChangeQueue(pipeline)
project = model.Project('project', self.source)
- tenant.addUntrustedProject(project)
+ tpc = model.TenantProjectConfig(project)
+ tenant.addUntrustedProject(tpc)
base = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': self.context,
@@ -604,7 +608,8 @@
self.layout.addJob(job)
project2 = model.Project('project2', self.source)
- self.tenant.addUntrustedProject(project2)
+ tpc2 = model.TenantProjectConfig(project2)
+ self.tenant.addUntrustedProject(tpc2)
context2 = model.SourceContext(project2, 'master',
'test', True)
@@ -805,7 +810,8 @@
connection=connection1)
source1_project1 = model.Project('project1', source1)
- tenant.addConfigProject(source1_project1)
+ source1_project1_tpc = model.TenantProjectConfig(source1_project1)
+ tenant.addConfigProject(source1_project1_tpc)
d = {'project1':
{'git1.example.com': source1_project1}}
self.assertEqual(d, tenant.projects)
@@ -815,7 +821,8 @@
tenant.getProject('git1.example.com/project1'))
source1_project2 = model.Project('project2', source1)
- tenant.addUntrustedProject(source1_project2)
+ tpc = model.TenantProjectConfig(source1_project2)
+ tenant.addUntrustedProject(tpc)
d = {'project1':
{'git1.example.com': source1_project1},
'project2':
@@ -832,7 +839,8 @@
connection=connection2)
source2_project1 = model.Project('project1', source2)
- tenant.addUntrustedProject(source2_project1)
+ tpc = model.TenantProjectConfig(source2_project1)
+ tenant.addUntrustedProject(tpc)
d = {'project1':
{'git1.example.com': source1_project1,
'git2.example.com': source2_project1},
@@ -851,7 +859,8 @@
tenant.getProject('git2.example.com/project1'))
source2_project2 = model.Project('project2', source2)
- tenant.addConfigProject(source2_project2)
+ tpc = model.TenantProjectConfig(source2_project2)
+ tenant.addConfigProject(tpc)
d = {'project1':
{'git1.example.com': source1_project1,
'git2.example.com': source2_project1},
@@ -877,7 +886,8 @@
tenant.getProject('git2.example.com/project2'))
source1_project2b = model.Project('subpath/project2', source1)
- tenant.addConfigProject(source1_project2b)
+ tpc = model.TenantProjectConfig(source1_project2b)
+ tenant.addConfigProject(tpc)
d = {'project1':
{'git1.example.com': source1_project1,
'git2.example.com': source2_project1},
@@ -898,7 +908,8 @@
tenant.getProject('git1.example.com/subpath/project2'))
source2_project2b = model.Project('subpath/project2', source2)
- tenant.addConfigProject(source2_project2b)
+ tpc = model.TenantProjectConfig(source2_project2b)
+ tenant.addConfigProject(tpc)
d = {'project1':
{'git1.example.com': source1_project1,
'git2.example.com': source2_project1},
@@ -927,4 +938,4 @@
with testtools.ExpectedException(
Exception,
"Project project1 is already in project index"):
- tenant._addProject(source1_project1)
+ tenant._addProject(source1_project1_tpc)
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 627ebdd..ccf35da 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -932,13 +932,14 @@
tenant = model.Tenant(conf['name'])
tenant.unparsed_config = conf
unparsed_config = model.UnparsedTenantConfig()
- config_projects, untrusted_projects = \
+ # tpcs is TenantProjectConfigs
+ config_tpcs, untrusted_tpcs = \
TenantParser._loadTenantProjects(
project_key_dir, connections, conf)
- for project in config_projects:
- tenant.addConfigProject(project)
- for project in untrusted_projects:
- tenant.addUntrustedProject(project)
+ for tpc in config_tpcs:
+ tenant.addConfigProject(tpc)
+ for tpc in untrusted_tpcs:
+ tenant.addUntrustedProject(tpc)
tenant.config_projects_config, tenant.untrusted_projects_config = \
TenantParser._loadTenantInRepoLayouts(merger, connections,
tenant.config_projects,
@@ -1020,8 +1021,10 @@
if project_exclude:
project_include = frozenset(project_include - project_exclude)
- project.load_classes = frozenset(project_include)
- return project
+ tenant_project_config = model.TenantProjectConfig(project)
+ tenant_project_config.load_classes = frozenset(project_include)
+
+ return tenant_project_config
@staticmethod
def _getProjects(source, conf, current_include):
@@ -1065,21 +1068,22 @@
current_include = default_include
for conf_repo in conf_source.get('config-projects', []):
- projects = TenantParser._getProjects(source, conf_repo,
- current_include)
- for project in projects:
+ # tpcs = TenantProjectConfigs
+ tpcs = TenantParser._getProjects(source, conf_repo,
+ current_include)
+ for tpc in tpcs:
TenantParser._loadProjectKeys(
- project_key_dir, source_name, project)
- config_projects.append(project)
+ project_key_dir, source_name, tpc.project)
+ config_projects.append(tpc)
current_include = frozenset(default_include - set(['pipeline']))
for conf_repo in conf_source.get('untrusted-projects', []):
- projects = TenantParser._getProjects(source, conf_repo,
- current_include)
- for project in projects:
+ tpcs = TenantParser._getProjects(source, conf_repo,
+ current_include)
+ for tpc in tpcs:
TenantParser._loadProjectKeys(
- project_key_dir, source_name, project)
- untrusted_projects.append(project)
+ project_key_dir, source_name, tpc.project)
+ untrusted_projects.append(tpc)
return config_projects, untrusted_projects
@@ -1198,12 +1202,18 @@
return config
@staticmethod
+ def _getLoadClasses(tenant, conf_object):
+ project = conf_object['_source_context'].project
+ tpc = tenant.project_configs[project.canonical_name]
+ return tpc.load_classes
+
+ @staticmethod
def _parseLayoutItems(layout, tenant, data, scheduler, connections,
skip_pipelines=False, skip_semaphores=False):
if not skip_pipelines:
for config_pipeline in data.pipelines:
- classes = config_pipeline['_source_context'].\
- project.load_classes
+ classes = TenantParser._getLoadClasses(
+ tenant, config_pipeline)
if 'pipeline' not in classes:
continue
layout.addPipeline(PipelineParser.fromYaml(
@@ -1211,7 +1221,7 @@
scheduler, config_pipeline))
for config_nodeset in data.nodesets:
- classes = config_nodeset['_source_context'].project.load_classes
+ classes = TenantParser._getLoadClasses(tenant, config_nodeset)
if 'nodeset' not in classes:
continue
with configuration_exceptions('nodeset', config_nodeset):
@@ -1219,13 +1229,13 @@
layout, config_nodeset))
for config_secret in data.secrets:
- classes = config_secret['_source_context'].project.load_classes
+ classes = TenantParser._getLoadClasses(tenant, config_secret)
if 'secret' not in classes:
continue
layout.addSecret(SecretParser.fromYaml(layout, config_secret))
for config_job in data.jobs:
- classes = config_job['_source_context'].project.load_classes
+ classes = TenantParser._getLoadClasses(tenant, config_job)
if 'job' not in classes:
continue
with configuration_exceptions('job', config_job):
@@ -1234,14 +1244,14 @@
if not skip_semaphores:
for config_semaphore in data.semaphores:
- classes = config_semaphore['_source_context'].\
- project.load_classes
+ classes = TenantParser._getLoadClasses(
+ tenant, config_semaphore)
if 'semaphore' not in classes:
continue
layout.addSemaphore(SemaphoreParser.fromYaml(config_semaphore))
for config_template in data.project_templates:
- classes = config_template['_source_context'].project.load_classes
+ classes = TenantParser._getLoadClasses(tenant, config_template)
if 'project-template' not in classes:
continue
layout.addProjectTemplate(ProjectTemplateParser.fromYaml(
@@ -1255,10 +1265,11 @@
# each of the project stanzas. Each one may be (should
# be!) from a different repo, so filter them according to
# the include/exclude rules before parsing them.
- filtered_projects = [
- p for p in config_projects if
- 'project' in p['_source_context'].project.load_classes
- ]
+ filtered_projects = []
+ for config_project in config_projects:
+ classes = TenantParser._getLoadClasses(tenant, config_project)
+ if 'project' in classes:
+ filtered_projects.append(config_project)
if not filtered_projects:
continue
diff --git a/zuul/model.py b/zuul/model.py
index f58ecbe..d233415 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -331,9 +331,6 @@
self.foreign = foreign
self.unparsed_config = None
self.unparsed_branch_config = {} # branch -> UnparsedTenantConfig
- # Configuration object classes to include or exclude when
- # loading zuul config files.
- self.load_classes = frozenset()
def __str__(self):
return self.name
@@ -2001,6 +1998,19 @@
self.merge_mode = None
+class TenantProjectConfig(object):
+ """A project in the context of a tenant.
+
+ A Project is globally unique in the system, however, when used in
+ a tenant, some metadata about the project local to the tenant is
+ stored in a TenantProjectConfig.
+ """
+
+ def __init__(self, project):
+ self.project = project
+ self.load_classes = set()
+
+
class ProjectConfig(object):
# Represents a project cofiguration
def __init__(self, name):
@@ -2012,6 +2022,7 @@
class UnparsedAbideConfig(object):
+
"""A collection of yaml lists that has not yet been parsed into objects.
An Abide is a collection of tenants.
@@ -2359,6 +2370,9 @@
# The unparsed config from those projects.
self.untrusted_projects_config = None
self.semaphore_handler = SemaphoreHandler()
+ # Metadata about projects for this tenant
+ # canonical project name -> TenantProjectConfig
+ self.project_configs = {}
# A mapping of project names to projects. project_name ->
# VALUE where VALUE is a further dictionary of
@@ -2366,17 +2380,21 @@
self.projects = {}
self.canonical_hostnames = set()
- def _addProject(self, project):
+ def _addProject(self, tpc):
"""Add a project to the project index
- :arg Project project: The project to add.
+ :arg TenantProjectConfig tpc: The TenantProjectConfig (with
+ associated project) to add.
+
"""
+ project = tpc.project
self.canonical_hostnames.add(project.canonical_hostname)
hostname_dict = self.projects.setdefault(project.name, {})
if project.canonical_hostname in hostname_dict:
raise Exception("Project %s is already in project index" %
(project,))
hostname_dict[project.canonical_hostname] = project
+ self.project_configs[project.canonical_name] = tpc
def getProject(self, name):
"""Return a project given its name.
@@ -2423,13 +2441,13 @@
raise Exception("Project %s is neither trusted nor untrusted" %
(project,))
- def addConfigProject(self, project):
- self.config_projects.append(project)
- self._addProject(project)
+ def addConfigProject(self, tpc):
+ self.config_projects.append(tpc.project)
+ self._addProject(tpc)
- def addUntrustedProject(self, project):
- self.untrusted_projects.append(project)
- self._addProject(project)
+ def addUntrustedProject(self, tpc):
+ self.untrusted_projects.append(tpc.project)
+ self._addProject(tpc)
def getSafeAttributes(self):
return Attributes(name=self.name)