Permit config shadowing
To support the idea that diverse zuul installations can share common
job definitions in a 'standard library' project, but still be able to
override some job definitions with their own local versions if needed,
add a tenant config option to permit a repo to shadow another one.
Place the local project first in configuration, then on the remote project,
indicate that it shadows the local one. Then, any definitions in the
remote repository which conflict with the local will be ignored.
Change-Id: Ia715c5fa45141eacbb11449404ee3a3ec948d27f
diff --git a/tests/base.py b/tests/base.py
index e605c87..696156f 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -2139,6 +2139,8 @@
# Make sure we set up an RSA key for the project so that we
# don't spend time generating one:
+ if isinstance(project, dict):
+ project = list(project.keys())[0]
key_root = os.path.join(self.state_root, 'keys')
if not os.path.isdir(key_root):
os.mkdir(key_root, 0o700)
diff --git a/tests/fixtures/config/shadow/git/local-config/playbooks/base.yaml b/tests/fixtures/config/shadow/git/local-config/playbooks/base.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/shadow/git/local-config/playbooks/base.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/shadow/git/local-config/playbooks/test2.yaml b/tests/fixtures/config/shadow/git/local-config/playbooks/test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/shadow/git/local-config/playbooks/test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/shadow/git/local-config/zuul.yaml b/tests/fixtures/config/shadow/git/local-config/zuul.yaml
new file mode 100644
index 0000000..756e843
--- /dev/null
+++ b/tests/fixtures/config/shadow/git/local-config/zuul.yaml
@@ -0,0 +1,25 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- job:
+ name: base
+
+- job:
+ name: test2
+
+- project:
+ name: org/project
+ check:
+ jobs:
+ - test1
+ - test2
diff --git a/tests/fixtures/config/shadow/git/org_project/README b/tests/fixtures/config/shadow/git/org_project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/shadow/git/org_project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/shadow/git/stdlib/.zuul.yaml b/tests/fixtures/config/shadow/git/stdlib/.zuul.yaml
new file mode 100644
index 0000000..6a6f9c9
--- /dev/null
+++ b/tests/fixtures/config/shadow/git/stdlib/.zuul.yaml
@@ -0,0 +1,10 @@
+- job:
+ name: base
+
+- job:
+ name: test1
+ parent: base
+
+- job:
+ name: test2
+ parent: base
diff --git a/tests/fixtures/config/shadow/git/stdlib/README b/tests/fixtures/config/shadow/git/stdlib/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/shadow/git/stdlib/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/shadow/git/stdlib/playbooks/base.yaml b/tests/fixtures/config/shadow/git/stdlib/playbooks/base.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/shadow/git/stdlib/playbooks/base.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/shadow/git/stdlib/playbooks/test1.yaml b/tests/fixtures/config/shadow/git/stdlib/playbooks/test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/shadow/git/stdlib/playbooks/test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/shadow/git/stdlib/playbooks/test2.yaml b/tests/fixtures/config/shadow/git/stdlib/playbooks/test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/shadow/git/stdlib/playbooks/test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/shadow/main.yaml b/tests/fixtures/config/shadow/main.yaml
new file mode 100644
index 0000000..f148a84
--- /dev/null
+++ b/tests/fixtures/config/shadow/main.yaml
@@ -0,0 +1,10 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - local-config
+ untrusted-projects:
+ - stdlib:
+ shadow: local-config
+ - org/project
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index 3ab3305..7fe101e 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -40,7 +40,7 @@
self.source = Dummy(canonical_hostname='git.example.com',
connection=self.connection)
self.tenant = model.Tenant('tenant')
- self.layout = model.Layout()
+ self.layout = model.Layout(self.tenant)
self.project = model.Project('project', self.source)
self.tpc = model.TenantProjectConfig(self.project)
self.tenant.addUntrustedProject(self.tpc)
@@ -59,7 +59,7 @@
@property
def job(self):
tenant = model.Tenant('tenant')
- layout = model.Layout()
+ layout = model.Layout(tenant)
job = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': self.context,
'_start_mark': self.start_mark,
@@ -170,7 +170,7 @@
def test_job_inheritance_configloader(self):
# TODO(jeblair): move this to a configloader test
tenant = model.Tenant('tenant')
- layout = model.Layout()
+ layout = model.Layout(tenant)
pipeline = model.Pipeline('gate', layout)
layout.addPipeline(pipeline)
@@ -333,8 +333,8 @@
'playbooks/base'])
def test_job_auth_inheritance(self):
- tenant = model.Tenant('tenant')
- layout = model.Layout()
+ tenant = self.tenant
+ layout = self.layout
conf = yaml.safe_load('''
- secret:
@@ -359,7 +359,7 @@
secret = configloader.SecretParser.fromYaml(layout, conf)
layout.addSecret(secret)
- base = configloader.JobParser.fromYaml(tenant, layout, {
+ base = configloader.JobParser.fromYaml(self.tenant, self.layout, {
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'base',
@@ -443,7 +443,7 @@
def test_job_inheritance_job_tree(self):
tenant = model.Tenant('tenant')
- layout = model.Layout()
+ layout = model.Layout(tenant)
tpc = model.TenantProjectConfig(self.project)
tenant.addUntrustedProject(tpc)
@@ -520,7 +520,7 @@
def test_inheritance_keeps_matchers(self):
tenant = model.Tenant('tenant')
- layout = model.Layout()
+ layout = model.Layout(tenant)
pipeline = model.Pipeline('gate', layout)
layout.addPipeline(pipeline)
@@ -571,11 +571,13 @@
self.assertEqual([], item.getJobs())
def test_job_source_project(self):
- tenant = model.Tenant('tenant')
- layout = model.Layout()
+ tenant = self.tenant
+ layout = self.layout
base_project = model.Project('base_project', self.source)
base_context = model.SourceContext(base_project, 'master',
'test', True)
+ tpc = model.TenantProjectConfig(base_project)
+ tenant.addUntrustedProject(tpc)
base = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': base_context,
@@ -587,6 +589,8 @@
other_project = model.Project('other_project', self.source)
other_context = model.SourceContext(other_project, 'master',
'test', True)
+ tpc = model.TenantProjectConfig(other_project)
+ tenant.addUntrustedProject(tpc)
base2 = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': other_context,
'_start_mark': self.start_mark,
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 7c5fa70..f765a53 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -617,3 +617,17 @@
self.assertHistory([
dict(name='project-test', result='SUCCESS', changes='1,1 2,1'),
])
+
+
+class TestShadow(ZuulTestCase):
+ tenant_config_file = 'config/shadow/main.yaml'
+
+ def test_shadow(self):
+ # Test that a repo is allowed to shadow another's job definitions.
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertHistory([
+ dict(name='test1', result='SUCCESS', changes='1,1'),
+ dict(name='test2', result='SUCCESS', changes='1,1'),
+ ])
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 18fca83..256a859 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -887,6 +887,7 @@
project_dict = {str: {
'include': to_list(classes),
'exclude': to_list(classes),
+ 'shadow': to_list(str),
}}
project = vs.Any(str, project_dict)
@@ -940,6 +941,10 @@
tenant.addConfigProject(tpc)
for tpc in untrusted_tpcs:
tenant.addUntrustedProject(tpc)
+
+ for tpc in config_tpcs + untrusted_tpcs:
+ TenantParser._resolveShadowProjects(tenant, tpc)
+
tenant.config_projects_config, tenant.untrusted_projects_config = \
TenantParser._loadTenantInRepoLayouts(merger, connections,
tenant.config_projects,
@@ -954,6 +959,13 @@
return tenant
@staticmethod
+ def _resolveShadowProjects(tenant, tpc):
+ shadow_projects = []
+ for sp in tpc.shadow_projects:
+ shadow_projects.append(tenant.getProject(sp)[1])
+ tpc.shadow_projects = frozenset(shadow_projects)
+
+ @staticmethod
def _loadProjectKeys(project_key_dir, connection_name, project):
project.private_key_file = (
os.path.join(project_key_dir, connection_name,
@@ -1008,9 +1020,11 @@
# Return a project object whether conf is a dict or a str
project = source.getProject(conf)
project_include = current_include
+ shadow_projects = []
else:
project_name = list(conf.keys())[0]
project = source.getProject(project_name)
+ shadow_projects = as_list(conf[project_name].get('shadow', []))
project_include = frozenset(
as_list(conf[project_name].get('include', [])))
@@ -1023,6 +1037,7 @@
tenant_project_config = model.TenantProjectConfig(project)
tenant_project_config.load_classes = frozenset(project_include)
+ tenant_project_config.shadow_projects = shadow_projects
return tenant_project_config
@@ -1234,7 +1249,11 @@
continue
with configuration_exceptions('job', config_job):
job = JobParser.fromYaml(tenant, layout, config_job)
- layout.addJob(job)
+ added = layout.addJob(job)
+ if not added:
+ TenantParser.log.debug(
+ "Skipped adding job %s which shadows an existing job" %
+ (job,))
if not skip_semaphores:
for config_semaphore in data.semaphores:
@@ -1273,13 +1292,11 @@
@staticmethod
def _parseLayout(base, tenant, data, scheduler, connections):
- layout = model.Layout()
+ layout = model.Layout(tenant)
TenantParser._parseLayoutItems(layout, tenant, data,
scheduler, connections)
- layout.tenant = tenant
-
for pipeline in layout.pipelines.values():
pipeline.manager._postConfig(layout)
@@ -1372,7 +1389,7 @@
for project in tenant.untrusted_projects:
self._loadDynamicProjectData(config, project, files, False)
- layout = model.Layout()
+ layout = model.Layout(tenant)
# NOTE: the actual pipeline objects (complete with queues and
# enqueued items) are copied by reference here. This allows
# our shadow dynamic configuration to continue to interact
diff --git a/zuul/model.py b/zuul/model.py
index 3cf4984..17301b7 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -2006,6 +2006,7 @@
def __init__(self, project):
self.project = project
self.load_classes = set()
+ self.shadow_projects = set()
class ProjectConfig(object):
@@ -2129,8 +2130,8 @@
class Layout(object):
"""Holds all of the Pipelines."""
- def __init__(self):
- self.tenant = None
+ def __init__(self, tenant):
+ self.tenant = tenant
self.project_configs = {}
self.project_templates = {}
self.pipelines = OrderedDict()
@@ -2159,6 +2160,18 @@
prior_jobs = [j for j in self.getJobs(job.name) if
j.source_context.project !=
job.source_context.project]
+ # Unless the repo is permitted to shadow another. If so, and
+ # the job we are adding is from a repo that is permitted to
+ # shadow the one with the older jobs, skip adding this job.
+ job_project = job.source_context.project
+ job_tpc = self.tenant.project_configs[job_project.canonical_name]
+ skip_add = False
+ for prior_job in prior_jobs[:]:
+ prior_project = prior_job.source_context.project
+ if prior_project in job_tpc.shadow_projects:
+ prior_jobs.remove(prior_job)
+ skip_add = True
+
if prior_jobs:
raise Exception("Job %s in %s is not permitted to shadow "
"job %s in %s" % (
@@ -2166,11 +2179,13 @@
job.source_context.project,
prior_jobs[0],
prior_jobs[0].source_context.project))
-
+ if skip_add:
+ return False
if job.name in self.jobs:
self.jobs[job.name].append(job)
else:
self.jobs[job.name] = [job]
+ return True
def addNodeSet(self, nodeset):
if nodeset.name in self.nodesets: