Merge "Permit config shadowing" into feature/zuulv3
diff --git a/tests/base.py b/tests/base.py
index f210591..2e4e9a5 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -2148,6 +2148,8 @@
# Make sure we set up an RSA key for the project so that we
# don't spend time generating one:
+ if isinstance(project, dict):
+ project = list(project.keys())[0]
key_root = os.path.join(self.state_root, 'keys')
if not os.path.isdir(key_root):
os.mkdir(key_root, 0o700)
diff --git a/tests/fixtures/config/shadow/git/local-config/playbooks/base.yaml b/tests/fixtures/config/shadow/git/local-config/playbooks/base.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/shadow/git/local-config/playbooks/base.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/shadow/git/local-config/playbooks/test2.yaml b/tests/fixtures/config/shadow/git/local-config/playbooks/test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/shadow/git/local-config/playbooks/test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/shadow/git/local-config/zuul.yaml b/tests/fixtures/config/shadow/git/local-config/zuul.yaml
new file mode 100644
index 0000000..756e843
--- /dev/null
+++ b/tests/fixtures/config/shadow/git/local-config/zuul.yaml
@@ -0,0 +1,25 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- job:
+ name: base
+
+- job:
+ name: test2
+
+- project:
+ name: org/project
+ check:
+ jobs:
+ - test1
+ - test2
diff --git a/tests/fixtures/config/shadow/git/org_project/README b/tests/fixtures/config/shadow/git/org_project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/shadow/git/org_project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/shadow/git/stdlib/.zuul.yaml b/tests/fixtures/config/shadow/git/stdlib/.zuul.yaml
new file mode 100644
index 0000000..6a6f9c9
--- /dev/null
+++ b/tests/fixtures/config/shadow/git/stdlib/.zuul.yaml
@@ -0,0 +1,10 @@
+- job:
+ name: base
+
+- job:
+ name: test1
+ parent: base
+
+- job:
+ name: test2
+ parent: base
diff --git a/tests/fixtures/config/shadow/git/stdlib/README b/tests/fixtures/config/shadow/git/stdlib/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/shadow/git/stdlib/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/shadow/git/stdlib/playbooks/base.yaml b/tests/fixtures/config/shadow/git/stdlib/playbooks/base.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/shadow/git/stdlib/playbooks/base.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/shadow/git/stdlib/playbooks/test1.yaml b/tests/fixtures/config/shadow/git/stdlib/playbooks/test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/shadow/git/stdlib/playbooks/test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/shadow/git/stdlib/playbooks/test2.yaml b/tests/fixtures/config/shadow/git/stdlib/playbooks/test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/shadow/git/stdlib/playbooks/test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/shadow/main.yaml b/tests/fixtures/config/shadow/main.yaml
new file mode 100644
index 0000000..f148a84
--- /dev/null
+++ b/tests/fixtures/config/shadow/main.yaml
@@ -0,0 +1,10 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - local-config
+ untrusted-projects:
+ - stdlib:
+ shadow: local-config
+ - org/project
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index 3ab3305..7fe101e 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -40,7 +40,7 @@
self.source = Dummy(canonical_hostname='git.example.com',
connection=self.connection)
self.tenant = model.Tenant('tenant')
- self.layout = model.Layout()
+ self.layout = model.Layout(self.tenant)
self.project = model.Project('project', self.source)
self.tpc = model.TenantProjectConfig(self.project)
self.tenant.addUntrustedProject(self.tpc)
@@ -59,7 +59,7 @@
@property
def job(self):
tenant = model.Tenant('tenant')
- layout = model.Layout()
+ layout = model.Layout(tenant)
job = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': self.context,
'_start_mark': self.start_mark,
@@ -170,7 +170,7 @@
def test_job_inheritance_configloader(self):
# TODO(jeblair): move this to a configloader test
tenant = model.Tenant('tenant')
- layout = model.Layout()
+ layout = model.Layout(tenant)
pipeline = model.Pipeline('gate', layout)
layout.addPipeline(pipeline)
@@ -333,8 +333,8 @@
'playbooks/base'])
def test_job_auth_inheritance(self):
- tenant = model.Tenant('tenant')
- layout = model.Layout()
+ tenant = self.tenant
+ layout = self.layout
conf = yaml.safe_load('''
- secret:
@@ -359,7 +359,7 @@
secret = configloader.SecretParser.fromYaml(layout, conf)
layout.addSecret(secret)
- base = configloader.JobParser.fromYaml(tenant, layout, {
+ base = configloader.JobParser.fromYaml(self.tenant, self.layout, {
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'base',
@@ -443,7 +443,7 @@
def test_job_inheritance_job_tree(self):
tenant = model.Tenant('tenant')
- layout = model.Layout()
+ layout = model.Layout(tenant)
tpc = model.TenantProjectConfig(self.project)
tenant.addUntrustedProject(tpc)
@@ -520,7 +520,7 @@
def test_inheritance_keeps_matchers(self):
tenant = model.Tenant('tenant')
- layout = model.Layout()
+ layout = model.Layout(tenant)
pipeline = model.Pipeline('gate', layout)
layout.addPipeline(pipeline)
@@ -571,11 +571,13 @@
self.assertEqual([], item.getJobs())
def test_job_source_project(self):
- tenant = model.Tenant('tenant')
- layout = model.Layout()
+ tenant = self.tenant
+ layout = self.layout
base_project = model.Project('base_project', self.source)
base_context = model.SourceContext(base_project, 'master',
'test', True)
+ tpc = model.TenantProjectConfig(base_project)
+ tenant.addUntrustedProject(tpc)
base = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': base_context,
@@ -587,6 +589,8 @@
other_project = model.Project('other_project', self.source)
other_context = model.SourceContext(other_project, 'master',
'test', True)
+ tpc = model.TenantProjectConfig(other_project)
+ tenant.addUntrustedProject(tpc)
base2 = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': other_context,
'_start_mark': self.start_mark,
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 327f745..112f48c 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -630,3 +630,17 @@
self.assertHistory([
dict(name='project-test', result='SUCCESS', changes='1,1 2,1'),
])
+
+
+class TestShadow(ZuulTestCase):
+ tenant_config_file = 'config/shadow/main.yaml'
+
+ def test_shadow(self):
+ # Test that a repo is allowed to shadow another's job definitions.
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertHistory([
+ dict(name='test1', result='SUCCESS', changes='1,1'),
+ dict(name='test2', result='SUCCESS', changes='1,1'),
+ ])
diff --git a/zuul/configloader.py b/zuul/configloader.py
index ccf35da..735fe38 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -887,6 +887,7 @@
project_dict = {str: {
'include': to_list(classes),
'exclude': to_list(classes),
+ 'shadow': to_list(str),
}}
project = vs.Any(str, project_dict)
@@ -940,6 +941,10 @@
tenant.addConfigProject(tpc)
for tpc in untrusted_tpcs:
tenant.addUntrustedProject(tpc)
+
+ for tpc in config_tpcs + untrusted_tpcs:
+ TenantParser._resolveShadowProjects(tenant, tpc)
+
tenant.config_projects_config, tenant.untrusted_projects_config = \
TenantParser._loadTenantInRepoLayouts(merger, connections,
tenant.config_projects,
@@ -954,6 +959,13 @@
return tenant
@staticmethod
+ def _resolveShadowProjects(tenant, tpc):
+ shadow_projects = []
+ for sp in tpc.shadow_projects:
+ shadow_projects.append(tenant.getProject(sp)[1])
+ tpc.shadow_projects = frozenset(shadow_projects)
+
+ @staticmethod
def _loadProjectKeys(project_key_dir, connection_name, project):
project.private_key_file = (
os.path.join(project_key_dir, connection_name,
@@ -1008,9 +1020,11 @@
# Return a project object whether conf is a dict or a str
project = source.getProject(conf)
project_include = current_include
+ shadow_projects = []
else:
project_name = list(conf.keys())[0]
project = source.getProject(project_name)
+ shadow_projects = as_list(conf[project_name].get('shadow', []))
project_include = frozenset(
as_list(conf[project_name].get('include', [])))
@@ -1023,6 +1037,7 @@
tenant_project_config = model.TenantProjectConfig(project)
tenant_project_config.load_classes = frozenset(project_include)
+ tenant_project_config.shadow_projects = shadow_projects
return tenant_project_config
@@ -1240,7 +1255,11 @@
continue
with configuration_exceptions('job', config_job):
job = JobParser.fromYaml(tenant, layout, config_job)
- layout.addJob(job)
+ added = layout.addJob(job)
+ if not added:
+ TenantParser.log.debug(
+ "Skipped adding job %s which shadows an existing job" %
+ (job,))
if not skip_semaphores:
for config_semaphore in data.semaphores:
@@ -1279,13 +1298,11 @@
@staticmethod
def _parseLayout(base, tenant, data, scheduler, connections):
- layout = model.Layout()
+ layout = model.Layout(tenant)
TenantParser._parseLayoutItems(layout, tenant, data,
scheduler, connections)
- layout.tenant = tenant
-
for pipeline in layout.pipelines.values():
pipeline.manager._postConfig(layout)
@@ -1401,7 +1418,7 @@
for project in tenant.untrusted_projects:
self._loadDynamicProjectData(config, project, files, False)
- layout = model.Layout()
+ layout = model.Layout(tenant)
# NOTE: the actual pipeline objects (complete with queues and
# enqueued items) are copied by reference here. This allows
# our shadow dynamic configuration to continue to interact
diff --git a/zuul/model.py b/zuul/model.py
index d233415..4744bbe 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -2009,6 +2009,7 @@
def __init__(self, project):
self.project = project
self.load_classes = set()
+ self.shadow_projects = set()
class ProjectConfig(object):
@@ -2132,8 +2133,8 @@
class Layout(object):
"""Holds all of the Pipelines."""
- def __init__(self):
- self.tenant = None
+ def __init__(self, tenant):
+ self.tenant = tenant
self.project_configs = {}
self.project_templates = {}
self.pipelines = OrderedDict()
@@ -2162,6 +2163,18 @@
prior_jobs = [j for j in self.getJobs(job.name) if
j.source_context.project !=
job.source_context.project]
+ # Unless the repo is permitted to shadow another. If so, and
+ # the job we are adding is from a repo that is permitted to
+ # shadow the one with the older jobs, skip adding this job.
+ job_project = job.source_context.project
+ job_tpc = self.tenant.project_configs[job_project.canonical_name]
+ skip_add = False
+ for prior_job in prior_jobs[:]:
+ prior_project = prior_job.source_context.project
+ if prior_project in job_tpc.shadow_projects:
+ prior_jobs.remove(prior_job)
+ skip_add = True
+
if prior_jobs:
raise Exception("Job %s in %s is not permitted to shadow "
"job %s in %s" % (
@@ -2169,11 +2182,13 @@
job.source_context.project,
prior_jobs[0],
prior_jobs[0].source_context.project))
-
+ if skip_add:
+ return False
if job.name in self.jobs:
self.jobs[job.name].append(job)
else:
self.jobs[job.name] = [job]
+ return True
def addNodeSet(self, nodeset):
if nodeset.name in self.nodesets: