Add support for zuul.d configuration split
This change implements the zuul_split spec to support configuration split in
a zuul.d directory.
Change-Id: I6bc7250b2045b73dfba109aa0b2f1ba5d66752b2
diff --git a/tests/fixtures/config/split-config/git/common-config/playbooks/project-test1.yaml b/tests/fixtures/config/split-config/git/common-config/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/split-config/git/common-config/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/split-config/git/common-config/zuul.d/jobs.yaml b/tests/fixtures/config/split-config/git/common-config/zuul.d/jobs.yaml
new file mode 100644
index 0000000..280342c
--- /dev/null
+++ b/tests/fixtures/config/split-config/git/common-config/zuul.d/jobs.yaml
@@ -0,0 +1,2 @@
+- job:
+ name: project-test1
diff --git a/tests/fixtures/config/split-config/git/common-config/zuul.d/org-project.yaml b/tests/fixtures/config/split-config/git/common-config/zuul.d/org-project.yaml
new file mode 100644
index 0000000..872e126
--- /dev/null
+++ b/tests/fixtures/config/split-config/git/common-config/zuul.d/org-project.yaml
@@ -0,0 +1,5 @@
+- project:
+ name: org/project
+ check:
+ jobs:
+ - project-test1
diff --git a/tests/fixtures/config/split-config/git/common-config/zuul.d/pipelines.yaml b/tests/fixtures/config/split-config/git/common-config/zuul.d/pipelines.yaml
new file mode 100644
index 0000000..ba91fb5
--- /dev/null
+++ b/tests/fixtures/config/split-config/git/common-config/zuul.d/pipelines.yaml
@@ -0,0 +1,12 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
diff --git a/tests/fixtures/config/split-config/git/org_project/README b/tests/fixtures/config/split-config/git/org_project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/split-config/git/org_project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/split-config/git/org_project1/.zuul.d/gate.yaml b/tests/fixtures/config/split-config/git/org_project1/.zuul.d/gate.yaml
new file mode 100644
index 0000000..4bc0d81
--- /dev/null
+++ b/tests/fixtures/config/split-config/git/org_project1/.zuul.d/gate.yaml
@@ -0,0 +1,7 @@
+- project:
+ name: org/project1
+ check:
+ jobs:
+ - project-test1
+ - project1-project2-integration:
+ dependencies: project-test1
diff --git a/tests/fixtures/config/split-config/git/org_project1/.zuul.d/jobs.yaml b/tests/fixtures/config/split-config/git/org_project1/.zuul.d/jobs.yaml
new file mode 100644
index 0000000..33d74f3
--- /dev/null
+++ b/tests/fixtures/config/split-config/git/org_project1/.zuul.d/jobs.yaml
@@ -0,0 +1,2 @@
+- job:
+ name: project1-project2-integration
diff --git a/tests/fixtures/config/split-config/git/org_project1/README b/tests/fixtures/config/split-config/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/split-config/git/org_project1/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/split-config/git/org_project1/playbooks/project1-project2-integration.yaml b/tests/fixtures/config/split-config/git/org_project1/playbooks/project1-project2-integration.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/split-config/git/org_project1/playbooks/project1-project2-integration.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/split-config/main.yaml b/tests/fixtures/config/split-config/main.yaml
new file mode 100644
index 0000000..5f57245
--- /dev/null
+++ b/tests/fixtures/config/split-config/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - org/project
+ - org/project1
diff --git a/tests/fixtures/config/templated-project/git/common-config/zuul.d/jobs.yaml b/tests/fixtures/config/templated-project/git/common-config/zuul.d/jobs.yaml
new file mode 100644
index 0000000..e051871
--- /dev/null
+++ b/tests/fixtures/config/templated-project/git/common-config/zuul.d/jobs.yaml
@@ -0,0 +1,17 @@
+- job:
+ name: project-test1
+
+- job:
+ name: project-test2
+
+- job:
+ name: layered-project-test3
+
+- job:
+ name: layered-project-test4
+
+- job:
+ name: layered-project-foo-test5
+
+- job:
+ name: project-test6
diff --git a/tests/fixtures/config/templated-project/git/common-config/zuul.d/pipelines.yaml b/tests/fixtures/config/templated-project/git/common-config/zuul.d/pipelines.yaml
new file mode 100644
index 0000000..4a19796
--- /dev/null
+++ b/tests/fixtures/config/templated-project/git/common-config/zuul.d/pipelines.yaml
@@ -0,0 +1,41 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- pipeline:
+ name: gate
+ manager: dependent
+ success-message: Build succeeded (gate).
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
+
+- pipeline:
+ name: post
+ manager: independent
+ trigger:
+ gerrit:
+ - event: ref-updated
+ ref: ^(?!refs/).*$
diff --git a/tests/fixtures/config/templated-project/git/common-config/zuul.d/projects.yaml b/tests/fixtures/config/templated-project/git/common-config/zuul.d/projects.yaml
new file mode 100644
index 0000000..891c863
--- /dev/null
+++ b/tests/fixtures/config/templated-project/git/common-config/zuul.d/projects.yaml
@@ -0,0 +1,14 @@
+- project:
+ name: org/templated-project
+ templates:
+ - test-one-and-two
+
+- project:
+ name: org/layered-project
+ templates:
+ - test-one-and-two
+ - test-three-and-four
+ - test-five
+ check:
+ jobs:
+ - project-test6
diff --git a/tests/fixtures/config/templated-project/git/common-config/zuul.d/templates.yaml b/tests/fixtures/config/templated-project/git/common-config/zuul.d/templates.yaml
new file mode 100644
index 0000000..27d2f16
--- /dev/null
+++ b/tests/fixtures/config/templated-project/git/common-config/zuul.d/templates.yaml
@@ -0,0 +1,19 @@
+- project-template:
+ name: test-one-and-two
+ check:
+ jobs:
+ - project-test1
+ - project-test2
+
+- project-template:
+ name: test-three-and-four
+ check:
+ jobs:
+ - layered-project-test3
+ - layered-project-test4
+
+- project-template:
+ name: test-five
+ check:
+ jobs:
+ - layered-project-foo-test5
diff --git a/tests/fixtures/config/templated-project/git/common-config/zuul.yaml b/tests/fixtures/config/templated-project/git/common-config/zuul.yaml
deleted file mode 100644
index 251a3cd..0000000
--- a/tests/fixtures/config/templated-project/git/common-config/zuul.yaml
+++ /dev/null
@@ -1,94 +0,0 @@
-- pipeline:
- name: check
- manager: independent
- trigger:
- gerrit:
- - event: patchset-created
- success:
- gerrit:
- verified: 1
- failure:
- gerrit:
- verified: -1
-
-- pipeline:
- name: gate
- manager: dependent
- success-message: Build succeeded (gate).
- trigger:
- gerrit:
- - event: comment-added
- approval:
- - approved: 1
- success:
- gerrit:
- verified: 2
- submit: true
- failure:
- gerrit:
- verified: -2
- start:
- gerrit:
- verified: 0
- precedence: high
-
-- pipeline:
- name: post
- manager: independent
- trigger:
- gerrit:
- - event: ref-updated
- ref: ^(?!refs/).*$
-
-- project-template:
- name: test-one-and-two
- check:
- jobs:
- - project-test1
- - project-test2
-
-- project-template:
- name: test-three-and-four
- check:
- jobs:
- - layered-project-test3
- - layered-project-test4
-
-- project-template:
- name: test-five
- check:
- jobs:
- - layered-project-foo-test5
-
-- job:
- name: project-test1
-
-- job:
- name: project-test2
-
-- job:
- name: layered-project-test3
-
-- job:
- name: layered-project-test4
-
-- job:
- name: layered-project-foo-test5
-
-- job:
- name: project-test6
-
-- project:
- name: org/templated-project
- templates:
- - test-one-and-two
-
-- project:
- name: org/layered-project
- templates:
- - test-one-and-two
- - test-three-and-four
- - test-five
- check:
- jobs:
- - project-test6
diff --git a/tests/unit/test_configloader.py b/tests/unit/test_configloader.py
index faa2f61..f0e606a 100644
--- a/tests/unit/test_configloader.py
+++ b/tests/unit/test_configloader.py
@@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
+import textwrap
from tests.base import ZuulTestCase
@@ -186,3 +187,40 @@
project2_config.pipelines['check'].job_list.jobs)
self.assertTrue('project2-job' in
project2_config.pipelines['check'].job_list.jobs)
+
+
+class TestSplitConfig(ZuulTestCase):
+ tenant_config_file = 'config/split-config/main.yaml'
+
+ def setup_config(self):
+ super(TestSplitConfig, self).setup_config()
+
+ def test_split_config(self):
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ self.assertIn('project-test1', tenant.layout.jobs)
+ project_config = tenant.layout.project_configs.get(
+ 'review.example.com/org/project')
+ self.assertIn('project-test1',
+ project_config.pipelines['check'].job_list.jobs)
+ project1_config = tenant.layout.project_configs.get(
+ 'review.example.com/org/project1')
+ self.assertIn('project1-project2-integration',
+ project1_config.pipelines['check'].job_list.jobs)
+
+ def test_dynamic_split_config(self):
+ in_repo_conf = textwrap.dedent(
+ """
+ - project:
+ name: org/project1
+ check:
+ jobs:
+ - project-test1
+ """)
+ file_dict = {'.zuul.d/gate.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ # project1-project2-integration test removed, only want project-test1
+ self.assertHistory([
+ dict(name='project-test1', result='SUCCESS', changes='1,1')])
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 4246206..627ebdd 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -1106,7 +1106,8 @@
job = merger.getFiles(
project.source.connection.connection_name,
project.name, 'master',
- files=['zuul.yaml', '.zuul.yaml'])
+ files=['zuul.yaml', '.zuul.yaml'],
+ dirs=['zuul.d', '.zuul.d'])
job.source_context = model.SourceContext(project, 'master',
'', True)
jobs.append(job)
@@ -1134,7 +1135,8 @@
job = merger.getFiles(
project.source.connection.connection_name,
project.name, branch,
- files=['zuul.yaml', '.zuul.yaml'])
+ files=['zuul.yaml', '.zuul.yaml'],
+ dirs=['zuul.d', '.zuul.d'])
job.source_context = model.SourceContext(
project, branch, '', False)
jobs.append(job)
@@ -1147,15 +1149,19 @@
TenantParser.log.debug("Waiting for cat job %s" % (job,))
job.wait()
loaded = False
- for fn in ['zuul.yaml', '.zuul.yaml']:
- if job.files.get(fn):
- # Don't load from more than one file in a repo-branch
- if loaded:
+ files = sorted(job.files.keys())
+ for conf_root in ['zuul.yaml', '.zuul.yaml', 'zuul.d', '.zuul.d']:
+ for fn in files:
+ fn_root = fn.split('/')[0]
+ if fn_root != conf_root or not job.files.get(fn):
+ continue
+ # Don't load from more than configuration in a repo-branch
+ if loaded and loaded != conf_root:
TenantParser.log.warning(
"Multiple configuration files in %s" %
(job.source_context,))
continue
- loaded = True
+ loaded = conf_root
job.source_context.path = fn
TenantParser.log.info(
"Loading configuration from %s" %
@@ -1328,28 +1334,50 @@
branches = project.source.getProjectBranches(project)
for branch in branches:
+ fns1 = []
+ fns2 = []
+ files_list = files.connections.get(
+ project.source.connection.connection_name, {}).get(
+ project.name, {}).get(branch, {}).keys()
+ for fn in files_list:
+ if fn.startswith("zuul.d/"):
+ fns1.append(fn)
+ if fn.startswith(".zuul.d/"):
+ fns2.append(fn)
+
+ fns = ['zuul.yaml', '.zuul.yaml'] + sorted(fns1) + sorted(fns2)
incdata = None
- for fn in ['zuul.yaml', '.zuul.yaml']:
+ loaded = None
+ for fn in fns:
data = files.getFile(project.source.connection.connection_name,
project.name, branch, fn)
if data:
- break
- if data:
- source_context = model.SourceContext(project, branch,
- fn, trusted)
- if trusted:
- incdata = TenantParser._parseConfigProjectLayout(
- data, source_context)
- else:
- incdata = TenantParser._parseUntrustedProjectLayout(
- data, source_context)
- else:
+ source_context = model.SourceContext(project, branch,
+ fn, trusted)
+ # Prevent mixing configuration source
+ conf_root = fn.split('/')[0]
+ if loaded and loaded != conf_root:
+ TenantParser.log.warning(
+ "Multiple configuration in %s" % source_context)
+ continue
+ loaded = conf_root
+
+ if trusted:
+ incdata = TenantParser._parseConfigProjectLayout(
+ data, source_context)
+ else:
+ incdata = TenantParser._parseUntrustedProjectLayout(
+ data, source_context)
+
+ config.extend(incdata)
+
+ if not loaded:
if trusted:
incdata = project.unparsed_config
else:
incdata = project.unparsed_branch_config.get(branch)
- if incdata:
- config.extend(incdata)
+ if incdata:
+ config.extend(incdata)
def createDynamicLayout(self, tenant, files,
include_config_projects=False):
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 6c390db..1ebe55d 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -641,7 +641,8 @@
task.wait()
with self.merger_lock:
files = self.merger.getFiles(args['connection'], args['project'],
- args['branch'], args['files'])
+ args['branch'], args['files'],
+ args.get('dirs', []))
result = dict(updated=True,
files=files,
zuul_url=self.zuul_url)
@@ -651,6 +652,7 @@
args = json.loads(job.arguments)
with self.merger_lock:
ret = self.merger.mergeChanges(args['items'], args.get('files'),
+ args.get('dirs', []),
args.get('repo_state'))
result = dict(merged=(ret is not None),
zuul_url=self.zuul_url)
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index 01429ce..09b09d7 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -480,7 +480,7 @@
self.log.debug("Preparing dynamic layout for: %s" % item.change)
return self._loadDynamicLayout(item)
- def scheduleMerge(self, item, files=None):
+ def scheduleMerge(self, item, files=None, dirs=None):
build_set = item.current_build_set
if not hasattr(item.change, 'branch'):
@@ -490,12 +490,12 @@
build_set.merge_state = build_set.COMPLETE
return True
- self.log.debug("Scheduling merge for item %s (files: %s)" %
- (item, files))
+ self.log.debug("Scheduling merge for item %s (files: %s, dirs: %s)" %
+ (item, files, dirs))
build_set = item.current_build_set
build_set.merge_state = build_set.PENDING
self.sched.merger.mergeChanges(build_set.merger_items,
- item.current_build_set, files,
+ item.current_build_set, files, dirs,
precedence=self.pipeline.precedence)
return False
@@ -506,7 +506,9 @@
if not build_set.ref:
build_set.setConfiguration()
if build_set.merge_state == build_set.NEW:
- return self.scheduleMerge(item, ['zuul.yaml', '.zuul.yaml'])
+ return self.scheduleMerge(item,
+ files=['zuul.yaml', '.zuul.yaml'],
+ dirs=['zuul.d', '.zuul.d'])
if build_set.merge_state == build_set.PENDING:
return False
if build_set.unable_to_merge:
diff --git a/zuul/merger/client.py b/zuul/merger/client.py
index e92d9fd..e354d5d 100644
--- a/zuul/merger/client.py
+++ b/zuul/merger/client.py
@@ -108,19 +108,21 @@
timeout=300)
return job
- def mergeChanges(self, items, build_set, files=None, repo_state=None,
- precedence=zuul.model.PRECEDENCE_NORMAL):
+ def mergeChanges(self, items, build_set, files=None, dirs=None,
+ repo_state=None, precedence=zuul.model.PRECEDENCE_NORMAL):
data = dict(items=items,
files=files,
+ dirs=dirs,
repo_state=repo_state)
self.submitJob('merger:merge', data, build_set, precedence)
- def getFiles(self, connection_name, project_name, branch, files,
+ def getFiles(self, connection_name, project_name, branch, files, dirs=[],
precedence=zuul.model.PRECEDENCE_HIGH):
data = dict(connection=connection_name,
project=project_name,
branch=branch,
- files=files)
+ files=files,
+ dirs=dirs)
job = self.submitJob('merger:cat', data, None, precedence)
return job
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index 2ac0de8..93340fa 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -254,7 +254,7 @@
origin.fetch()
origin.fetch(tags=True)
- def getFiles(self, files, branch=None, commit=None):
+ def getFiles(self, files, dirs=[], branch=None, commit=None):
ret = {}
repo = self.createRepoObject()
if branch:
@@ -266,6 +266,14 @@
ret[fn] = tree[fn].data_stream.read().decode('utf8')
else:
ret[fn] = None
+ if dirs:
+ for dn in dirs:
+ if dn not in tree:
+ continue
+ for blob in tree[dn].traverse():
+ if blob.path.endswith(".yaml"):
+ ret[blob.path] = blob.data_stream.read().decode(
+ 'utf-8')
return ret
def deleteRemote(self, remote):
@@ -452,7 +460,7 @@
return None
return commit
- def mergeChanges(self, items, files=None, repo_state=None):
+ def mergeChanges(self, items, files=None, dirs=None, repo_state=None):
# connection+project+branch -> commit
recent = {}
commit = None
@@ -470,9 +478,9 @@
commit = self._mergeItem(item, recent, repo_state)
if not commit:
return None
- if files:
+ if files or dirs:
repo = self.getRepo(item['connection'], item['project'])
- repo_files = repo.getFiles(files, commit=commit)
+ repo_files = repo.getFiles(files, dirs, commit=commit)
read_files.append(dict(
connection=item['connection'],
project=item['project'],
@@ -483,6 +491,6 @@
ret_recent[k] = v.hexsha
return commit.hexsha, read_files, repo_state, ret_recent
- def getFiles(self, connection_name, project_name, branch, files):
+ def getFiles(self, connection_name, project_name, branch, files, dirs=[]):
repo = self.getRepo(connection_name, project_name)
- return repo.getFiles(files, branch=branch)
+ return repo.getFiles(files, dirs, branch=branch)
diff --git a/zuul/merger/server.py b/zuul/merger/server.py
index cbc4cb8..555a4bc 100644
--- a/zuul/merger/server.py
+++ b/zuul/merger/server.py
@@ -94,8 +94,9 @@
def merge(self, job):
args = json.loads(job.arguments)
- ret = self.merger.mergeChanges(args['items'], args.get('files'),
- args.get('repo_state'))
+ ret = self.merger.mergeChanges(
+ args['items'], args.get('files'),
+ args.get('dirs'), args.get('repo_state'))
result = dict(merged=(ret is not None),
zuul_url=self.zuul_url)
if ret is None:
@@ -109,7 +110,8 @@
args = json.loads(job.arguments)
self.merger.updateRepo(args['connection'], args['project'])
files = self.merger.getFiles(args['connection'], args['project'],
- args['branch'], args['files'])
+ args['branch'], args['files'],
+ args.get('dirs'))
result = dict(updated=True,
files=files,
zuul_url=self.zuul_url)
diff --git a/zuul/model.py b/zuul/model.py
index 436a9c8..f901c55 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -1848,7 +1848,9 @@
return set()
def updatesConfig(self):
- if 'zuul.yaml' in self.files or '.zuul.yaml' in self.files:
+ if 'zuul.yaml' in self.files or '.zuul.yaml' in self.files or \
+ [True for fn in self.files if fn.startswith("zuul.d/") or
+ fn.startswith(".zuul.d/")]:
return True
return False