Add dynamic reconfiguration
If a change alters .zuul.yaml in a repo that is permitted to use in-repo
configuration, create a shadow configuration layout specifically for that
and any following changes with the new configuration in place.
Such configuration changes extend only to altering jobs and job trees.
More substantial changes such as altering pipelines will be ignored. This
only applies to "project" repos (ie, the repositories under test which may
incidentally have .zuul.yaml files) rather than "config" repos (repositories
specifically designed to hold Zuul configuration in zuul.yaml files). This
is to avoid the situation where a user might propose a change to a config
repository (and Zuul would therefore run) that would perform actions that
the gatekeepers of that repository would not normally permit.
This change also corrects an issue with job inheritance in that the Job
instances attached to the project pipeline job trees (ie, those that
represent the job as invoked in the specific pipeline configuration for
a project) were inheriting attributes at configuration time rather than
when job trees are frozen when a change is enqueued. This could mean that
they would inherit attributes from the wrong variant of a job.
Change-Id: If3cd47094e6c6914abf0ffaeca45997c132b8e32
diff --git a/tests/base.py b/tests/base.py
index 6321fe9..a75d36b 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -108,7 +108,7 @@
'VRFY': ('Verified', -2, 2)}
def __init__(self, gerrit, number, project, branch, subject,
- status='NEW', upstream_root=None):
+ status='NEW', upstream_root=None, files={}):
self.gerrit = gerrit
self.reported = 0
self.queried = 0
@@ -142,11 +142,11 @@
'url': 'https://hostname/%s' % number}
self.upstream_root = upstream_root
- self.addPatchset()
+ self.addPatchset(files=files)
self.data['submitRecords'] = self.getSubmitRecords()
self.open = status == 'NEW'
- def add_fake_change_to_repo(self, msg, fn, large):
+ def addFakeChangeToRepo(self, msg, files, large):
path = os.path.join(self.upstream_root, self.project)
repo = git.Repo(path)
ref = ChangeReference.create(repo, '1/%s/%s' % (self.number,
@@ -158,12 +158,11 @@
path = os.path.join(self.upstream_root, self.project)
if not large:
- fn = os.path.join(path, fn)
- f = open(fn, 'w')
- f.write("test %s %s %s\n" %
- (self.branch, self.number, self.latest_patchset))
- f.close()
- repo.index.add([fn])
+ for fn, content in files.items():
+ fn = os.path.join(path, fn)
+ with open(fn, 'w') as f:
+ f.write(content)
+ repo.index.add([fn])
else:
for fni in range(100):
fn = os.path.join(path, str(fni))
@@ -180,19 +179,20 @@
repo.heads['master'].checkout()
return r
- def addPatchset(self, files=[], large=False):
+ def addPatchset(self, files=None, large=False):
self.latest_patchset += 1
- if files:
- fn = files[0]
- else:
+ if not files:
fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
+ data = ("test %s %s %s\n" %
+ (self.branch, self.number, self.latest_patchset))
+ files = {fn: data}
msg = self.subject + '-' + str(self.latest_patchset)
- c = self.add_fake_change_to_repo(msg, fn, large)
+ c = self.addFakeChangeToRepo(msg, files, large)
ps_files = [{'file': '/COMMIT_MSG',
'type': 'ADDED'},
{'file': 'README',
'type': 'MODIFIED'}]
- for f in files:
+ for f in files.keys():
ps_files.append({'file': f, 'type': 'ADDED'})
d = {'approvals': [],
'createdOn': time.time(),
@@ -400,11 +400,12 @@
self.queries = []
self.upstream_root = upstream_root
- def addFakeChange(self, project, branch, subject, status='NEW'):
+ def addFakeChange(self, project, branch, subject, status='NEW',
+ files=None):
self.change_number += 1
c = FakeChange(self, self.change_number, project, branch, subject,
upstream_root=self.upstream_root,
- status=status)
+ status=status, files=files)
self.changes[self.change_number] = c
return c
@@ -937,9 +938,8 @@
self.config.set('zuul', 'state_dir', self.state_root)
# For each project in config:
- self.init_repo("org/project")
- self.init_repo("org/project1")
- self.init_repo("org/project2")
+ # TODOv3(jeblair): remove these and replace with new git
+ # filesystem fixtures
self.init_repo("org/project3")
self.init_repo("org/project4")
self.init_repo("org/project5")
@@ -1107,13 +1107,12 @@
'git')
if os.path.exists(git_path):
for reponame in os.listdir(git_path):
- self.copyDirToRepo(reponame,
+ project = reponame.replace('_', '/')
+ self.copyDirToRepo(project,
os.path.join(git_path, reponame))
def copyDirToRepo(self, project, source_path):
- repo_path = os.path.join(self.upstream_root, project)
- if not os.path.exists(repo_path):
- self.init_repo(project)
+ self.init_repo(project)
files = {}
for (dirpath, dirnames, filenames) in os.walk(source_path):
@@ -1126,7 +1125,7 @@
content = f.read()
files[relative_filepath] = content
self.addCommitToRepo(project, 'add content from fixture',
- files, branch='master')
+ files, branch='master', tag='init')
def setup_repos(self):
"""Subclasses can override to manipulate repos before tests"""
@@ -1176,21 +1175,13 @@
config_writer.set_value('user', 'email', 'user@example.com')
config_writer.set_value('user', 'name', 'User Name')
- fn = os.path.join(path, 'README')
- f = open(fn, 'w')
- f.write("test\n")
- f.close()
- repo.index.add([fn])
repo.index.commit('initial commit')
master = repo.create_head('master')
- repo.create_tag('init')
repo.head.reference = master
zuul.merger.merger.reset_repo_to_head(repo)
repo.git.clean('-x', '-f', '-d')
- self.create_branch(project, 'mp')
-
def create_branch(self, project, branch):
path = os.path.join(self.upstream_root, project)
repo = git.Repo.init(path)
@@ -1452,7 +1443,8 @@
f.close()
self.config.set('zuul', 'tenant_config', f.name)
- def addCommitToRepo(self, project, message, files, branch='master'):
+ def addCommitToRepo(self, project, message, files,
+ branch='master', tag=None):
path = os.path.join(self.upstream_root, project)
repo = git.Repo(path)
repo.head.reference = branch
@@ -1467,3 +1459,5 @@
repo.head.reference = branch
repo.git.clean('-x', '-f', '-d')
repo.heads[branch].checkout()
+ if tag:
+ repo.create_tag(tag)
diff --git a/tests/fixtures/config/in-repo/git/org_project/.zuul.yaml b/tests/fixtures/config/in-repo/git/org_project/.zuul.yaml
new file mode 100644
index 0000000..d6f083d
--- /dev/null
+++ b/tests/fixtures/config/in-repo/git/org_project/.zuul.yaml
@@ -0,0 +1,8 @@
+- job:
+ name: project-test1
+
+- project:
+ name: org/project
+ tenant-one-gate:
+ jobs:
+ - project-test1
diff --git a/tests/fixtures/config/in-repo/git/org_project/README b/tests/fixtures/config/in-repo/git/org_project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/in-repo/git/org_project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/multi-tenant/git/org_project1/README b/tests/fixtures/config/multi-tenant/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/multi-tenant/git/org_project1/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/multi-tenant/git/org_project2/README b/tests/fixtures/config/multi-tenant/git/org_project2/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/multi-tenant/git/org_project2/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/project-template/git/org_project/README b/tests/fixtures/config/project-template/git/org_project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/project-template/git/org_project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/test_model.py b/tests/test_model.py
index 145c119..e22351b 100644
--- a/tests/test_model.py
+++ b/tests/test_model.py
@@ -51,6 +51,11 @@
def test_job_inheritance(self):
layout = model.Layout()
+
+ pipeline = model.Pipeline('gate', layout)
+ layout.addPipeline(pipeline)
+ queue = model.ChangeQueue(pipeline)
+
base = configloader.JobParser.fromYaml(layout, {
'name': 'base',
'timeout': 30,
@@ -71,17 +76,21 @@
})
layout.addJob(python27diablo)
- pipeline = model.Pipeline('gate', layout)
- layout.addPipeline(pipeline)
- queue = model.ChangeQueue(pipeline)
+ project_config = configloader.ProjectParser.fromYaml(layout, {
+ 'name': 'project',
+ 'gate': {
+ 'jobs': [
+ 'python27'
+ ]
+ }
+ })
+ layout.addProjectConfig(project_config, update_pipeline=False)
project = model.Project('project')
- tree = pipeline.addProject(project)
- tree.addJob(layout.getJob('python27'))
-
change = model.Change(project)
change.branch = 'master'
item = queue.enqueueChange(change)
+ item.current_build_set.layout = layout
self.assertTrue(base.changeMatches(change))
self.assertTrue(python27.changeMatches(change))
@@ -94,6 +103,8 @@
self.assertEqual(job.timeout, 40)
change.branch = 'stable/diablo'
+ item = queue.enqueueChange(change)
+ item.current_build_set.layout = layout
self.assertTrue(base.changeMatches(change))
self.assertTrue(python27.changeMatches(change))
@@ -105,6 +116,73 @@
self.assertEqual(job.name, 'python27')
self.assertEqual(job.timeout, 50)
+ def test_job_inheritance_job_tree(self):
+ layout = model.Layout()
+
+ pipeline = model.Pipeline('gate', layout)
+ layout.addPipeline(pipeline)
+ queue = model.ChangeQueue(pipeline)
+
+ base = configloader.JobParser.fromYaml(layout, {
+ 'name': 'base',
+ 'timeout': 30,
+ })
+ layout.addJob(base)
+ python27 = configloader.JobParser.fromYaml(layout, {
+ 'name': 'python27',
+ 'parent': 'base',
+ 'timeout': 40,
+ })
+ layout.addJob(python27)
+ python27diablo = configloader.JobParser.fromYaml(layout, {
+ 'name': 'python27',
+ 'branches': [
+ 'stable/diablo'
+ ],
+ 'timeout': 50,
+ })
+ layout.addJob(python27diablo)
+
+ project_config = configloader.ProjectParser.fromYaml(layout, {
+ 'name': 'project',
+ 'gate': {
+ 'jobs': [
+ {'python27': {'timeout': 70}}
+ ]
+ }
+ })
+ layout.addProjectConfig(project_config, update_pipeline=False)
+
+ project = model.Project('project')
+ change = model.Change(project)
+ change.branch = 'master'
+ item = queue.enqueueChange(change)
+ item.current_build_set.layout = layout
+
+ self.assertTrue(base.changeMatches(change))
+ self.assertTrue(python27.changeMatches(change))
+ self.assertFalse(python27diablo.changeMatches(change))
+
+ item.freezeJobTree()
+ self.assertEqual(len(item.getJobs()), 1)
+ job = item.getJobs()[0]
+ self.assertEqual(job.name, 'python27')
+ self.assertEqual(job.timeout, 70)
+
+ change.branch = 'stable/diablo'
+ item = queue.enqueueChange(change)
+ item.current_build_set.layout = layout
+
+ self.assertTrue(base.changeMatches(change))
+ self.assertTrue(python27.changeMatches(change))
+ self.assertTrue(python27diablo.changeMatches(change))
+
+ item.freezeJobTree()
+ self.assertEqual(len(item.getJobs()), 1)
+ job = item.getJobs()[0]
+ self.assertEqual(job.name, 'python27')
+ self.assertEqual(job.timeout, 70)
+
class TestJobTimeData(BaseTestCase):
def setUp(self):
diff --git a/tests/test_v3.py b/tests/test_v3.py
index 8874015..50e20c8 100644
--- a/tests/test_v3.py
+++ b/tests/test_v3.py
@@ -74,22 +74,6 @@
tenant_config_file = 'config/in-repo/main.yaml'
- def setup_repos(self):
- in_repo_conf = textwrap.dedent(
- """
- - job:
- name: project-test1
-
- - project:
- name: org/project
- tenant-one-gate:
- jobs:
- - project-test1
- """)
-
- self.addCommitToRepo('org/project', 'add zuul conf',
- {'.zuul.yaml': in_repo_conf})
-
def test_in_repo_config(self):
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('CRVW', 2)
@@ -103,6 +87,32 @@
self.assertIn('tenant-one-gate', A.messages[1],
"A should transit tenant-one gate")
+ def test_dynamic_config(self):
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ name: project-test2
+
+ - project:
+ name: org/project
+ tenant-one-gate:
+ jobs:
+ - project-test2
+ """)
+
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files={'.zuul.yaml': in_repo_conf})
+ A.addApproval('CRVW', 2)
+ self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+ self.waitUntilSettled()
+ self.assertEqual(self.getJobFromHistory('project-test2').result,
+ 'SUCCESS')
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2,
+ "A should report start and success")
+ self.assertIn('tenant-one-gate', A.messages[1],
+ "A should transit tenant-one gate")
+
class TestProjectTemplate(ZuulTestCase):
tenant_config_file = 'config/project-template/main.yaml'