Inherit playbooks and modify job variance
An earlier change dealt with inheritance for pre and post playbooks;
they are nested so that parent job pre and post playbooks run first
and last respectively.
As for the actual playbook, since it's implied by the job name, it's
not clear whether it should be overidden or not. We could drop that
and say that if you specify a 'run' attribute, it means you want to
set the playbook for a job, but if you omit it, you want to use the
parent's playbook.
However, we could keep the implied playbook behavior by making the
'run' attribute a list and adding a playbook context to the list each
time a job is inherited. Then the launcher can walk the list in order
and the first playbook it finds, it runs.
This is what is implemented here.
However, we need to restrict playbooks or other execution-related
job attributes from being overidden by out-of-repo variants (such
as the implicit variant which is created by every entry in a
project-pipeline). To do this, we make more of a distinction
between inheritance and variance, implementing each with its own
method on Job. This way we can better control when certain
attributes are allowed to be set. The 'final' job attribute is
added to indicate that a job should not accept any further
modifications to execution-related attributes.
The attribute storage in Job is altered so that each Job object
explicitly stores whether an attribute was set on it. This makes
it easier to start with a job and apply only the specified
attributes of each variant in turn. Default values are still
handled.
Essentially, each "job" appearance in the configuration will
create a new Job entry with exactly those attributes (with the
exception that a job where "parent" is set will first copy
attributes which are explicitly set on its parent).
When a job is frozen after an item is enqueued, the first
matching job is copied, and each subsequent matching job is
applied as a varient. When that is completed, if the job has
un-inheritable auth information, it is set as final, and then the
project-pipeline variant is applied.
New tests are added to exercise the new methods on Job.
Change-Id: Iaf6d661a7bd0085e55bc301f83fe158fd0a70166
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index b7dc706..b54eb5f 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -27,6 +27,11 @@
class TestJob(BaseTestCase):
+ def setUp(self):
+ super(TestJob, self).setUp()
+ self.project = model.Project('project', None)
+ self.context = model.SourceContext(self.project, 'master', True)
+
@property
def job(self):
layout = model.Layout()
@@ -54,6 +59,81 @@
self.assertIsNotNone(self.job.voting)
def test_job_inheritance(self):
+ # This is standard job inheritance.
+
+ base_pre = model.PlaybookContext(self.context, 'base-pre')
+ base_run = model.PlaybookContext(self.context, 'base-run')
+ base_post = model.PlaybookContext(self.context, 'base-post')
+
+ base = model.Job('base')
+ base.timeout = 30
+ base.pre_run = [base_pre]
+ base.run = [base_run]
+ base.post_run = [base_post]
+ base.auth = dict(foo='bar', inherit=False)
+
+ py27 = model.Job('py27')
+ self.assertEqual(None, py27.timeout)
+ py27.inheritFrom(base)
+ self.assertEqual(30, py27.timeout)
+ self.assertEqual(['base-pre'],
+ [x.path for x in py27.pre_run])
+ self.assertEqual(['base-run'],
+ [x.path for x in py27.run])
+ self.assertEqual(['base-post'],
+ [x.path for x in py27.post_run])
+ self.assertEqual({}, py27.auth)
+
+ def test_job_variants(self):
+ # This simulates freezing a job.
+
+ py27_pre = model.PlaybookContext(self.context, 'py27-pre')
+ py27_run = model.PlaybookContext(self.context, 'py27-run')
+ py27_post = model.PlaybookContext(self.context, 'py27-post')
+
+ py27 = model.Job('py27')
+ py27.timeout = 30
+ py27.pre_run = [py27_pre]
+ py27.run = [py27_run]
+ py27.post_run = [py27_post]
+ auth = dict(foo='bar', inherit=False)
+ py27.auth = auth
+
+ job = py27.copy()
+ self.assertEqual(30, job.timeout)
+
+ # Apply the diablo variant
+ diablo = model.Job('py27')
+ diablo.timeout = 40
+ job.applyVariant(diablo)
+
+ self.assertEqual(40, job.timeout)
+ self.assertEqual(['py27-pre'],
+ [x.path for x in job.pre_run])
+ self.assertEqual(['py27-run'],
+ [x.path for x in job.run])
+ self.assertEqual(['py27-post'],
+ [x.path for x in job.post_run])
+ self.assertEqual(auth, job.auth)
+
+ # Set the job to final for the following checks
+ job.final = True
+ self.assertTrue(job.voting)
+
+ good_final = model.Job('py27')
+ good_final.voting = False
+ job.applyVariant(good_final)
+ self.assertFalse(job.voting)
+
+ bad_final = model.Job('py27')
+ bad_final.timeout = 600
+ with testtools.ExpectedException(
+ Exception,
+ "Unable to modify final job"):
+ job.applyVariant(bad_final)
+
+ def test_job_inheritance_configloader(self):
+ # TODO(jeblair): move this to a configloader test
layout = model.Layout()
pipeline = model.Pipeline('gate', layout)
@@ -66,6 +146,8 @@
'_source_context': context,
'name': 'base',
'timeout': 30,
+ 'pre-run': 'base-pre',
+ 'post-run': 'base-post',
'nodes': [{
'name': 'controller',
'image': 'base',
@@ -76,6 +158,8 @@
'_source_context': context,
'name': 'python27',
'parent': 'base',
+ 'pre-run': 'py27-pre',
+ 'post-run': 'py27-post',
'nodes': [{
'name': 'controller',
'image': 'new',
@@ -89,6 +173,9 @@
'branches': [
'stable/diablo'
],
+ 'pre-run': 'py27-diablo-pre',
+ 'run': 'py27-diablo',
+ 'post-run': 'py27-diablo-post',
'nodes': [{
'name': 'controller',
'image': 'old',
@@ -97,6 +184,17 @@
})
layout.addJob(python27diablo)
+ python27essex = configloader.JobParser.fromYaml(layout, {
+ '_source_context': context,
+ 'name': 'python27',
+ 'branches': [
+ 'stable/essex'
+ ],
+ 'pre-run': 'py27-essex-pre',
+ 'post-run': 'py27-essex-post',
+ })
+ layout.addJob(python27essex)
+
project_config = configloader.ProjectParser.fromYaml(layout, {
'_source_context': context,
'name': 'project',
@@ -117,6 +215,7 @@
self.assertTrue(base.changeMatches(change))
self.assertTrue(python27.changeMatches(change))
self.assertFalse(python27diablo.changeMatches(change))
+ self.assertFalse(python27essex.changeMatches(change))
item.freezeJobTree()
self.assertEqual(len(item.getJobs()), 1)
@@ -126,6 +225,15 @@
nodes = job.nodeset.getNodes()
self.assertEqual(len(nodes), 1)
self.assertEqual(nodes[0].image, 'new')
+ self.assertEqual([x.path for x in job.pre_run],
+ ['playbooks/base-pre',
+ 'playbooks/py27-pre'])
+ self.assertEqual([x.path for x in job.post_run],
+ ['playbooks/py27-post',
+ 'playbooks/base-post'])
+ self.assertEqual([x.path for x in job.run],
+ ['playbooks/python27',
+ 'playbooks/base'])
# Test diablo
change.branch = 'stable/diablo'
@@ -135,6 +243,7 @@
self.assertTrue(base.changeMatches(change))
self.assertTrue(python27.changeMatches(change))
self.assertTrue(python27diablo.changeMatches(change))
+ self.assertFalse(python27essex.changeMatches(change))
item.freezeJobTree()
self.assertEqual(len(item.getJobs()), 1)
@@ -144,6 +253,42 @@
nodes = job.nodeset.getNodes()
self.assertEqual(len(nodes), 1)
self.assertEqual(nodes[0].image, 'old')
+ self.assertEqual([x.path for x in job.pre_run],
+ ['playbooks/base-pre',
+ 'playbooks/py27-pre',
+ 'playbooks/py27-diablo-pre'])
+ self.assertEqual([x.path for x in job.post_run],
+ ['playbooks/py27-diablo-post',
+ 'playbooks/py27-post',
+ 'playbooks/base-post'])
+ self.assertEqual([x.path for x in job.run],
+ ['playbooks/py27-diablo']),
+
+ # Test essex
+ change.branch = 'stable/essex'
+ item = queue.enqueueChange(change)
+ item.current_build_set.layout = layout
+
+ self.assertTrue(base.changeMatches(change))
+ self.assertTrue(python27.changeMatches(change))
+ self.assertFalse(python27diablo.changeMatches(change))
+ self.assertTrue(python27essex.changeMatches(change))
+
+ item.freezeJobTree()
+ self.assertEqual(len(item.getJobs()), 1)
+ job = item.getJobs()[0]
+ self.assertEqual(job.name, 'python27')
+ self.assertEqual([x.path for x in job.pre_run],
+ ['playbooks/base-pre',
+ 'playbooks/py27-pre',
+ 'playbooks/py27-essex-pre'])
+ self.assertEqual([x.path for x in job.post_run],
+ ['playbooks/py27-essex-post',
+ 'playbooks/py27-post',
+ 'playbooks/base-post'])
+ self.assertEqual([x.path for x in job.run],
+ ['playbooks/python27',
+ 'playbooks/base'])
def test_job_auth_inheritance(self):
layout = model.Layout()