blob: e7e53c4d5af51c5915938edc948332e34138f007 [file] [log] [blame]
Maru Newby3fe5f852015-01-13 04:22:14 +00001# Copyright 2015 Red Hat, Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
Joshua Hesketh0aa7e8b2016-07-14 00:12:25 +100015
James E. Blairce8a2132016-05-19 15:21:52 -070016import os
17import random
18
19import fixtures
James E. Blair4317e9f2016-07-15 10:05:47 -070020import testtools
James E. Blairce8a2132016-05-19 15:21:52 -070021
Maru Newby3fe5f852015-01-13 04:22:14 +000022from zuul import model
James E. Blair83005782015-12-11 14:46:03 -080023from zuul import configloader
James E. Blairbf1a4f22017-03-17 10:59:37 -070024from zuul.lib import encryption
Clint Byrum88fdfde2017-03-28 16:22:42 -070025from zuul.lib import yamlutil as yaml
Maru Newby3fe5f852015-01-13 04:22:14 +000026
James E. Blair18f86a32017-03-15 14:43:26 -070027from tests.base import BaseTestCase, FIXTURE_DIR
Maru Newby3fe5f852015-01-13 04:22:14 +000028
29
James E. Blair0a899752017-03-29 13:22:16 -070030class Dummy(object):
31 def __init__(self, **kw):
32 for k, v in kw.items():
33 setattr(self, k, v)
James E. Blairb3f5db12017-03-17 12:57:39 -070034
35
Maru Newby3fe5f852015-01-13 04:22:14 +000036class TestJob(BaseTestCase):
James E. Blaira7f51ca2017-02-07 16:01:26 -080037 def setUp(self):
38 super(TestJob, self).setUp()
James E. Blair0a899752017-03-29 13:22:16 -070039 self.connection = Dummy(connection_name='dummy_connection')
40 self.source = Dummy(canonical_hostname='git.example.com',
James E. Blair0a899752017-03-29 13:22:16 -070041 connection=self.connection)
James E. Blairb3f5db12017-03-17 12:57:39 -070042 self.tenant = model.Tenant('tenant')
43 self.layout = model.Layout()
James E. Blair0a899752017-03-29 13:22:16 -070044 self.project = model.Project('project', self.source)
James E. Blair109da3f2017-04-04 14:39:43 -070045 self.tenant.addUntrustedProject(self.project)
James E. Blairb3f5db12017-03-17 12:57:39 -070046 self.pipeline = model.Pipeline('gate', self.layout)
47 self.layout.addPipeline(self.pipeline)
48 self.queue = model.ChangeQueue(self.pipeline)
49
James E. Blair18f86a32017-03-15 14:43:26 -070050 private_key_file = os.path.join(FIXTURE_DIR, 'private.pem')
51 with open(private_key_file, "rb") as f:
James E. Blairbf1a4f22017-03-17 10:59:37 -070052 self.project.private_key, self.project.public_key = \
53 encryption.deserialize_rsa_keypair(f.read())
James E. Blair6f140c72017-03-03 10:32:07 -080054 self.context = model.SourceContext(self.project, 'master',
55 'test', True)
James E. Blairec7ff302017-03-04 07:31:32 -080056 self.start_mark = yaml.Mark('name', 0, 0, 0, '', 0)
James E. Blaira7f51ca2017-02-07 16:01:26 -080057
Maru Newby3fe5f852015-01-13 04:22:14 +000058 @property
59 def job(self):
James E. Blair5ac93842017-01-20 06:47:34 -080060 tenant = model.Tenant('tenant')
James E. Blair83005782015-12-11 14:46:03 -080061 layout = model.Layout()
James E. Blair5ac93842017-01-20 06:47:34 -080062 job = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -080063 '_source_context': self.context,
64 '_start_mark': self.start_mark,
James E. Blair83005782015-12-11 14:46:03 -080065 'name': 'job',
66 'irrelevant-files': [
67 '^docs/.*$'
68 ]})
Maru Newby3fe5f852015-01-13 04:22:14 +000069 return job
70
71 def test_change_matches_returns_false_for_matched_skip_if(self):
72 change = model.Change('project')
Alexander Evseevdbe6fab2015-11-19 12:46:34 +030073 change.files = ['/COMMIT_MSG', 'docs/foo']
Maru Newby3fe5f852015-01-13 04:22:14 +000074 self.assertFalse(self.job.changeMatches(change))
75
Jan Hruban570d01c2016-03-10 21:51:32 +010076 def test_change_matches_returns_false_for_single_matched_skip_if(self):
77 change = model.Change('project')
78 change.files = ['docs/foo']
79 self.assertFalse(self.job.changeMatches(change))
80
Maru Newby3fe5f852015-01-13 04:22:14 +000081 def test_change_matches_returns_true_for_unmatched_skip_if(self):
82 change = model.Change('project')
Alexander Evseevdbe6fab2015-11-19 12:46:34 +030083 change.files = ['/COMMIT_MSG', 'foo']
Maru Newby3fe5f852015-01-13 04:22:14 +000084 self.assertTrue(self.job.changeMatches(change))
85
Jan Hruban570d01c2016-03-10 21:51:32 +010086 def test_change_matches_returns_true_for_single_unmatched_skip_if(self):
87 change = model.Change('project')
88 change.files = ['foo']
89 self.assertTrue(self.job.changeMatches(change))
90
Maru Newby79427a42015-02-17 17:54:45 +000091 def test_job_sets_defaults_for_boolean_attributes(self):
James E. Blair83005782015-12-11 14:46:03 -080092 self.assertIsNotNone(self.job.voting)
93
94 def test_job_inheritance(self):
James E. Blaira7f51ca2017-02-07 16:01:26 -080095 # This is standard job inheritance.
96
97 base_pre = model.PlaybookContext(self.context, 'base-pre')
98 base_run = model.PlaybookContext(self.context, 'base-run')
99 base_post = model.PlaybookContext(self.context, 'base-post')
100
101 base = model.Job('base')
102 base.timeout = 30
103 base.pre_run = [base_pre]
104 base.run = [base_run]
105 base.post_run = [base_post]
James E. Blair8525e2b2017-03-15 14:05:47 -0700106 base.auth = model.AuthContext()
James E. Blaira7f51ca2017-02-07 16:01:26 -0800107
108 py27 = model.Job('py27')
Monty Taylor38b553a2017-06-05 13:06:10 -0500109 self.assertIsNone(py27.timeout)
James E. Blaira7f51ca2017-02-07 16:01:26 -0800110 py27.inheritFrom(base)
111 self.assertEqual(30, py27.timeout)
112 self.assertEqual(['base-pre'],
113 [x.path for x in py27.pre_run])
114 self.assertEqual(['base-run'],
115 [x.path for x in py27.run])
116 self.assertEqual(['base-post'],
117 [x.path for x in py27.post_run])
Monty Taylor38b553a2017-06-05 13:06:10 -0500118 self.assertIsNone(py27.auth)
James E. Blaira7f51ca2017-02-07 16:01:26 -0800119
120 def test_job_variants(self):
121 # This simulates freezing a job.
122
123 py27_pre = model.PlaybookContext(self.context, 'py27-pre')
124 py27_run = model.PlaybookContext(self.context, 'py27-run')
125 py27_post = model.PlaybookContext(self.context, 'py27-post')
126
127 py27 = model.Job('py27')
128 py27.timeout = 30
129 py27.pre_run = [py27_pre]
130 py27.run = [py27_run]
131 py27.post_run = [py27_post]
James E. Blair8525e2b2017-03-15 14:05:47 -0700132 auth = model.AuthContext()
133 auth.secrets.append('foo')
James E. Blaira7f51ca2017-02-07 16:01:26 -0800134 py27.auth = auth
135
136 job = py27.copy()
137 self.assertEqual(30, job.timeout)
138
139 # Apply the diablo variant
140 diablo = model.Job('py27')
141 diablo.timeout = 40
142 job.applyVariant(diablo)
143
144 self.assertEqual(40, job.timeout)
145 self.assertEqual(['py27-pre'],
146 [x.path for x in job.pre_run])
147 self.assertEqual(['py27-run'],
148 [x.path for x in job.run])
149 self.assertEqual(['py27-post'],
150 [x.path for x in job.post_run])
151 self.assertEqual(auth, job.auth)
152
153 # Set the job to final for the following checks
154 job.final = True
155 self.assertTrue(job.voting)
156
157 good_final = model.Job('py27')
158 good_final.voting = False
159 job.applyVariant(good_final)
160 self.assertFalse(job.voting)
161
162 bad_final = model.Job('py27')
163 bad_final.timeout = 600
164 with testtools.ExpectedException(
165 Exception,
166 "Unable to modify final job"):
167 job.applyVariant(bad_final)
168
169 def test_job_inheritance_configloader(self):
170 # TODO(jeblair): move this to a configloader test
James E. Blair5ac93842017-01-20 06:47:34 -0800171 tenant = model.Tenant('tenant')
James E. Blair83005782015-12-11 14:46:03 -0800172 layout = model.Layout()
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700173
174 pipeline = model.Pipeline('gate', layout)
175 layout.addPipeline(pipeline)
176 queue = model.ChangeQueue(pipeline)
James E. Blair0a899752017-03-29 13:22:16 -0700177 project = model.Project('project', self.source)
James E. Blair109da3f2017-04-04 14:39:43 -0700178 tenant.addUntrustedProject(project)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700179
James E. Blair5ac93842017-01-20 06:47:34 -0800180 base = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800181 '_source_context': self.context,
182 '_start_mark': self.start_mark,
James E. Blair83005782015-12-11 14:46:03 -0800183 'name': 'base',
184 'timeout': 30,
James E. Blaira7f51ca2017-02-07 16:01:26 -0800185 'pre-run': 'base-pre',
186 'post-run': 'base-post',
James E. Blair1774dd52017-02-03 10:52:32 -0800187 'nodes': [{
188 'name': 'controller',
189 'image': 'base',
190 }],
James E. Blair83005782015-12-11 14:46:03 -0800191 })
192 layout.addJob(base)
James E. Blair5ac93842017-01-20 06:47:34 -0800193 python27 = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800194 '_source_context': self.context,
195 '_start_mark': self.start_mark,
James E. Blair83005782015-12-11 14:46:03 -0800196 'name': 'python27',
197 'parent': 'base',
James E. Blaira7f51ca2017-02-07 16:01:26 -0800198 'pre-run': 'py27-pre',
199 'post-run': 'py27-post',
James E. Blair1774dd52017-02-03 10:52:32 -0800200 'nodes': [{
201 'name': 'controller',
202 'image': 'new',
203 }],
James E. Blair83005782015-12-11 14:46:03 -0800204 'timeout': 40,
205 })
206 layout.addJob(python27)
James E. Blair5ac93842017-01-20 06:47:34 -0800207 python27diablo = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800208 '_source_context': self.context,
209 '_start_mark': self.start_mark,
James E. Blair83005782015-12-11 14:46:03 -0800210 'name': 'python27',
211 'branches': [
212 'stable/diablo'
213 ],
James E. Blaira7f51ca2017-02-07 16:01:26 -0800214 'pre-run': 'py27-diablo-pre',
215 'run': 'py27-diablo',
216 'post-run': 'py27-diablo-post',
James E. Blair1774dd52017-02-03 10:52:32 -0800217 'nodes': [{
218 'name': 'controller',
219 'image': 'old',
220 }],
James E. Blair83005782015-12-11 14:46:03 -0800221 'timeout': 50,
222 })
223 layout.addJob(python27diablo)
224
James E. Blair5ac93842017-01-20 06:47:34 -0800225 python27essex = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800226 '_source_context': self.context,
227 '_start_mark': self.start_mark,
James E. Blaira7f51ca2017-02-07 16:01:26 -0800228 'name': 'python27',
229 'branches': [
230 'stable/essex'
231 ],
232 'pre-run': 'py27-essex-pre',
233 'post-run': 'py27-essex-post',
234 })
235 layout.addJob(python27essex)
236
James E. Blairff555742017-02-19 11:34:27 -0800237 project_config = configloader.ProjectParser.fromYaml(tenant, layout, [{
James E. Blairec7ff302017-03-04 07:31:32 -0800238 '_source_context': self.context,
239 '_start_mark': self.start_mark,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700240 'name': 'project',
241 'gate': {
242 'jobs': [
243 'python27'
244 ]
245 }
James E. Blairff555742017-02-19 11:34:27 -0800246 }])
James E. Blairf59f3cf2017-02-19 14:50:26 -0800247 layout.addProjectConfig(project_config)
James E. Blair83005782015-12-11 14:46:03 -0800248
James E. Blair83005782015-12-11 14:46:03 -0800249 change = model.Change(project)
James E. Blair1774dd52017-02-03 10:52:32 -0800250 # Test master
James E. Blair83005782015-12-11 14:46:03 -0800251 change.branch = 'master'
252 item = queue.enqueueChange(change)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700253 item.current_build_set.layout = layout
James E. Blair83005782015-12-11 14:46:03 -0800254
255 self.assertTrue(base.changeMatches(change))
256 self.assertTrue(python27.changeMatches(change))
257 self.assertFalse(python27diablo.changeMatches(change))
James E. Blaira7f51ca2017-02-07 16:01:26 -0800258 self.assertFalse(python27essex.changeMatches(change))
James E. Blair83005782015-12-11 14:46:03 -0800259
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200260 item.freezeJobGraph()
James E. Blair83005782015-12-11 14:46:03 -0800261 self.assertEqual(len(item.getJobs()), 1)
262 job = item.getJobs()[0]
263 self.assertEqual(job.name, 'python27')
264 self.assertEqual(job.timeout, 40)
James E. Blair1774dd52017-02-03 10:52:32 -0800265 nodes = job.nodeset.getNodes()
266 self.assertEqual(len(nodes), 1)
267 self.assertEqual(nodes[0].image, 'new')
James E. Blaira7f51ca2017-02-07 16:01:26 -0800268 self.assertEqual([x.path for x in job.pre_run],
269 ['playbooks/base-pre',
270 'playbooks/py27-pre'])
271 self.assertEqual([x.path for x in job.post_run],
272 ['playbooks/py27-post',
273 'playbooks/base-post'])
274 self.assertEqual([x.path for x in job.run],
275 ['playbooks/python27',
276 'playbooks/base'])
James E. Blair83005782015-12-11 14:46:03 -0800277
James E. Blair1774dd52017-02-03 10:52:32 -0800278 # Test diablo
James E. Blair83005782015-12-11 14:46:03 -0800279 change.branch = 'stable/diablo'
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700280 item = queue.enqueueChange(change)
281 item.current_build_set.layout = layout
James E. Blair83005782015-12-11 14:46:03 -0800282
283 self.assertTrue(base.changeMatches(change))
284 self.assertTrue(python27.changeMatches(change))
285 self.assertTrue(python27diablo.changeMatches(change))
James E. Blaira7f51ca2017-02-07 16:01:26 -0800286 self.assertFalse(python27essex.changeMatches(change))
James E. Blair83005782015-12-11 14:46:03 -0800287
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200288 item.freezeJobGraph()
James E. Blair83005782015-12-11 14:46:03 -0800289 self.assertEqual(len(item.getJobs()), 1)
290 job = item.getJobs()[0]
291 self.assertEqual(job.name, 'python27')
292 self.assertEqual(job.timeout, 50)
James E. Blair1774dd52017-02-03 10:52:32 -0800293 nodes = job.nodeset.getNodes()
294 self.assertEqual(len(nodes), 1)
295 self.assertEqual(nodes[0].image, 'old')
James E. Blaira7f51ca2017-02-07 16:01:26 -0800296 self.assertEqual([x.path for x in job.pre_run],
297 ['playbooks/base-pre',
298 'playbooks/py27-pre',
299 'playbooks/py27-diablo-pre'])
300 self.assertEqual([x.path for x in job.post_run],
301 ['playbooks/py27-diablo-post',
302 'playbooks/py27-post',
303 'playbooks/base-post'])
304 self.assertEqual([x.path for x in job.run],
305 ['playbooks/py27-diablo']),
306
307 # Test essex
308 change.branch = 'stable/essex'
309 item = queue.enqueueChange(change)
310 item.current_build_set.layout = layout
311
312 self.assertTrue(base.changeMatches(change))
313 self.assertTrue(python27.changeMatches(change))
314 self.assertFalse(python27diablo.changeMatches(change))
315 self.assertTrue(python27essex.changeMatches(change))
316
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200317 item.freezeJobGraph()
James E. Blaira7f51ca2017-02-07 16:01:26 -0800318 self.assertEqual(len(item.getJobs()), 1)
319 job = item.getJobs()[0]
320 self.assertEqual(job.name, 'python27')
321 self.assertEqual([x.path for x in job.pre_run],
322 ['playbooks/base-pre',
323 'playbooks/py27-pre',
324 'playbooks/py27-essex-pre'])
325 self.assertEqual([x.path for x in job.post_run],
326 ['playbooks/py27-essex-post',
327 'playbooks/py27-post',
328 'playbooks/base-post'])
329 self.assertEqual([x.path for x in job.run],
330 ['playbooks/python27',
331 'playbooks/base'])
James E. Blairce8a2132016-05-19 15:21:52 -0700332
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000333 def test_job_auth_inheritance(self):
James E. Blair5ac93842017-01-20 06:47:34 -0800334 tenant = model.Tenant('tenant')
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000335 layout = model.Layout()
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000336
James E. Blair01f83b72017-03-15 13:03:40 -0700337 conf = yaml.safe_load('''
338- secret:
339 name: pypi-credentials
340 data:
341 username: test-username
James E. Blair717e8e92017-03-17 11:03:27 -0700342 password: !encrypted/pkcs1-oaep |
James E. Blair01f83b72017-03-15 13:03:40 -0700343 BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ
344 L0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4joeusC9drN3AA8a4o
345 ykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CRgd0QBMPl6VDoFgBPB8vxtJw+
346 3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzibDsSXsfJt1y+5n7yOURsC7lovMg4GF/v
347 Cl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCYceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qt
348 xhbpjTxG4U5Q/SoppOJ60WqEkQvbXs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYr
349 aI+AKYsMYx3RBlfAmCeC1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFW
350 Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd
351 +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs=
352''')[0]['secret']
353
354 conf['_source_context'] = self.context
355 conf['_start_mark'] = self.start_mark
356
357 secret = configloader.SecretParser.fromYaml(layout, conf)
358 layout.addSecret(secret)
359
James E. Blair5ac93842017-01-20 06:47:34 -0800360 base = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800361 '_source_context': self.context,
362 '_start_mark': self.start_mark,
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000363 'name': 'base',
364 'timeout': 30,
365 })
366 layout.addJob(base)
James E. Blair5ac93842017-01-20 06:47:34 -0800367 pypi_upload_without_inherit = configloader.JobParser.fromYaml(
368 tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800369 '_source_context': self.context,
370 '_start_mark': self.start_mark,
James E. Blair5ac93842017-01-20 06:47:34 -0800371 'name': 'pypi-upload-without-inherit',
372 'parent': 'base',
373 'timeout': 40,
374 'auth': {
375 'secrets': [
376 'pypi-credentials',
377 ]
378 }
379 })
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000380 layout.addJob(pypi_upload_without_inherit)
James E. Blair5ac93842017-01-20 06:47:34 -0800381 pypi_upload_with_inherit = configloader.JobParser.fromYaml(
382 tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800383 '_source_context': self.context,
384 '_start_mark': self.start_mark,
James E. Blair5ac93842017-01-20 06:47:34 -0800385 'name': 'pypi-upload-with-inherit',
386 'parent': 'base',
387 'timeout': 40,
388 'auth': {
389 'inherit': True,
390 'secrets': [
391 'pypi-credentials',
392 ]
393 }
394 })
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000395 layout.addJob(pypi_upload_with_inherit)
396 pypi_upload_with_inherit_false = configloader.JobParser.fromYaml(
James E. Blair5ac93842017-01-20 06:47:34 -0800397 tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800398 '_source_context': self.context,
399 '_start_mark': self.start_mark,
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000400 'name': 'pypi-upload-with-inherit-false',
401 'parent': 'base',
402 'timeout': 40,
403 'auth': {
404 'inherit': False,
Ricardo Carrillo Cruz12c892b2016-11-18 15:35:49 +0000405 'secrets': [
406 'pypi-credentials',
407 ]
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000408 }
409 })
410 layout.addJob(pypi_upload_with_inherit_false)
James E. Blair5ac93842017-01-20 06:47:34 -0800411 in_repo_job_without_inherit = configloader.JobParser.fromYaml(
412 tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800413 '_source_context': self.context,
414 '_start_mark': self.start_mark,
James E. Blair5ac93842017-01-20 06:47:34 -0800415 'name': 'in-repo-job-without-inherit',
416 'parent': 'pypi-upload-without-inherit',
417 })
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000418 layout.addJob(in_repo_job_without_inherit)
James E. Blair5ac93842017-01-20 06:47:34 -0800419 in_repo_job_with_inherit = configloader.JobParser.fromYaml(
420 tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800421 '_source_context': self.context,
422 '_start_mark': self.start_mark,
James E. Blair5ac93842017-01-20 06:47:34 -0800423 'name': 'in-repo-job-with-inherit',
424 'parent': 'pypi-upload-with-inherit',
425 })
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000426 layout.addJob(in_repo_job_with_inherit)
427 in_repo_job_with_inherit_false = configloader.JobParser.fromYaml(
James E. Blair5ac93842017-01-20 06:47:34 -0800428 tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800429 '_source_context': self.context,
430 '_start_mark': self.start_mark,
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000431 'name': 'in-repo-job-with-inherit-false',
432 'parent': 'pypi-upload-with-inherit-false',
433 })
434 layout.addJob(in_repo_job_with_inherit_false)
435
Monty Taylor38b553a2017-06-05 13:06:10 -0500436 self.assertIsNone(in_repo_job_without_inherit.auth)
James E. Blair8525e2b2017-03-15 14:05:47 -0700437 self.assertEqual(1, len(in_repo_job_with_inherit.auth.secrets))
438 self.assertEqual(in_repo_job_with_inherit.auth.secrets[0].name,
439 'pypi-credentials')
Monty Taylor38b553a2017-06-05 13:06:10 -0500440 self.assertIsNone(in_repo_job_with_inherit_false.auth)
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000441
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700442 def test_job_inheritance_job_tree(self):
James E. Blair5ac93842017-01-20 06:47:34 -0800443 tenant = model.Tenant('tenant')
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700444 layout = model.Layout()
James E. Blair109da3f2017-04-04 14:39:43 -0700445 tenant.addUntrustedProject(self.project)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700446
447 pipeline = model.Pipeline('gate', layout)
448 layout.addPipeline(pipeline)
449 queue = model.ChangeQueue(pipeline)
450
James E. Blair5ac93842017-01-20 06:47:34 -0800451 base = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800452 '_source_context': self.context,
453 '_start_mark': self.start_mark,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700454 'name': 'base',
455 'timeout': 30,
456 })
457 layout.addJob(base)
James E. Blair5ac93842017-01-20 06:47:34 -0800458 python27 = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800459 '_source_context': self.context,
460 '_start_mark': self.start_mark,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700461 'name': 'python27',
462 'parent': 'base',
463 'timeout': 40,
464 })
465 layout.addJob(python27)
James E. Blair5ac93842017-01-20 06:47:34 -0800466 python27diablo = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800467 '_source_context': self.context,
468 '_start_mark': self.start_mark,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700469 'name': 'python27',
470 'branches': [
471 'stable/diablo'
472 ],
473 'timeout': 50,
474 })
475 layout.addJob(python27diablo)
476
James E. Blairff555742017-02-19 11:34:27 -0800477 project_config = configloader.ProjectParser.fromYaml(tenant, layout, [{
James E. Blairec7ff302017-03-04 07:31:32 -0800478 '_source_context': self.context,
479 '_start_mark': self.start_mark,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700480 'name': 'project',
481 'gate': {
482 'jobs': [
483 {'python27': {'timeout': 70}}
484 ]
485 }
James E. Blairff555742017-02-19 11:34:27 -0800486 }])
James E. Blairf59f3cf2017-02-19 14:50:26 -0800487 layout.addProjectConfig(project_config)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700488
James E. Blairec7ff302017-03-04 07:31:32 -0800489 change = model.Change(self.project)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700490 change.branch = 'master'
491 item = queue.enqueueChange(change)
492 item.current_build_set.layout = layout
493
494 self.assertTrue(base.changeMatches(change))
495 self.assertTrue(python27.changeMatches(change))
496 self.assertFalse(python27diablo.changeMatches(change))
497
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200498 item.freezeJobGraph()
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700499 self.assertEqual(len(item.getJobs()), 1)
500 job = item.getJobs()[0]
501 self.assertEqual(job.name, 'python27')
502 self.assertEqual(job.timeout, 70)
503
504 change.branch = 'stable/diablo'
505 item = queue.enqueueChange(change)
506 item.current_build_set.layout = layout
507
508 self.assertTrue(base.changeMatches(change))
509 self.assertTrue(python27.changeMatches(change))
510 self.assertTrue(python27diablo.changeMatches(change))
511
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200512 item.freezeJobGraph()
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700513 self.assertEqual(len(item.getJobs()), 1)
514 job = item.getJobs()[0]
515 self.assertEqual(job.name, 'python27')
516 self.assertEqual(job.timeout, 70)
517
Clint Byrum85493602016-11-18 11:59:47 -0800518 def test_inheritance_keeps_matchers(self):
James E. Blair5ac93842017-01-20 06:47:34 -0800519 tenant = model.Tenant('tenant')
Clint Byrum85493602016-11-18 11:59:47 -0800520 layout = model.Layout()
521
522 pipeline = model.Pipeline('gate', layout)
523 layout.addPipeline(pipeline)
524 queue = model.ChangeQueue(pipeline)
James E. Blair0a899752017-03-29 13:22:16 -0700525 project = model.Project('project', self.source)
James E. Blair109da3f2017-04-04 14:39:43 -0700526 tenant.addUntrustedProject(project)
Clint Byrum85493602016-11-18 11:59:47 -0800527
James E. Blair5ac93842017-01-20 06:47:34 -0800528 base = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800529 '_source_context': self.context,
530 '_start_mark': self.start_mark,
Clint Byrum85493602016-11-18 11:59:47 -0800531 'name': 'base',
532 'timeout': 30,
533 })
534 layout.addJob(base)
James E. Blair5ac93842017-01-20 06:47:34 -0800535 python27 = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800536 '_source_context': self.context,
537 '_start_mark': self.start_mark,
Clint Byrum85493602016-11-18 11:59:47 -0800538 'name': 'python27',
539 'parent': 'base',
540 'timeout': 40,
541 'irrelevant-files': ['^ignored-file$'],
542 })
543 layout.addJob(python27)
544
James E. Blairff555742017-02-19 11:34:27 -0800545 project_config = configloader.ProjectParser.fromYaml(tenant, layout, [{
James E. Blairec7ff302017-03-04 07:31:32 -0800546 '_source_context': self.context,
547 '_start_mark': self.start_mark,
Clint Byrum85493602016-11-18 11:59:47 -0800548 'name': 'project',
549 'gate': {
550 'jobs': [
551 'python27',
552 ]
553 }
James E. Blairff555742017-02-19 11:34:27 -0800554 }])
James E. Blairf59f3cf2017-02-19 14:50:26 -0800555 layout.addProjectConfig(project_config)
Clint Byrum85493602016-11-18 11:59:47 -0800556
557 change = model.Change(project)
558 change.branch = 'master'
559 change.files = ['/COMMIT_MSG', 'ignored-file']
560 item = queue.enqueueChange(change)
561 item.current_build_set.layout = layout
562
563 self.assertTrue(base.changeMatches(change))
564 self.assertFalse(python27.changeMatches(change))
565
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200566 item.freezeJobGraph()
Clint Byrum85493602016-11-18 11:59:47 -0800567 self.assertEqual([], item.getJobs())
568
James E. Blair4317e9f2016-07-15 10:05:47 -0700569 def test_job_source_project(self):
James E. Blair5ac93842017-01-20 06:47:34 -0800570 tenant = model.Tenant('tenant')
James E. Blair4317e9f2016-07-15 10:05:47 -0700571 layout = model.Layout()
James E. Blair0a899752017-03-29 13:22:16 -0700572 base_project = model.Project('base_project', self.source)
James E. Blair6f140c72017-03-03 10:32:07 -0800573 base_context = model.SourceContext(base_project, 'master',
574 'test', True)
James E. Blaircdab2032017-02-01 09:09:29 -0800575
James E. Blair5ac93842017-01-20 06:47:34 -0800576 base = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blaircdab2032017-02-01 09:09:29 -0800577 '_source_context': base_context,
James E. Blairec7ff302017-03-04 07:31:32 -0800578 '_start_mark': self.start_mark,
James E. Blair4317e9f2016-07-15 10:05:47 -0700579 'name': 'base',
580 })
581 layout.addJob(base)
582
James E. Blair0a899752017-03-29 13:22:16 -0700583 other_project = model.Project('other_project', self.source)
James E. Blair6f140c72017-03-03 10:32:07 -0800584 other_context = model.SourceContext(other_project, 'master',
585 'test', True)
James E. Blair5ac93842017-01-20 06:47:34 -0800586 base2 = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blaircdab2032017-02-01 09:09:29 -0800587 '_source_context': other_context,
James E. Blairec7ff302017-03-04 07:31:32 -0800588 '_start_mark': self.start_mark,
James E. Blair4317e9f2016-07-15 10:05:47 -0700589 'name': 'base',
590 })
591 with testtools.ExpectedException(
592 Exception,
593 "Job base in other_project is not permitted "
594 "to shadow job base in base_project"):
595 layout.addJob(base2)
596
James E. Blairb3f5db12017-03-17 12:57:39 -0700597 def test_job_allowed_projects(self):
598 job = configloader.JobParser.fromYaml(self.tenant, self.layout, {
599 '_source_context': self.context,
600 '_start_mark': self.start_mark,
601 'name': 'job',
602 'allowed-projects': ['project'],
603 })
604 self.layout.addJob(job)
605
James E. Blair0a899752017-03-29 13:22:16 -0700606 project2 = model.Project('project2', self.source)
James E. Blair109da3f2017-04-04 14:39:43 -0700607 self.tenant.addUntrustedProject(project2)
James E. Blairb3f5db12017-03-17 12:57:39 -0700608 context2 = model.SourceContext(project2, 'master',
609 'test', True)
610
611 project2_config = configloader.ProjectParser.fromYaml(
612 self.tenant, self.layout, [{
613 '_source_context': context2,
614 '_start_mark': self.start_mark,
615 'name': 'project2',
616 'gate': {
617 'jobs': [
618 'job'
619 ]
620 }
621 }]
622 )
623 self.layout.addProjectConfig(project2_config)
624
625 change = model.Change(project2)
626 # Test master
627 change.branch = 'master'
628 item = self.queue.enqueueChange(change)
629 item.current_build_set.layout = self.layout
630 with testtools.ExpectedException(
631 Exception,
632 "Project project2 is not allowed to run job job"):
633 item.freezeJobGraph()
634
James E. Blaird2348362017-03-17 13:59:35 -0700635 def test_job_pipeline_allow_secrets(self):
636 self.pipeline.allow_secrets = False
637 job = configloader.JobParser.fromYaml(self.tenant, self.layout, {
638 '_source_context': self.context,
639 '_start_mark': self.start_mark,
640 'name': 'job',
641 })
642 auth = model.AuthContext()
643 auth.secrets.append('foo')
644 job.auth = auth
645
646 self.layout.addJob(job)
647
648 project_config = configloader.ProjectParser.fromYaml(
649 self.tenant, self.layout, [{
650 '_source_context': self.context,
651 '_start_mark': self.start_mark,
652 'name': 'project',
653 'gate': {
654 'jobs': [
655 'job'
656 ]
657 }
658 }]
659 )
660 self.layout.addProjectConfig(project_config)
661
662 change = model.Change(self.project)
663 # Test master
664 change.branch = 'master'
665 item = self.queue.enqueueChange(change)
666 item.current_build_set.layout = self.layout
667 with testtools.ExpectedException(
668 Exception,
669 "Pipeline gate does not allow jobs with secrets"):
670 item.freezeJobGraph()
671
James E. Blairce8a2132016-05-19 15:21:52 -0700672
673class TestJobTimeData(BaseTestCase):
674 def setUp(self):
675 super(TestJobTimeData, self).setUp()
676 self.tmp_root = self.useFixture(fixtures.TempDir(
677 rootdir=os.environ.get("ZUUL_TEST_ROOT"))
678 ).path
679
680 def test_empty_timedata(self):
681 path = os.path.join(self.tmp_root, 'job-name')
682 self.assertFalse(os.path.exists(path))
683 self.assertFalse(os.path.exists(path + '.tmp'))
684 td = model.JobTimeData(path)
685 self.assertEqual(td.success_times, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
686 self.assertEqual(td.failure_times, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
687 self.assertEqual(td.results, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
688
689 def test_save_reload(self):
690 path = os.path.join(self.tmp_root, 'job-name')
691 self.assertFalse(os.path.exists(path))
692 self.assertFalse(os.path.exists(path + '.tmp'))
693 td = model.JobTimeData(path)
694 self.assertEqual(td.success_times, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
695 self.assertEqual(td.failure_times, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
696 self.assertEqual(td.results, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
697 success_times = []
698 failure_times = []
699 results = []
700 for x in range(10):
701 success_times.append(int(random.random() * 1000))
702 failure_times.append(int(random.random() * 1000))
703 results.append(0)
704 results.append(1)
705 random.shuffle(results)
706 s = f = 0
707 for result in results:
708 if result:
709 td.add(failure_times[f], 'FAILURE')
710 f += 1
711 else:
712 td.add(success_times[s], 'SUCCESS')
713 s += 1
714 self.assertEqual(td.success_times, success_times)
715 self.assertEqual(td.failure_times, failure_times)
716 self.assertEqual(td.results, results[10:])
717 td.save()
718 self.assertTrue(os.path.exists(path))
719 self.assertFalse(os.path.exists(path + '.tmp'))
720 td = model.JobTimeData(path)
721 td.load()
722 self.assertEqual(td.success_times, success_times)
723 self.assertEqual(td.failure_times, failure_times)
724 self.assertEqual(td.results, results[10:])
725
726
727class TestTimeDataBase(BaseTestCase):
728 def setUp(self):
729 super(TestTimeDataBase, self).setUp()
730 self.tmp_root = self.useFixture(fixtures.TempDir(
731 rootdir=os.environ.get("ZUUL_TEST_ROOT"))
732 ).path
733 self.db = model.TimeDataBase(self.tmp_root)
734
735 def test_timedatabase(self):
736 self.assertEqual(self.db.getEstimatedTime('job-name'), 0)
737 self.db.update('job-name', 50, 'SUCCESS')
738 self.assertEqual(self.db.getEstimatedTime('job-name'), 50)
739 self.db.update('job-name', 100, 'SUCCESS')
740 self.assertEqual(self.db.getEstimatedTime('job-name'), 75)
741 for x in range(10):
742 self.db.update('job-name', 100, 'SUCCESS')
743 self.assertEqual(self.db.getEstimatedTime('job-name'), 100)
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200744
745
746class TestGraph(BaseTestCase):
747 def test_job_graph_disallows_multiple_jobs_with_same_name(self):
748 graph = model.JobGraph()
749 job1 = model.Job('job')
750 job2 = model.Job('job')
751 graph.addJob(job1)
752 with testtools.ExpectedException(Exception,
753 "Job job already added"):
754 graph.addJob(job2)
755
756 def test_job_graph_disallows_circular_dependencies(self):
757 graph = model.JobGraph()
758 jobs = [model.Job('job%d' % i) for i in range(0, 10)]
759 prevjob = None
760 for j in jobs[:3]:
761 if prevjob:
762 j.dependencies = frozenset([prevjob.name])
763 graph.addJob(j)
764 prevjob = j
765 # 0 triggers 1 triggers 2 triggers 3...
766
767 # Cannot depend on itself
768 with testtools.ExpectedException(
769 Exception,
770 "Dependency cycle detected in job jobX"):
771 j = model.Job('jobX')
772 j.dependencies = frozenset([j.name])
773 graph.addJob(j)
774
775 # Disallow circular dependencies
776 with testtools.ExpectedException(
777 Exception,
778 "Dependency cycle detected in job job3"):
779 jobs[4].dependencies = frozenset([jobs[3].name])
780 graph.addJob(jobs[4])
781 jobs[3].dependencies = frozenset([jobs[4].name])
782 graph.addJob(jobs[3])
783
784 jobs[5].dependencies = frozenset([jobs[4].name])
785 graph.addJob(jobs[5])
786
787 with testtools.ExpectedException(
788 Exception,
789 "Dependency cycle detected in job job3"):
790 jobs[3].dependencies = frozenset([jobs[5].name])
791 graph.addJob(jobs[3])
792
793 jobs[3].dependencies = frozenset([jobs[2].name])
794 graph.addJob(jobs[3])
795 jobs[6].dependencies = frozenset([jobs[2].name])
796 graph.addJob(jobs[6])
James E. Blairc2a54fd2017-03-29 15:19:26 -0700797
798
799class TestTenant(BaseTestCase):
800 def test_add_project(self):
801 tenant = model.Tenant('tenant')
802 connection1 = Dummy(connection_name='dummy_connection1')
803 source1 = Dummy(canonical_hostname='git1.example.com',
804 name='dummy', # TODOv3(jeblair): remove
805 connection=connection1)
806
807 source1_project1 = model.Project('project1', source1)
James E. Blair109da3f2017-04-04 14:39:43 -0700808 tenant.addConfigProject(source1_project1)
James E. Blairc2a54fd2017-03-29 15:19:26 -0700809 d = {'project1':
810 {'git1.example.com': source1_project1}}
811 self.assertEqual(d, tenant.projects)
812 self.assertEqual((True, source1_project1),
813 tenant.getProject('project1'))
814 self.assertEqual((True, source1_project1),
815 tenant.getProject('git1.example.com/project1'))
816
817 source1_project2 = model.Project('project2', source1)
James E. Blair109da3f2017-04-04 14:39:43 -0700818 tenant.addUntrustedProject(source1_project2)
James E. Blairc2a54fd2017-03-29 15:19:26 -0700819 d = {'project1':
820 {'git1.example.com': source1_project1},
821 'project2':
822 {'git1.example.com': source1_project2}}
823 self.assertEqual(d, tenant.projects)
824 self.assertEqual((False, source1_project2),
825 tenant.getProject('project2'))
826 self.assertEqual((False, source1_project2),
827 tenant.getProject('git1.example.com/project2'))
828
829 connection2 = Dummy(connection_name='dummy_connection2')
830 source2 = Dummy(canonical_hostname='git2.example.com',
831 name='dummy', # TODOv3(jeblair): remove
832 connection=connection2)
833
834 source2_project1 = model.Project('project1', source2)
James E. Blair109da3f2017-04-04 14:39:43 -0700835 tenant.addUntrustedProject(source2_project1)
James E. Blairc2a54fd2017-03-29 15:19:26 -0700836 d = {'project1':
837 {'git1.example.com': source1_project1,
838 'git2.example.com': source2_project1},
839 'project2':
840 {'git1.example.com': source1_project2}}
841 self.assertEqual(d, tenant.projects)
842 with testtools.ExpectedException(
843 Exception,
844 "Project name 'project1' is ambiguous"):
845 tenant.getProject('project1')
846 self.assertEqual((False, source1_project2),
847 tenant.getProject('project2'))
848 self.assertEqual((True, source1_project1),
849 tenant.getProject('git1.example.com/project1'))
850 self.assertEqual((False, source2_project1),
851 tenant.getProject('git2.example.com/project1'))
852
853 source2_project2 = model.Project('project2', source2)
James E. Blair109da3f2017-04-04 14:39:43 -0700854 tenant.addConfigProject(source2_project2)
James E. Blairc2a54fd2017-03-29 15:19:26 -0700855 d = {'project1':
856 {'git1.example.com': source1_project1,
857 'git2.example.com': source2_project1},
858 'project2':
859 {'git1.example.com': source1_project2,
860 'git2.example.com': source2_project2}}
861 self.assertEqual(d, tenant.projects)
862 with testtools.ExpectedException(
863 Exception,
864 "Project name 'project1' is ambiguous"):
865 tenant.getProject('project1')
866 with testtools.ExpectedException(
867 Exception,
868 "Project name 'project2' is ambiguous"):
869 tenant.getProject('project2')
870 self.assertEqual((True, source1_project1),
871 tenant.getProject('git1.example.com/project1'))
872 self.assertEqual((False, source2_project1),
873 tenant.getProject('git2.example.com/project1'))
874 self.assertEqual((False, source1_project2),
875 tenant.getProject('git1.example.com/project2'))
876 self.assertEqual((True, source2_project2),
877 tenant.getProject('git2.example.com/project2'))
878
879 source1_project2b = model.Project('subpath/project2', source1)
James E. Blair109da3f2017-04-04 14:39:43 -0700880 tenant.addConfigProject(source1_project2b)
James E. Blairc2a54fd2017-03-29 15:19:26 -0700881 d = {'project1':
882 {'git1.example.com': source1_project1,
883 'git2.example.com': source2_project1},
884 'project2':
885 {'git1.example.com': source1_project2,
886 'git2.example.com': source2_project2},
887 'subpath/project2':
888 {'git1.example.com': source1_project2b}}
889 self.assertEqual(d, tenant.projects)
890 self.assertEqual((False, source1_project2),
891 tenant.getProject('git1.example.com/project2'))
892 self.assertEqual((True, source2_project2),
893 tenant.getProject('git2.example.com/project2'))
894 self.assertEqual((True, source1_project2b),
895 tenant.getProject('subpath/project2'))
896 self.assertEqual(
897 (True, source1_project2b),
898 tenant.getProject('git1.example.com/subpath/project2'))
899
900 source2_project2b = model.Project('subpath/project2', source2)
James E. Blair109da3f2017-04-04 14:39:43 -0700901 tenant.addConfigProject(source2_project2b)
James E. Blairc2a54fd2017-03-29 15:19:26 -0700902 d = {'project1':
903 {'git1.example.com': source1_project1,
904 'git2.example.com': source2_project1},
905 'project2':
906 {'git1.example.com': source1_project2,
907 'git2.example.com': source2_project2},
908 'subpath/project2':
909 {'git1.example.com': source1_project2b,
910 'git2.example.com': source2_project2b}}
911 self.assertEqual(d, tenant.projects)
912 self.assertEqual((False, source1_project2),
913 tenant.getProject('git1.example.com/project2'))
914 self.assertEqual((True, source2_project2),
915 tenant.getProject('git2.example.com/project2'))
916 with testtools.ExpectedException(
917 Exception,
918 "Project name 'subpath/project2' is ambiguous"):
919 tenant.getProject('subpath/project2')
920 self.assertEqual(
921 (True, source1_project2b),
922 tenant.getProject('git1.example.com/subpath/project2'))
923 self.assertEqual(
924 (True, source2_project2b),
925 tenant.getProject('git2.example.com/subpath/project2'))
926
927 with testtools.ExpectedException(
928 Exception,
929 "Project project1 is already in project index"):
930 tenant._addProject(source1_project1)