blob: 7d8a0580dd44d556df430b17c8cd7352e514bc1b [file] [log] [blame]
Maru Newby3fe5f852015-01-13 04:22:14 +00001# Copyright 2015 Red Hat, Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
Joshua Hesketh0aa7e8b2016-07-14 00:12:25 +100015
James E. Blairce8a2132016-05-19 15:21:52 -070016import os
17import random
18
19import fixtures
James E. Blair4317e9f2016-07-15 10:05:47 -070020import testtools
James E. Blairce8a2132016-05-19 15:21:52 -070021
Maru Newby3fe5f852015-01-13 04:22:14 +000022from zuul import model
James E. Blair83005782015-12-11 14:46:03 -080023from zuul import configloader
James E. Blairbf1a4f22017-03-17 10:59:37 -070024from zuul.lib import encryption
Clint Byrum88fdfde2017-03-28 16:22:42 -070025from zuul.lib import yamlutil as yaml
Maru Newby3fe5f852015-01-13 04:22:14 +000026
James E. Blair18f86a32017-03-15 14:43:26 -070027from tests.base import BaseTestCase, FIXTURE_DIR
Maru Newby3fe5f852015-01-13 04:22:14 +000028
29
James E. Blair0a899752017-03-29 13:22:16 -070030class Dummy(object):
31 def __init__(self, **kw):
32 for k, v in kw.items():
33 setattr(self, k, v)
James E. Blairb3f5db12017-03-17 12:57:39 -070034
35
Maru Newby3fe5f852015-01-13 04:22:14 +000036class TestJob(BaseTestCase):
James E. Blaira7f51ca2017-02-07 16:01:26 -080037 def setUp(self):
38 super(TestJob, self).setUp()
James E. Blair0a899752017-03-29 13:22:16 -070039 self.connection = Dummy(connection_name='dummy_connection')
40 self.source = Dummy(canonical_hostname='git.example.com',
41 name='dummy_connection', # TODOv3(jeblair): remove
42 connection=self.connection)
James E. Blairb3f5db12017-03-17 12:57:39 -070043 self.tenant = model.Tenant('tenant')
44 self.layout = model.Layout()
James E. Blair0a899752017-03-29 13:22:16 -070045 self.project = model.Project('project', self.source)
James E. Blairb3f5db12017-03-17 12:57:39 -070046 self.tenant.addProjectRepo(self.source, self.project)
47 self.pipeline = model.Pipeline('gate', self.layout)
48 self.layout.addPipeline(self.pipeline)
49 self.queue = model.ChangeQueue(self.pipeline)
50
James E. Blair18f86a32017-03-15 14:43:26 -070051 private_key_file = os.path.join(FIXTURE_DIR, 'private.pem')
52 with open(private_key_file, "rb") as f:
James E. Blairbf1a4f22017-03-17 10:59:37 -070053 self.project.private_key, self.project.public_key = \
54 encryption.deserialize_rsa_keypair(f.read())
James E. Blair6f140c72017-03-03 10:32:07 -080055 self.context = model.SourceContext(self.project, 'master',
56 'test', True)
James E. Blairec7ff302017-03-04 07:31:32 -080057 self.start_mark = yaml.Mark('name', 0, 0, 0, '', 0)
James E. Blaira7f51ca2017-02-07 16:01:26 -080058
Maru Newby3fe5f852015-01-13 04:22:14 +000059 @property
60 def job(self):
James E. Blair5ac93842017-01-20 06:47:34 -080061 tenant = model.Tenant('tenant')
James E. Blair83005782015-12-11 14:46:03 -080062 layout = model.Layout()
James E. Blair5ac93842017-01-20 06:47:34 -080063 job = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -080064 '_source_context': self.context,
65 '_start_mark': self.start_mark,
James E. Blair83005782015-12-11 14:46:03 -080066 'name': 'job',
67 'irrelevant-files': [
68 '^docs/.*$'
69 ]})
Maru Newby3fe5f852015-01-13 04:22:14 +000070 return job
71
72 def test_change_matches_returns_false_for_matched_skip_if(self):
73 change = model.Change('project')
Alexander Evseevdbe6fab2015-11-19 12:46:34 +030074 change.files = ['/COMMIT_MSG', 'docs/foo']
Maru Newby3fe5f852015-01-13 04:22:14 +000075 self.assertFalse(self.job.changeMatches(change))
76
77 def test_change_matches_returns_true_for_unmatched_skip_if(self):
78 change = model.Change('project')
Alexander Evseevdbe6fab2015-11-19 12:46:34 +030079 change.files = ['/COMMIT_MSG', 'foo']
Maru Newby3fe5f852015-01-13 04:22:14 +000080 self.assertTrue(self.job.changeMatches(change))
81
Maru Newby79427a42015-02-17 17:54:45 +000082 def test_job_sets_defaults_for_boolean_attributes(self):
James E. Blair83005782015-12-11 14:46:03 -080083 self.assertIsNotNone(self.job.voting)
84
85 def test_job_inheritance(self):
James E. Blaira7f51ca2017-02-07 16:01:26 -080086 # This is standard job inheritance.
87
88 base_pre = model.PlaybookContext(self.context, 'base-pre')
89 base_run = model.PlaybookContext(self.context, 'base-run')
90 base_post = model.PlaybookContext(self.context, 'base-post')
91
92 base = model.Job('base')
93 base.timeout = 30
94 base.pre_run = [base_pre]
95 base.run = [base_run]
96 base.post_run = [base_post]
James E. Blair8525e2b2017-03-15 14:05:47 -070097 base.auth = model.AuthContext()
James E. Blaira7f51ca2017-02-07 16:01:26 -080098
99 py27 = model.Job('py27')
100 self.assertEqual(None, py27.timeout)
101 py27.inheritFrom(base)
102 self.assertEqual(30, py27.timeout)
103 self.assertEqual(['base-pre'],
104 [x.path for x in py27.pre_run])
105 self.assertEqual(['base-run'],
106 [x.path for x in py27.run])
107 self.assertEqual(['base-post'],
108 [x.path for x in py27.post_run])
James E. Blair8525e2b2017-03-15 14:05:47 -0700109 self.assertEqual(None, py27.auth)
James E. Blaira7f51ca2017-02-07 16:01:26 -0800110
111 def test_job_variants(self):
112 # This simulates freezing a job.
113
114 py27_pre = model.PlaybookContext(self.context, 'py27-pre')
115 py27_run = model.PlaybookContext(self.context, 'py27-run')
116 py27_post = model.PlaybookContext(self.context, 'py27-post')
117
118 py27 = model.Job('py27')
119 py27.timeout = 30
120 py27.pre_run = [py27_pre]
121 py27.run = [py27_run]
122 py27.post_run = [py27_post]
James E. Blair8525e2b2017-03-15 14:05:47 -0700123 auth = model.AuthContext()
124 auth.secrets.append('foo')
James E. Blaira7f51ca2017-02-07 16:01:26 -0800125 py27.auth = auth
126
127 job = py27.copy()
128 self.assertEqual(30, job.timeout)
129
130 # Apply the diablo variant
131 diablo = model.Job('py27')
132 diablo.timeout = 40
133 job.applyVariant(diablo)
134
135 self.assertEqual(40, job.timeout)
136 self.assertEqual(['py27-pre'],
137 [x.path for x in job.pre_run])
138 self.assertEqual(['py27-run'],
139 [x.path for x in job.run])
140 self.assertEqual(['py27-post'],
141 [x.path for x in job.post_run])
142 self.assertEqual(auth, job.auth)
143
144 # Set the job to final for the following checks
145 job.final = True
146 self.assertTrue(job.voting)
147
148 good_final = model.Job('py27')
149 good_final.voting = False
150 job.applyVariant(good_final)
151 self.assertFalse(job.voting)
152
153 bad_final = model.Job('py27')
154 bad_final.timeout = 600
155 with testtools.ExpectedException(
156 Exception,
157 "Unable to modify final job"):
158 job.applyVariant(bad_final)
159
160 def test_job_inheritance_configloader(self):
161 # TODO(jeblair): move this to a configloader test
James E. Blair5ac93842017-01-20 06:47:34 -0800162 tenant = model.Tenant('tenant')
James E. Blair83005782015-12-11 14:46:03 -0800163 layout = model.Layout()
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700164
165 pipeline = model.Pipeline('gate', layout)
166 layout.addPipeline(pipeline)
167 queue = model.ChangeQueue(pipeline)
James E. Blair0a899752017-03-29 13:22:16 -0700168 project = model.Project('project', self.source)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700169
James E. Blair5ac93842017-01-20 06:47:34 -0800170 base = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800171 '_source_context': self.context,
172 '_start_mark': self.start_mark,
James E. Blair83005782015-12-11 14:46:03 -0800173 'name': 'base',
174 'timeout': 30,
James E. Blaira7f51ca2017-02-07 16:01:26 -0800175 'pre-run': 'base-pre',
176 'post-run': 'base-post',
James E. Blair1774dd52017-02-03 10:52:32 -0800177 'nodes': [{
178 'name': 'controller',
179 'image': 'base',
180 }],
James E. Blair83005782015-12-11 14:46:03 -0800181 })
182 layout.addJob(base)
James E. Blair5ac93842017-01-20 06:47:34 -0800183 python27 = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800184 '_source_context': self.context,
185 '_start_mark': self.start_mark,
James E. Blair83005782015-12-11 14:46:03 -0800186 'name': 'python27',
187 'parent': 'base',
James E. Blaira7f51ca2017-02-07 16:01:26 -0800188 'pre-run': 'py27-pre',
189 'post-run': 'py27-post',
James E. Blair1774dd52017-02-03 10:52:32 -0800190 'nodes': [{
191 'name': 'controller',
192 'image': 'new',
193 }],
James E. Blair83005782015-12-11 14:46:03 -0800194 'timeout': 40,
195 })
196 layout.addJob(python27)
James E. Blair5ac93842017-01-20 06:47:34 -0800197 python27diablo = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800198 '_source_context': self.context,
199 '_start_mark': self.start_mark,
James E. Blair83005782015-12-11 14:46:03 -0800200 'name': 'python27',
201 'branches': [
202 'stable/diablo'
203 ],
James E. Blaira7f51ca2017-02-07 16:01:26 -0800204 'pre-run': 'py27-diablo-pre',
205 'run': 'py27-diablo',
206 'post-run': 'py27-diablo-post',
James E. Blair1774dd52017-02-03 10:52:32 -0800207 'nodes': [{
208 'name': 'controller',
209 'image': 'old',
210 }],
James E. Blair83005782015-12-11 14:46:03 -0800211 'timeout': 50,
212 })
213 layout.addJob(python27diablo)
214
James E. Blair5ac93842017-01-20 06:47:34 -0800215 python27essex = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800216 '_source_context': self.context,
217 '_start_mark': self.start_mark,
James E. Blaira7f51ca2017-02-07 16:01:26 -0800218 'name': 'python27',
219 'branches': [
220 'stable/essex'
221 ],
222 'pre-run': 'py27-essex-pre',
223 'post-run': 'py27-essex-post',
224 })
225 layout.addJob(python27essex)
226
James E. Blairff555742017-02-19 11:34:27 -0800227 project_config = configloader.ProjectParser.fromYaml(tenant, layout, [{
James E. Blairec7ff302017-03-04 07:31:32 -0800228 '_source_context': self.context,
229 '_start_mark': self.start_mark,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700230 'name': 'project',
231 'gate': {
232 'jobs': [
233 'python27'
234 ]
235 }
James E. Blairff555742017-02-19 11:34:27 -0800236 }])
James E. Blairf59f3cf2017-02-19 14:50:26 -0800237 layout.addProjectConfig(project_config)
James E. Blair83005782015-12-11 14:46:03 -0800238
James E. Blair83005782015-12-11 14:46:03 -0800239 change = model.Change(project)
James E. Blair1774dd52017-02-03 10:52:32 -0800240 # Test master
James E. Blair83005782015-12-11 14:46:03 -0800241 change.branch = 'master'
242 item = queue.enqueueChange(change)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700243 item.current_build_set.layout = layout
James E. Blair83005782015-12-11 14:46:03 -0800244
245 self.assertTrue(base.changeMatches(change))
246 self.assertTrue(python27.changeMatches(change))
247 self.assertFalse(python27diablo.changeMatches(change))
James E. Blaira7f51ca2017-02-07 16:01:26 -0800248 self.assertFalse(python27essex.changeMatches(change))
James E. Blair83005782015-12-11 14:46:03 -0800249
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200250 item.freezeJobGraph()
James E. Blair83005782015-12-11 14:46:03 -0800251 self.assertEqual(len(item.getJobs()), 1)
252 job = item.getJobs()[0]
253 self.assertEqual(job.name, 'python27')
254 self.assertEqual(job.timeout, 40)
James E. Blair1774dd52017-02-03 10:52:32 -0800255 nodes = job.nodeset.getNodes()
256 self.assertEqual(len(nodes), 1)
257 self.assertEqual(nodes[0].image, 'new')
James E. Blaira7f51ca2017-02-07 16:01:26 -0800258 self.assertEqual([x.path for x in job.pre_run],
259 ['playbooks/base-pre',
260 'playbooks/py27-pre'])
261 self.assertEqual([x.path for x in job.post_run],
262 ['playbooks/py27-post',
263 'playbooks/base-post'])
264 self.assertEqual([x.path for x in job.run],
265 ['playbooks/python27',
266 'playbooks/base'])
James E. Blair83005782015-12-11 14:46:03 -0800267
James E. Blair1774dd52017-02-03 10:52:32 -0800268 # Test diablo
James E. Blair83005782015-12-11 14:46:03 -0800269 change.branch = 'stable/diablo'
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700270 item = queue.enqueueChange(change)
271 item.current_build_set.layout = layout
James E. Blair83005782015-12-11 14:46:03 -0800272
273 self.assertTrue(base.changeMatches(change))
274 self.assertTrue(python27.changeMatches(change))
275 self.assertTrue(python27diablo.changeMatches(change))
James E. Blaira7f51ca2017-02-07 16:01:26 -0800276 self.assertFalse(python27essex.changeMatches(change))
James E. Blair83005782015-12-11 14:46:03 -0800277
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200278 item.freezeJobGraph()
James E. Blair83005782015-12-11 14:46:03 -0800279 self.assertEqual(len(item.getJobs()), 1)
280 job = item.getJobs()[0]
281 self.assertEqual(job.name, 'python27')
282 self.assertEqual(job.timeout, 50)
James E. Blair1774dd52017-02-03 10:52:32 -0800283 nodes = job.nodeset.getNodes()
284 self.assertEqual(len(nodes), 1)
285 self.assertEqual(nodes[0].image, 'old')
James E. Blaira7f51ca2017-02-07 16:01:26 -0800286 self.assertEqual([x.path for x in job.pre_run],
287 ['playbooks/base-pre',
288 'playbooks/py27-pre',
289 'playbooks/py27-diablo-pre'])
290 self.assertEqual([x.path for x in job.post_run],
291 ['playbooks/py27-diablo-post',
292 'playbooks/py27-post',
293 'playbooks/base-post'])
294 self.assertEqual([x.path for x in job.run],
295 ['playbooks/py27-diablo']),
296
297 # Test essex
298 change.branch = 'stable/essex'
299 item = queue.enqueueChange(change)
300 item.current_build_set.layout = layout
301
302 self.assertTrue(base.changeMatches(change))
303 self.assertTrue(python27.changeMatches(change))
304 self.assertFalse(python27diablo.changeMatches(change))
305 self.assertTrue(python27essex.changeMatches(change))
306
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200307 item.freezeJobGraph()
James E. Blaira7f51ca2017-02-07 16:01:26 -0800308 self.assertEqual(len(item.getJobs()), 1)
309 job = item.getJobs()[0]
310 self.assertEqual(job.name, 'python27')
311 self.assertEqual([x.path for x in job.pre_run],
312 ['playbooks/base-pre',
313 'playbooks/py27-pre',
314 'playbooks/py27-essex-pre'])
315 self.assertEqual([x.path for x in job.post_run],
316 ['playbooks/py27-essex-post',
317 'playbooks/py27-post',
318 'playbooks/base-post'])
319 self.assertEqual([x.path for x in job.run],
320 ['playbooks/python27',
321 'playbooks/base'])
James E. Blairce8a2132016-05-19 15:21:52 -0700322
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000323 def test_job_auth_inheritance(self):
James E. Blair5ac93842017-01-20 06:47:34 -0800324 tenant = model.Tenant('tenant')
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000325 layout = model.Layout()
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000326
James E. Blair01f83b72017-03-15 13:03:40 -0700327 conf = yaml.safe_load('''
328- secret:
329 name: pypi-credentials
330 data:
331 username: test-username
James E. Blair717e8e92017-03-17 11:03:27 -0700332 password: !encrypted/pkcs1-oaep |
James E. Blair01f83b72017-03-15 13:03:40 -0700333 BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ
334 L0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4joeusC9drN3AA8a4o
335 ykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CRgd0QBMPl6VDoFgBPB8vxtJw+
336 3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzibDsSXsfJt1y+5n7yOURsC7lovMg4GF/v
337 Cl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCYceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qt
338 xhbpjTxG4U5Q/SoppOJ60WqEkQvbXs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYr
339 aI+AKYsMYx3RBlfAmCeC1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFW
340 Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd
341 +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs=
342''')[0]['secret']
343
344 conf['_source_context'] = self.context
345 conf['_start_mark'] = self.start_mark
346
347 secret = configloader.SecretParser.fromYaml(layout, conf)
348 layout.addSecret(secret)
349
James E. Blair5ac93842017-01-20 06:47:34 -0800350 base = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800351 '_source_context': self.context,
352 '_start_mark': self.start_mark,
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000353 'name': 'base',
354 'timeout': 30,
355 })
356 layout.addJob(base)
James E. Blair5ac93842017-01-20 06:47:34 -0800357 pypi_upload_without_inherit = configloader.JobParser.fromYaml(
358 tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800359 '_source_context': self.context,
360 '_start_mark': self.start_mark,
James E. Blair5ac93842017-01-20 06:47:34 -0800361 'name': 'pypi-upload-without-inherit',
362 'parent': 'base',
363 'timeout': 40,
364 'auth': {
365 'secrets': [
366 'pypi-credentials',
367 ]
368 }
369 })
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000370 layout.addJob(pypi_upload_without_inherit)
James E. Blair5ac93842017-01-20 06:47:34 -0800371 pypi_upload_with_inherit = configloader.JobParser.fromYaml(
372 tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800373 '_source_context': self.context,
374 '_start_mark': self.start_mark,
James E. Blair5ac93842017-01-20 06:47:34 -0800375 'name': 'pypi-upload-with-inherit',
376 'parent': 'base',
377 'timeout': 40,
378 'auth': {
379 'inherit': True,
380 'secrets': [
381 'pypi-credentials',
382 ]
383 }
384 })
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000385 layout.addJob(pypi_upload_with_inherit)
386 pypi_upload_with_inherit_false = configloader.JobParser.fromYaml(
James E. Blair5ac93842017-01-20 06:47:34 -0800387 tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800388 '_source_context': self.context,
389 '_start_mark': self.start_mark,
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000390 'name': 'pypi-upload-with-inherit-false',
391 'parent': 'base',
392 'timeout': 40,
393 'auth': {
394 'inherit': False,
Ricardo Carrillo Cruz12c892b2016-11-18 15:35:49 +0000395 'secrets': [
396 'pypi-credentials',
397 ]
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000398 }
399 })
400 layout.addJob(pypi_upload_with_inherit_false)
James E. Blair5ac93842017-01-20 06:47:34 -0800401 in_repo_job_without_inherit = configloader.JobParser.fromYaml(
402 tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800403 '_source_context': self.context,
404 '_start_mark': self.start_mark,
James E. Blair5ac93842017-01-20 06:47:34 -0800405 'name': 'in-repo-job-without-inherit',
406 'parent': 'pypi-upload-without-inherit',
407 })
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000408 layout.addJob(in_repo_job_without_inherit)
James E. Blair5ac93842017-01-20 06:47:34 -0800409 in_repo_job_with_inherit = configloader.JobParser.fromYaml(
410 tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800411 '_source_context': self.context,
412 '_start_mark': self.start_mark,
James E. Blair5ac93842017-01-20 06:47:34 -0800413 'name': 'in-repo-job-with-inherit',
414 'parent': 'pypi-upload-with-inherit',
415 })
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000416 layout.addJob(in_repo_job_with_inherit)
417 in_repo_job_with_inherit_false = configloader.JobParser.fromYaml(
James E. Blair5ac93842017-01-20 06:47:34 -0800418 tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800419 '_source_context': self.context,
420 '_start_mark': self.start_mark,
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000421 'name': 'in-repo-job-with-inherit-false',
422 'parent': 'pypi-upload-with-inherit-false',
423 })
424 layout.addJob(in_repo_job_with_inherit_false)
425
James E. Blair8525e2b2017-03-15 14:05:47 -0700426 self.assertEqual(None, in_repo_job_without_inherit.auth)
427 self.assertEqual(1, len(in_repo_job_with_inherit.auth.secrets))
428 self.assertEqual(in_repo_job_with_inherit.auth.secrets[0].name,
429 'pypi-credentials')
430 self.assertEqual(None, in_repo_job_with_inherit_false.auth)
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000431
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700432 def test_job_inheritance_job_tree(self):
James E. Blair5ac93842017-01-20 06:47:34 -0800433 tenant = model.Tenant('tenant')
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700434 layout = model.Layout()
435
436 pipeline = model.Pipeline('gate', layout)
437 layout.addPipeline(pipeline)
438 queue = model.ChangeQueue(pipeline)
439
James E. Blair5ac93842017-01-20 06:47:34 -0800440 base = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800441 '_source_context': self.context,
442 '_start_mark': self.start_mark,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700443 'name': 'base',
444 'timeout': 30,
445 })
446 layout.addJob(base)
James E. Blair5ac93842017-01-20 06:47:34 -0800447 python27 = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800448 '_source_context': self.context,
449 '_start_mark': self.start_mark,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700450 'name': 'python27',
451 'parent': 'base',
452 'timeout': 40,
453 })
454 layout.addJob(python27)
James E. Blair5ac93842017-01-20 06:47:34 -0800455 python27diablo = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800456 '_source_context': self.context,
457 '_start_mark': self.start_mark,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700458 'name': 'python27',
459 'branches': [
460 'stable/diablo'
461 ],
462 'timeout': 50,
463 })
464 layout.addJob(python27diablo)
465
James E. Blairff555742017-02-19 11:34:27 -0800466 project_config = configloader.ProjectParser.fromYaml(tenant, layout, [{
James E. Blairec7ff302017-03-04 07:31:32 -0800467 '_source_context': self.context,
468 '_start_mark': self.start_mark,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700469 'name': 'project',
470 'gate': {
471 'jobs': [
472 {'python27': {'timeout': 70}}
473 ]
474 }
James E. Blairff555742017-02-19 11:34:27 -0800475 }])
James E. Blairf59f3cf2017-02-19 14:50:26 -0800476 layout.addProjectConfig(project_config)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700477
James E. Blairec7ff302017-03-04 07:31:32 -0800478 change = model.Change(self.project)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700479 change.branch = 'master'
480 item = queue.enqueueChange(change)
481 item.current_build_set.layout = layout
482
483 self.assertTrue(base.changeMatches(change))
484 self.assertTrue(python27.changeMatches(change))
485 self.assertFalse(python27diablo.changeMatches(change))
486
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200487 item.freezeJobGraph()
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700488 self.assertEqual(len(item.getJobs()), 1)
489 job = item.getJobs()[0]
490 self.assertEqual(job.name, 'python27')
491 self.assertEqual(job.timeout, 70)
492
493 change.branch = 'stable/diablo'
494 item = queue.enqueueChange(change)
495 item.current_build_set.layout = layout
496
497 self.assertTrue(base.changeMatches(change))
498 self.assertTrue(python27.changeMatches(change))
499 self.assertTrue(python27diablo.changeMatches(change))
500
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200501 item.freezeJobGraph()
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700502 self.assertEqual(len(item.getJobs()), 1)
503 job = item.getJobs()[0]
504 self.assertEqual(job.name, 'python27')
505 self.assertEqual(job.timeout, 70)
506
Clint Byrum85493602016-11-18 11:59:47 -0800507 def test_inheritance_keeps_matchers(self):
James E. Blair5ac93842017-01-20 06:47:34 -0800508 tenant = model.Tenant('tenant')
Clint Byrum85493602016-11-18 11:59:47 -0800509 layout = model.Layout()
510
511 pipeline = model.Pipeline('gate', layout)
512 layout.addPipeline(pipeline)
513 queue = model.ChangeQueue(pipeline)
James E. Blair0a899752017-03-29 13:22:16 -0700514 project = model.Project('project', self.source)
Clint Byrum85493602016-11-18 11:59:47 -0800515
James E. Blair5ac93842017-01-20 06:47:34 -0800516 base = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800517 '_source_context': self.context,
518 '_start_mark': self.start_mark,
Clint Byrum85493602016-11-18 11:59:47 -0800519 'name': 'base',
520 'timeout': 30,
521 })
522 layout.addJob(base)
James E. Blair5ac93842017-01-20 06:47:34 -0800523 python27 = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800524 '_source_context': self.context,
525 '_start_mark': self.start_mark,
Clint Byrum85493602016-11-18 11:59:47 -0800526 'name': 'python27',
527 'parent': 'base',
528 'timeout': 40,
529 'irrelevant-files': ['^ignored-file$'],
530 })
531 layout.addJob(python27)
532
James E. Blairff555742017-02-19 11:34:27 -0800533 project_config = configloader.ProjectParser.fromYaml(tenant, layout, [{
James E. Blairec7ff302017-03-04 07:31:32 -0800534 '_source_context': self.context,
535 '_start_mark': self.start_mark,
Clint Byrum85493602016-11-18 11:59:47 -0800536 'name': 'project',
537 'gate': {
538 'jobs': [
539 'python27',
540 ]
541 }
James E. Blairff555742017-02-19 11:34:27 -0800542 }])
James E. Blairf59f3cf2017-02-19 14:50:26 -0800543 layout.addProjectConfig(project_config)
Clint Byrum85493602016-11-18 11:59:47 -0800544
545 change = model.Change(project)
546 change.branch = 'master'
547 change.files = ['/COMMIT_MSG', 'ignored-file']
548 item = queue.enqueueChange(change)
549 item.current_build_set.layout = layout
550
551 self.assertTrue(base.changeMatches(change))
552 self.assertFalse(python27.changeMatches(change))
553
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200554 item.freezeJobGraph()
Clint Byrum85493602016-11-18 11:59:47 -0800555 self.assertEqual([], item.getJobs())
556
James E. Blair4317e9f2016-07-15 10:05:47 -0700557 def test_job_source_project(self):
James E. Blair5ac93842017-01-20 06:47:34 -0800558 tenant = model.Tenant('tenant')
James E. Blair4317e9f2016-07-15 10:05:47 -0700559 layout = model.Layout()
James E. Blair0a899752017-03-29 13:22:16 -0700560 base_project = model.Project('base_project', self.source)
James E. Blair6f140c72017-03-03 10:32:07 -0800561 base_context = model.SourceContext(base_project, 'master',
562 'test', True)
James E. Blaircdab2032017-02-01 09:09:29 -0800563
James E. Blair5ac93842017-01-20 06:47:34 -0800564 base = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blaircdab2032017-02-01 09:09:29 -0800565 '_source_context': base_context,
James E. Blairec7ff302017-03-04 07:31:32 -0800566 '_start_mark': self.start_mark,
James E. Blair4317e9f2016-07-15 10:05:47 -0700567 'name': 'base',
568 })
569 layout.addJob(base)
570
James E. Blair0a899752017-03-29 13:22:16 -0700571 other_project = model.Project('other_project', self.source)
James E. Blair6f140c72017-03-03 10:32:07 -0800572 other_context = model.SourceContext(other_project, 'master',
573 'test', True)
James E. Blair5ac93842017-01-20 06:47:34 -0800574 base2 = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blaircdab2032017-02-01 09:09:29 -0800575 '_source_context': other_context,
James E. Blairec7ff302017-03-04 07:31:32 -0800576 '_start_mark': self.start_mark,
James E. Blair4317e9f2016-07-15 10:05:47 -0700577 'name': 'base',
578 })
579 with testtools.ExpectedException(
580 Exception,
581 "Job base in other_project is not permitted "
582 "to shadow job base in base_project"):
583 layout.addJob(base2)
584
James E. Blairb3f5db12017-03-17 12:57:39 -0700585 def test_job_allowed_projects(self):
586 job = configloader.JobParser.fromYaml(self.tenant, self.layout, {
587 '_source_context': self.context,
588 '_start_mark': self.start_mark,
589 'name': 'job',
590 'allowed-projects': ['project'],
591 })
592 self.layout.addJob(job)
593
James E. Blair0a899752017-03-29 13:22:16 -0700594 project2 = model.Project('project2', self.source)
James E. Blairb3f5db12017-03-17 12:57:39 -0700595 context2 = model.SourceContext(project2, 'master',
596 'test', True)
597
598 project2_config = configloader.ProjectParser.fromYaml(
599 self.tenant, self.layout, [{
600 '_source_context': context2,
601 '_start_mark': self.start_mark,
602 'name': 'project2',
603 'gate': {
604 'jobs': [
605 'job'
606 ]
607 }
608 }]
609 )
610 self.layout.addProjectConfig(project2_config)
611
612 change = model.Change(project2)
613 # Test master
614 change.branch = 'master'
615 item = self.queue.enqueueChange(change)
616 item.current_build_set.layout = self.layout
617 with testtools.ExpectedException(
618 Exception,
619 "Project project2 is not allowed to run job job"):
620 item.freezeJobGraph()
621
James E. Blaird2348362017-03-17 13:59:35 -0700622 def test_job_pipeline_allow_secrets(self):
623 self.pipeline.allow_secrets = False
624 job = configloader.JobParser.fromYaml(self.tenant, self.layout, {
625 '_source_context': self.context,
626 '_start_mark': self.start_mark,
627 'name': 'job',
628 })
629 auth = model.AuthContext()
630 auth.secrets.append('foo')
631 job.auth = auth
632
633 self.layout.addJob(job)
634
635 project_config = configloader.ProjectParser.fromYaml(
636 self.tenant, self.layout, [{
637 '_source_context': self.context,
638 '_start_mark': self.start_mark,
639 'name': 'project',
640 'gate': {
641 'jobs': [
642 'job'
643 ]
644 }
645 }]
646 )
647 self.layout.addProjectConfig(project_config)
648
649 change = model.Change(self.project)
650 # Test master
651 change.branch = 'master'
652 item = self.queue.enqueueChange(change)
653 item.current_build_set.layout = self.layout
654 with testtools.ExpectedException(
655 Exception,
656 "Pipeline gate does not allow jobs with secrets"):
657 item.freezeJobGraph()
658
James E. Blairce8a2132016-05-19 15:21:52 -0700659
660class TestJobTimeData(BaseTestCase):
661 def setUp(self):
662 super(TestJobTimeData, self).setUp()
663 self.tmp_root = self.useFixture(fixtures.TempDir(
664 rootdir=os.environ.get("ZUUL_TEST_ROOT"))
665 ).path
666
667 def test_empty_timedata(self):
668 path = os.path.join(self.tmp_root, 'job-name')
669 self.assertFalse(os.path.exists(path))
670 self.assertFalse(os.path.exists(path + '.tmp'))
671 td = model.JobTimeData(path)
672 self.assertEqual(td.success_times, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
673 self.assertEqual(td.failure_times, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
674 self.assertEqual(td.results, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
675
676 def test_save_reload(self):
677 path = os.path.join(self.tmp_root, 'job-name')
678 self.assertFalse(os.path.exists(path))
679 self.assertFalse(os.path.exists(path + '.tmp'))
680 td = model.JobTimeData(path)
681 self.assertEqual(td.success_times, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
682 self.assertEqual(td.failure_times, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
683 self.assertEqual(td.results, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
684 success_times = []
685 failure_times = []
686 results = []
687 for x in range(10):
688 success_times.append(int(random.random() * 1000))
689 failure_times.append(int(random.random() * 1000))
690 results.append(0)
691 results.append(1)
692 random.shuffle(results)
693 s = f = 0
694 for result in results:
695 if result:
696 td.add(failure_times[f], 'FAILURE')
697 f += 1
698 else:
699 td.add(success_times[s], 'SUCCESS')
700 s += 1
701 self.assertEqual(td.success_times, success_times)
702 self.assertEqual(td.failure_times, failure_times)
703 self.assertEqual(td.results, results[10:])
704 td.save()
705 self.assertTrue(os.path.exists(path))
706 self.assertFalse(os.path.exists(path + '.tmp'))
707 td = model.JobTimeData(path)
708 td.load()
709 self.assertEqual(td.success_times, success_times)
710 self.assertEqual(td.failure_times, failure_times)
711 self.assertEqual(td.results, results[10:])
712
713
714class TestTimeDataBase(BaseTestCase):
715 def setUp(self):
716 super(TestTimeDataBase, self).setUp()
717 self.tmp_root = self.useFixture(fixtures.TempDir(
718 rootdir=os.environ.get("ZUUL_TEST_ROOT"))
719 ).path
720 self.db = model.TimeDataBase(self.tmp_root)
721
722 def test_timedatabase(self):
723 self.assertEqual(self.db.getEstimatedTime('job-name'), 0)
724 self.db.update('job-name', 50, 'SUCCESS')
725 self.assertEqual(self.db.getEstimatedTime('job-name'), 50)
726 self.db.update('job-name', 100, 'SUCCESS')
727 self.assertEqual(self.db.getEstimatedTime('job-name'), 75)
728 for x in range(10):
729 self.db.update('job-name', 100, 'SUCCESS')
730 self.assertEqual(self.db.getEstimatedTime('job-name'), 100)
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200731
732
733class TestGraph(BaseTestCase):
734 def test_job_graph_disallows_multiple_jobs_with_same_name(self):
735 graph = model.JobGraph()
736 job1 = model.Job('job')
737 job2 = model.Job('job')
738 graph.addJob(job1)
739 with testtools.ExpectedException(Exception,
740 "Job job already added"):
741 graph.addJob(job2)
742
743 def test_job_graph_disallows_circular_dependencies(self):
744 graph = model.JobGraph()
745 jobs = [model.Job('job%d' % i) for i in range(0, 10)]
746 prevjob = None
747 for j in jobs[:3]:
748 if prevjob:
749 j.dependencies = frozenset([prevjob.name])
750 graph.addJob(j)
751 prevjob = j
752 # 0 triggers 1 triggers 2 triggers 3...
753
754 # Cannot depend on itself
755 with testtools.ExpectedException(
756 Exception,
757 "Dependency cycle detected in job jobX"):
758 j = model.Job('jobX')
759 j.dependencies = frozenset([j.name])
760 graph.addJob(j)
761
762 # Disallow circular dependencies
763 with testtools.ExpectedException(
764 Exception,
765 "Dependency cycle detected in job job3"):
766 jobs[4].dependencies = frozenset([jobs[3].name])
767 graph.addJob(jobs[4])
768 jobs[3].dependencies = frozenset([jobs[4].name])
769 graph.addJob(jobs[3])
770
771 jobs[5].dependencies = frozenset([jobs[4].name])
772 graph.addJob(jobs[5])
773
774 with testtools.ExpectedException(
775 Exception,
776 "Dependency cycle detected in job job3"):
777 jobs[3].dependencies = frozenset([jobs[5].name])
778 graph.addJob(jobs[3])
779
780 jobs[3].dependencies = frozenset([jobs[2].name])
781 graph.addJob(jobs[3])
782 jobs[6].dependencies = frozenset([jobs[2].name])
783 graph.addJob(jobs[6])