blob: d8480ea0a7a1a721424fe6ea9d23b12cd929340d [file] [log] [blame]
Maru Newby3fe5f852015-01-13 04:22:14 +00001# Copyright 2015 Red Hat, Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
Joshua Hesketh0aa7e8b2016-07-14 00:12:25 +100015
James E. Blairce8a2132016-05-19 15:21:52 -070016import os
17import random
18
19import fixtures
James E. Blair4317e9f2016-07-15 10:05:47 -070020import testtools
James E. Blairce8a2132016-05-19 15:21:52 -070021
Maru Newby3fe5f852015-01-13 04:22:14 +000022from zuul import model
James E. Blair83005782015-12-11 14:46:03 -080023from zuul import configloader
James E. Blairbf1a4f22017-03-17 10:59:37 -070024from zuul.lib import encryption
Clint Byrum88fdfde2017-03-28 16:22:42 -070025from zuul.lib import yamlutil as yaml
Maru Newby3fe5f852015-01-13 04:22:14 +000026
James E. Blair18f86a32017-03-15 14:43:26 -070027from tests.base import BaseTestCase, FIXTURE_DIR
Maru Newby3fe5f852015-01-13 04:22:14 +000028
29
James E. Blair0a899752017-03-29 13:22:16 -070030class Dummy(object):
31 def __init__(self, **kw):
32 for k, v in kw.items():
33 setattr(self, k, v)
James E. Blairb3f5db12017-03-17 12:57:39 -070034
35
Maru Newby3fe5f852015-01-13 04:22:14 +000036class TestJob(BaseTestCase):
James E. Blaira7f51ca2017-02-07 16:01:26 -080037 def setUp(self):
38 super(TestJob, self).setUp()
James E. Blair0a899752017-03-29 13:22:16 -070039 self.connection = Dummy(connection_name='dummy_connection')
40 self.source = Dummy(canonical_hostname='git.example.com',
James E. Blair0a899752017-03-29 13:22:16 -070041 connection=self.connection)
James E. Blairb3f5db12017-03-17 12:57:39 -070042 self.tenant = model.Tenant('tenant')
43 self.layout = model.Layout()
James E. Blair0a899752017-03-29 13:22:16 -070044 self.project = model.Project('project', self.source)
James E. Blair109da3f2017-04-04 14:39:43 -070045 self.tenant.addUntrustedProject(self.project)
James E. Blairb3f5db12017-03-17 12:57:39 -070046 self.pipeline = model.Pipeline('gate', self.layout)
47 self.layout.addPipeline(self.pipeline)
48 self.queue = model.ChangeQueue(self.pipeline)
49
James E. Blair18f86a32017-03-15 14:43:26 -070050 private_key_file = os.path.join(FIXTURE_DIR, 'private.pem')
51 with open(private_key_file, "rb") as f:
James E. Blairbf1a4f22017-03-17 10:59:37 -070052 self.project.private_key, self.project.public_key = \
53 encryption.deserialize_rsa_keypair(f.read())
James E. Blair6f140c72017-03-03 10:32:07 -080054 self.context = model.SourceContext(self.project, 'master',
55 'test', True)
James E. Blairec7ff302017-03-04 07:31:32 -080056 self.start_mark = yaml.Mark('name', 0, 0, 0, '', 0)
James E. Blaira7f51ca2017-02-07 16:01:26 -080057
Maru Newby3fe5f852015-01-13 04:22:14 +000058 @property
59 def job(self):
James E. Blair5ac93842017-01-20 06:47:34 -080060 tenant = model.Tenant('tenant')
James E. Blair83005782015-12-11 14:46:03 -080061 layout = model.Layout()
James E. Blair5ac93842017-01-20 06:47:34 -080062 job = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -080063 '_source_context': self.context,
64 '_start_mark': self.start_mark,
James E. Blair83005782015-12-11 14:46:03 -080065 'name': 'job',
66 'irrelevant-files': [
67 '^docs/.*$'
68 ]})
Maru Newby3fe5f852015-01-13 04:22:14 +000069 return job
70
71 def test_change_matches_returns_false_for_matched_skip_if(self):
72 change = model.Change('project')
Alexander Evseevdbe6fab2015-11-19 12:46:34 +030073 change.files = ['/COMMIT_MSG', 'docs/foo']
Maru Newby3fe5f852015-01-13 04:22:14 +000074 self.assertFalse(self.job.changeMatches(change))
75
76 def test_change_matches_returns_true_for_unmatched_skip_if(self):
77 change = model.Change('project')
Alexander Evseevdbe6fab2015-11-19 12:46:34 +030078 change.files = ['/COMMIT_MSG', 'foo']
Maru Newby3fe5f852015-01-13 04:22:14 +000079 self.assertTrue(self.job.changeMatches(change))
80
Maru Newby79427a42015-02-17 17:54:45 +000081 def test_job_sets_defaults_for_boolean_attributes(self):
James E. Blair83005782015-12-11 14:46:03 -080082 self.assertIsNotNone(self.job.voting)
83
84 def test_job_inheritance(self):
James E. Blaira7f51ca2017-02-07 16:01:26 -080085 # This is standard job inheritance.
86
87 base_pre = model.PlaybookContext(self.context, 'base-pre')
88 base_run = model.PlaybookContext(self.context, 'base-run')
89 base_post = model.PlaybookContext(self.context, 'base-post')
90
91 base = model.Job('base')
92 base.timeout = 30
93 base.pre_run = [base_pre]
94 base.run = [base_run]
95 base.post_run = [base_post]
James E. Blair8525e2b2017-03-15 14:05:47 -070096 base.auth = model.AuthContext()
James E. Blaira7f51ca2017-02-07 16:01:26 -080097
98 py27 = model.Job('py27')
99 self.assertEqual(None, py27.timeout)
100 py27.inheritFrom(base)
101 self.assertEqual(30, py27.timeout)
102 self.assertEqual(['base-pre'],
103 [x.path for x in py27.pre_run])
104 self.assertEqual(['base-run'],
105 [x.path for x in py27.run])
106 self.assertEqual(['base-post'],
107 [x.path for x in py27.post_run])
James E. Blair8525e2b2017-03-15 14:05:47 -0700108 self.assertEqual(None, py27.auth)
James E. Blaira7f51ca2017-02-07 16:01:26 -0800109
110 def test_job_variants(self):
111 # This simulates freezing a job.
112
113 py27_pre = model.PlaybookContext(self.context, 'py27-pre')
114 py27_run = model.PlaybookContext(self.context, 'py27-run')
115 py27_post = model.PlaybookContext(self.context, 'py27-post')
116
117 py27 = model.Job('py27')
118 py27.timeout = 30
119 py27.pre_run = [py27_pre]
120 py27.run = [py27_run]
121 py27.post_run = [py27_post]
James E. Blair8525e2b2017-03-15 14:05:47 -0700122 auth = model.AuthContext()
123 auth.secrets.append('foo')
James E. Blaira7f51ca2017-02-07 16:01:26 -0800124 py27.auth = auth
125
126 job = py27.copy()
127 self.assertEqual(30, job.timeout)
128
129 # Apply the diablo variant
130 diablo = model.Job('py27')
131 diablo.timeout = 40
132 job.applyVariant(diablo)
133
134 self.assertEqual(40, job.timeout)
135 self.assertEqual(['py27-pre'],
136 [x.path for x in job.pre_run])
137 self.assertEqual(['py27-run'],
138 [x.path for x in job.run])
139 self.assertEqual(['py27-post'],
140 [x.path for x in job.post_run])
141 self.assertEqual(auth, job.auth)
142
143 # Set the job to final for the following checks
144 job.final = True
145 self.assertTrue(job.voting)
146
147 good_final = model.Job('py27')
148 good_final.voting = False
149 job.applyVariant(good_final)
150 self.assertFalse(job.voting)
151
152 bad_final = model.Job('py27')
153 bad_final.timeout = 600
154 with testtools.ExpectedException(
155 Exception,
156 "Unable to modify final job"):
157 job.applyVariant(bad_final)
158
159 def test_job_inheritance_configloader(self):
160 # TODO(jeblair): move this to a configloader test
James E. Blair5ac93842017-01-20 06:47:34 -0800161 tenant = model.Tenant('tenant')
James E. Blair83005782015-12-11 14:46:03 -0800162 layout = model.Layout()
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700163
164 pipeline = model.Pipeline('gate', layout)
165 layout.addPipeline(pipeline)
166 queue = model.ChangeQueue(pipeline)
James E. Blair0a899752017-03-29 13:22:16 -0700167 project = model.Project('project', self.source)
James E. Blair109da3f2017-04-04 14:39:43 -0700168 tenant.addUntrustedProject(project)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700169
James E. Blair5ac93842017-01-20 06:47:34 -0800170 base = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800171 '_source_context': self.context,
172 '_start_mark': self.start_mark,
James E. Blair83005782015-12-11 14:46:03 -0800173 'name': 'base',
174 'timeout': 30,
James E. Blaira7f51ca2017-02-07 16:01:26 -0800175 'pre-run': 'base-pre',
176 'post-run': 'base-post',
James E. Blair1774dd52017-02-03 10:52:32 -0800177 'nodes': [{
178 'name': 'controller',
179 'image': 'base',
180 }],
James E. Blair83005782015-12-11 14:46:03 -0800181 })
182 layout.addJob(base)
James E. Blair5ac93842017-01-20 06:47:34 -0800183 python27 = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800184 '_source_context': self.context,
185 '_start_mark': self.start_mark,
James E. Blair83005782015-12-11 14:46:03 -0800186 'name': 'python27',
187 'parent': 'base',
James E. Blaira7f51ca2017-02-07 16:01:26 -0800188 'pre-run': 'py27-pre',
189 'post-run': 'py27-post',
James E. Blair1774dd52017-02-03 10:52:32 -0800190 'nodes': [{
191 'name': 'controller',
192 'image': 'new',
193 }],
James E. Blair83005782015-12-11 14:46:03 -0800194 'timeout': 40,
195 })
196 layout.addJob(python27)
James E. Blair5ac93842017-01-20 06:47:34 -0800197 python27diablo = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800198 '_source_context': self.context,
199 '_start_mark': self.start_mark,
James E. Blair83005782015-12-11 14:46:03 -0800200 'name': 'python27',
201 'branches': [
202 'stable/diablo'
203 ],
James E. Blaira7f51ca2017-02-07 16:01:26 -0800204 'pre-run': 'py27-diablo-pre',
205 'run': 'py27-diablo',
206 'post-run': 'py27-diablo-post',
James E. Blair1774dd52017-02-03 10:52:32 -0800207 'nodes': [{
208 'name': 'controller',
209 'image': 'old',
210 }],
James E. Blair83005782015-12-11 14:46:03 -0800211 'timeout': 50,
212 })
213 layout.addJob(python27diablo)
214
James E. Blair5ac93842017-01-20 06:47:34 -0800215 python27essex = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800216 '_source_context': self.context,
217 '_start_mark': self.start_mark,
James E. Blaira7f51ca2017-02-07 16:01:26 -0800218 'name': 'python27',
219 'branches': [
220 'stable/essex'
221 ],
222 'pre-run': 'py27-essex-pre',
223 'post-run': 'py27-essex-post',
224 })
225 layout.addJob(python27essex)
226
James E. Blairff555742017-02-19 11:34:27 -0800227 project_config = configloader.ProjectParser.fromYaml(tenant, layout, [{
James E. Blairec7ff302017-03-04 07:31:32 -0800228 '_source_context': self.context,
229 '_start_mark': self.start_mark,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700230 'name': 'project',
231 'gate': {
232 'jobs': [
233 'python27'
234 ]
235 }
James E. Blairff555742017-02-19 11:34:27 -0800236 }])
James E. Blairf59f3cf2017-02-19 14:50:26 -0800237 layout.addProjectConfig(project_config)
James E. Blair83005782015-12-11 14:46:03 -0800238
James E. Blair83005782015-12-11 14:46:03 -0800239 change = model.Change(project)
James E. Blair1774dd52017-02-03 10:52:32 -0800240 # Test master
James E. Blair83005782015-12-11 14:46:03 -0800241 change.branch = 'master'
242 item = queue.enqueueChange(change)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700243 item.current_build_set.layout = layout
James E. Blair83005782015-12-11 14:46:03 -0800244
245 self.assertTrue(base.changeMatches(change))
246 self.assertTrue(python27.changeMatches(change))
247 self.assertFalse(python27diablo.changeMatches(change))
James E. Blaira7f51ca2017-02-07 16:01:26 -0800248 self.assertFalse(python27essex.changeMatches(change))
James E. Blair83005782015-12-11 14:46:03 -0800249
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200250 item.freezeJobGraph()
James E. Blair83005782015-12-11 14:46:03 -0800251 self.assertEqual(len(item.getJobs()), 1)
252 job = item.getJobs()[0]
253 self.assertEqual(job.name, 'python27')
254 self.assertEqual(job.timeout, 40)
James E. Blair1774dd52017-02-03 10:52:32 -0800255 nodes = job.nodeset.getNodes()
256 self.assertEqual(len(nodes), 1)
257 self.assertEqual(nodes[0].image, 'new')
James E. Blaira7f51ca2017-02-07 16:01:26 -0800258 self.assertEqual([x.path for x in job.pre_run],
259 ['playbooks/base-pre',
260 'playbooks/py27-pre'])
261 self.assertEqual([x.path for x in job.post_run],
262 ['playbooks/py27-post',
263 'playbooks/base-post'])
264 self.assertEqual([x.path for x in job.run],
265 ['playbooks/python27',
266 'playbooks/base'])
James E. Blair83005782015-12-11 14:46:03 -0800267
James E. Blair1774dd52017-02-03 10:52:32 -0800268 # Test diablo
James E. Blair83005782015-12-11 14:46:03 -0800269 change.branch = 'stable/diablo'
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700270 item = queue.enqueueChange(change)
271 item.current_build_set.layout = layout
James E. Blair83005782015-12-11 14:46:03 -0800272
273 self.assertTrue(base.changeMatches(change))
274 self.assertTrue(python27.changeMatches(change))
275 self.assertTrue(python27diablo.changeMatches(change))
James E. Blaira7f51ca2017-02-07 16:01:26 -0800276 self.assertFalse(python27essex.changeMatches(change))
James E. Blair83005782015-12-11 14:46:03 -0800277
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200278 item.freezeJobGraph()
James E. Blair83005782015-12-11 14:46:03 -0800279 self.assertEqual(len(item.getJobs()), 1)
280 job = item.getJobs()[0]
281 self.assertEqual(job.name, 'python27')
282 self.assertEqual(job.timeout, 50)
James E. Blair1774dd52017-02-03 10:52:32 -0800283 nodes = job.nodeset.getNodes()
284 self.assertEqual(len(nodes), 1)
285 self.assertEqual(nodes[0].image, 'old')
James E. Blaira7f51ca2017-02-07 16:01:26 -0800286 self.assertEqual([x.path for x in job.pre_run],
287 ['playbooks/base-pre',
288 'playbooks/py27-pre',
289 'playbooks/py27-diablo-pre'])
290 self.assertEqual([x.path for x in job.post_run],
291 ['playbooks/py27-diablo-post',
292 'playbooks/py27-post',
293 'playbooks/base-post'])
294 self.assertEqual([x.path for x in job.run],
295 ['playbooks/py27-diablo']),
296
297 # Test essex
298 change.branch = 'stable/essex'
299 item = queue.enqueueChange(change)
300 item.current_build_set.layout = layout
301
302 self.assertTrue(base.changeMatches(change))
303 self.assertTrue(python27.changeMatches(change))
304 self.assertFalse(python27diablo.changeMatches(change))
305 self.assertTrue(python27essex.changeMatches(change))
306
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200307 item.freezeJobGraph()
James E. Blaira7f51ca2017-02-07 16:01:26 -0800308 self.assertEqual(len(item.getJobs()), 1)
309 job = item.getJobs()[0]
310 self.assertEqual(job.name, 'python27')
311 self.assertEqual([x.path for x in job.pre_run],
312 ['playbooks/base-pre',
313 'playbooks/py27-pre',
314 'playbooks/py27-essex-pre'])
315 self.assertEqual([x.path for x in job.post_run],
316 ['playbooks/py27-essex-post',
317 'playbooks/py27-post',
318 'playbooks/base-post'])
319 self.assertEqual([x.path for x in job.run],
320 ['playbooks/python27',
321 'playbooks/base'])
James E. Blairce8a2132016-05-19 15:21:52 -0700322
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000323 def test_job_auth_inheritance(self):
James E. Blair5ac93842017-01-20 06:47:34 -0800324 tenant = model.Tenant('tenant')
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000325 layout = model.Layout()
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000326
James E. Blair01f83b72017-03-15 13:03:40 -0700327 conf = yaml.safe_load('''
328- secret:
329 name: pypi-credentials
330 data:
331 username: test-username
James E. Blair717e8e92017-03-17 11:03:27 -0700332 password: !encrypted/pkcs1-oaep |
James E. Blair01f83b72017-03-15 13:03:40 -0700333 BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ
334 L0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4joeusC9drN3AA8a4o
335 ykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CRgd0QBMPl6VDoFgBPB8vxtJw+
336 3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzibDsSXsfJt1y+5n7yOURsC7lovMg4GF/v
337 Cl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCYceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qt
338 xhbpjTxG4U5Q/SoppOJ60WqEkQvbXs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYr
339 aI+AKYsMYx3RBlfAmCeC1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFW
340 Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd
341 +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs=
342''')[0]['secret']
343
344 conf['_source_context'] = self.context
345 conf['_start_mark'] = self.start_mark
346
347 secret = configloader.SecretParser.fromYaml(layout, conf)
348 layout.addSecret(secret)
349
James E. Blair5ac93842017-01-20 06:47:34 -0800350 base = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800351 '_source_context': self.context,
352 '_start_mark': self.start_mark,
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000353 'name': 'base',
354 'timeout': 30,
355 })
356 layout.addJob(base)
James E. Blair5ac93842017-01-20 06:47:34 -0800357 pypi_upload_without_inherit = configloader.JobParser.fromYaml(
358 tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800359 '_source_context': self.context,
360 '_start_mark': self.start_mark,
James E. Blair5ac93842017-01-20 06:47:34 -0800361 'name': 'pypi-upload-without-inherit',
362 'parent': 'base',
363 'timeout': 40,
364 'auth': {
365 'secrets': [
366 'pypi-credentials',
367 ]
368 }
369 })
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000370 layout.addJob(pypi_upload_without_inherit)
James E. Blair5ac93842017-01-20 06:47:34 -0800371 pypi_upload_with_inherit = configloader.JobParser.fromYaml(
372 tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800373 '_source_context': self.context,
374 '_start_mark': self.start_mark,
James E. Blair5ac93842017-01-20 06:47:34 -0800375 'name': 'pypi-upload-with-inherit',
376 'parent': 'base',
377 'timeout': 40,
378 'auth': {
379 'inherit': True,
380 'secrets': [
381 'pypi-credentials',
382 ]
383 }
384 })
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000385 layout.addJob(pypi_upload_with_inherit)
386 pypi_upload_with_inherit_false = configloader.JobParser.fromYaml(
James E. Blair5ac93842017-01-20 06:47:34 -0800387 tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800388 '_source_context': self.context,
389 '_start_mark': self.start_mark,
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000390 'name': 'pypi-upload-with-inherit-false',
391 'parent': 'base',
392 'timeout': 40,
393 'auth': {
394 'inherit': False,
Ricardo Carrillo Cruz12c892b2016-11-18 15:35:49 +0000395 'secrets': [
396 'pypi-credentials',
397 ]
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000398 }
399 })
400 layout.addJob(pypi_upload_with_inherit_false)
James E. Blair5ac93842017-01-20 06:47:34 -0800401 in_repo_job_without_inherit = configloader.JobParser.fromYaml(
402 tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800403 '_source_context': self.context,
404 '_start_mark': self.start_mark,
James E. Blair5ac93842017-01-20 06:47:34 -0800405 'name': 'in-repo-job-without-inherit',
406 'parent': 'pypi-upload-without-inherit',
407 })
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000408 layout.addJob(in_repo_job_without_inherit)
James E. Blair5ac93842017-01-20 06:47:34 -0800409 in_repo_job_with_inherit = configloader.JobParser.fromYaml(
410 tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800411 '_source_context': self.context,
412 '_start_mark': self.start_mark,
James E. Blair5ac93842017-01-20 06:47:34 -0800413 'name': 'in-repo-job-with-inherit',
414 'parent': 'pypi-upload-with-inherit',
415 })
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000416 layout.addJob(in_repo_job_with_inherit)
417 in_repo_job_with_inherit_false = configloader.JobParser.fromYaml(
James E. Blair5ac93842017-01-20 06:47:34 -0800418 tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800419 '_source_context': self.context,
420 '_start_mark': self.start_mark,
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000421 'name': 'in-repo-job-with-inherit-false',
422 'parent': 'pypi-upload-with-inherit-false',
423 })
424 layout.addJob(in_repo_job_with_inherit_false)
425
James E. Blair8525e2b2017-03-15 14:05:47 -0700426 self.assertEqual(None, in_repo_job_without_inherit.auth)
427 self.assertEqual(1, len(in_repo_job_with_inherit.auth.secrets))
428 self.assertEqual(in_repo_job_with_inherit.auth.secrets[0].name,
429 'pypi-credentials')
430 self.assertEqual(None, in_repo_job_with_inherit_false.auth)
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000431
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700432 def test_job_inheritance_job_tree(self):
James E. Blair5ac93842017-01-20 06:47:34 -0800433 tenant = model.Tenant('tenant')
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700434 layout = model.Layout()
James E. Blair109da3f2017-04-04 14:39:43 -0700435 tenant.addUntrustedProject(self.project)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700436
437 pipeline = model.Pipeline('gate', layout)
438 layout.addPipeline(pipeline)
439 queue = model.ChangeQueue(pipeline)
440
James E. Blair5ac93842017-01-20 06:47:34 -0800441 base = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800442 '_source_context': self.context,
443 '_start_mark': self.start_mark,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700444 'name': 'base',
445 'timeout': 30,
446 })
447 layout.addJob(base)
James E. Blair5ac93842017-01-20 06:47:34 -0800448 python27 = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800449 '_source_context': self.context,
450 '_start_mark': self.start_mark,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700451 'name': 'python27',
452 'parent': 'base',
453 'timeout': 40,
454 })
455 layout.addJob(python27)
James E. Blair5ac93842017-01-20 06:47:34 -0800456 python27diablo = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800457 '_source_context': self.context,
458 '_start_mark': self.start_mark,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700459 'name': 'python27',
460 'branches': [
461 'stable/diablo'
462 ],
463 'timeout': 50,
464 })
465 layout.addJob(python27diablo)
466
James E. Blairff555742017-02-19 11:34:27 -0800467 project_config = configloader.ProjectParser.fromYaml(tenant, layout, [{
James E. Blairec7ff302017-03-04 07:31:32 -0800468 '_source_context': self.context,
469 '_start_mark': self.start_mark,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700470 'name': 'project',
471 'gate': {
472 'jobs': [
473 {'python27': {'timeout': 70}}
474 ]
475 }
James E. Blairff555742017-02-19 11:34:27 -0800476 }])
James E. Blairf59f3cf2017-02-19 14:50:26 -0800477 layout.addProjectConfig(project_config)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700478
James E. Blairec7ff302017-03-04 07:31:32 -0800479 change = model.Change(self.project)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700480 change.branch = 'master'
481 item = queue.enqueueChange(change)
482 item.current_build_set.layout = layout
483
484 self.assertTrue(base.changeMatches(change))
485 self.assertTrue(python27.changeMatches(change))
486 self.assertFalse(python27diablo.changeMatches(change))
487
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200488 item.freezeJobGraph()
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700489 self.assertEqual(len(item.getJobs()), 1)
490 job = item.getJobs()[0]
491 self.assertEqual(job.name, 'python27')
492 self.assertEqual(job.timeout, 70)
493
494 change.branch = 'stable/diablo'
495 item = queue.enqueueChange(change)
496 item.current_build_set.layout = layout
497
498 self.assertTrue(base.changeMatches(change))
499 self.assertTrue(python27.changeMatches(change))
500 self.assertTrue(python27diablo.changeMatches(change))
501
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200502 item.freezeJobGraph()
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700503 self.assertEqual(len(item.getJobs()), 1)
504 job = item.getJobs()[0]
505 self.assertEqual(job.name, 'python27')
506 self.assertEqual(job.timeout, 70)
507
Clint Byrum85493602016-11-18 11:59:47 -0800508 def test_inheritance_keeps_matchers(self):
James E. Blair5ac93842017-01-20 06:47:34 -0800509 tenant = model.Tenant('tenant')
Clint Byrum85493602016-11-18 11:59:47 -0800510 layout = model.Layout()
511
512 pipeline = model.Pipeline('gate', layout)
513 layout.addPipeline(pipeline)
514 queue = model.ChangeQueue(pipeline)
James E. Blair0a899752017-03-29 13:22:16 -0700515 project = model.Project('project', self.source)
James E. Blair109da3f2017-04-04 14:39:43 -0700516 tenant.addUntrustedProject(project)
Clint Byrum85493602016-11-18 11:59:47 -0800517
James E. Blair5ac93842017-01-20 06:47:34 -0800518 base = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800519 '_source_context': self.context,
520 '_start_mark': self.start_mark,
Clint Byrum85493602016-11-18 11:59:47 -0800521 'name': 'base',
522 'timeout': 30,
523 })
524 layout.addJob(base)
James E. Blair5ac93842017-01-20 06:47:34 -0800525 python27 = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blairec7ff302017-03-04 07:31:32 -0800526 '_source_context': self.context,
527 '_start_mark': self.start_mark,
Clint Byrum85493602016-11-18 11:59:47 -0800528 'name': 'python27',
529 'parent': 'base',
530 'timeout': 40,
531 'irrelevant-files': ['^ignored-file$'],
532 })
533 layout.addJob(python27)
534
James E. Blairff555742017-02-19 11:34:27 -0800535 project_config = configloader.ProjectParser.fromYaml(tenant, layout, [{
James E. Blairec7ff302017-03-04 07:31:32 -0800536 '_source_context': self.context,
537 '_start_mark': self.start_mark,
Clint Byrum85493602016-11-18 11:59:47 -0800538 'name': 'project',
539 'gate': {
540 'jobs': [
541 'python27',
542 ]
543 }
James E. Blairff555742017-02-19 11:34:27 -0800544 }])
James E. Blairf59f3cf2017-02-19 14:50:26 -0800545 layout.addProjectConfig(project_config)
Clint Byrum85493602016-11-18 11:59:47 -0800546
547 change = model.Change(project)
548 change.branch = 'master'
549 change.files = ['/COMMIT_MSG', 'ignored-file']
550 item = queue.enqueueChange(change)
551 item.current_build_set.layout = layout
552
553 self.assertTrue(base.changeMatches(change))
554 self.assertFalse(python27.changeMatches(change))
555
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200556 item.freezeJobGraph()
Clint Byrum85493602016-11-18 11:59:47 -0800557 self.assertEqual([], item.getJobs())
558
James E. Blair4317e9f2016-07-15 10:05:47 -0700559 def test_job_source_project(self):
James E. Blair5ac93842017-01-20 06:47:34 -0800560 tenant = model.Tenant('tenant')
James E. Blair4317e9f2016-07-15 10:05:47 -0700561 layout = model.Layout()
James E. Blair0a899752017-03-29 13:22:16 -0700562 base_project = model.Project('base_project', self.source)
James E. Blair6f140c72017-03-03 10:32:07 -0800563 base_context = model.SourceContext(base_project, 'master',
564 'test', True)
James E. Blaircdab2032017-02-01 09:09:29 -0800565
James E. Blair5ac93842017-01-20 06:47:34 -0800566 base = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blaircdab2032017-02-01 09:09:29 -0800567 '_source_context': base_context,
James E. Blairec7ff302017-03-04 07:31:32 -0800568 '_start_mark': self.start_mark,
James E. Blair4317e9f2016-07-15 10:05:47 -0700569 'name': 'base',
570 })
571 layout.addJob(base)
572
James E. Blair0a899752017-03-29 13:22:16 -0700573 other_project = model.Project('other_project', self.source)
James E. Blair6f140c72017-03-03 10:32:07 -0800574 other_context = model.SourceContext(other_project, 'master',
575 'test', True)
James E. Blair5ac93842017-01-20 06:47:34 -0800576 base2 = configloader.JobParser.fromYaml(tenant, layout, {
James E. Blaircdab2032017-02-01 09:09:29 -0800577 '_source_context': other_context,
James E. Blairec7ff302017-03-04 07:31:32 -0800578 '_start_mark': self.start_mark,
James E. Blair4317e9f2016-07-15 10:05:47 -0700579 'name': 'base',
580 })
581 with testtools.ExpectedException(
582 Exception,
583 "Job base in other_project is not permitted "
584 "to shadow job base in base_project"):
585 layout.addJob(base2)
586
James E. Blairb3f5db12017-03-17 12:57:39 -0700587 def test_job_allowed_projects(self):
588 job = configloader.JobParser.fromYaml(self.tenant, self.layout, {
589 '_source_context': self.context,
590 '_start_mark': self.start_mark,
591 'name': 'job',
592 'allowed-projects': ['project'],
593 })
594 self.layout.addJob(job)
595
James E. Blair0a899752017-03-29 13:22:16 -0700596 project2 = model.Project('project2', self.source)
James E. Blair109da3f2017-04-04 14:39:43 -0700597 self.tenant.addUntrustedProject(project2)
James E. Blairb3f5db12017-03-17 12:57:39 -0700598 context2 = model.SourceContext(project2, 'master',
599 'test', True)
600
601 project2_config = configloader.ProjectParser.fromYaml(
602 self.tenant, self.layout, [{
603 '_source_context': context2,
604 '_start_mark': self.start_mark,
605 'name': 'project2',
606 'gate': {
607 'jobs': [
608 'job'
609 ]
610 }
611 }]
612 )
613 self.layout.addProjectConfig(project2_config)
614
615 change = model.Change(project2)
616 # Test master
617 change.branch = 'master'
618 item = self.queue.enqueueChange(change)
619 item.current_build_set.layout = self.layout
620 with testtools.ExpectedException(
621 Exception,
622 "Project project2 is not allowed to run job job"):
623 item.freezeJobGraph()
624
James E. Blaird2348362017-03-17 13:59:35 -0700625 def test_job_pipeline_allow_secrets(self):
626 self.pipeline.allow_secrets = False
627 job = configloader.JobParser.fromYaml(self.tenant, self.layout, {
628 '_source_context': self.context,
629 '_start_mark': self.start_mark,
630 'name': 'job',
631 })
632 auth = model.AuthContext()
633 auth.secrets.append('foo')
634 job.auth = auth
635
636 self.layout.addJob(job)
637
638 project_config = configloader.ProjectParser.fromYaml(
639 self.tenant, self.layout, [{
640 '_source_context': self.context,
641 '_start_mark': self.start_mark,
642 'name': 'project',
643 'gate': {
644 'jobs': [
645 'job'
646 ]
647 }
648 }]
649 )
650 self.layout.addProjectConfig(project_config)
651
652 change = model.Change(self.project)
653 # Test master
654 change.branch = 'master'
655 item = self.queue.enqueueChange(change)
656 item.current_build_set.layout = self.layout
657 with testtools.ExpectedException(
658 Exception,
659 "Pipeline gate does not allow jobs with secrets"):
660 item.freezeJobGraph()
661
James E. Blairce8a2132016-05-19 15:21:52 -0700662
663class TestJobTimeData(BaseTestCase):
664 def setUp(self):
665 super(TestJobTimeData, self).setUp()
666 self.tmp_root = self.useFixture(fixtures.TempDir(
667 rootdir=os.environ.get("ZUUL_TEST_ROOT"))
668 ).path
669
670 def test_empty_timedata(self):
671 path = os.path.join(self.tmp_root, 'job-name')
672 self.assertFalse(os.path.exists(path))
673 self.assertFalse(os.path.exists(path + '.tmp'))
674 td = model.JobTimeData(path)
675 self.assertEqual(td.success_times, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
676 self.assertEqual(td.failure_times, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
677 self.assertEqual(td.results, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
678
679 def test_save_reload(self):
680 path = os.path.join(self.tmp_root, 'job-name')
681 self.assertFalse(os.path.exists(path))
682 self.assertFalse(os.path.exists(path + '.tmp'))
683 td = model.JobTimeData(path)
684 self.assertEqual(td.success_times, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
685 self.assertEqual(td.failure_times, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
686 self.assertEqual(td.results, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
687 success_times = []
688 failure_times = []
689 results = []
690 for x in range(10):
691 success_times.append(int(random.random() * 1000))
692 failure_times.append(int(random.random() * 1000))
693 results.append(0)
694 results.append(1)
695 random.shuffle(results)
696 s = f = 0
697 for result in results:
698 if result:
699 td.add(failure_times[f], 'FAILURE')
700 f += 1
701 else:
702 td.add(success_times[s], 'SUCCESS')
703 s += 1
704 self.assertEqual(td.success_times, success_times)
705 self.assertEqual(td.failure_times, failure_times)
706 self.assertEqual(td.results, results[10:])
707 td.save()
708 self.assertTrue(os.path.exists(path))
709 self.assertFalse(os.path.exists(path + '.tmp'))
710 td = model.JobTimeData(path)
711 td.load()
712 self.assertEqual(td.success_times, success_times)
713 self.assertEqual(td.failure_times, failure_times)
714 self.assertEqual(td.results, results[10:])
715
716
717class TestTimeDataBase(BaseTestCase):
718 def setUp(self):
719 super(TestTimeDataBase, self).setUp()
720 self.tmp_root = self.useFixture(fixtures.TempDir(
721 rootdir=os.environ.get("ZUUL_TEST_ROOT"))
722 ).path
723 self.db = model.TimeDataBase(self.tmp_root)
724
725 def test_timedatabase(self):
726 self.assertEqual(self.db.getEstimatedTime('job-name'), 0)
727 self.db.update('job-name', 50, 'SUCCESS')
728 self.assertEqual(self.db.getEstimatedTime('job-name'), 50)
729 self.db.update('job-name', 100, 'SUCCESS')
730 self.assertEqual(self.db.getEstimatedTime('job-name'), 75)
731 for x in range(10):
732 self.db.update('job-name', 100, 'SUCCESS')
733 self.assertEqual(self.db.getEstimatedTime('job-name'), 100)
Fredrik Medleyf8aec832015-09-28 13:40:20 +0200734
735
736class TestGraph(BaseTestCase):
737 def test_job_graph_disallows_multiple_jobs_with_same_name(self):
738 graph = model.JobGraph()
739 job1 = model.Job('job')
740 job2 = model.Job('job')
741 graph.addJob(job1)
742 with testtools.ExpectedException(Exception,
743 "Job job already added"):
744 graph.addJob(job2)
745
746 def test_job_graph_disallows_circular_dependencies(self):
747 graph = model.JobGraph()
748 jobs = [model.Job('job%d' % i) for i in range(0, 10)]
749 prevjob = None
750 for j in jobs[:3]:
751 if prevjob:
752 j.dependencies = frozenset([prevjob.name])
753 graph.addJob(j)
754 prevjob = j
755 # 0 triggers 1 triggers 2 triggers 3...
756
757 # Cannot depend on itself
758 with testtools.ExpectedException(
759 Exception,
760 "Dependency cycle detected in job jobX"):
761 j = model.Job('jobX')
762 j.dependencies = frozenset([j.name])
763 graph.addJob(j)
764
765 # Disallow circular dependencies
766 with testtools.ExpectedException(
767 Exception,
768 "Dependency cycle detected in job job3"):
769 jobs[4].dependencies = frozenset([jobs[3].name])
770 graph.addJob(jobs[4])
771 jobs[3].dependencies = frozenset([jobs[4].name])
772 graph.addJob(jobs[3])
773
774 jobs[5].dependencies = frozenset([jobs[4].name])
775 graph.addJob(jobs[5])
776
777 with testtools.ExpectedException(
778 Exception,
779 "Dependency cycle detected in job job3"):
780 jobs[3].dependencies = frozenset([jobs[5].name])
781 graph.addJob(jobs[3])
782
783 jobs[3].dependencies = frozenset([jobs[2].name])
784 graph.addJob(jobs[3])
785 jobs[6].dependencies = frozenset([jobs[2].name])
786 graph.addJob(jobs[6])
James E. Blairc2a54fd2017-03-29 15:19:26 -0700787
788
789class TestTenant(BaseTestCase):
790 def test_add_project(self):
791 tenant = model.Tenant('tenant')
792 connection1 = Dummy(connection_name='dummy_connection1')
793 source1 = Dummy(canonical_hostname='git1.example.com',
794 name='dummy', # TODOv3(jeblair): remove
795 connection=connection1)
796
797 source1_project1 = model.Project('project1', source1)
James E. Blair109da3f2017-04-04 14:39:43 -0700798 tenant.addConfigProject(source1_project1)
James E. Blairc2a54fd2017-03-29 15:19:26 -0700799 d = {'project1':
800 {'git1.example.com': source1_project1}}
801 self.assertEqual(d, tenant.projects)
802 self.assertEqual((True, source1_project1),
803 tenant.getProject('project1'))
804 self.assertEqual((True, source1_project1),
805 tenant.getProject('git1.example.com/project1'))
806
807 source1_project2 = model.Project('project2', source1)
James E. Blair109da3f2017-04-04 14:39:43 -0700808 tenant.addUntrustedProject(source1_project2)
James E. Blairc2a54fd2017-03-29 15:19:26 -0700809 d = {'project1':
810 {'git1.example.com': source1_project1},
811 'project2':
812 {'git1.example.com': source1_project2}}
813 self.assertEqual(d, tenant.projects)
814 self.assertEqual((False, source1_project2),
815 tenant.getProject('project2'))
816 self.assertEqual((False, source1_project2),
817 tenant.getProject('git1.example.com/project2'))
818
819 connection2 = Dummy(connection_name='dummy_connection2')
820 source2 = Dummy(canonical_hostname='git2.example.com',
821 name='dummy', # TODOv3(jeblair): remove
822 connection=connection2)
823
824 source2_project1 = model.Project('project1', source2)
James E. Blair109da3f2017-04-04 14:39:43 -0700825 tenant.addUntrustedProject(source2_project1)
James E. Blairc2a54fd2017-03-29 15:19:26 -0700826 d = {'project1':
827 {'git1.example.com': source1_project1,
828 'git2.example.com': source2_project1},
829 'project2':
830 {'git1.example.com': source1_project2}}
831 self.assertEqual(d, tenant.projects)
832 with testtools.ExpectedException(
833 Exception,
834 "Project name 'project1' is ambiguous"):
835 tenant.getProject('project1')
836 self.assertEqual((False, source1_project2),
837 tenant.getProject('project2'))
838 self.assertEqual((True, source1_project1),
839 tenant.getProject('git1.example.com/project1'))
840 self.assertEqual((False, source2_project1),
841 tenant.getProject('git2.example.com/project1'))
842
843 source2_project2 = model.Project('project2', source2)
James E. Blair109da3f2017-04-04 14:39:43 -0700844 tenant.addConfigProject(source2_project2)
James E. Blairc2a54fd2017-03-29 15:19:26 -0700845 d = {'project1':
846 {'git1.example.com': source1_project1,
847 'git2.example.com': source2_project1},
848 'project2':
849 {'git1.example.com': source1_project2,
850 'git2.example.com': source2_project2}}
851 self.assertEqual(d, tenant.projects)
852 with testtools.ExpectedException(
853 Exception,
854 "Project name 'project1' is ambiguous"):
855 tenant.getProject('project1')
856 with testtools.ExpectedException(
857 Exception,
858 "Project name 'project2' is ambiguous"):
859 tenant.getProject('project2')
860 self.assertEqual((True, source1_project1),
861 tenant.getProject('git1.example.com/project1'))
862 self.assertEqual((False, source2_project1),
863 tenant.getProject('git2.example.com/project1'))
864 self.assertEqual((False, source1_project2),
865 tenant.getProject('git1.example.com/project2'))
866 self.assertEqual((True, source2_project2),
867 tenant.getProject('git2.example.com/project2'))
868
869 source1_project2b = model.Project('subpath/project2', source1)
James E. Blair109da3f2017-04-04 14:39:43 -0700870 tenant.addConfigProject(source1_project2b)
James E. Blairc2a54fd2017-03-29 15:19:26 -0700871 d = {'project1':
872 {'git1.example.com': source1_project1,
873 'git2.example.com': source2_project1},
874 'project2':
875 {'git1.example.com': source1_project2,
876 'git2.example.com': source2_project2},
877 'subpath/project2':
878 {'git1.example.com': source1_project2b}}
879 self.assertEqual(d, tenant.projects)
880 self.assertEqual((False, source1_project2),
881 tenant.getProject('git1.example.com/project2'))
882 self.assertEqual((True, source2_project2),
883 tenant.getProject('git2.example.com/project2'))
884 self.assertEqual((True, source1_project2b),
885 tenant.getProject('subpath/project2'))
886 self.assertEqual(
887 (True, source1_project2b),
888 tenant.getProject('git1.example.com/subpath/project2'))
889
890 source2_project2b = model.Project('subpath/project2', source2)
James E. Blair109da3f2017-04-04 14:39:43 -0700891 tenant.addConfigProject(source2_project2b)
James E. Blairc2a54fd2017-03-29 15:19:26 -0700892 d = {'project1':
893 {'git1.example.com': source1_project1,
894 'git2.example.com': source2_project1},
895 'project2':
896 {'git1.example.com': source1_project2,
897 'git2.example.com': source2_project2},
898 'subpath/project2':
899 {'git1.example.com': source1_project2b,
900 'git2.example.com': source2_project2b}}
901 self.assertEqual(d, tenant.projects)
902 self.assertEqual((False, source1_project2),
903 tenant.getProject('git1.example.com/project2'))
904 self.assertEqual((True, source2_project2),
905 tenant.getProject('git2.example.com/project2'))
906 with testtools.ExpectedException(
907 Exception,
908 "Project name 'subpath/project2' is ambiguous"):
909 tenant.getProject('subpath/project2')
910 self.assertEqual(
911 (True, source1_project2b),
912 tenant.getProject('git1.example.com/subpath/project2'))
913 self.assertEqual(
914 (True, source2_project2b),
915 tenant.getProject('git2.example.com/subpath/project2'))
916
917 with testtools.ExpectedException(
918 Exception,
919 "Project project1 is already in project index"):
920 tenant._addProject(source1_project1)