blob: 95cccee52da7645cd995f177c7bedc41904e653e [file] [log] [blame]
Antoine Musso45dd2cb2014-01-29 17:17:43 +01001#!/usr/bin/env python
2
3# Copyright 2012 Hewlett-Packard Development Company, L.P.
4# Copyright 2014 Wikimedia Foundation Inc.
5#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may
7# not use this file except in compliance with the License. You may obtain
8# a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15# License for the specific language governing permissions and limitations
16# under the License.
17
18import logging
James E. Blair217b10d2015-01-08 16:25:28 -080019import time
Antoine Musso45dd2cb2014-01-29 17:17:43 +010020
Tristan Cacqueray80954402017-05-28 00:33:55 +000021import zuul.executor.server
22import zuul.model
23
James E. Blaird8af5422017-05-24 13:59:40 -070024from tests.base import ZuulTestCase, simple_layout
Antoine Musso45dd2cb2014-01-29 17:17:43 +010025
Antoine Musso45dd2cb2014-01-29 17:17:43 +010026
James E. Blairb3b5b382017-06-02 11:29:57 -070027class TestExecutorRepos(ZuulTestCase):
James E. Blaird8af5422017-05-24 13:59:40 -070028 tenant_config_file = 'config/single-tenant/main.yaml'
Antoine Musso45dd2cb2014-01-29 17:17:43 +010029
James E. Blairb3b5b382017-06-02 11:29:57 -070030 log = logging.getLogger("zuul.test.executor")
Antoine Musso45dd2cb2014-01-29 17:17:43 +010031
James E. Blairaf3d01e2017-05-26 14:30:18 -070032 def assertRepoState(self, repo, state, project, build, number):
James E. Blairbcaa2d42017-05-26 15:44:23 -070033 if 'branch' in state:
34 self.assertFalse(repo.head.is_detached,
35 'Project %s commit for build %s #%s should '
36 'not have a detached HEAD' % (
37 project, build, number))
38 self.assertEquals(repo.active_branch.name,
39 state['branch'],
40 'Project %s commit for build %s #%s should '
41 'be on the correct branch' % (
42 project, build, number))
43 if 'commit' in state:
44 self.assertEquals(state['commit'],
James E. Blairaf3d01e2017-05-26 14:30:18 -070045 str(repo.commit('HEAD')),
46 'Project %s commit for build %s #%s should '
47 'be correct' % (
48 project, build, number))
James E. Blairbcaa2d42017-05-26 15:44:23 -070049 ref = repo.commit('HEAD')
50 repo_messages = set(
51 [c.message.strip() for c in repo.iter_commits(ref)])
52 if 'present' in state:
53 for change in state['present']:
James E. Blairaf3d01e2017-05-26 14:30:18 -070054 msg = '%s-1' % change.subject
55 self.assertTrue(msg in repo_messages,
56 'Project %s for build %s #%s should '
57 'have change %s' % (
58 project, build, number, change.subject))
James E. Blairbcaa2d42017-05-26 15:44:23 -070059 if 'absent' in state:
60 for change in state['absent']:
James E. Blairaf3d01e2017-05-26 14:30:18 -070061 msg = '%s-1' % change.subject
62 self.assertTrue(msg not in repo_messages,
63 'Project %s for build %s #%s should '
64 'not have change %s' % (
65 project, build, number, change.subject))
66
James E. Blair987855f2017-06-06 14:41:58 -070067 def assertBuildStates(self, states, projects):
68 for number, build in enumerate(self.builds):
69 work = build.getWorkspaceRepos(projects)
70 state = states[number]
71
72 for project in projects:
73 self.assertRepoState(work[project], state[project],
74 project, build, number)
75
76 self.executor_server.hold_jobs_in_build = False
77 self.executor_server.release()
78 self.waitUntilSettled()
79
James E. Blaird8af5422017-05-24 13:59:40 -070080 @simple_layout('layouts/repo-checkout-two-project.yaml')
James E. Blair97d902e2014-08-21 13:25:56 -070081 def test_one_branch(self):
James E. Blaird8af5422017-05-24 13:59:40 -070082 self.executor_server.hold_jobs_in_build = True
Antoine Musso45dd2cb2014-01-29 17:17:43 +010083
James E. Blaird8af5422017-05-24 13:59:40 -070084 p1 = 'review.example.com/org/project1'
85 p2 = 'review.example.com/org/project2'
86 projects = [p1, p2]
Antoine Musso45dd2cb2014-01-29 17:17:43 +010087 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
88 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
James E. Blaird8af5422017-05-24 13:59:40 -070089 A.addApproval('code-review', 2)
90 B.addApproval('code-review', 2)
91 self.fake_gerrit.addEvent(A.addApproval('approved', 1))
92 self.fake_gerrit.addEvent(B.addApproval('approved', 1))
Antoine Musso45dd2cb2014-01-29 17:17:43 +010093
94 self.waitUntilSettled()
95
96 self.assertEquals(2, len(self.builds), "Two builds are running")
97
James E. Blair97d902e2014-08-21 13:25:56 -070098 upstream = self.getUpstreamRepos(projects)
99 states = [
James E. Blairbcaa2d42017-05-26 15:44:23 -0700100 {p1: dict(present=[A], absent=[B], branch='master'),
101 p2: dict(commit=str(upstream[p2].commit('master')),
102 branch='master'),
James E. Blair97d902e2014-08-21 13:25:56 -0700103 },
James E. Blairbcaa2d42017-05-26 15:44:23 -0700104 {p1: dict(present=[A], absent=[B], branch='master'),
105 p2: dict(present=[B], absent=[A], branch='master'),
James E. Blair97d902e2014-08-21 13:25:56 -0700106 },
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000107 ]
James E. Blair97d902e2014-08-21 13:25:56 -0700108
James E. Blair987855f2017-06-06 14:41:58 -0700109 self.assertBuildStates(states, projects)
James E. Blair97d902e2014-08-21 13:25:56 -0700110
James E. Blairaf3d01e2017-05-26 14:30:18 -0700111 @simple_layout('layouts/repo-checkout-four-project.yaml')
James E. Blair97d902e2014-08-21 13:25:56 -0700112 def test_multi_branch(self):
James E. Blairaf3d01e2017-05-26 14:30:18 -0700113 self.executor_server.hold_jobs_in_build = True
114
115 p1 = 'review.example.com/org/project1'
116 p2 = 'review.example.com/org/project2'
117 p3 = 'review.example.com/org/project3'
118 p4 = 'review.example.com/org/project4'
119 projects = [p1, p2, p3, p4]
James E. Blair97d902e2014-08-21 13:25:56 -0700120
121 self.create_branch('org/project2', 'stable/havana')
122 self.create_branch('org/project4', 'stable/havana')
123 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000124 B = self.fake_gerrit.addFakeChange('org/project2', 'stable/havana',
125 'B')
James E. Blair97d902e2014-08-21 13:25:56 -0700126 C = self.fake_gerrit.addFakeChange('org/project3', 'master', 'C')
James E. Blairaf3d01e2017-05-26 14:30:18 -0700127 A.addApproval('code-review', 2)
128 B.addApproval('code-review', 2)
129 C.addApproval('code-review', 2)
130 self.fake_gerrit.addEvent(A.addApproval('approved', 1))
131 self.fake_gerrit.addEvent(B.addApproval('approved', 1))
132 self.fake_gerrit.addEvent(C.addApproval('approved', 1))
James E. Blair97d902e2014-08-21 13:25:56 -0700133
134 self.waitUntilSettled()
135
136 self.assertEquals(3, len(self.builds), "Three builds are running")
137
138 upstream = self.getUpstreamRepos(projects)
139 states = [
James E. Blairbcaa2d42017-05-26 15:44:23 -0700140 {p1: dict(present=[A], absent=[B, C], branch='master'),
141 p2: dict(commit=str(upstream[p2].commit('master')),
142 branch='master'),
143 p3: dict(commit=str(upstream[p3].commit('master')),
144 branch='master'),
145 p4: dict(commit=str(upstream[p4].commit('master')),
146 branch='master'),
James E. Blair97d902e2014-08-21 13:25:56 -0700147 },
James E. Blairbcaa2d42017-05-26 15:44:23 -0700148 {p1: dict(present=[A], absent=[B, C], branch='master'),
149 p2: dict(present=[B], absent=[A, C], branch='stable/havana'),
150 p3: dict(commit=str(upstream[p3].commit('master')),
151 branch='master'),
152 p4: dict(commit=str(upstream[p4].commit('stable/havana')),
153 branch='stable/havana'),
James E. Blair97d902e2014-08-21 13:25:56 -0700154 },
James E. Blairbcaa2d42017-05-26 15:44:23 -0700155 {p1: dict(present=[A], absent=[B, C], branch='master'),
156 p2: dict(commit=str(upstream[p2].commit('master')),
157 branch='master'),
158 p3: dict(present=[C], absent=[A, B], branch='master'),
159 p4: dict(commit=str(upstream[p4].commit('master')),
160 branch='master'),
James E. Blair97d902e2014-08-21 13:25:56 -0700161 },
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000162 ]
James E. Blair97d902e2014-08-21 13:25:56 -0700163
James E. Blair987855f2017-06-06 14:41:58 -0700164 self.assertBuildStates(states, projects)
James E. Blairbce35e12014-08-21 14:31:17 -0700165
James E. Blair4de8d9e2017-06-01 15:41:40 -0700166 @simple_layout('layouts/repo-checkout-six-project.yaml')
James E. Blairf0420222014-08-21 16:02:17 -0700167 def test_project_override(self):
James E. Blair4de8d9e2017-06-01 15:41:40 -0700168 self.executor_server.hold_jobs_in_build = True
169
170 p1 = 'review.example.com/org/project1'
171 p2 = 'review.example.com/org/project2'
172 p3 = 'review.example.com/org/project3'
173 p4 = 'review.example.com/org/project4'
174 p5 = 'review.example.com/org/project5'
175 p6 = 'review.example.com/org/project6'
176 projects = [p1, p2, p3, p4, p5, p6]
James E. Blairf0420222014-08-21 16:02:17 -0700177
178 self.create_branch('org/project3', 'stable/havana')
179 self.create_branch('org/project4', 'stable/havana')
180 self.create_branch('org/project6', 'stable/havana')
181 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
182 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
183 C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000184 D = self.fake_gerrit.addFakeChange('org/project3', 'stable/havana',
185 'D')
James E. Blair4de8d9e2017-06-01 15:41:40 -0700186 A.addApproval('code-review', 2)
187 B.addApproval('code-review', 2)
188 C.addApproval('code-review', 2)
189 D.addApproval('code-review', 2)
190 self.fake_gerrit.addEvent(A.addApproval('approved', 1))
191 self.fake_gerrit.addEvent(B.addApproval('approved', 1))
192 self.fake_gerrit.addEvent(C.addApproval('approved', 1))
193 self.fake_gerrit.addEvent(D.addApproval('approved', 1))
James E. Blairf0420222014-08-21 16:02:17 -0700194
195 self.waitUntilSettled()
196
197 self.assertEquals(4, len(self.builds), "Four builds are running")
198
199 upstream = self.getUpstreamRepos(projects)
200 states = [
James E. Blair4de8d9e2017-06-01 15:41:40 -0700201 {p1: dict(present=[A], absent=[B, C, D], branch='master'),
202 p2: dict(commit=str(upstream[p2].commit('master')),
203 branch='master'),
204 p3: dict(commit=str(upstream[p3].commit('master')),
205 branch='master'),
206 p4: dict(commit=str(upstream[p4].commit('master')),
207 branch='master'),
208 p5: dict(commit=str(upstream[p5].commit('master')),
209 branch='master'),
210 p6: dict(commit=str(upstream[p6].commit('master')),
211 branch='master'),
James E. Blairf0420222014-08-21 16:02:17 -0700212 },
James E. Blair4de8d9e2017-06-01 15:41:40 -0700213 {p1: dict(present=[A, B], absent=[C, D], branch='master'),
214 p2: dict(commit=str(upstream[p2].commit('master')),
215 branch='master'),
216 p3: dict(commit=str(upstream[p3].commit('master')),
217 branch='master'),
218 p4: dict(commit=str(upstream[p4].commit('master')),
219 branch='master'),
220 p5: dict(commit=str(upstream[p5].commit('master')),
221 branch='master'),
222 p6: dict(commit=str(upstream[p6].commit('master')),
223 branch='master'),
James E. Blairf0420222014-08-21 16:02:17 -0700224 },
James E. Blair4de8d9e2017-06-01 15:41:40 -0700225 {p1: dict(present=[A, B], absent=[C, D], branch='master'),
226 p2: dict(present=[C], absent=[A, B, D], branch='master'),
227 p3: dict(commit=str(upstream[p3].commit('master')),
228 branch='master'),
229 p4: dict(commit=str(upstream[p4].commit('master')),
230 branch='master'),
231 p5: dict(commit=str(upstream[p5].commit('master')),
232 branch='master'),
233 p6: dict(commit=str(upstream[p6].commit('master')),
234 branch='master'),
James E. Blairf0420222014-08-21 16:02:17 -0700235 },
James E. Blair4de8d9e2017-06-01 15:41:40 -0700236 {p1: dict(present=[A, B], absent=[C, D], branch='master'),
237 p2: dict(present=[C], absent=[A, B, D], branch='master'),
238 p3: dict(present=[D], absent=[A, B, C],
239 branch='stable/havana'),
240 p4: dict(commit=str(upstream[p4].commit('master')),
241 branch='master'),
242 p5: dict(commit=str(upstream[p5].commit('master')),
243 branch='master'),
244 p6: dict(commit=str(upstream[p6].commit('stable/havana')),
245 branch='stable/havana'),
James E. Blairf0420222014-08-21 16:02:17 -0700246 },
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000247 ]
James E. Blairf0420222014-08-21 16:02:17 -0700248
James E. Blair987855f2017-06-06 14:41:58 -0700249 self.assertBuildStates(states, projects)
James E. Blair217b10d2015-01-08 16:25:28 -0800250
James E. Blair21037782017-07-19 11:56:55 -0700251 def test_periodic_override(self):
252 # This test can not use simple_layout because it must start
253 # with a configuration which does not include a
254 # timer-triggered job so that we have an opportunity to set
255 # the hold flag before the first job.
256
257 # This tests that we can override the branch in a timer
258 # trigger (mostly to ensure backwards compatability for jobs).
259 self.executor_server.hold_jobs_in_build = True
260 # Start timer trigger - also org/project
261 self.commitConfigUpdate('common-config',
262 'layouts/repo-checkout-timer-override.yaml')
263 self.sched.reconfigure(self.config)
264
265 p1 = 'review.example.com/org/project1'
266 projects = [p1]
267 self.create_branch('org/project1', 'stable/havana')
268
269 # The pipeline triggers every second, so we should have seen
270 # several by now.
271 time.sleep(5)
272 self.waitUntilSettled()
273
274 # Stop queuing timer triggered jobs so that the assertions
275 # below don't race against more jobs being queued.
276 self.commitConfigUpdate('common-config',
James E. Blair0b137b42017-07-27 08:51:25 -0700277 'layouts/repo-checkout-no-timer-override.yaml')
James E. Blair21037782017-07-19 11:56:55 -0700278 self.sched.reconfigure(self.config)
James E. Blair0b137b42017-07-27 08:51:25 -0700279 self.waitUntilSettled()
James E. Blair21037782017-07-19 11:56:55 -0700280
281 self.assertEquals(1, len(self.builds), "One build is running")
282
283 upstream = self.getUpstreamRepos(projects)
284 states = [
285 {p1: dict(commit=str(upstream[p1].commit('stable/havana')),
286 branch='stable/havana'),
287 },
288 ]
289
290 self.assertBuildStates(states, projects)
291
James E. Blair217b10d2015-01-08 16:25:28 -0800292 def test_periodic(self):
James E. Blair95ae4fc2017-06-01 15:54:53 -0700293 # This test can not use simple_layout because it must start
294 # with a configuration which does not include a
295 # timer-triggered job so that we have an opportunity to set
296 # the hold flag before the first job.
297 self.executor_server.hold_jobs_in_build = True
298 # Start timer trigger - also org/project
299 self.commitConfigUpdate('common-config',
300 'layouts/repo-checkout-timer.yaml')
James E. Blair217b10d2015-01-08 16:25:28 -0800301 self.sched.reconfigure(self.config)
James E. Blair95ae4fc2017-06-01 15:54:53 -0700302
303 p1 = 'review.example.com/org/project1'
304 projects = [p1]
305 self.create_branch('org/project1', 'stable/havana')
James E. Blair217b10d2015-01-08 16:25:28 -0800306
307 # The pipeline triggers every second, so we should have seen
308 # several by now.
309 time.sleep(5)
310 self.waitUntilSettled()
311
James E. Blair217b10d2015-01-08 16:25:28 -0800312 # Stop queuing timer triggered jobs so that the assertions
313 # below don't race against more jobs being queued.
James E. Blair95ae4fc2017-06-01 15:54:53 -0700314 self.commitConfigUpdate('common-config',
315 'layouts/repo-checkout-no-timer.yaml')
James E. Blair217b10d2015-01-08 16:25:28 -0800316 self.sched.reconfigure(self.config)
James E. Blair0b137b42017-07-27 08:51:25 -0700317 self.waitUntilSettled()
James E. Blair217b10d2015-01-08 16:25:28 -0800318
James E. Blair21037782017-07-19 11:56:55 -0700319 self.assertEquals(2, len(self.builds), "Two builds are running")
James E. Blair217b10d2015-01-08 16:25:28 -0800320
321 upstream = self.getUpstreamRepos(projects)
322 states = [
James E. Blair95ae4fc2017-06-01 15:54:53 -0700323 {p1: dict(commit=str(upstream[p1].commit('stable/havana')),
324 branch='stable/havana'),
James E. Blair217b10d2015-01-08 16:25:28 -0800325 },
James E. Blair21037782017-07-19 11:56:55 -0700326 {p1: dict(commit=str(upstream[p1].commit('master')),
327 branch='master'),
328 },
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000329 ]
James E. Blair21037782017-07-19 11:56:55 -0700330 if self.builds[0].parameters['zuul']['ref'] == 'refs/heads/master':
331 states = list(reversed(states))
James E. Blair217b10d2015-01-08 16:25:28 -0800332
James E. Blair987855f2017-06-06 14:41:58 -0700333 self.assertBuildStates(states, projects)
Sachi King9f16d522016-03-16 12:20:45 +1100334
James E. Blairbbe811e2017-06-02 10:47:44 -0700335 @simple_layout('layouts/repo-checkout-post.yaml')
Sachi King9f16d522016-03-16 12:20:45 +1100336 def test_post_and_master_checkout(self):
James E. Blair1de2e492017-06-02 11:04:42 -0700337 self.executor_server.hold_jobs_in_build = True
338 p1 = "review.example.com/org/project1"
339 p2 = "review.example.com/org/project2"
340 projects = [p1, p2]
James E. Blair8cce42e2016-10-18 08:18:36 -0700341
James E. Blair1de2e492017-06-02 11:04:42 -0700342 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
James E. Blair8cce42e2016-10-18 08:18:36 -0700343 event = A.getRefUpdatedEvent()
344 A.setMerged()
345 self.fake_gerrit.addEvent(event)
346 self.waitUntilSettled()
347
James E. Blair8cce42e2016-10-18 08:18:36 -0700348 upstream = self.getUpstreamRepos(projects)
James E. Blair1de2e492017-06-02 11:04:42 -0700349 states = [
350 {p1: dict(commit=str(upstream[p1].commit('master')),
351 present=[A], branch='master'),
352 p2: dict(commit=str(upstream[p2].commit('master')),
353 absent=[A], branch='master'),
354 },
355 ]
James E. Blair8cce42e2016-10-18 08:18:36 -0700356
James E. Blair987855f2017-06-06 14:41:58 -0700357 self.assertBuildStates(states, projects)
Tristan Cacqueray80954402017-05-28 00:33:55 +0000358
359
360class TestAnsibleJob(ZuulTestCase):
361 tenant_config_file = 'config/ansible/main.yaml'
362
363 def setUp(self):
364 super(TestAnsibleJob, self).setUp()
365 job = zuul.model.Job('test')
366 job.unique = 'test'
367 self.test_job = zuul.executor.server.AnsibleJob(self.executor_server,
368 job)
369
370 def test_getHostList_host_keys(self):
371 # Test without ssh_port set
372 node = {'name': 'fake-host',
373 'host_keys': ['fake-host-key'],
374 'interface_ip': 'localhost'}
375 keys = self.test_job.getHostList({'nodes': [node]})[0]['host_keys']
376 self.assertEqual(keys[0], 'localhost fake-host-key')
377
378 # Test with custom ssh_port set
379 node['ssh_port'] = 22022
380 keys = self.test_job.getHostList({'nodes': [node]})[0]['host_keys']
381 self.assertEqual(keys[0], '[localhost]:22022 fake-host-key')