blob: 6ab1bccf13aa68c601ca7908cc7d9432687820df [file] [log] [blame]
James E. Blairb0fcae42012-07-17 11:12:10 -07001#!/usr/bin/env python
2
3# Copyright 2012 Hewlett-Packard Development Company, L.P.
4#
5# Licensed under the Apache License, Version 2.0 (the "License"); you may
6# not use this file except in compliance with the License. You may obtain
7# a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14# License for the specific language governing permissions and limitations
15# under the License.
16
James E. Blaira002b032017-04-18 10:35:48 -070017import gc
James E. Blairb0fcae42012-07-17 11:12:10 -070018import json
Tobias Henkel9a0e1942017-03-20 16:16:02 +010019import textwrap
20
Monty Taylorbc758832013-06-17 17:22:42 -040021import os
James E. Blairb0fcae42012-07-17 11:12:10 -070022import re
James E. Blair4886cc12012-07-18 15:39:41 -070023import shutil
Monty Taylorbc758832013-06-17 17:22:42 -040024import time
Clint Byrum03454a52017-05-26 17:14:02 -070025from unittest import skip
Monty Taylorbc758832013-06-17 17:22:42 -040026
James E. Blair4886cc12012-07-18 15:39:41 -070027import git
Monty Taylorbc758832013-06-17 17:22:42 -040028import testtools
Monty Taylorb934c1a2017-06-16 19:31:47 -050029import urllib
James E. Blairb0fcae42012-07-17 11:12:10 -070030
Maru Newby3fe5f852015-01-13 04:22:14 +000031import zuul.change_matcher
Clint Byrumb9de8be2017-05-27 13:49:19 -070032from zuul.driver.gerrit import gerritreporter
James E. Blairb0fcae42012-07-17 11:12:10 -070033import zuul.scheduler
James E. Blairad28e912013-11-27 10:43:22 -080034import zuul.rpcclient
Paul Belanger9bba4902016-11-02 16:07:33 -040035import zuul.model
James E. Blairb0fcae42012-07-17 11:12:10 -070036
Maru Newby3fe5f852015-01-13 04:22:14 +000037from tests.base import (
Paul Belanger0a21f0a2017-06-13 13:14:42 -040038 SSLZuulTestCase,
Maru Newby3fe5f852015-01-13 04:22:14 +000039 ZuulTestCase,
40 repack_repo,
James E. Blair06cc3922017-04-19 10:08:10 -070041 simple_layout,
Maru Newby3fe5f852015-01-13 04:22:14 +000042)
James E. Blairb0fcae42012-07-17 11:12:10 -070043
James E. Blairb0fcae42012-07-17 11:12:10 -070044
Paul Belanger0a21f0a2017-06-13 13:14:42 -040045class TestSchedulerSSL(SSLZuulTestCase):
46 tenant_config_file = 'config/single-tenant/main.yaml'
47
48 def test_jobs_executed(self):
49 "Test that jobs are executed and a change is merged"
50
51 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +020052 A.addApproval('Code-Review', 2)
53 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
Paul Belanger0a21f0a2017-06-13 13:14:42 -040054 self.waitUntilSettled()
55 self.assertEqual(self.getJobFromHistory('project-merge').result,
56 'SUCCESS')
57 self.assertEqual(self.getJobFromHistory('project-test1').result,
58 'SUCCESS')
59 self.assertEqual(self.getJobFromHistory('project-test2').result,
60 'SUCCESS')
61 self.assertEqual(A.data['status'], 'MERGED')
62 self.assertEqual(A.reported, 2)
63 self.assertEqual(self.getJobFromHistory('project-test1').node,
64 'label1')
James E. Blairfcbf95d2017-07-25 16:38:09 -070065 self.assertEqual(self.getJobFromHistory('project-test2').node,
66 'label1')
Paul Belanger0a21f0a2017-06-13 13:14:42 -040067
68
Clark Boylanb640e052014-04-03 16:41:46 -070069class TestScheduler(ZuulTestCase):
James E. Blair552b54f2016-07-22 13:55:32 -070070 tenant_config_file = 'config/single-tenant/main.yaml'
Antoine Mussobd86a312014-01-08 14:51:33 +010071
Paul Belanger174a8272017-03-14 13:20:10 -040072 def test_jobs_executed(self):
73 "Test that jobs are executed and a change is merged"
James E. Blair1f4c2bb2013-04-26 08:40:46 -070074
James E. Blairb0fcae42012-07-17 11:12:10 -070075 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +020076 A.addApproval('Code-Review', 2)
77 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blairb0fcae42012-07-17 11:12:10 -070078 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -040079 self.assertEqual(self.getJobFromHistory('project-merge').result,
80 'SUCCESS')
81 self.assertEqual(self.getJobFromHistory('project-test1').result,
82 'SUCCESS')
83 self.assertEqual(self.getJobFromHistory('project-test2').result,
84 'SUCCESS')
85 self.assertEqual(A.data['status'], 'MERGED')
86 self.assertEqual(A.reported, 2)
James E. Blair34776ee2016-08-25 13:53:54 -070087 self.assertEqual(self.getJobFromHistory('project-test1').node,
James E. Blair16d96a02017-06-08 11:32:56 -070088 'label1')
James E. Blairfcbf95d2017-07-25 16:38:09 -070089 self.assertEqual(self.getJobFromHistory('project-test2').node,
90 'label1')
James E. Blairb0fcae42012-07-17 11:12:10 -070091
James E. Blair552b54f2016-07-22 13:55:32 -070092 # TODOv3(jeblair): we may want to report stats by tenant (also?).
Monty Taylorf1385042017-05-27 12:25:05 -050093 # Per-driver
94 self.assertReportedStat('zuul.event.gerrit.comment-added', value='1|c')
95 # Per-driver per-connection
96 self.assertReportedStat('zuul.event.gerrit.gerrit.comment-added',
97 value='1|c')
James E. Blair66eeebf2013-07-27 17:44:32 -070098 self.assertReportedStat('zuul.pipeline.gate.current_changes',
99 value='1|g')
100 self.assertReportedStat('zuul.pipeline.gate.job.project-merge.SUCCESS',
101 kind='ms')
102 self.assertReportedStat('zuul.pipeline.gate.job.project-merge.SUCCESS',
103 value='1|c')
104 self.assertReportedStat('zuul.pipeline.gate.resident_time', kind='ms')
105 self.assertReportedStat('zuul.pipeline.gate.total_changes',
106 value='1|c')
James E. Blair412e5582013-04-22 15:50:12 -0700107 self.assertReportedStat(
James E. Blair66eeebf2013-07-27 17:44:32 -0700108 'zuul.pipeline.gate.org.project.resident_time', kind='ms')
James E. Blair412e5582013-04-22 15:50:12 -0700109 self.assertReportedStat(
James E. Blair66eeebf2013-07-27 17:44:32 -0700110 'zuul.pipeline.gate.org.project.total_changes', value='1|c')
James E. Blair412e5582013-04-22 15:50:12 -0700111
James E. Blair2ef29e92017-07-21 15:25:05 -0700112 for build in self.history:
113 self.assertTrue(build.parameters['zuul']['voting'])
James E. Blair5821bd92015-09-16 08:48:15 -0700114
James E. Blair3cb10702013-08-24 08:56:03 -0700115 def test_initial_pipeline_gauges(self):
116 "Test that each pipeline reported its length on start"
James E. Blair7da51b72016-07-25 15:29:43 -0700117 self.assertReportedStat('zuul.pipeline.gate.current_changes',
118 value='0|g')
119 self.assertReportedStat('zuul.pipeline.check.current_changes',
120 value='0|g')
James E. Blair3cb10702013-08-24 08:56:03 -0700121
James E. Blair34776ee2016-08-25 13:53:54 -0700122 def test_job_branch(self):
123 "Test the correct variant of a job runs on a branch"
124 self.create_branch('org/project', 'stable')
125 A = self.fake_gerrit.addFakeChange('org/project', 'stable', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200126 A.addApproval('Code-Review', 2)
127 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair34776ee2016-08-25 13:53:54 -0700128 self.waitUntilSettled()
129 self.assertEqual(self.getJobFromHistory('project-test1').result,
130 'SUCCESS')
131 self.assertEqual(self.getJobFromHistory('project-test2').result,
132 'SUCCESS')
133 self.assertEqual(A.data['status'], 'MERGED')
134 self.assertEqual(A.reported, 2,
135 "A should report start and success")
136 self.assertIn('gate', A.messages[1],
137 "A should transit gate")
138 self.assertEqual(self.getJobFromHistory('project-test1').node,
James E. Blair16d96a02017-06-08 11:32:56 -0700139 'label2')
James E. Blair34776ee2016-08-25 13:53:54 -0700140
James E. Blairb0fcae42012-07-17 11:12:10 -0700141 def test_parallel_changes(self):
142 "Test that changes are tested in parallel and merged in series"
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700143
Paul Belanger174a8272017-03-14 13:20:10 -0400144 self.executor_server.hold_jobs_in_build = True
James E. Blairb0fcae42012-07-17 11:12:10 -0700145 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
146 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
147 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200148 A.addApproval('Code-Review', 2)
149 B.addApproval('Code-Review', 2)
150 C.addApproval('Code-Review', 2)
James E. Blairb0fcae42012-07-17 11:12:10 -0700151
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200152 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
153 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
154 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blairb0fcae42012-07-17 11:12:10 -0700155
156 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400157 self.assertEqual(len(self.builds), 1)
158 self.assertEqual(self.builds[0].name, 'project-merge')
James E. Blair791b5392016-08-03 11:25:56 -0700159 self.assertTrue(self.builds[0].hasChanges(A))
James E. Blairb0fcae42012-07-17 11:12:10 -0700160
Paul Belanger174a8272017-03-14 13:20:10 -0400161 self.executor_server.release('.*-merge')
James E. Blairb0fcae42012-07-17 11:12:10 -0700162 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400163 self.assertEqual(len(self.builds), 3)
164 self.assertEqual(self.builds[0].name, 'project-test1')
James E. Blair791b5392016-08-03 11:25:56 -0700165 self.assertTrue(self.builds[0].hasChanges(A))
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400166 self.assertEqual(self.builds[1].name, 'project-test2')
James E. Blair791b5392016-08-03 11:25:56 -0700167 self.assertTrue(self.builds[1].hasChanges(A))
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400168 self.assertEqual(self.builds[2].name, 'project-merge')
James E. Blair791b5392016-08-03 11:25:56 -0700169 self.assertTrue(self.builds[2].hasChanges(A, B))
James E. Blairb0fcae42012-07-17 11:12:10 -0700170
Paul Belanger174a8272017-03-14 13:20:10 -0400171 self.executor_server.release('.*-merge')
James E. Blairb0fcae42012-07-17 11:12:10 -0700172 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400173 self.assertEqual(len(self.builds), 5)
174 self.assertEqual(self.builds[0].name, 'project-test1')
James E. Blair791b5392016-08-03 11:25:56 -0700175 self.assertTrue(self.builds[0].hasChanges(A))
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400176 self.assertEqual(self.builds[1].name, 'project-test2')
James E. Blair791b5392016-08-03 11:25:56 -0700177 self.assertTrue(self.builds[1].hasChanges(A))
James E. Blairb0fcae42012-07-17 11:12:10 -0700178
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400179 self.assertEqual(self.builds[2].name, 'project-test1')
James E. Blair791b5392016-08-03 11:25:56 -0700180 self.assertTrue(self.builds[2].hasChanges(A, B))
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400181 self.assertEqual(self.builds[3].name, 'project-test2')
James E. Blair791b5392016-08-03 11:25:56 -0700182 self.assertTrue(self.builds[3].hasChanges(A, B))
James E. Blairb0fcae42012-07-17 11:12:10 -0700183
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400184 self.assertEqual(self.builds[4].name, 'project-merge')
James E. Blair791b5392016-08-03 11:25:56 -0700185 self.assertTrue(self.builds[4].hasChanges(A, B, C))
James E. Blairb0fcae42012-07-17 11:12:10 -0700186
Paul Belanger174a8272017-03-14 13:20:10 -0400187 self.executor_server.release('.*-merge')
James E. Blairb0fcae42012-07-17 11:12:10 -0700188 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400189 self.assertEqual(len(self.builds), 6)
190 self.assertEqual(self.builds[0].name, 'project-test1')
James E. Blair791b5392016-08-03 11:25:56 -0700191 self.assertTrue(self.builds[0].hasChanges(A))
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400192 self.assertEqual(self.builds[1].name, 'project-test2')
James E. Blair791b5392016-08-03 11:25:56 -0700193 self.assertTrue(self.builds[1].hasChanges(A))
James E. Blairb0fcae42012-07-17 11:12:10 -0700194
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400195 self.assertEqual(self.builds[2].name, 'project-test1')
James E. Blair791b5392016-08-03 11:25:56 -0700196 self.assertTrue(self.builds[2].hasChanges(A, B))
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400197 self.assertEqual(self.builds[3].name, 'project-test2')
James E. Blair791b5392016-08-03 11:25:56 -0700198 self.assertTrue(self.builds[3].hasChanges(A, B))
James E. Blairb0fcae42012-07-17 11:12:10 -0700199
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400200 self.assertEqual(self.builds[4].name, 'project-test1')
James E. Blair791b5392016-08-03 11:25:56 -0700201 self.assertTrue(self.builds[4].hasChanges(A, B, C))
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400202 self.assertEqual(self.builds[5].name, 'project-test2')
James E. Blair791b5392016-08-03 11:25:56 -0700203 self.assertTrue(self.builds[5].hasChanges(A, B, C))
James E. Blairb0fcae42012-07-17 11:12:10 -0700204
Paul Belanger174a8272017-03-14 13:20:10 -0400205 self.executor_server.hold_jobs_in_build = False
206 self.executor_server.release()
James E. Blairb0fcae42012-07-17 11:12:10 -0700207 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400208 self.assertEqual(len(self.builds), 0)
James E. Blairb0fcae42012-07-17 11:12:10 -0700209
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400210 self.assertEqual(len(self.history), 9)
211 self.assertEqual(A.data['status'], 'MERGED')
212 self.assertEqual(B.data['status'], 'MERGED')
213 self.assertEqual(C.data['status'], 'MERGED')
214 self.assertEqual(A.reported, 2)
215 self.assertEqual(B.reported, 2)
216 self.assertEqual(C.reported, 2)
James E. Blairb02a3bb2012-07-30 17:49:55 -0700217
218 def test_failed_changes(self):
219 "Test that a change behind a failed change is retested"
Paul Belanger174a8272017-03-14 13:20:10 -0400220 self.executor_server.hold_jobs_in_build = True
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700221
James E. Blairb02a3bb2012-07-30 17:49:55 -0700222 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
223 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200224 A.addApproval('Code-Review', 2)
225 B.addApproval('Code-Review', 2)
James E. Blairb02a3bb2012-07-30 17:49:55 -0700226
Paul Belanger174a8272017-03-14 13:20:10 -0400227 self.executor_server.failJob('project-test1', A)
James E. Blairb02a3bb2012-07-30 17:49:55 -0700228
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200229 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
230 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blairb02a3bb2012-07-30 17:49:55 -0700231 self.waitUntilSettled()
James E. Blair2b2a8ab2016-08-11 14:39:11 -0700232 self.assertBuilds([dict(name='project-merge', changes='1,1')])
James E. Blaire2819012013-06-28 17:17:26 -0400233
Paul Belanger174a8272017-03-14 13:20:10 -0400234 self.executor_server.release('.*-merge')
James E. Blaire2819012013-06-28 17:17:26 -0400235 self.waitUntilSettled()
James E. Blair2b2a8ab2016-08-11 14:39:11 -0700236 # A/project-merge is complete
237 self.assertBuilds([
238 dict(name='project-test1', changes='1,1'),
239 dict(name='project-test2', changes='1,1'),
240 dict(name='project-merge', changes='1,1 2,1'),
241 ])
James E. Blaire2819012013-06-28 17:17:26 -0400242
Paul Belanger174a8272017-03-14 13:20:10 -0400243 self.executor_server.release('.*-merge')
James E. Blaire2819012013-06-28 17:17:26 -0400244 self.waitUntilSettled()
James E. Blair2b2a8ab2016-08-11 14:39:11 -0700245 # A/project-merge is complete
246 # B/project-merge is complete
247 self.assertBuilds([
248 dict(name='project-test1', changes='1,1'),
249 dict(name='project-test2', changes='1,1'),
250 dict(name='project-test1', changes='1,1 2,1'),
251 dict(name='project-test2', changes='1,1 2,1'),
252 ])
253
254 # Release project-test1 for A which will fail. This will
Paul Belanger174a8272017-03-14 13:20:10 -0400255 # abort both running B jobs and reexecute project-merge for B.
James E. Blair2b2a8ab2016-08-11 14:39:11 -0700256 self.builds[0].release()
257 self.waitUntilSettled()
258
259 self.orderedRelease()
260 self.assertHistory([
261 dict(name='project-merge', result='SUCCESS', changes='1,1'),
262 dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'),
263 dict(name='project-test1', result='FAILURE', changes='1,1'),
264 dict(name='project-test1', result='ABORTED', changes='1,1 2,1'),
265 dict(name='project-test2', result='ABORTED', changes='1,1 2,1'),
266 dict(name='project-test2', result='SUCCESS', changes='1,1'),
267 dict(name='project-merge', result='SUCCESS', changes='2,1'),
268 dict(name='project-test1', result='SUCCESS', changes='2,1'),
269 dict(name='project-test2', result='SUCCESS', changes='2,1'),
James E. Blairb536ecc2016-08-31 10:11:42 -0700270 ], ordered=False)
James E. Blair2b2a8ab2016-08-11 14:39:11 -0700271
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400272 self.assertEqual(A.data['status'], 'NEW')
273 self.assertEqual(B.data['status'], 'MERGED')
274 self.assertEqual(A.reported, 2)
275 self.assertEqual(B.reported, 2)
James E. Blairb02a3bb2012-07-30 17:49:55 -0700276
277 def test_independent_queues(self):
278 "Test that changes end up in the right queues"
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700279
Paul Belanger174a8272017-03-14 13:20:10 -0400280 self.executor_server.hold_jobs_in_build = True
Zhongyue Luo5d556072012-09-21 02:00:47 +0900281 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
James E. Blairb02a3bb2012-07-30 17:49:55 -0700282 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
283 C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200284 A.addApproval('Code-Review', 2)
285 B.addApproval('Code-Review', 2)
286 C.addApproval('Code-Review', 2)
James E. Blairb02a3bb2012-07-30 17:49:55 -0700287
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200288 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blairf59f3cf2017-02-19 14:50:26 -0800289 self.waitUntilSettled()
James E. Blairb02a3bb2012-07-30 17:49:55 -0700290
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200291 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blairf59f3cf2017-02-19 14:50:26 -0800292 self.waitUntilSettled()
293
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200294 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blairb02a3bb2012-07-30 17:49:55 -0700295 self.waitUntilSettled()
296
297 # There should be one merge job at the head of each queue running
James E. Blair5f0d49e2016-08-19 09:36:50 -0700298 self.assertBuilds([
299 dict(name='project-merge', changes='1,1'),
300 dict(name='project-merge', changes='2,1'),
301 ])
James E. Blairb02a3bb2012-07-30 17:49:55 -0700302
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700303 # Release the current merge builds
James E. Blair5f0d49e2016-08-19 09:36:50 -0700304 self.builds[0].release()
305 self.waitUntilSettled()
306 self.builds[0].release()
James E. Blairb02a3bb2012-07-30 17:49:55 -0700307 self.waitUntilSettled()
308 # Release the merge job for project2 which is behind project1
Paul Belanger174a8272017-03-14 13:20:10 -0400309 self.executor_server.release('.*-merge')
James E. Blairb02a3bb2012-07-30 17:49:55 -0700310 self.waitUntilSettled()
311
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700312 # All the test builds should be running:
James E. Blair5f0d49e2016-08-19 09:36:50 -0700313 self.assertBuilds([
314 dict(name='project-test1', changes='1,1'),
315 dict(name='project-test2', changes='1,1'),
316 dict(name='project-test1', changes='2,1'),
317 dict(name='project-test2', changes='2,1'),
Paul Belanger58bf6912016-11-11 19:36:01 -0500318 dict(name='project1-project2-integration', changes='2,1'),
James E. Blair5f0d49e2016-08-19 09:36:50 -0700319 dict(name='project-test1', changes='2,1 3,1'),
320 dict(name='project-test2', changes='2,1 3,1'),
Clint Byrumcd9393f2016-12-21 12:21:07 -0800321 dict(name='project1-project2-integration', changes='2,1 3,1'),
James E. Blair5f0d49e2016-08-19 09:36:50 -0700322 ])
James E. Blairb02a3bb2012-07-30 17:49:55 -0700323
James E. Blair5f0d49e2016-08-19 09:36:50 -0700324 self.orderedRelease()
325 self.assertHistory([
326 dict(name='project-merge', result='SUCCESS', changes='1,1'),
327 dict(name='project-merge', result='SUCCESS', changes='2,1'),
328 dict(name='project-merge', result='SUCCESS', changes='2,1 3,1'),
329 dict(name='project-test1', result='SUCCESS', changes='1,1'),
330 dict(name='project-test2', result='SUCCESS', changes='1,1'),
331 dict(name='project-test1', result='SUCCESS', changes='2,1'),
332 dict(name='project-test2', result='SUCCESS', changes='2,1'),
Paul Belanger58bf6912016-11-11 19:36:01 -0500333 dict(
334 name='project1-project2-integration',
335 result='SUCCESS',
336 changes='2,1'),
James E. Blair5f0d49e2016-08-19 09:36:50 -0700337 dict(name='project-test1', result='SUCCESS', changes='2,1 3,1'),
338 dict(name='project-test2', result='SUCCESS', changes='2,1 3,1'),
Clint Byrumcd9393f2016-12-21 12:21:07 -0800339 dict(name='project1-project2-integration',
340 result='SUCCESS',
341 changes='2,1 3,1'),
James E. Blair5f0d49e2016-08-19 09:36:50 -0700342 ])
James E. Blairb02a3bb2012-07-30 17:49:55 -0700343
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400344 self.assertEqual(A.data['status'], 'MERGED')
345 self.assertEqual(B.data['status'], 'MERGED')
346 self.assertEqual(C.data['status'], 'MERGED')
347 self.assertEqual(A.reported, 2)
348 self.assertEqual(B.reported, 2)
349 self.assertEqual(C.reported, 2)
James E. Blaird466dc42012-07-31 10:42:56 -0700350
351 def test_failed_change_at_head(self):
352 "Test that if a change at the head fails, jobs behind it are canceled"
James E. Blaird466dc42012-07-31 10:42:56 -0700353
Paul Belanger174a8272017-03-14 13:20:10 -0400354 self.executor_server.hold_jobs_in_build = True
James E. Blaird466dc42012-07-31 10:42:56 -0700355 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
356 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
357 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200358 A.addApproval('Code-Review', 2)
359 B.addApproval('Code-Review', 2)
360 C.addApproval('Code-Review', 2)
James E. Blaird466dc42012-07-31 10:42:56 -0700361
Paul Belanger174a8272017-03-14 13:20:10 -0400362 self.executor_server.failJob('project-test1', A)
James E. Blaird466dc42012-07-31 10:42:56 -0700363
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200364 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
365 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
366 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blaird466dc42012-07-31 10:42:56 -0700367
368 self.waitUntilSettled()
James E. Blaird466dc42012-07-31 10:42:56 -0700369
James E. Blairb536ecc2016-08-31 10:11:42 -0700370 self.assertBuilds([
371 dict(name='project-merge', changes='1,1'),
372 ])
James E. Blaird466dc42012-07-31 10:42:56 -0700373
Paul Belanger174a8272017-03-14 13:20:10 -0400374 self.executor_server.release('.*-merge')
James E. Blaird466dc42012-07-31 10:42:56 -0700375 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -0400376 self.executor_server.release('.*-merge')
James E. Blaird466dc42012-07-31 10:42:56 -0700377 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -0400378 self.executor_server.release('.*-merge')
James E. Blaird466dc42012-07-31 10:42:56 -0700379 self.waitUntilSettled()
380
James E. Blairb536ecc2016-08-31 10:11:42 -0700381 self.assertBuilds([
382 dict(name='project-test1', changes='1,1'),
383 dict(name='project-test2', changes='1,1'),
384 dict(name='project-test1', changes='1,1 2,1'),
385 dict(name='project-test2', changes='1,1 2,1'),
386 dict(name='project-test1', changes='1,1 2,1 3,1'),
387 dict(name='project-test2', changes='1,1 2,1 3,1'),
388 ])
James E. Blaird466dc42012-07-31 10:42:56 -0700389
Monty Taylor6bef8ef2013-06-02 08:17:12 -0400390 self.release(self.builds[0])
James E. Blaird466dc42012-07-31 10:42:56 -0700391 self.waitUntilSettled()
392
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400393 # project-test2, project-merge for B
James E. Blairb536ecc2016-08-31 10:11:42 -0700394 self.assertBuilds([
395 dict(name='project-test2', changes='1,1'),
396 dict(name='project-merge', changes='2,1'),
397 ])
398 # Unordered history comparison because the aborts can finish
399 # in any order.
400 self.assertHistory([
401 dict(name='project-merge', result='SUCCESS',
402 changes='1,1'),
403 dict(name='project-merge', result='SUCCESS',
404 changes='1,1 2,1'),
405 dict(name='project-merge', result='SUCCESS',
406 changes='1,1 2,1 3,1'),
407 dict(name='project-test1', result='FAILURE',
408 changes='1,1'),
409 dict(name='project-test1', result='ABORTED',
410 changes='1,1 2,1'),
411 dict(name='project-test2', result='ABORTED',
412 changes='1,1 2,1'),
413 dict(name='project-test1', result='ABORTED',
414 changes='1,1 2,1 3,1'),
415 dict(name='project-test2', result='ABORTED',
416 changes='1,1 2,1 3,1'),
417 ], ordered=False)
James E. Blaird466dc42012-07-31 10:42:56 -0700418
Paul Belanger174a8272017-03-14 13:20:10 -0400419 self.executor_server.release('.*-merge')
James E. Blaird466dc42012-07-31 10:42:56 -0700420 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -0400421 self.executor_server.release('.*-merge')
James E. Blairb536ecc2016-08-31 10:11:42 -0700422 self.waitUntilSettled()
423 self.orderedRelease()
James E. Blaird466dc42012-07-31 10:42:56 -0700424
James E. Blairb536ecc2016-08-31 10:11:42 -0700425 self.assertBuilds([])
426 self.assertHistory([
427 dict(name='project-merge', result='SUCCESS',
428 changes='1,1'),
429 dict(name='project-merge', result='SUCCESS',
430 changes='1,1 2,1'),
431 dict(name='project-merge', result='SUCCESS',
432 changes='1,1 2,1 3,1'),
433 dict(name='project-test1', result='FAILURE',
434 changes='1,1'),
435 dict(name='project-test1', result='ABORTED',
436 changes='1,1 2,1'),
437 dict(name='project-test2', result='ABORTED',
438 changes='1,1 2,1'),
439 dict(name='project-test1', result='ABORTED',
440 changes='1,1 2,1 3,1'),
441 dict(name='project-test2', result='ABORTED',
442 changes='1,1 2,1 3,1'),
443 dict(name='project-merge', result='SUCCESS',
444 changes='2,1'),
445 dict(name='project-merge', result='SUCCESS',
446 changes='2,1 3,1'),
447 dict(name='project-test2', result='SUCCESS',
448 changes='1,1'),
449 dict(name='project-test1', result='SUCCESS',
450 changes='2,1'),
451 dict(name='project-test2', result='SUCCESS',
452 changes='2,1'),
453 dict(name='project-test1', result='SUCCESS',
454 changes='2,1 3,1'),
455 dict(name='project-test2', result='SUCCESS',
456 changes='2,1 3,1'),
457 ], ordered=False)
458
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400459 self.assertEqual(A.data['status'], 'NEW')
460 self.assertEqual(B.data['status'], 'MERGED')
461 self.assertEqual(C.data['status'], 'MERGED')
462 self.assertEqual(A.reported, 2)
463 self.assertEqual(B.reported, 2)
464 self.assertEqual(C.reported, 2)
James E. Blaird466dc42012-07-31 10:42:56 -0700465
James E. Blair0aac4872013-08-23 14:02:38 -0700466 def test_failed_change_in_middle(self):
467 "Test a failed change in the middle of the queue"
468
Paul Belanger174a8272017-03-14 13:20:10 -0400469 self.executor_server.hold_jobs_in_build = True
James E. Blair0aac4872013-08-23 14:02:38 -0700470 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
471 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
472 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200473 A.addApproval('Code-Review', 2)
474 B.addApproval('Code-Review', 2)
475 C.addApproval('Code-Review', 2)
James E. Blair0aac4872013-08-23 14:02:38 -0700476
Paul Belanger174a8272017-03-14 13:20:10 -0400477 self.executor_server.failJob('project-test1', B)
James E. Blair0aac4872013-08-23 14:02:38 -0700478
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200479 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
480 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
481 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blair0aac4872013-08-23 14:02:38 -0700482
483 self.waitUntilSettled()
484
Paul Belanger174a8272017-03-14 13:20:10 -0400485 self.executor_server.release('.*-merge')
James E. Blair0aac4872013-08-23 14:02:38 -0700486 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -0400487 self.executor_server.release('.*-merge')
James E. Blair0aac4872013-08-23 14:02:38 -0700488 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -0400489 self.executor_server.release('.*-merge')
James E. Blair0aac4872013-08-23 14:02:38 -0700490 self.waitUntilSettled()
491
492 self.assertEqual(len(self.builds), 6)
493 self.assertEqual(self.builds[0].name, 'project-test1')
494 self.assertEqual(self.builds[1].name, 'project-test2')
495 self.assertEqual(self.builds[2].name, 'project-test1')
496 self.assertEqual(self.builds[3].name, 'project-test2')
497 self.assertEqual(self.builds[4].name, 'project-test1')
498 self.assertEqual(self.builds[5].name, 'project-test2')
499
500 self.release(self.builds[2])
501 self.waitUntilSettled()
502
James E. Blair972e3c72013-08-29 12:04:55 -0700503 # project-test1 and project-test2 for A
504 # project-test2 for B
505 # project-merge for C (without B)
506 self.assertEqual(len(self.builds), 4)
James E. Blair0aac4872013-08-23 14:02:38 -0700507 self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 2)
508
Paul Belanger174a8272017-03-14 13:20:10 -0400509 self.executor_server.release('.*-merge')
James E. Blair972e3c72013-08-29 12:04:55 -0700510 self.waitUntilSettled()
511
512 # project-test1 and project-test2 for A
513 # project-test2 for B
514 # project-test1 and project-test2 for C
515 self.assertEqual(len(self.builds), 5)
516
Paul Belanger2e2a0492016-10-30 18:33:37 -0400517 tenant = self.sched.abide.tenants.get('tenant-one')
518 items = tenant.layout.pipelines['gate'].getAllItems()
James E. Blair0aac4872013-08-23 14:02:38 -0700519 builds = items[0].current_build_set.getBuilds()
520 self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1)
521 self.assertEqual(self.countJobResults(builds, None), 2)
522 builds = items[1].current_build_set.getBuilds()
523 self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1)
524 self.assertEqual(self.countJobResults(builds, 'FAILURE'), 1)
525 self.assertEqual(self.countJobResults(builds, None), 1)
526 builds = items[2].current_build_set.getBuilds()
527 self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1)
James E. Blair972e3c72013-08-29 12:04:55 -0700528 self.assertEqual(self.countJobResults(builds, None), 2)
James E. Blair0aac4872013-08-23 14:02:38 -0700529
Paul Belanger174a8272017-03-14 13:20:10 -0400530 self.executor_server.hold_jobs_in_build = False
531 self.executor_server.release()
James E. Blair0aac4872013-08-23 14:02:38 -0700532 self.waitUntilSettled()
533
534 self.assertEqual(len(self.builds), 0)
535 self.assertEqual(len(self.history), 12)
536 self.assertEqual(A.data['status'], 'MERGED')
537 self.assertEqual(B.data['status'], 'NEW')
538 self.assertEqual(C.data['status'], 'MERGED')
539 self.assertEqual(A.reported, 2)
540 self.assertEqual(B.reported, 2)
541 self.assertEqual(C.reported, 2)
542
James E. Blaird466dc42012-07-31 10:42:56 -0700543 def test_failed_change_at_head_with_queue(self):
544 "Test that if a change at the head fails, queued jobs are canceled"
James E. Blaird466dc42012-07-31 10:42:56 -0700545
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700546 self.gearman_server.hold_jobs_in_queue = True
James E. Blaird466dc42012-07-31 10:42:56 -0700547 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
548 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
549 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200550 A.addApproval('Code-Review', 2)
551 B.addApproval('Code-Review', 2)
552 C.addApproval('Code-Review', 2)
James E. Blaird466dc42012-07-31 10:42:56 -0700553
Paul Belanger174a8272017-03-14 13:20:10 -0400554 self.executor_server.failJob('project-test1', A)
James E. Blaird466dc42012-07-31 10:42:56 -0700555
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200556 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
557 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
558 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blaird466dc42012-07-31 10:42:56 -0700559
560 self.waitUntilSettled()
James E. Blair701c5b42013-06-06 09:34:59 -0700561 queue = self.gearman_server.getQueue()
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400562 self.assertEqual(len(self.builds), 0)
563 self.assertEqual(len(queue), 1)
Clint Byrumf322fe22017-05-10 20:53:12 -0700564 self.assertEqual(queue[0].name, b'executor:execute')
565 job_args = json.loads(queue[0].arguments.decode('utf8'))
Clint Byrum8b2bc0c2016-11-16 16:28:50 -0800566 self.assertEqual(job_args['job'], 'project-merge')
567 self.assertEqual(job_args['items'][0]['number'], '%d' % A.number)
James E. Blaird466dc42012-07-31 10:42:56 -0700568
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700569 self.gearman_server.release('.*-merge')
James E. Blaird466dc42012-07-31 10:42:56 -0700570 self.waitUntilSettled()
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700571 self.gearman_server.release('.*-merge')
James E. Blaird466dc42012-07-31 10:42:56 -0700572 self.waitUntilSettled()
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700573 self.gearman_server.release('.*-merge')
James E. Blaird466dc42012-07-31 10:42:56 -0700574 self.waitUntilSettled()
James E. Blair701c5b42013-06-06 09:34:59 -0700575 queue = self.gearman_server.getQueue()
James E. Blaird466dc42012-07-31 10:42:56 -0700576
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400577 self.assertEqual(len(self.builds), 0)
578 self.assertEqual(len(queue), 6)
Clint Byrum8b2bc0c2016-11-16 16:28:50 -0800579
580 self.assertEqual(
Clint Byrumf322fe22017-05-10 20:53:12 -0700581 json.loads(queue[0].arguments.decode('utf8'))['job'],
582 'project-test1')
Clint Byrum8b2bc0c2016-11-16 16:28:50 -0800583 self.assertEqual(
Clint Byrumf322fe22017-05-10 20:53:12 -0700584 json.loads(queue[1].arguments.decode('utf8'))['job'],
585 'project-test2')
Clint Byrum8b2bc0c2016-11-16 16:28:50 -0800586 self.assertEqual(
Clint Byrumf322fe22017-05-10 20:53:12 -0700587 json.loads(queue[2].arguments.decode('utf8'))['job'],
588 'project-test1')
Clint Byrum8b2bc0c2016-11-16 16:28:50 -0800589 self.assertEqual(
Clint Byrumf322fe22017-05-10 20:53:12 -0700590 json.loads(queue[3].arguments.decode('utf8'))['job'],
591 'project-test2')
Clint Byrum8b2bc0c2016-11-16 16:28:50 -0800592 self.assertEqual(
Clint Byrumf322fe22017-05-10 20:53:12 -0700593 json.loads(queue[4].arguments.decode('utf8'))['job'],
594 'project-test1')
Clint Byrum8b2bc0c2016-11-16 16:28:50 -0800595 self.assertEqual(
Clint Byrumf322fe22017-05-10 20:53:12 -0700596 json.loads(queue[5].arguments.decode('utf8'))['job'],
597 'project-test2')
James E. Blaird466dc42012-07-31 10:42:56 -0700598
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700599 self.release(queue[0])
James E. Blaird466dc42012-07-31 10:42:56 -0700600 self.waitUntilSettled()
601
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400602 self.assertEqual(len(self.builds), 0)
James E. Blair701c5b42013-06-06 09:34:59 -0700603 queue = self.gearman_server.getQueue()
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400604 self.assertEqual(len(queue), 2) # project-test2, project-merge for B
605 self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 0)
James E. Blaird466dc42012-07-31 10:42:56 -0700606
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700607 self.gearman_server.hold_jobs_in_queue = False
608 self.gearman_server.release()
James E. Blaird466dc42012-07-31 10:42:56 -0700609 self.waitUntilSettled()
610
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400611 self.assertEqual(len(self.builds), 0)
612 self.assertEqual(len(self.history), 11)
613 self.assertEqual(A.data['status'], 'NEW')
614 self.assertEqual(B.data['status'], 'MERGED')
615 self.assertEqual(C.data['status'], 'MERGED')
616 self.assertEqual(A.reported, 2)
617 self.assertEqual(B.reported, 2)
618 self.assertEqual(C.reported, 2)
James E. Blair8c803f82012-07-31 16:25:42 -0700619
James E. Blairce8a2132016-05-19 15:21:52 -0700620 def _test_time_database(self, iteration):
Paul Belanger174a8272017-03-14 13:20:10 -0400621 self.executor_server.hold_jobs_in_build = True
James E. Blairce8a2132016-05-19 15:21:52 -0700622 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200623 A.addApproval('Code-Review', 2)
624 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blairce8a2132016-05-19 15:21:52 -0700625 self.waitUntilSettled()
626 time.sleep(2)
627
James E. Blairae0f23c2017-09-13 10:55:15 -0600628 data = json.loads(self.sched.formatStatusJSON('tenant-one'))
James E. Blairce8a2132016-05-19 15:21:52 -0700629 found_job = None
630 for pipeline in data['pipelines']:
631 if pipeline['name'] != 'gate':
632 continue
633 for queue in pipeline['change_queues']:
634 for head in queue['heads']:
635 for item in head:
636 for job in item['jobs']:
637 if job['name'] == 'project-merge':
638 found_job = job
639 break
640
641 self.assertIsNotNone(found_job)
642 if iteration == 1:
643 self.assertIsNotNone(found_job['estimated_time'])
644 self.assertIsNone(found_job['remaining_time'])
645 else:
646 self.assertIsNotNone(found_job['estimated_time'])
647 self.assertTrue(found_job['estimated_time'] >= 2)
648 self.assertIsNotNone(found_job['remaining_time'])
649
Paul Belanger174a8272017-03-14 13:20:10 -0400650 self.executor_server.hold_jobs_in_build = False
651 self.executor_server.release()
James E. Blairce8a2132016-05-19 15:21:52 -0700652 self.waitUntilSettled()
653
654 def test_time_database(self):
655 "Test the time database"
656
657 self._test_time_database(1)
658 self._test_time_database(2)
659
James E. Blairfef71632013-09-23 11:15:47 -0700660 def test_two_failed_changes_at_head(self):
661 "Test that changes are reparented correctly if 2 fail at head"
662
Paul Belanger174a8272017-03-14 13:20:10 -0400663 self.executor_server.hold_jobs_in_build = True
James E. Blairfef71632013-09-23 11:15:47 -0700664 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
665 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
666 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200667 A.addApproval('Code-Review', 2)
668 B.addApproval('Code-Review', 2)
669 C.addApproval('Code-Review', 2)
James E. Blairfef71632013-09-23 11:15:47 -0700670
Paul Belanger174a8272017-03-14 13:20:10 -0400671 self.executor_server.failJob('project-test1', A)
672 self.executor_server.failJob('project-test1', B)
James E. Blairfef71632013-09-23 11:15:47 -0700673
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200674 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
675 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
676 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blairfef71632013-09-23 11:15:47 -0700677 self.waitUntilSettled()
678
Paul Belanger174a8272017-03-14 13:20:10 -0400679 self.executor_server.release('.*-merge')
James E. Blairfef71632013-09-23 11:15:47 -0700680 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -0400681 self.executor_server.release('.*-merge')
James E. Blairfef71632013-09-23 11:15:47 -0700682 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -0400683 self.executor_server.release('.*-merge')
James E. Blairfef71632013-09-23 11:15:47 -0700684 self.waitUntilSettled()
685
686 self.assertEqual(len(self.builds), 6)
687 self.assertEqual(self.builds[0].name, 'project-test1')
688 self.assertEqual(self.builds[1].name, 'project-test2')
689 self.assertEqual(self.builds[2].name, 'project-test1')
690 self.assertEqual(self.builds[3].name, 'project-test2')
691 self.assertEqual(self.builds[4].name, 'project-test1')
692 self.assertEqual(self.builds[5].name, 'project-test2')
693
Paul Belanger71dd2fc2016-11-04 19:05:55 -0400694 self.assertTrue(self.builds[0].hasChanges(A))
695 self.assertTrue(self.builds[2].hasChanges(A))
696 self.assertTrue(self.builds[2].hasChanges(B))
697 self.assertTrue(self.builds[4].hasChanges(A))
698 self.assertTrue(self.builds[4].hasChanges(B))
699 self.assertTrue(self.builds[4].hasChanges(C))
James E. Blairfef71632013-09-23 11:15:47 -0700700
701 # Fail change B first
702 self.release(self.builds[2])
703 self.waitUntilSettled()
704
705 # restart of C after B failure
Paul Belanger174a8272017-03-14 13:20:10 -0400706 self.executor_server.release('.*-merge')
James E. Blairfef71632013-09-23 11:15:47 -0700707 self.waitUntilSettled()
708
709 self.assertEqual(len(self.builds), 5)
710 self.assertEqual(self.builds[0].name, 'project-test1')
711 self.assertEqual(self.builds[1].name, 'project-test2')
712 self.assertEqual(self.builds[2].name, 'project-test2')
713 self.assertEqual(self.builds[3].name, 'project-test1')
714 self.assertEqual(self.builds[4].name, 'project-test2')
715
Paul Belanger71dd2fc2016-11-04 19:05:55 -0400716 self.assertTrue(self.builds[1].hasChanges(A))
717 self.assertTrue(self.builds[2].hasChanges(A))
718 self.assertTrue(self.builds[2].hasChanges(B))
719 self.assertTrue(self.builds[4].hasChanges(A))
720 self.assertFalse(self.builds[4].hasChanges(B))
721 self.assertTrue(self.builds[4].hasChanges(C))
James E. Blairfef71632013-09-23 11:15:47 -0700722
723 # Finish running all passing jobs for change A
724 self.release(self.builds[1])
725 self.waitUntilSettled()
726 # Fail and report change A
727 self.release(self.builds[0])
728 self.waitUntilSettled()
729
730 # restart of B,C after A failure
Paul Belanger174a8272017-03-14 13:20:10 -0400731 self.executor_server.release('.*-merge')
James E. Blairfef71632013-09-23 11:15:47 -0700732 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -0400733 self.executor_server.release('.*-merge')
James E. Blairfef71632013-09-23 11:15:47 -0700734 self.waitUntilSettled()
735
736 self.assertEqual(len(self.builds), 4)
737 self.assertEqual(self.builds[0].name, 'project-test1') # B
738 self.assertEqual(self.builds[1].name, 'project-test2') # B
739 self.assertEqual(self.builds[2].name, 'project-test1') # C
740 self.assertEqual(self.builds[3].name, 'project-test2') # C
741
Paul Belanger71dd2fc2016-11-04 19:05:55 -0400742 self.assertFalse(self.builds[1].hasChanges(A))
743 self.assertTrue(self.builds[1].hasChanges(B))
744 self.assertFalse(self.builds[1].hasChanges(C))
James E. Blairfef71632013-09-23 11:15:47 -0700745
Paul Belanger71dd2fc2016-11-04 19:05:55 -0400746 self.assertFalse(self.builds[2].hasChanges(A))
James E. Blairfef71632013-09-23 11:15:47 -0700747 # After A failed and B and C restarted, B should be back in
748 # C's tests because it has not failed yet.
Paul Belanger71dd2fc2016-11-04 19:05:55 -0400749 self.assertTrue(self.builds[2].hasChanges(B))
750 self.assertTrue(self.builds[2].hasChanges(C))
James E. Blairfef71632013-09-23 11:15:47 -0700751
Paul Belanger174a8272017-03-14 13:20:10 -0400752 self.executor_server.hold_jobs_in_build = False
753 self.executor_server.release()
James E. Blairfef71632013-09-23 11:15:47 -0700754 self.waitUntilSettled()
755
756 self.assertEqual(len(self.builds), 0)
757 self.assertEqual(len(self.history), 21)
758 self.assertEqual(A.data['status'], 'NEW')
759 self.assertEqual(B.data['status'], 'NEW')
760 self.assertEqual(C.data['status'], 'MERGED')
761 self.assertEqual(A.reported, 2)
762 self.assertEqual(B.reported, 2)
763 self.assertEqual(C.reported, 2)
764
James E. Blair8c803f82012-07-31 16:25:42 -0700765 def test_patch_order(self):
766 "Test that dependent patches are tested in the right order"
767 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
768 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
769 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200770 A.addApproval('Code-Review', 2)
771 B.addApproval('Code-Review', 2)
772 C.addApproval('Code-Review', 2)
James E. Blair8c803f82012-07-31 16:25:42 -0700773
774 M2 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M2')
775 M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1')
776 M2.setMerged()
777 M1.setMerged()
778
779 # C -> B -> A -> M1 -> M2
780 # M2 is here to make sure it is never queried. If it is, it
781 # means zuul is walking down the entire history of merged
782 # changes.
783
784 C.setDependsOn(B, 1)
785 B.setDependsOn(A, 1)
786 A.setDependsOn(M1, 1)
787 M1.setDependsOn(M2, 1)
788
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200789 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blair8c803f82012-07-31 16:25:42 -0700790
791 self.waitUntilSettled()
792
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400793 self.assertEqual(A.data['status'], 'NEW')
794 self.assertEqual(B.data['status'], 'NEW')
795 self.assertEqual(C.data['status'], 'NEW')
James E. Blair8c803f82012-07-31 16:25:42 -0700796
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200797 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
798 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair8c803f82012-07-31 16:25:42 -0700799
800 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400801 self.assertEqual(M2.queried, 0)
802 self.assertEqual(A.data['status'], 'MERGED')
803 self.assertEqual(B.data['status'], 'MERGED')
804 self.assertEqual(C.data['status'], 'MERGED')
805 self.assertEqual(A.reported, 2)
806 self.assertEqual(B.reported, 2)
807 self.assertEqual(C.reported, 2)
James E. Blair8c803f82012-07-31 16:25:42 -0700808
James E. Blair063672f2015-01-29 13:09:12 -0800809 def test_needed_changes_enqueue(self):
810 "Test that a needed change is enqueued ahead"
811 # A Given a git tree like this, if we enqueue
812 # / \ change C, we should walk up and down the tree
813 # B G and enqueue changes in the order ABCDEFG.
814 # /|\ This is also the order that you would get if
815 # *C E F you enqueued changes in the order ABCDEFG, so
816 # / the ordering is stable across re-enqueue events.
817 # D
818
819 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
820 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
821 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
822 D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
823 E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')
824 F = self.fake_gerrit.addFakeChange('org/project', 'master', 'F')
825 G = self.fake_gerrit.addFakeChange('org/project', 'master', 'G')
826 B.setDependsOn(A, 1)
827 C.setDependsOn(B, 1)
828 D.setDependsOn(C, 1)
829 E.setDependsOn(B, 1)
830 F.setDependsOn(B, 1)
831 G.setDependsOn(A, 1)
832
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200833 A.addApproval('Code-Review', 2)
834 B.addApproval('Code-Review', 2)
835 C.addApproval('Code-Review', 2)
836 D.addApproval('Code-Review', 2)
837 E.addApproval('Code-Review', 2)
838 F.addApproval('Code-Review', 2)
839 G.addApproval('Code-Review', 2)
840 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blair063672f2015-01-29 13:09:12 -0800841
842 self.waitUntilSettled()
843
844 self.assertEqual(A.data['status'], 'NEW')
845 self.assertEqual(B.data['status'], 'NEW')
846 self.assertEqual(C.data['status'], 'NEW')
847 self.assertEqual(D.data['status'], 'NEW')
848 self.assertEqual(E.data['status'], 'NEW')
849 self.assertEqual(F.data['status'], 'NEW')
850 self.assertEqual(G.data['status'], 'NEW')
851
852 # We're about to add approvals to changes without adding the
853 # triggering events to Zuul, so that we can be sure that it is
854 # enqueing the changes based on dependencies, not because of
855 # triggering events. Since it will have the changes cached
856 # already (without approvals), we need to clear the cache
857 # first.
Paul Belanger593480a2016-11-02 19:06:28 -0400858 for connection in self.connections.connections.values():
Joshua Hesketh4bd7da32016-02-17 20:58:47 +1100859 connection.maintainCache([])
James E. Blair063672f2015-01-29 13:09:12 -0800860
Paul Belanger174a8272017-03-14 13:20:10 -0400861 self.executor_server.hold_jobs_in_build = True
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200862 A.addApproval('Approved', 1)
863 B.addApproval('Approved', 1)
864 D.addApproval('Approved', 1)
865 E.addApproval('Approved', 1)
866 F.addApproval('Approved', 1)
867 G.addApproval('Approved', 1)
868 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blair063672f2015-01-29 13:09:12 -0800869
870 for x in range(8):
Paul Belanger174a8272017-03-14 13:20:10 -0400871 self.executor_server.release('.*-merge')
James E. Blair063672f2015-01-29 13:09:12 -0800872 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -0400873 self.executor_server.hold_jobs_in_build = False
874 self.executor_server.release()
James E. Blair063672f2015-01-29 13:09:12 -0800875 self.waitUntilSettled()
876
877 self.assertEqual(A.data['status'], 'MERGED')
878 self.assertEqual(B.data['status'], 'MERGED')
879 self.assertEqual(C.data['status'], 'MERGED')
880 self.assertEqual(D.data['status'], 'MERGED')
881 self.assertEqual(E.data['status'], 'MERGED')
882 self.assertEqual(F.data['status'], 'MERGED')
883 self.assertEqual(G.data['status'], 'MERGED')
884 self.assertEqual(A.reported, 2)
885 self.assertEqual(B.reported, 2)
886 self.assertEqual(C.reported, 2)
887 self.assertEqual(D.reported, 2)
888 self.assertEqual(E.reported, 2)
889 self.assertEqual(F.reported, 2)
890 self.assertEqual(G.reported, 2)
891 self.assertEqual(self.history[6].changes,
892 '1,1 2,1 3,1 4,1 5,1 6,1 7,1')
893
Joshua Hesketh850ccb62014-11-27 11:31:02 +1100894 def test_source_cache(self):
895 "Test that the source cache operates correctly"
Paul Belanger174a8272017-03-14 13:20:10 -0400896 self.executor_server.hold_jobs_in_build = True
James E. Blair0e933c52013-07-11 10:18:52 -0700897
898 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
899 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
900 X = self.fake_gerrit.addFakeChange('org/project', 'master', 'X')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200901 A.addApproval('Code-Review', 2)
902 B.addApproval('Code-Review', 2)
James E. Blair0e933c52013-07-11 10:18:52 -0700903
904 M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1')
905 M1.setMerged()
906
907 B.setDependsOn(A, 1)
908 A.setDependsOn(M1, 1)
909
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200910 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair0e933c52013-07-11 10:18:52 -0700911 self.fake_gerrit.addEvent(X.getPatchsetCreatedEvent(1))
912
913 self.waitUntilSettled()
914
915 for build in self.builds:
James E. Blaire675d682017-07-21 15:29:35 -0700916 if build.pipeline == 'check':
James E. Blair0e933c52013-07-11 10:18:52 -0700917 build.release()
918 self.waitUntilSettled()
919 for build in self.builds:
James E. Blaire675d682017-07-21 15:29:35 -0700920 if build.pipeline == 'check':
James E. Blair0e933c52013-07-11 10:18:52 -0700921 build.release()
922 self.waitUntilSettled()
923
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200924 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blair0e933c52013-07-11 10:18:52 -0700925 self.waitUntilSettled()
926
Joshua Hesketh352264b2015-08-11 23:42:08 +1000927 self.log.debug("len %s" % self.fake_gerrit._change_cache.keys())
James E. Blair0e933c52013-07-11 10:18:52 -0700928 # there should still be changes in the cache
Joshua Hesketh352264b2015-08-11 23:42:08 +1000929 self.assertNotEqual(len(self.fake_gerrit._change_cache.keys()), 0)
James E. Blair0e933c52013-07-11 10:18:52 -0700930
Paul Belanger174a8272017-03-14 13:20:10 -0400931 self.executor_server.hold_jobs_in_build = False
932 self.executor_server.release()
James E. Blair0e933c52013-07-11 10:18:52 -0700933 self.waitUntilSettled()
934
935 self.assertEqual(A.data['status'], 'MERGED')
936 self.assertEqual(B.data['status'], 'MERGED')
937 self.assertEqual(A.queried, 2) # Initial and isMerged
938 self.assertEqual(B.queried, 3) # Initial A, refresh from B, isMerged
939
James E. Blair8c803f82012-07-31 16:25:42 -0700940 def test_can_merge(self):
James E. Blair4886cc12012-07-18 15:39:41 -0700941 "Test whether a change is ready to merge"
James E. Blair8c803f82012-07-31 16:25:42 -0700942 # TODO: move to test_gerrit (this is a unit test!)
943 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Paul Belanger9bba4902016-11-02 16:07:33 -0400944 tenant = self.sched.abide.tenants.get('tenant-one')
James E. Blair6053de42017-04-05 11:27:11 -0700945 (trusted, project) = tenant.getProject('org/project')
946 source = project.source
Paul Belanger9bba4902016-11-02 16:07:33 -0400947
948 # TODO(pabelanger): As we add more source / trigger APIs we should make
949 # it easier for users to create events for testing.
950 event = zuul.model.TriggerEvent()
951 event.trigger_name = 'gerrit'
952 event.change_number = '1'
953 event.patch_number = '2'
954
955 a = source.getChange(event)
956 mgr = tenant.layout.pipelines['gate'].manager
James E. Blairc0dedf82014-08-06 09:37:52 -0700957 self.assertFalse(source.canMerge(a, mgr.getSubmitAllowNeeds()))
James E. Blair8c803f82012-07-31 16:25:42 -0700958
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200959 A.addApproval('Code-Review', 2)
Paul Belanger9bba4902016-11-02 16:07:33 -0400960 a = source.getChange(event, refresh=True)
James E. Blairc0dedf82014-08-06 09:37:52 -0700961 self.assertFalse(source.canMerge(a, mgr.getSubmitAllowNeeds()))
James E. Blair8c803f82012-07-31 16:25:42 -0700962
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200963 A.addApproval('Approved', 1)
Paul Belanger9bba4902016-11-02 16:07:33 -0400964 a = source.getChange(event, refresh=True)
James E. Blairc0dedf82014-08-06 09:37:52 -0700965 self.assertTrue(source.canMerge(a, mgr.getSubmitAllowNeeds()))
James E. Blair4886cc12012-07-18 15:39:41 -0700966
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -0700967 def test_project_merge_conflict(self):
968 "Test that gate merge conflicts are handled properly"
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700969
970 self.gearman_server.hold_jobs_in_queue = True
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -0700971 A = self.fake_gerrit.addFakeChange('org/project',
972 'master', 'A',
973 files={'conflict': 'foo'})
974 B = self.fake_gerrit.addFakeChange('org/project',
975 'master', 'B',
976 files={'conflict': 'bar'})
977 C = self.fake_gerrit.addFakeChange('org/project',
James E. Blair6736beb2013-07-11 15:18:15 -0700978 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200979 A.addApproval('Code-Review', 2)
980 B.addApproval('Code-Review', 2)
981 C.addApproval('Code-Review', 2)
982 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
983 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
984 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blair973721f2012-08-15 10:19:43 -0700985 self.waitUntilSettled()
986
James E. Blair6736beb2013-07-11 15:18:15 -0700987 self.assertEqual(A.reported, 1)
James E. Blair6736beb2013-07-11 15:18:15 -0700988 self.assertEqual(C.reported, 1)
989
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -0700990 self.gearman_server.release('project-merge')
James E. Blair973721f2012-08-15 10:19:43 -0700991 self.waitUntilSettled()
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -0700992 self.gearman_server.release('project-merge')
James E. Blair973721f2012-08-15 10:19:43 -0700993 self.waitUntilSettled()
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -0700994 self.gearman_server.release('project-merge')
James E. Blair973721f2012-08-15 10:19:43 -0700995 self.waitUntilSettled()
James E. Blair972e3c72013-08-29 12:04:55 -0700996
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700997 self.gearman_server.hold_jobs_in_queue = False
998 self.gearman_server.release()
James E. Blair973721f2012-08-15 10:19:43 -0700999 self.waitUntilSettled()
1000
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001001 self.assertEqual(A.data['status'], 'MERGED')
1002 self.assertEqual(B.data['status'], 'NEW')
1003 self.assertEqual(C.data['status'], 'MERGED')
1004 self.assertEqual(A.reported, 2)
Tobias Henkel9842bd72017-05-16 13:40:03 +02001005 self.assertIn('Merge Failed', B.messages[-1])
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001006 self.assertEqual(C.reported, 2)
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -07001007
1008 self.assertHistory([
1009 dict(name='project-merge', result='SUCCESS', changes='1,1'),
1010 dict(name='project-test1', result='SUCCESS', changes='1,1'),
1011 dict(name='project-test2', result='SUCCESS', changes='1,1'),
1012 dict(name='project-merge', result='SUCCESS', changes='1,1 3,1'),
1013 dict(name='project-test1', result='SUCCESS', changes='1,1 3,1'),
1014 dict(name='project-test2', result='SUCCESS', changes='1,1 3,1'),
1015 ], ordered=False)
1016
1017 def test_delayed_merge_conflict(self):
1018 "Test that delayed check merge conflicts are handled properly"
1019
1020 # Hold jobs in the gearman queue so that we can test whether
James E. Blair1960d682017-04-28 15:44:14 -07001021 # the executor sucesfully merges a change based on an old
1022 # repo state (frozen by the scheduler) which would otherwise
1023 # conflict.
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -07001024 self.gearman_server.hold_jobs_in_queue = True
1025 A = self.fake_gerrit.addFakeChange('org/project',
1026 'master', 'A',
1027 files={'conflict': 'foo'})
1028 B = self.fake_gerrit.addFakeChange('org/project',
1029 'master', 'B',
1030 files={'conflict': 'bar'})
1031 C = self.fake_gerrit.addFakeChange('org/project',
1032 'master', 'C')
1033 C.setDependsOn(B, 1)
1034
1035 # A enters the gate queue; B and C enter the check queue
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001036 A.addApproval('Code-Review', 2)
1037 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -07001038 self.waitUntilSettled()
1039 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
James E. Blair8d144dc2017-05-05 10:13:45 -07001040 self.waitUntilSettled()
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -07001041 self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
1042 self.waitUntilSettled()
1043
1044 self.assertEqual(A.reported, 1)
1045 self.assertEqual(B.reported, 0) # Check does not report start
1046 self.assertEqual(C.reported, 0) # Check does not report start
1047
1048 # A merges while B and C are queued in check
1049 # Release A project-merge
1050 queue = self.gearman_server.getQueue()
1051 self.release(queue[0])
1052 self.waitUntilSettled()
1053
1054 # Release A project-test*
1055 # gate has higher precedence, so A's test jobs are added in
1056 # front of the merge jobs for B and C
1057 queue = self.gearman_server.getQueue()
1058 self.release(queue[0])
1059 self.release(queue[1])
1060 self.waitUntilSettled()
1061
1062 self.assertEqual(A.data['status'], 'MERGED')
1063 self.assertEqual(B.data['status'], 'NEW')
1064 self.assertEqual(C.data['status'], 'NEW')
1065 self.assertEqual(A.reported, 2)
1066 self.assertEqual(B.reported, 0)
1067 self.assertEqual(C.reported, 0)
1068 self.assertHistory([
1069 dict(name='project-merge', result='SUCCESS', changes='1,1'),
1070 dict(name='project-test1', result='SUCCESS', changes='1,1'),
1071 dict(name='project-test2', result='SUCCESS', changes='1,1'),
1072 ], ordered=False)
1073
1074 # B and C report merge conflicts
1075 # Release B project-merge
1076 queue = self.gearman_server.getQueue()
1077 self.release(queue[0])
1078 self.waitUntilSettled()
1079
1080 # Release C
1081 self.gearman_server.hold_jobs_in_queue = False
1082 self.gearman_server.release()
1083 self.waitUntilSettled()
1084
1085 self.assertEqual(A.data['status'], 'MERGED')
1086 self.assertEqual(B.data['status'], 'NEW')
1087 self.assertEqual(C.data['status'], 'NEW')
1088 self.assertEqual(A.reported, 2)
1089 self.assertEqual(B.reported, 1)
1090 self.assertEqual(C.reported, 1)
1091
1092 self.assertHistory([
1093 dict(name='project-merge', result='SUCCESS', changes='1,1'),
1094 dict(name='project-test1', result='SUCCESS', changes='1,1'),
1095 dict(name='project-test2', result='SUCCESS', changes='1,1'),
James E. Blair1960d682017-04-28 15:44:14 -07001096 dict(name='project-merge', result='SUCCESS', changes='2,1'),
1097 dict(name='project-test1', result='SUCCESS', changes='2,1'),
1098 dict(name='project-test2', result='SUCCESS', changes='2,1'),
1099 dict(name='project-merge', result='SUCCESS', changes='2,1 3,1'),
1100 dict(name='project-test1', result='SUCCESS', changes='2,1 3,1'),
1101 dict(name='project-test2', result='SUCCESS', changes='2,1 3,1'),
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -07001102 ], ordered=False)
James E. Blair6736beb2013-07-11 15:18:15 -07001103
James E. Blairdaabed22012-08-15 15:38:57 -07001104 def test_post(self):
1105 "Test that post jobs run"
James E. Blair289f5932017-07-27 15:02:29 -07001106 p = "review.example.com/org/project"
1107 upstream = self.getUpstreamRepos([p])
1108 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1109 A.setMerged()
1110 A_commit = str(upstream[p].commit('master'))
1111 self.log.debug("A commit: %s" % A_commit)
James E. Blair1f4c2bb2013-04-26 08:40:46 -07001112
Zhongyue Luo5d556072012-09-21 02:00:47 +09001113 e = {
1114 "type": "ref-updated",
1115 "submitter": {
1116 "name": "User Name",
1117 },
1118 "refUpdate": {
1119 "oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
James E. Blair289f5932017-07-27 15:02:29 -07001120 "newRev": A_commit,
Zhongyue Luo5d556072012-09-21 02:00:47 +09001121 "refName": "master",
1122 "project": "org/project",
1123 }
1124 }
James E. Blairdaabed22012-08-15 15:38:57 -07001125 self.fake_gerrit.addEvent(e)
1126 self.waitUntilSettled()
1127
Monty Taylor6bef8ef2013-06-02 08:17:12 -04001128 job_names = [x.name for x in self.history]
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001129 self.assertEqual(len(self.history), 1)
1130 self.assertIn('project-post', job_names)
James E. Blairc6294a52012-08-17 10:19:48 -07001131
K Jonathan Harkerf95e7232015-04-29 13:33:16 -07001132 def test_post_ignore_deletes(self):
1133 "Test that deleting refs does not trigger post jobs"
1134
1135 e = {
1136 "type": "ref-updated",
1137 "submitter": {
1138 "name": "User Name",
1139 },
1140 "refUpdate": {
1141 "oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
1142 "newRev": "0000000000000000000000000000000000000000",
1143 "refName": "master",
1144 "project": "org/project",
1145 }
1146 }
1147 self.fake_gerrit.addEvent(e)
1148 self.waitUntilSettled()
1149
1150 job_names = [x.name for x in self.history]
1151 self.assertEqual(len(self.history), 0)
1152 self.assertNotIn('project-post', job_names)
1153
James E. Blaircc324192017-04-19 14:36:08 -07001154 @simple_layout('layouts/dont-ignore-ref-deletes.yaml')
K Jonathan Harkerf95e7232015-04-29 13:33:16 -07001155 def test_post_ignore_deletes_negative(self):
1156 "Test that deleting refs does trigger post jobs"
K Jonathan Harkerf95e7232015-04-29 13:33:16 -07001157 e = {
1158 "type": "ref-updated",
1159 "submitter": {
1160 "name": "User Name",
1161 },
1162 "refUpdate": {
1163 "oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
1164 "newRev": "0000000000000000000000000000000000000000",
James E. Blair289f5932017-07-27 15:02:29 -07001165 "refName": "testbranch",
K Jonathan Harkerf95e7232015-04-29 13:33:16 -07001166 "project": "org/project",
1167 }
1168 }
1169 self.fake_gerrit.addEvent(e)
1170 self.waitUntilSettled()
1171
1172 job_names = [x.name for x in self.history]
1173 self.assertEqual(len(self.history), 1)
1174 self.assertIn('project-post', job_names)
1175
James E. Blairec056492016-07-22 09:45:56 -07001176 @skip("Disabled for early v3 development")
James E. Blairc6294a52012-08-17 10:19:48 -07001177 def test_build_configuration_branch_interaction(self):
1178 "Test that switching between branches works"
1179 self.test_build_configuration()
1180 self.test_build_configuration_branch()
1181 # C has been merged, undo that
Monty Taylorbc758832013-06-17 17:22:42 -04001182 path = os.path.join(self.upstream_root, "org/project")
James E. Blairc6294a52012-08-17 10:19:48 -07001183 repo = git.Repo(path)
1184 repo.heads.master.commit = repo.commit('init')
1185 self.test_build_configuration()
1186
James E. Blairdb93b302017-07-19 15:33:11 -07001187 def test_dependent_changes_rebase(self):
1188 # Test that no errors occur when we walk a dependency tree
1189 # with an unused leaf node due to a rebase.
1190 # Start by constructing: C -> B -> A
1191 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1192 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
1193 B.setDependsOn(A, 1)
1194
1195 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
1196 C.setDependsOn(B, 1)
1197
1198 # Then rebase to form: D -> C -> A
1199 C.addPatchset() # C,2
1200 C.setDependsOn(A, 1)
1201
1202 D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
1203 D.setDependsOn(C, 2)
1204
1205 # Walk the entire tree
1206 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
1207 self.waitUntilSettled()
1208 self.assertEqual(len(self.history), 3)
1209
1210 # Verify that walking just part of the tree still works
1211 self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
1212 self.waitUntilSettled()
1213 self.assertEqual(len(self.history), 6)
1214
James E. Blaircaec0c52012-08-22 14:52:22 -07001215 def test_dependent_changes_dequeue(self):
1216 "Test that dependent patches are not needlessly tested"
James E. Blair1f4c2bb2013-04-26 08:40:46 -07001217
James E. Blaircaec0c52012-08-22 14:52:22 -07001218 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1219 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
1220 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001221 A.addApproval('Code-Review', 2)
1222 B.addApproval('Code-Review', 2)
1223 C.addApproval('Code-Review', 2)
James E. Blaircaec0c52012-08-22 14:52:22 -07001224
1225 M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1')
1226 M1.setMerged()
1227
1228 # C -> B -> A -> M1
1229
1230 C.setDependsOn(B, 1)
1231 B.setDependsOn(A, 1)
1232 A.setDependsOn(M1, 1)
1233
Paul Belanger174a8272017-03-14 13:20:10 -04001234 self.executor_server.failJob('project-merge', A)
James E. Blaircaec0c52012-08-22 14:52:22 -07001235
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001236 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
1237 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
1238 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blaircaec0c52012-08-22 14:52:22 -07001239
1240 self.waitUntilSettled()
1241
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001242 self.assertEqual(A.data['status'], 'NEW')
1243 self.assertEqual(A.reported, 2)
1244 self.assertEqual(B.data['status'], 'NEW')
1245 self.assertEqual(B.reported, 2)
1246 self.assertEqual(C.data['status'], 'NEW')
James E. Blair0d3e83b2017-06-05 13:51:57 -07001247 self.assertIn('This change depends on a change that failed to merge.',
1248 C.messages[-1])
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001249 self.assertEqual(len(self.history), 1)
James E. Blairec590122012-08-22 15:19:31 -07001250
James E. Blair972e3c72013-08-29 12:04:55 -07001251 def test_failing_dependent_changes(self):
1252 "Test that failing dependent patches are taken out of stream"
Paul Belanger174a8272017-03-14 13:20:10 -04001253 self.executor_server.hold_jobs_in_build = True
James E. Blair972e3c72013-08-29 12:04:55 -07001254 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1255 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
1256 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
1257 D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
1258 E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001259 A.addApproval('Code-Review', 2)
1260 B.addApproval('Code-Review', 2)
1261 C.addApproval('Code-Review', 2)
1262 D.addApproval('Code-Review', 2)
1263 E.addApproval('Code-Review', 2)
James E. Blair972e3c72013-08-29 12:04:55 -07001264
1265 # E, D -> C -> B, A
1266
1267 D.setDependsOn(C, 1)
1268 C.setDependsOn(B, 1)
1269
Paul Belanger174a8272017-03-14 13:20:10 -04001270 self.executor_server.failJob('project-test1', B)
James E. Blair972e3c72013-08-29 12:04:55 -07001271
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001272 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
1273 self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
1274 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
1275 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
1276 self.fake_gerrit.addEvent(E.addApproval('Approved', 1))
James E. Blair972e3c72013-08-29 12:04:55 -07001277
1278 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001279 self.executor_server.release('.*-merge')
James E. Blair972e3c72013-08-29 12:04:55 -07001280 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001281 self.executor_server.release('.*-merge')
James E. Blair972e3c72013-08-29 12:04:55 -07001282 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001283 self.executor_server.release('.*-merge')
James E. Blair972e3c72013-08-29 12:04:55 -07001284 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001285 self.executor_server.release('.*-merge')
James E. Blair972e3c72013-08-29 12:04:55 -07001286 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001287 self.executor_server.release('.*-merge')
James E. Blair972e3c72013-08-29 12:04:55 -07001288 self.waitUntilSettled()
1289
Paul Belanger174a8272017-03-14 13:20:10 -04001290 self.executor_server.hold_jobs_in_build = False
James E. Blair972e3c72013-08-29 12:04:55 -07001291 for build in self.builds:
James E. Blaire3db2952017-07-21 15:03:36 -07001292 if build.parameters['zuul']['change'] != '1':
James E. Blair972e3c72013-08-29 12:04:55 -07001293 build.release()
1294 self.waitUntilSettled()
1295
Paul Belanger174a8272017-03-14 13:20:10 -04001296 self.executor_server.release()
James E. Blair972e3c72013-08-29 12:04:55 -07001297 self.waitUntilSettled()
1298
1299 self.assertEqual(A.data['status'], 'MERGED')
1300 self.assertEqual(A.reported, 2)
Jeremy Stanley10837132014-08-02 16:10:56 +00001301 self.assertIn('Build succeeded', A.messages[1])
James E. Blair972e3c72013-08-29 12:04:55 -07001302 self.assertEqual(B.data['status'], 'NEW')
1303 self.assertEqual(B.reported, 2)
Jeremy Stanley10837132014-08-02 16:10:56 +00001304 self.assertIn('Build failed', B.messages[1])
James E. Blair972e3c72013-08-29 12:04:55 -07001305 self.assertEqual(C.data['status'], 'NEW')
1306 self.assertEqual(C.reported, 2)
Jeremy Stanley10837132014-08-02 16:10:56 +00001307 self.assertIn('depends on a change', C.messages[1])
James E. Blair972e3c72013-08-29 12:04:55 -07001308 self.assertEqual(D.data['status'], 'NEW')
1309 self.assertEqual(D.reported, 2)
Jeremy Stanley10837132014-08-02 16:10:56 +00001310 self.assertIn('depends on a change', D.messages[1])
James E. Blair972e3c72013-08-29 12:04:55 -07001311 self.assertEqual(E.data['status'], 'MERGED')
1312 self.assertEqual(E.reported, 2)
Jeremy Stanley10837132014-08-02 16:10:56 +00001313 self.assertIn('Build succeeded', E.messages[1])
James E. Blair972e3c72013-08-29 12:04:55 -07001314 self.assertEqual(len(self.history), 18)
1315
James E. Blairec590122012-08-22 15:19:31 -07001316 def test_head_is_dequeued_once(self):
James E. Blair2fa50962013-01-30 21:50:41 -08001317 "Test that if a change at the head fails it is dequeued only once"
James E. Blairec590122012-08-22 15:19:31 -07001318 # If it's dequeued more than once, we should see extra
1319 # aborted jobs.
James E. Blairec590122012-08-22 15:19:31 -07001320
Paul Belanger174a8272017-03-14 13:20:10 -04001321 self.executor_server.hold_jobs_in_build = True
James E. Blair6df67132017-04-19 13:53:20 -07001322 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1323 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
1324 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001325 A.addApproval('Code-Review', 2)
1326 B.addApproval('Code-Review', 2)
1327 C.addApproval('Code-Review', 2)
James E. Blairec590122012-08-22 15:19:31 -07001328
Paul Belanger174a8272017-03-14 13:20:10 -04001329 self.executor_server.failJob('project-test1', A)
1330 self.executor_server.failJob('project-test2', A)
James E. Blairec590122012-08-22 15:19:31 -07001331
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001332 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
1333 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
1334 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blairec590122012-08-22 15:19:31 -07001335
1336 self.waitUntilSettled()
James E. Blairec590122012-08-22 15:19:31 -07001337
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001338 self.assertEqual(len(self.builds), 1)
Paul Belanger58bf6912016-11-11 19:36:01 -05001339 self.assertEqual(self.builds[0].name, 'project-merge')
1340 self.assertTrue(self.builds[0].hasChanges(A))
James E. Blairec590122012-08-22 15:19:31 -07001341
Paul Belanger174a8272017-03-14 13:20:10 -04001342 self.executor_server.release('.*-merge')
James E. Blairec590122012-08-22 15:19:31 -07001343 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001344 self.executor_server.release('.*-merge')
James E. Blairec590122012-08-22 15:19:31 -07001345 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001346 self.executor_server.release('.*-merge')
James E. Blairec590122012-08-22 15:19:31 -07001347 self.waitUntilSettled()
1348
James E. Blair6df67132017-04-19 13:53:20 -07001349 self.assertEqual(len(self.builds), 6)
Paul Belanger58bf6912016-11-11 19:36:01 -05001350 self.assertEqual(self.builds[0].name, 'project-test1')
1351 self.assertEqual(self.builds[1].name, 'project-test2')
James E. Blair6df67132017-04-19 13:53:20 -07001352 self.assertEqual(self.builds[2].name, 'project-test1')
1353 self.assertEqual(self.builds[3].name, 'project-test2')
1354 self.assertEqual(self.builds[4].name, 'project-test1')
1355 self.assertEqual(self.builds[5].name, 'project-test2')
James E. Blairec590122012-08-22 15:19:31 -07001356
Monty Taylor6bef8ef2013-06-02 08:17:12 -04001357 self.release(self.builds[0])
James E. Blairec590122012-08-22 15:19:31 -07001358 self.waitUntilSettled()
1359
James E. Blair6df67132017-04-19 13:53:20 -07001360 self.assertEqual(len(self.builds), 2) # test2, merge for B
1361 self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 4)
James E. Blairec590122012-08-22 15:19:31 -07001362
Paul Belanger174a8272017-03-14 13:20:10 -04001363 self.executor_server.hold_jobs_in_build = False
1364 self.executor_server.release()
James E. Blairec590122012-08-22 15:19:31 -07001365 self.waitUntilSettled()
1366
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001367 self.assertEqual(len(self.builds), 0)
James E. Blair6df67132017-04-19 13:53:20 -07001368 self.assertEqual(len(self.history), 15)
James E. Blaircaec0c52012-08-22 14:52:22 -07001369
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001370 self.assertEqual(A.data['status'], 'NEW')
1371 self.assertEqual(B.data['status'], 'MERGED')
1372 self.assertEqual(C.data['status'], 'MERGED')
1373 self.assertEqual(A.reported, 2)
1374 self.assertEqual(B.reported, 2)
1375 self.assertEqual(C.reported, 2)
James E. Blair4ec821f2012-08-23 15:28:28 -07001376
James E. Blaircc020532017-04-19 13:04:44 -07001377 @simple_layout('layouts/nonvoting-job.yaml')
James E. Blair4ec821f2012-08-23 15:28:28 -07001378 def test_nonvoting_job(self):
1379 "Test that non-voting jobs don't vote."
James E. Blair1f4c2bb2013-04-26 08:40:46 -07001380
James E. Blair4ec821f2012-08-23 15:28:28 -07001381 A = self.fake_gerrit.addFakeChange('org/nonvoting-project',
1382 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001383 A.addApproval('Code-Review', 2)
Paul Belanger174a8272017-03-14 13:20:10 -04001384 self.executor_server.failJob('nonvoting-project-test2', A)
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001385 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair4ec821f2012-08-23 15:28:28 -07001386
1387 self.waitUntilSettled()
James E. Blair4ec821f2012-08-23 15:28:28 -07001388
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001389 self.assertEqual(A.data['status'], 'MERGED')
1390 self.assertEqual(A.reported, 2)
1391 self.assertEqual(
1392 self.getJobFromHistory('nonvoting-project-merge').result,
1393 'SUCCESS')
1394 self.assertEqual(
1395 self.getJobFromHistory('nonvoting-project-test1').result,
1396 'SUCCESS')
1397 self.assertEqual(
1398 self.getJobFromHistory('nonvoting-project-test2').result,
1399 'FAILURE')
James E. Blaire0487072012-08-29 17:38:31 -07001400
James E. Blair2ef29e92017-07-21 15:25:05 -07001401 self.assertTrue(self.getJobFromHistory('nonvoting-project-merge').
1402 parameters['zuul']['voting'])
1403 self.assertTrue(self.getJobFromHistory('nonvoting-project-test1').
1404 parameters['zuul']['voting'])
1405 self.assertFalse(self.getJobFromHistory('nonvoting-project-test2').
1406 parameters['zuul']['voting'])
James E. Blair5821bd92015-09-16 08:48:15 -07001407
James E. Blaire0487072012-08-29 17:38:31 -07001408 def test_check_queue_success(self):
1409 "Test successful check queue jobs."
James E. Blair1f4c2bb2013-04-26 08:40:46 -07001410
James E. Blaire0487072012-08-29 17:38:31 -07001411 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1412 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
1413
1414 self.waitUntilSettled()
James E. Blaire0487072012-08-29 17:38:31 -07001415
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001416 self.assertEqual(A.data['status'], 'NEW')
1417 self.assertEqual(A.reported, 1)
1418 self.assertEqual(self.getJobFromHistory('project-merge').result,
1419 'SUCCESS')
1420 self.assertEqual(self.getJobFromHistory('project-test1').result,
1421 'SUCCESS')
1422 self.assertEqual(self.getJobFromHistory('project-test2').result,
1423 'SUCCESS')
James E. Blaire0487072012-08-29 17:38:31 -07001424
1425 def test_check_queue_failure(self):
1426 "Test failed check queue jobs."
James E. Blair1f4c2bb2013-04-26 08:40:46 -07001427
James E. Blaire0487072012-08-29 17:38:31 -07001428 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Paul Belanger174a8272017-03-14 13:20:10 -04001429 self.executor_server.failJob('project-test2', A)
James E. Blaire0487072012-08-29 17:38:31 -07001430 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
1431
1432 self.waitUntilSettled()
James E. Blaire0487072012-08-29 17:38:31 -07001433
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001434 self.assertEqual(A.data['status'], 'NEW')
1435 self.assertEqual(A.reported, 1)
1436 self.assertEqual(self.getJobFromHistory('project-merge').result,
James E. Blair78e31b32013-07-09 09:11:34 -07001437 'SUCCESS')
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001438 self.assertEqual(self.getJobFromHistory('project-test1').result,
1439 'SUCCESS')
1440 self.assertEqual(self.getJobFromHistory('project-test2').result,
1441 'FAILURE')
James E. Blair127bc182012-08-28 15:55:15 -07001442
David Shrewsburyffab07a2017-07-24 12:45:07 -04001443 @simple_layout('layouts/autohold.yaml')
1444 def test_autohold(self):
1445 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1446
1447 client = zuul.rpcclient.RPCClient('127.0.0.1',
1448 self.gearman_server.port)
1449 self.addCleanup(client.shutdown)
David Shrewsbury36b2adf2017-07-31 15:40:13 -04001450 r = client.autohold('tenant-one', 'org/project', 'project-test2',
1451 "reason text", 1)
David Shrewsburyffab07a2017-07-24 12:45:07 -04001452 self.assertTrue(r)
1453
1454 self.executor_server.failJob('project-test2', A)
1455 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
1456
1457 self.waitUntilSettled()
1458
1459 self.assertEqual(A.data['status'], 'NEW')
1460 self.assertEqual(A.reported, 1)
1461 self.assertEqual(self.getJobFromHistory('project-test2').result,
1462 'FAILURE')
1463
1464 # Check nodepool for a held node
1465 held_node = None
1466 for node in self.fake_nodepool.getNodes():
1467 if node['state'] == zuul.model.STATE_HOLD:
1468 held_node = node
1469 break
1470 self.assertIsNotNone(held_node)
1471
1472 # Validate node has recorded the failed job
1473 self.assertEqual(
1474 held_node['hold_job'],
1475 " ".join(['tenant-one',
1476 'review.example.com/org/project',
1477 'project-test2'])
1478 )
David Shrewsburyf9af9df2017-08-01 15:19:26 -04001479 self.assertEqual(held_node['comment'], "reason text")
David Shrewsburyffab07a2017-07-24 12:45:07 -04001480
1481 # Another failed change should not hold any more nodes
1482 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
1483 self.executor_server.failJob('project-test2', B)
1484 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
1485 self.waitUntilSettled()
1486 self.assertEqual(B.data['status'], 'NEW')
1487 self.assertEqual(B.reported, 1)
1488 self.assertEqual(self.getJobFromHistory('project-test2').result,
1489 'FAILURE')
1490
1491 held_nodes = 0
1492 for node in self.fake_nodepool.getNodes():
1493 if node['state'] == zuul.model.STATE_HOLD:
1494 held_nodes += 1
1495 self.assertEqual(held_nodes, 1)
1496
James E. Blair7cb84542017-04-19 13:35:05 -07001497 @simple_layout('layouts/three-projects.yaml')
James E. Blair127bc182012-08-28 15:55:15 -07001498 def test_dependent_behind_dequeue(self):
Clint Byruma9626572017-02-22 14:04:00 -05001499 # This particular test does a large amount of merges and needs a little
1500 # more time to complete
James E. Blair267e5162017-04-07 10:08:20 -07001501 self.wait_timeout = 120
James E. Blair127bc182012-08-28 15:55:15 -07001502 "test that dependent changes behind dequeued changes work"
1503 # This complicated test is a reproduction of a real life bug
1504 self.sched.reconfigure(self.config)
James E. Blair127bc182012-08-28 15:55:15 -07001505
Paul Belanger174a8272017-03-14 13:20:10 -04001506 self.executor_server.hold_jobs_in_build = True
James E. Blair127bc182012-08-28 15:55:15 -07001507 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
1508 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
1509 C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
1510 D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')
1511 E = self.fake_gerrit.addFakeChange('org/project2', 'master', 'E')
1512 F = self.fake_gerrit.addFakeChange('org/project3', 'master', 'F')
1513 D.setDependsOn(C, 1)
1514 E.setDependsOn(D, 1)
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001515 A.addApproval('Code-Review', 2)
1516 B.addApproval('Code-Review', 2)
1517 C.addApproval('Code-Review', 2)
1518 D.addApproval('Code-Review', 2)
1519 E.addApproval('Code-Review', 2)
1520 F.addApproval('Code-Review', 2)
James E. Blair127bc182012-08-28 15:55:15 -07001521
1522 A.fail_merge = True
James E. Blair127bc182012-08-28 15:55:15 -07001523
1524 # Change object re-use in the gerrit trigger is hidden if
1525 # changes are added in quick succession; waiting makes it more
1526 # like real life.
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001527 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair127bc182012-08-28 15:55:15 -07001528 self.waitUntilSettled()
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001529 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blair127bc182012-08-28 15:55:15 -07001530 self.waitUntilSettled()
1531
Paul Belanger174a8272017-03-14 13:20:10 -04001532 self.executor_server.release('.*-merge')
James E. Blair127bc182012-08-28 15:55:15 -07001533 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001534 self.executor_server.release('.*-merge')
James E. Blair127bc182012-08-28 15:55:15 -07001535 self.waitUntilSettled()
1536
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001537 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blair127bc182012-08-28 15:55:15 -07001538 self.waitUntilSettled()
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001539 self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
James E. Blair127bc182012-08-28 15:55:15 -07001540 self.waitUntilSettled()
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001541 self.fake_gerrit.addEvent(E.addApproval('Approved', 1))
James E. Blair127bc182012-08-28 15:55:15 -07001542 self.waitUntilSettled()
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001543 self.fake_gerrit.addEvent(F.addApproval('Approved', 1))
James E. Blair127bc182012-08-28 15:55:15 -07001544 self.waitUntilSettled()
1545
Paul Belanger174a8272017-03-14 13:20:10 -04001546 self.executor_server.release('.*-merge')
James E. Blair127bc182012-08-28 15:55:15 -07001547 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001548 self.executor_server.release('.*-merge')
James E. Blair127bc182012-08-28 15:55:15 -07001549 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001550 self.executor_server.release('.*-merge')
James E. Blair127bc182012-08-28 15:55:15 -07001551 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001552 self.executor_server.release('.*-merge')
James E. Blair127bc182012-08-28 15:55:15 -07001553 self.waitUntilSettled()
1554
1555 # all jobs running
James E. Blaire955e062012-10-08 09:49:03 -07001556
1557 # Grab pointers to the jobs we want to release before
1558 # releasing any, because list indexes may change as
1559 # the jobs complete.
Monty Taylor6bef8ef2013-06-02 08:17:12 -04001560 a, b, c = self.builds[:3]
James E. Blaire955e062012-10-08 09:49:03 -07001561 a.release()
1562 b.release()
1563 c.release()
James E. Blair127bc182012-08-28 15:55:15 -07001564 self.waitUntilSettled()
1565
Paul Belanger174a8272017-03-14 13:20:10 -04001566 self.executor_server.hold_jobs_in_build = False
1567 self.executor_server.release()
James E. Blair127bc182012-08-28 15:55:15 -07001568 self.waitUntilSettled()
1569
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001570 self.assertEqual(A.data['status'], 'NEW')
1571 self.assertEqual(B.data['status'], 'MERGED')
1572 self.assertEqual(C.data['status'], 'MERGED')
1573 self.assertEqual(D.data['status'], 'MERGED')
1574 self.assertEqual(E.data['status'], 'MERGED')
1575 self.assertEqual(F.data['status'], 'MERGED')
James E. Blair127bc182012-08-28 15:55:15 -07001576
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001577 self.assertEqual(A.reported, 2)
1578 self.assertEqual(B.reported, 2)
1579 self.assertEqual(C.reported, 2)
1580 self.assertEqual(D.reported, 2)
1581 self.assertEqual(E.reported, 2)
1582 self.assertEqual(F.reported, 2)
James E. Blair127bc182012-08-28 15:55:15 -07001583
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001584 self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 15)
1585 self.assertEqual(len(self.history), 44)
James E. Blair05fed602012-09-07 12:45:24 -07001586
1587 def test_merger_repack(self):
1588 "Test that the merger works after a repack"
James E. Blair1f4c2bb2013-04-26 08:40:46 -07001589
James E. Blair05fed602012-09-07 12:45:24 -07001590 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001591 A.addApproval('Code-Review', 2)
1592 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair05fed602012-09-07 12:45:24 -07001593 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001594 self.assertEqual(self.getJobFromHistory('project-merge').result,
1595 'SUCCESS')
1596 self.assertEqual(self.getJobFromHistory('project-test1').result,
1597 'SUCCESS')
1598 self.assertEqual(self.getJobFromHistory('project-test2').result,
1599 'SUCCESS')
1600 self.assertEqual(A.data['status'], 'MERGED')
1601 self.assertEqual(A.reported, 2)
James E. Blair05fed602012-09-07 12:45:24 -07001602 self.assertEmptyQueues()
Paul Belanger5bfe9e72016-11-04 19:20:04 -04001603 self.build_history = []
James E. Blair05fed602012-09-07 12:45:24 -07001604
James E. Blair2a535672017-04-27 12:03:15 -07001605 path = os.path.join(self.merger_src_root, "review.example.com",
1606 "org/project")
James E. Blair8c1be532017-02-07 14:04:12 -08001607 if os.path.exists(path):
1608 repack_repo(path)
James E. Blair2a535672017-04-27 12:03:15 -07001609 path = os.path.join(self.executor_src_root, "review.example.com",
1610 "org/project")
James E. Blair8c1be532017-02-07 14:04:12 -08001611 if os.path.exists(path):
1612 repack_repo(path)
James E. Blair05fed602012-09-07 12:45:24 -07001613
1614 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001615 A.addApproval('Code-Review', 2)
1616 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair05fed602012-09-07 12:45:24 -07001617 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001618 self.assertEqual(self.getJobFromHistory('project-merge').result,
1619 'SUCCESS')
1620 self.assertEqual(self.getJobFromHistory('project-test1').result,
1621 'SUCCESS')
1622 self.assertEqual(self.getJobFromHistory('project-test2').result,
1623 'SUCCESS')
1624 self.assertEqual(A.data['status'], 'MERGED')
1625 self.assertEqual(A.reported, 2)
James E. Blair7ee88a22012-09-12 18:59:31 +02001626
James E. Blair4886f282012-11-15 09:27:33 -08001627 def test_merger_repack_large_change(self):
1628 "Test that the merger works with large changes after a repack"
Paul Belanger174a8272017-03-14 13:20:10 -04001629 # https://bugs.executepad.net/zuul/+bug/1078946
James E. Blairac2c3242014-01-24 13:38:51 -08001630 # This test assumes the repo is already cloned; make sure it is
Paul Belanger503bded2016-11-11 19:43:38 -05001631 tenant = self.sched.abide.tenants.get('tenant-one')
James E. Blair6df67132017-04-19 13:53:20 -07001632 trusted, project = tenant.getProject('org/project')
James E. Blair0ffa0102017-03-30 13:11:33 -07001633 url = self.fake_gerrit.getGitUrl(project)
James E. Blair8d144dc2017-05-05 10:13:45 -07001634 self.executor_server.merger._addProject('review.example.com',
James E. Blair197e8202017-06-09 12:54:28 -07001635 'org/project', url, None)
James E. Blair6df67132017-04-19 13:53:20 -07001636 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
James E. Blair4886f282012-11-15 09:27:33 -08001637 A.addPatchset(large=True)
James E. Blair2a535672017-04-27 12:03:15 -07001638 # TODOv3(jeblair): add hostname to upstream root
1639 path = os.path.join(self.upstream_root, 'org/project')
James E. Blair8c1be532017-02-07 14:04:12 -08001640 repack_repo(path)
James E. Blair2a535672017-04-27 12:03:15 -07001641 path = os.path.join(self.merger_src_root, 'review.example.com',
1642 'org/project')
James E. Blair8c1be532017-02-07 14:04:12 -08001643 if os.path.exists(path):
1644 repack_repo(path)
James E. Blair2a535672017-04-27 12:03:15 -07001645 path = os.path.join(self.executor_src_root, 'review.example.com',
1646 'org/project')
James E. Blair8c1be532017-02-07 14:04:12 -08001647 if os.path.exists(path):
1648 repack_repo(path)
James E. Blair4886f282012-11-15 09:27:33 -08001649
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001650 A.addApproval('Code-Review', 2)
1651 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair4886f282012-11-15 09:27:33 -08001652 self.waitUntilSettled()
Paul Belanger503bded2016-11-11 19:43:38 -05001653 self.assertEqual(self.getJobFromHistory('project-merge').result,
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001654 'SUCCESS')
Paul Belanger503bded2016-11-11 19:43:38 -05001655 self.assertEqual(self.getJobFromHistory('project-test1').result,
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001656 'SUCCESS')
Paul Belanger503bded2016-11-11 19:43:38 -05001657 self.assertEqual(self.getJobFromHistory('project-test2').result,
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001658 'SUCCESS')
1659 self.assertEqual(A.data['status'], 'MERGED')
1660 self.assertEqual(A.reported, 2)
James E. Blair4886f282012-11-15 09:27:33 -08001661
James E. Blair2fa50962013-01-30 21:50:41 -08001662 def test_new_patchset_dequeues_old(self):
1663 "Test that a new patchset causes the old to be dequeued"
1664 # D -> C (depends on B) -> B (depends on A) -> A -> M
Paul Belanger174a8272017-03-14 13:20:10 -04001665 self.executor_server.hold_jobs_in_build = True
James E. Blair2fa50962013-01-30 21:50:41 -08001666 M = self.fake_gerrit.addFakeChange('org/project', 'master', 'M')
1667 M.setMerged()
1668
1669 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1670 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
1671 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
1672 D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001673 A.addApproval('Code-Review', 2)
1674 B.addApproval('Code-Review', 2)
1675 C.addApproval('Code-Review', 2)
1676 D.addApproval('Code-Review', 2)
James E. Blair2fa50962013-01-30 21:50:41 -08001677
1678 C.setDependsOn(B, 1)
1679 B.setDependsOn(A, 1)
1680 A.setDependsOn(M, 1)
1681
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001682 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
1683 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
1684 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
1685 self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
James E. Blair2fa50962013-01-30 21:50:41 -08001686 self.waitUntilSettled()
1687
1688 B.addPatchset()
1689 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
1690 self.waitUntilSettled()
1691
Paul Belanger174a8272017-03-14 13:20:10 -04001692 self.executor_server.hold_jobs_in_build = False
1693 self.executor_server.release()
James E. Blair2fa50962013-01-30 21:50:41 -08001694 self.waitUntilSettled()
1695
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001696 self.assertEqual(A.data['status'], 'MERGED')
1697 self.assertEqual(A.reported, 2)
1698 self.assertEqual(B.data['status'], 'NEW')
1699 self.assertEqual(B.reported, 2)
1700 self.assertEqual(C.data['status'], 'NEW')
1701 self.assertEqual(C.reported, 2)
1702 self.assertEqual(D.data['status'], 'MERGED')
1703 self.assertEqual(D.reported, 2)
1704 self.assertEqual(len(self.history), 9) # 3 each for A, B, D.
James E. Blair2fa50962013-01-30 21:50:41 -08001705
James E. Blairba437362015-02-07 11:41:52 -08001706 def test_new_patchset_check(self):
1707 "Test a new patchset in check"
Antoine Mussobd86a312014-01-08 14:51:33 +01001708
Paul Belanger174a8272017-03-14 13:20:10 -04001709 self.executor_server.hold_jobs_in_build = True
Antoine Mussobd86a312014-01-08 14:51:33 +01001710
1711 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
James E. Blairba437362015-02-07 11:41:52 -08001712 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
Paul Belanger18e5f632016-11-03 15:24:16 -04001713 tenant = self.sched.abide.tenants.get('tenant-one')
1714 check_pipeline = tenant.layout.pipelines['check']
James E. Blairba437362015-02-07 11:41:52 -08001715
1716 # Add two git-dependent changes
1717 B.setDependsOn(A, 1)
1718 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
1719 self.waitUntilSettled()
Antoine Mussobd86a312014-01-08 14:51:33 +01001720 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
1721 self.waitUntilSettled()
James E. Blairba437362015-02-07 11:41:52 -08001722
1723 # A live item, and a non-live/live pair
1724 items = check_pipeline.getAllItems()
1725 self.assertEqual(len(items), 3)
1726
1727 self.assertEqual(items[0].change.number, '1')
1728 self.assertEqual(items[0].change.patchset, '1')
1729 self.assertFalse(items[0].live)
1730
1731 self.assertEqual(items[1].change.number, '2')
1732 self.assertEqual(items[1].change.patchset, '1')
1733 self.assertTrue(items[1].live)
1734
1735 self.assertEqual(items[2].change.number, '1')
1736 self.assertEqual(items[2].change.patchset, '1')
1737 self.assertTrue(items[2].live)
1738
1739 # Add a new patchset to A
1740 A.addPatchset()
1741 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
1742 self.waitUntilSettled()
1743
1744 # The live copy of A,1 should be gone, but the non-live and B
1745 # should continue, and we should have a new A,2
1746 items = check_pipeline.getAllItems()
1747 self.assertEqual(len(items), 3)
1748
1749 self.assertEqual(items[0].change.number, '1')
1750 self.assertEqual(items[0].change.patchset, '1')
1751 self.assertFalse(items[0].live)
1752
1753 self.assertEqual(items[1].change.number, '2')
1754 self.assertEqual(items[1].change.patchset, '1')
1755 self.assertTrue(items[1].live)
1756
1757 self.assertEqual(items[2].change.number, '1')
1758 self.assertEqual(items[2].change.patchset, '2')
1759 self.assertTrue(items[2].live)
1760
1761 # Add a new patchset to B
1762 B.addPatchset()
1763 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
1764 self.waitUntilSettled()
1765
1766 # The live copy of B,1 should be gone, and it's non-live copy of A,1
1767 # but we should have a new B,2 (still based on A,1)
1768 items = check_pipeline.getAllItems()
1769 self.assertEqual(len(items), 3)
1770
1771 self.assertEqual(items[0].change.number, '1')
1772 self.assertEqual(items[0].change.patchset, '2')
1773 self.assertTrue(items[0].live)
1774
1775 self.assertEqual(items[1].change.number, '1')
1776 self.assertEqual(items[1].change.patchset, '1')
1777 self.assertFalse(items[1].live)
1778
1779 self.assertEqual(items[2].change.number, '2')
1780 self.assertEqual(items[2].change.patchset, '2')
1781 self.assertTrue(items[2].live)
1782
1783 self.builds[0].release()
1784 self.waitUntilSettled()
1785 self.builds[0].release()
1786 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001787 self.executor_server.hold_jobs_in_build = False
1788 self.executor_server.release()
James E. Blairba437362015-02-07 11:41:52 -08001789 self.waitUntilSettled()
1790
1791 self.assertEqual(A.reported, 1)
1792 self.assertEqual(B.reported, 1)
1793 self.assertEqual(self.history[0].result, 'ABORTED')
1794 self.assertEqual(self.history[0].changes, '1,1')
1795 self.assertEqual(self.history[1].result, 'ABORTED')
1796 self.assertEqual(self.history[1].changes, '1,1 2,1')
1797 self.assertEqual(self.history[2].result, 'SUCCESS')
1798 self.assertEqual(self.history[2].changes, '1,2')
1799 self.assertEqual(self.history[3].result, 'SUCCESS')
1800 self.assertEqual(self.history[3].changes, '1,1 2,2')
1801
1802 def test_abandoned_gate(self):
1803 "Test that an abandoned change is dequeued from gate"
1804
Paul Belanger174a8272017-03-14 13:20:10 -04001805 self.executor_server.hold_jobs_in_build = True
James E. Blairba437362015-02-07 11:41:52 -08001806
1807 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001808 A.addApproval('Code-Review', 2)
1809 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blairba437362015-02-07 11:41:52 -08001810 self.waitUntilSettled()
Antoine Mussobd86a312014-01-08 14:51:33 +01001811 self.assertEqual(len(self.builds), 1, "One job being built (on hold)")
1812 self.assertEqual(self.builds[0].name, 'project-merge')
1813
1814 self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
1815 self.waitUntilSettled()
1816
Paul Belanger174a8272017-03-14 13:20:10 -04001817 self.executor_server.release('.*-merge')
Antoine Mussobd86a312014-01-08 14:51:33 +01001818 self.waitUntilSettled()
1819
Arie5b4048c2016-10-05 16:12:06 +03001820 self.assertBuilds([])
1821 self.assertHistory([
1822 dict(name='project-merge', result='ABORTED', changes='1,1')],
1823 ordered=False)
James E. Blairba437362015-02-07 11:41:52 -08001824 self.assertEqual(A.reported, 1,
1825 "Abandoned gate change should report only start")
1826
1827 def test_abandoned_check(self):
1828 "Test that an abandoned change is dequeued from check"
1829
Paul Belanger174a8272017-03-14 13:20:10 -04001830 self.executor_server.hold_jobs_in_build = True
James E. Blairba437362015-02-07 11:41:52 -08001831
1832 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1833 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
Paul Belanger18e5f632016-11-03 15:24:16 -04001834 tenant = self.sched.abide.tenants.get('tenant-one')
1835 check_pipeline = tenant.layout.pipelines['check']
James E. Blairba437362015-02-07 11:41:52 -08001836
1837 # Add two git-dependent changes
1838 B.setDependsOn(A, 1)
1839 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
1840 self.waitUntilSettled()
1841 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
1842 self.waitUntilSettled()
1843 # A live item, and a non-live/live pair
1844 items = check_pipeline.getAllItems()
1845 self.assertEqual(len(items), 3)
1846
1847 self.assertEqual(items[0].change.number, '1')
1848 self.assertFalse(items[0].live)
1849
1850 self.assertEqual(items[1].change.number, '2')
1851 self.assertTrue(items[1].live)
1852
1853 self.assertEqual(items[2].change.number, '1')
1854 self.assertTrue(items[2].live)
1855
1856 # Abandon A
1857 self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
1858 self.waitUntilSettled()
1859
1860 # The live copy of A should be gone, but the non-live and B
1861 # should continue
1862 items = check_pipeline.getAllItems()
1863 self.assertEqual(len(items), 2)
1864
1865 self.assertEqual(items[0].change.number, '1')
1866 self.assertFalse(items[0].live)
1867
1868 self.assertEqual(items[1].change.number, '2')
1869 self.assertTrue(items[1].live)
1870
Paul Belanger174a8272017-03-14 13:20:10 -04001871 self.executor_server.hold_jobs_in_build = False
1872 self.executor_server.release()
James E. Blairba437362015-02-07 11:41:52 -08001873 self.waitUntilSettled()
1874
1875 self.assertEqual(len(self.history), 4)
1876 self.assertEqual(self.history[0].result, 'ABORTED',
Antoine Mussobd86a312014-01-08 14:51:33 +01001877 'Build should have been aborted')
1878 self.assertEqual(A.reported, 0, "Abandoned change should not report")
James E. Blairba437362015-02-07 11:41:52 -08001879 self.assertEqual(B.reported, 1, "Change should report")
Antoine Mussobd86a312014-01-08 14:51:33 +01001880
Steve Varnau7b78b312015-04-03 14:49:46 -07001881 def test_abandoned_not_timer(self):
1882 "Test that an abandoned change does not cancel timer jobs"
James E. Blairdfdfcfc2017-04-20 10:19:20 -07001883 # This test can not use simple_layout because it must start
1884 # with a configuration which does not include a
1885 # timer-triggered job so that we have an opportunity to set
1886 # the hold flag before the first job.
Paul Belanger174a8272017-03-14 13:20:10 -04001887 self.executor_server.hold_jobs_in_build = True
Steve Varnau7b78b312015-04-03 14:49:46 -07001888 # Start timer trigger - also org/project
James E. Blairdfdfcfc2017-04-20 10:19:20 -07001889 self.commitConfigUpdate('common-config', 'layouts/idle.yaml')
Steve Varnau7b78b312015-04-03 14:49:46 -07001890 self.sched.reconfigure(self.config)
Steve Varnau7b78b312015-04-03 14:49:46 -07001891 # The pipeline triggers every second, so we should have seen
1892 # several by now.
1893 time.sleep(5)
1894 self.waitUntilSettled()
1895 # Stop queuing timer triggered jobs so that the assertions
1896 # below don't race against more jobs being queued.
Clint Byrumead6c562017-02-01 16:34:04 -08001897 # Must be in same repo, so overwrite config with another one
James E. Blairdfdfcfc2017-04-20 10:19:20 -07001898 self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')
Steve Varnau7b78b312015-04-03 14:49:46 -07001899 self.sched.reconfigure(self.config)
James E. Blair0b137b42017-07-27 08:51:25 -07001900 self.waitUntilSettled()
James E. Blair78ae4782017-08-02 14:19:10 -07001901 # If APScheduler is in mid-event when we remove the job, we
1902 # can end up with one more event firing, so give it an extra
1903 # second to settle.
1904 time.sleep(1)
1905 self.waitUntilSettled()
James E. Blairdfdfcfc2017-04-20 10:19:20 -07001906
James E. Blair21037782017-07-19 11:56:55 -07001907 self.assertEqual(len(self.builds), 1, "One timer job")
Steve Varnau7b78b312015-04-03 14:49:46 -07001908
1909 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1910 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
1911 self.waitUntilSettled()
James E. Blair21037782017-07-19 11:56:55 -07001912 self.assertEqual(len(self.builds), 2, "One change plus one timer job")
Steve Varnau7b78b312015-04-03 14:49:46 -07001913
1914 self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
1915 self.waitUntilSettled()
1916
James E. Blair21037782017-07-19 11:56:55 -07001917 self.assertEqual(len(self.builds), 1, "One timer job remains")
Steve Varnau7b78b312015-04-03 14:49:46 -07001918
Paul Belanger174a8272017-03-14 13:20:10 -04001919 self.executor_server.release()
Steve Varnau7b78b312015-04-03 14:49:46 -07001920 self.waitUntilSettled()
1921
James E. Blair2fa50962013-01-30 21:50:41 -08001922 def test_new_patchset_dequeues_old_on_head(self):
1923 "Test that a new patchset causes the old to be dequeued (at head)"
1924 # D -> C (depends on B) -> B (depends on A) -> A -> M
Paul Belanger174a8272017-03-14 13:20:10 -04001925 self.executor_server.hold_jobs_in_build = True
James E. Blair2fa50962013-01-30 21:50:41 -08001926 M = self.fake_gerrit.addFakeChange('org/project', 'master', 'M')
1927 M.setMerged()
1928 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1929 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
1930 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
1931 D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001932 A.addApproval('Code-Review', 2)
1933 B.addApproval('Code-Review', 2)
1934 C.addApproval('Code-Review', 2)
1935 D.addApproval('Code-Review', 2)
James E. Blair2fa50962013-01-30 21:50:41 -08001936
1937 C.setDependsOn(B, 1)
1938 B.setDependsOn(A, 1)
1939 A.setDependsOn(M, 1)
1940
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001941 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
1942 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
1943 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
1944 self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
James E. Blair2fa50962013-01-30 21:50:41 -08001945 self.waitUntilSettled()
1946
1947 A.addPatchset()
1948 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
1949 self.waitUntilSettled()
1950
Paul Belanger174a8272017-03-14 13:20:10 -04001951 self.executor_server.hold_jobs_in_build = False
1952 self.executor_server.release()
James E. Blair2fa50962013-01-30 21:50:41 -08001953 self.waitUntilSettled()
1954
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001955 self.assertEqual(A.data['status'], 'NEW')
1956 self.assertEqual(A.reported, 2)
1957 self.assertEqual(B.data['status'], 'NEW')
1958 self.assertEqual(B.reported, 2)
1959 self.assertEqual(C.data['status'], 'NEW')
1960 self.assertEqual(C.reported, 2)
1961 self.assertEqual(D.data['status'], 'MERGED')
1962 self.assertEqual(D.reported, 2)
1963 self.assertEqual(len(self.history), 7)
James E. Blair2fa50962013-01-30 21:50:41 -08001964
1965 def test_new_patchset_dequeues_old_without_dependents(self):
1966 "Test that a new patchset causes only the old to be dequeued"
Paul Belanger174a8272017-03-14 13:20:10 -04001967 self.executor_server.hold_jobs_in_build = True
James E. Blair2fa50962013-01-30 21:50:41 -08001968 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1969 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
1970 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001971 A.addApproval('Code-Review', 2)
1972 B.addApproval('Code-Review', 2)
1973 C.addApproval('Code-Review', 2)
James E. Blair2fa50962013-01-30 21:50:41 -08001974
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001975 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
1976 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
1977 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair2fa50962013-01-30 21:50:41 -08001978 self.waitUntilSettled()
1979
1980 B.addPatchset()
1981 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
1982 self.waitUntilSettled()
1983
Paul Belanger174a8272017-03-14 13:20:10 -04001984 self.executor_server.hold_jobs_in_build = False
1985 self.executor_server.release()
James E. Blair2fa50962013-01-30 21:50:41 -08001986 self.waitUntilSettled()
1987
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001988 self.assertEqual(A.data['status'], 'MERGED')
1989 self.assertEqual(A.reported, 2)
1990 self.assertEqual(B.data['status'], 'NEW')
1991 self.assertEqual(B.reported, 2)
1992 self.assertEqual(C.data['status'], 'MERGED')
1993 self.assertEqual(C.reported, 2)
1994 self.assertEqual(len(self.history), 9)
James E. Blair2fa50962013-01-30 21:50:41 -08001995
1996 def test_new_patchset_dequeues_old_independent_queue(self):
1997 "Test that a new patchset causes the old to be dequeued (independent)"
Paul Belanger174a8272017-03-14 13:20:10 -04001998 self.executor_server.hold_jobs_in_build = True
James E. Blair2fa50962013-01-30 21:50:41 -08001999 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
2000 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
2001 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
2002 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
2003 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
2004 self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
2005 self.waitUntilSettled()
2006
2007 B.addPatchset()
2008 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
2009 self.waitUntilSettled()
2010
Paul Belanger174a8272017-03-14 13:20:10 -04002011 self.executor_server.hold_jobs_in_build = False
2012 self.executor_server.release()
James E. Blair2fa50962013-01-30 21:50:41 -08002013 self.waitUntilSettled()
2014
Monty Taylor98f0f3e2013-07-06 16:02:31 -04002015 self.assertEqual(A.data['status'], 'NEW')
2016 self.assertEqual(A.reported, 1)
2017 self.assertEqual(B.data['status'], 'NEW')
2018 self.assertEqual(B.reported, 1)
2019 self.assertEqual(C.data['status'], 'NEW')
2020 self.assertEqual(C.reported, 1)
2021 self.assertEqual(len(self.history), 10)
2022 self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 1)
James E. Blair7d0dedc2013-02-21 17:26:09 -08002023
James E. Blaircc020532017-04-19 13:04:44 -07002024 @simple_layout('layouts/noop-job.yaml')
James E. Blair18c64442014-03-18 10:14:45 -07002025 def test_noop_job(self):
2026 "Test that the internal noop job works"
2027 A = self.fake_gerrit.addFakeChange('org/noop-project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002028 A.addApproval('Code-Review', 2)
2029 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair18c64442014-03-18 10:14:45 -07002030 self.waitUntilSettled()
2031
2032 self.assertEqual(len(self.gearman_server.getQueue()), 0)
2033 self.assertTrue(self.sched._areAllBuildsComplete())
2034 self.assertEqual(len(self.history), 0)
2035 self.assertEqual(A.data['status'], 'MERGED')
2036 self.assertEqual(A.reported, 2)
2037
James E. Blair06cc3922017-04-19 10:08:10 -07002038 @simple_layout('layouts/no-jobs-project.yaml')
Evgeny Antyshevd6e546c2015-06-11 15:13:57 +00002039 def test_no_job_project(self):
2040 "Test that reports with no jobs don't get sent"
2041 A = self.fake_gerrit.addFakeChange('org/no-jobs-project',
2042 'master', 'A')
2043 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
2044 self.waitUntilSettled()
2045
2046 # Change wasn't reported to
2047 self.assertEqual(A.reported, False)
2048
2049 # Check queue is empty afterwards
Clint Byrum883c30e2016-12-13 11:54:33 -08002050 tenant = self.sched.abide.tenants.get('tenant-one')
2051 check_pipeline = tenant.layout.pipelines['check']
Evgeny Antyshevd6e546c2015-06-11 15:13:57 +00002052 items = check_pipeline.getAllItems()
2053 self.assertEqual(len(items), 0)
2054
2055 self.assertEqual(len(self.history), 0)
2056
James E. Blair7d0dedc2013-02-21 17:26:09 -08002057 def test_zuul_refs(self):
2058 "Test that zuul refs exist and have the right changes"
Paul Belanger174a8272017-03-14 13:20:10 -04002059 self.executor_server.hold_jobs_in_build = True
James E. Blair7d0dedc2013-02-21 17:26:09 -08002060 M1 = self.fake_gerrit.addFakeChange('org/project1', 'master', 'M1')
2061 M1.setMerged()
2062 M2 = self.fake_gerrit.addFakeChange('org/project2', 'master', 'M2')
2063 M2.setMerged()
2064
2065 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
2066 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
2067 C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
2068 D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002069 A.addApproval('Code-Review', 2)
2070 B.addApproval('Code-Review', 2)
2071 C.addApproval('Code-Review', 2)
2072 D.addApproval('Code-Review', 2)
2073 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
2074 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
2075 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
2076 self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
James E. Blair7d0dedc2013-02-21 17:26:09 -08002077
2078 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002079 self.executor_server.release('.*-merge')
James E. Blair7d0dedc2013-02-21 17:26:09 -08002080 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002081 self.executor_server.release('.*-merge')
James E. Blair7d0dedc2013-02-21 17:26:09 -08002082 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002083 self.executor_server.release('.*-merge')
James E. Blair7d0dedc2013-02-21 17:26:09 -08002084 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002085 self.executor_server.release('.*-merge')
James E. Blair7d0dedc2013-02-21 17:26:09 -08002086 self.waitUntilSettled()
2087
Clint Byrum3343e3e2016-11-15 16:05:03 -08002088 a_build = b_build = c_build = d_build = None
Monty Taylor6bef8ef2013-06-02 08:17:12 -04002089 for x in self.builds:
James E. Blaire3db2952017-07-21 15:03:36 -07002090 if x.parameters['zuul']['change'] == '3':
Clint Byrum3343e3e2016-11-15 16:05:03 -08002091 a_build = x
James E. Blaire3db2952017-07-21 15:03:36 -07002092 elif x.parameters['zuul']['change'] == '4':
Clint Byrum3343e3e2016-11-15 16:05:03 -08002093 b_build = x
James E. Blaire3db2952017-07-21 15:03:36 -07002094 elif x.parameters['zuul']['change'] == '5':
Clint Byrum3343e3e2016-11-15 16:05:03 -08002095 c_build = x
James E. Blaire3db2952017-07-21 15:03:36 -07002096 elif x.parameters['zuul']['change'] == '6':
Clint Byrum3343e3e2016-11-15 16:05:03 -08002097 d_build = x
2098 if a_build and b_build and c_build and d_build:
2099 break
James E. Blair7d0dedc2013-02-21 17:26:09 -08002100
Clint Byrum3343e3e2016-11-15 16:05:03 -08002101 # should have a, not b, and should not be in project2
2102 self.assertTrue(a_build.hasChanges(A))
2103 self.assertFalse(a_build.hasChanges(B, M2))
James E. Blair7d0dedc2013-02-21 17:26:09 -08002104
Clint Byrum3343e3e2016-11-15 16:05:03 -08002105 # should have a and b, and should not be in project2
2106 self.assertTrue(b_build.hasChanges(A, B))
2107 self.assertFalse(b_build.hasChanges(M2))
James E. Blair7d0dedc2013-02-21 17:26:09 -08002108
Clint Byrum3343e3e2016-11-15 16:05:03 -08002109 # should have a and b in 1, c in 2
2110 self.assertTrue(c_build.hasChanges(A, B, C))
2111 self.assertFalse(c_build.hasChanges(D))
James E. Blair7d0dedc2013-02-21 17:26:09 -08002112
Clint Byrum3343e3e2016-11-15 16:05:03 -08002113 # should have a and b in 1, c and d in 2
2114 self.assertTrue(d_build.hasChanges(A, B, C, D))
James E. Blair7d0dedc2013-02-21 17:26:09 -08002115
Paul Belanger174a8272017-03-14 13:20:10 -04002116 self.executor_server.hold_jobs_in_build = False
2117 self.executor_server.release()
James E. Blair7d0dedc2013-02-21 17:26:09 -08002118 self.waitUntilSettled()
2119
Monty Taylor98f0f3e2013-07-06 16:02:31 -04002120 self.assertEqual(A.data['status'], 'MERGED')
2121 self.assertEqual(A.reported, 2)
2122 self.assertEqual(B.data['status'], 'MERGED')
2123 self.assertEqual(B.reported, 2)
2124 self.assertEqual(C.data['status'], 'MERGED')
2125 self.assertEqual(C.reported, 2)
2126 self.assertEqual(D.data['status'], 'MERGED')
2127 self.assertEqual(D.reported, 2)
James E. Blair70c71582013-03-06 08:50:50 -08002128
James E. Blair4a28a882013-08-23 15:17:33 -07002129 def test_rerun_on_error(self):
2130 "Test that if a worker fails to run a job, it is run again"
Paul Belanger174a8272017-03-14 13:20:10 -04002131 self.executor_server.hold_jobs_in_build = True
James E. Blair4a28a882013-08-23 15:17:33 -07002132 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002133 A.addApproval('Code-Review', 2)
2134 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair4a28a882013-08-23 15:17:33 -07002135 self.waitUntilSettled()
2136
James E. Blair412fba82017-01-26 15:00:50 -08002137 self.builds[0].requeue = True
Paul Belanger174a8272017-03-14 13:20:10 -04002138 self.executor_server.hold_jobs_in_build = False
2139 self.executor_server.release()
James E. Blair4a28a882013-08-23 15:17:33 -07002140 self.waitUntilSettled()
James E. Blair412fba82017-01-26 15:00:50 -08002141 self.assertEqual(self.countJobResults(self.history, None), 1)
James E. Blair4a28a882013-08-23 15:17:33 -07002142 self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 3)
2143
James E. Blair412e5582013-04-22 15:50:12 -07002144 def test_statsd(self):
2145 "Test each of the statsd methods used in the scheduler"
2146 import extras
2147 statsd = extras.try_import('statsd.statsd')
2148 statsd.incr('test-incr')
2149 statsd.timing('test-timing', 3)
Alex Gaynor813d39b2014-05-17 16:17:16 -07002150 statsd.gauge('test-gauge', 12)
James E. Blair412e5582013-04-22 15:50:12 -07002151 self.assertReportedStat('test-incr', '1|c')
2152 self.assertReportedStat('test-timing', '3|ms')
Alex Gaynor813d39b2014-05-17 16:17:16 -07002153 self.assertReportedStat('test-gauge', '12|g')
James E. Blair412e5582013-04-22 15:50:12 -07002154
James E. Blairdad52252014-02-07 16:59:17 -08002155 def test_stuck_job_cleanup(self):
2156 "Test that pending jobs are cleaned up if removed from layout"
Clint Byrum678e2c32017-03-16 16:27:21 -07002157
2158 # We want to hold the project-merge job that the fake change enqueues
James E. Blairdad52252014-02-07 16:59:17 -08002159 self.gearman_server.hold_jobs_in_queue = True
2160 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002161 A.addApproval('Code-Review', 2)
2162 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blairdad52252014-02-07 16:59:17 -08002163 self.waitUntilSettled()
Clint Byrum678e2c32017-03-16 16:27:21 -07002164 # The assertion is that we have one job in the queue, project-merge
James E. Blairdad52252014-02-07 16:59:17 -08002165 self.assertEqual(len(self.gearman_server.getQueue()), 1)
2166
James E. Blair31cb3e82017-04-20 13:08:56 -07002167 self.commitConfigUpdate('common-config', 'layouts/no-jobs.yaml')
James E. Blairdad52252014-02-07 16:59:17 -08002168 self.sched.reconfigure(self.config)
2169 self.waitUntilSettled()
2170
James E. Blair18c64442014-03-18 10:14:45 -07002171 self.gearman_server.release('gate-noop')
James E. Blairdad52252014-02-07 16:59:17 -08002172 self.waitUntilSettled()
Clint Byrum678e2c32017-03-16 16:27:21 -07002173 # asserting that project-merge is removed from queue
James E. Blairdad52252014-02-07 16:59:17 -08002174 self.assertEqual(len(self.gearman_server.getQueue()), 0)
2175 self.assertTrue(self.sched._areAllBuildsComplete())
2176
2177 self.assertEqual(len(self.history), 1)
James E. Blair18c64442014-03-18 10:14:45 -07002178 self.assertEqual(self.history[0].name, 'gate-noop')
James E. Blairdad52252014-02-07 16:59:17 -08002179 self.assertEqual(self.history[0].result, 'SUCCESS')
2180
James E. Blair879dafb2015-07-17 14:04:49 -07002181 def test_file_head(self):
2182 # This is a regression test for an observed bug. A change
2183 # with a file named "HEAD" in the root directory of the repo
2184 # was processed by a merger. It then was unable to reset the
2185 # repo because of:
2186 # GitCommandError: 'git reset --hard HEAD' returned
2187 # with exit code 128
2188 # stderr: 'fatal: ambiguous argument 'HEAD': both revision
2189 # and filename
2190 # Use '--' to separate filenames from revisions'
2191
2192 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Paul Belangera91095f2016-11-04 19:28:48 -04002193 A.addPatchset({'HEAD': ''})
James E. Blair879dafb2015-07-17 14:04:49 -07002194 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
2195
2196 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
2197 self.waitUntilSettled()
2198
2199 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
2200 self.waitUntilSettled()
2201
2202 self.assertIn('Build succeeded', A.messages[0])
2203 self.assertIn('Build succeeded', B.messages[0])
2204
James E. Blair70c71582013-03-06 08:50:50 -08002205 def test_file_jobs(self):
2206 "Test that file jobs run only when appropriate"
2207 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Adam Gandelman5661bda2017-02-03 15:58:03 -08002208 A.addPatchset({'pip-requires': 'foo'})
James E. Blair70c71582013-03-06 08:50:50 -08002209 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002210 A.addApproval('Code-Review', 2)
2211 B.addApproval('Code-Review', 2)
2212 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
2213 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blair70c71582013-03-06 08:50:50 -08002214 self.waitUntilSettled()
2215
Monty Taylor6bef8ef2013-06-02 08:17:12 -04002216 testfile_jobs = [x for x in self.history
James E. Blair70c71582013-03-06 08:50:50 -08002217 if x.name == 'project-testfile']
2218
Monty Taylor98f0f3e2013-07-06 16:02:31 -04002219 self.assertEqual(len(testfile_jobs), 1)
2220 self.assertEqual(testfile_jobs[0].changes, '1,2')
2221 self.assertEqual(A.data['status'], 'MERGED')
2222 self.assertEqual(A.reported, 2)
2223 self.assertEqual(B.data['status'], 'MERGED')
2224 self.assertEqual(B.reported, 2)
James E. Blair3c5e5b52013-04-26 11:17:03 -07002225
Clint Byrum3ee39f32016-11-17 23:45:07 -08002226 def _test_irrelevant_files_jobs(self, should_skip):
2227 "Test that jobs with irrelevant-files filter run only when appropriate"
Clint Byrum3ee39f32016-11-17 23:45:07 -08002228 if should_skip:
2229 files = {'ignoreme': 'ignored\n'}
2230 else:
2231 files = {'respectme': 'please!\n'}
Maru Newby3fe5f852015-01-13 04:22:14 +00002232
2233 change = self.fake_gerrit.addFakeChange('org/project',
Clint Byrum3ee39f32016-11-17 23:45:07 -08002234 'master',
2235 'test irrelevant-files',
2236 files=files)
Maru Newby3fe5f852015-01-13 04:22:14 +00002237 self.fake_gerrit.addEvent(change.getPatchsetCreatedEvent(1))
2238 self.waitUntilSettled()
2239
2240 tested_change_ids = [x.changes[0] for x in self.history
Clint Byrum3ee39f32016-11-17 23:45:07 -08002241 if x.name == 'project-test-irrelevant-files']
Maru Newby3fe5f852015-01-13 04:22:14 +00002242
2243 if should_skip:
2244 self.assertEqual([], tested_change_ids)
2245 else:
2246 self.assertIn(change.data['number'], tested_change_ids)
2247
James E. Blaircc324192017-04-19 14:36:08 -07002248 @simple_layout('layouts/irrelevant-files.yaml')
Clint Byrum3ee39f32016-11-17 23:45:07 -08002249 def test_irrelevant_files_match_skips_job(self):
2250 self._test_irrelevant_files_jobs(should_skip=True)
Maru Newby3fe5f852015-01-13 04:22:14 +00002251
James E. Blaircc324192017-04-19 14:36:08 -07002252 @simple_layout('layouts/irrelevant-files.yaml')
Clint Byrum3ee39f32016-11-17 23:45:07 -08002253 def test_irrelevant_files_no_match_runs_job(self):
2254 self._test_irrelevant_files_jobs(should_skip=False)
Maru Newby3fe5f852015-01-13 04:22:14 +00002255
James E. Blaircc324192017-04-19 14:36:08 -07002256 @simple_layout('layouts/inheritance.yaml')
Clint Byrum85493602016-11-18 11:59:47 -08002257 def test_inherited_jobs_keep_matchers(self):
Clint Byrum85493602016-11-18 11:59:47 -08002258 files = {'ignoreme': 'ignored\n'}
2259
2260 change = self.fake_gerrit.addFakeChange('org/project',
2261 'master',
2262 'test irrelevant-files',
2263 files=files)
2264 self.fake_gerrit.addEvent(change.getPatchsetCreatedEvent(1))
2265 self.waitUntilSettled()
2266
2267 run_jobs = set([build.name for build in self.history])
2268
2269 self.assertEqual(set(['project-test-nomatch-starts-empty',
2270 'project-test-nomatch-starts-full']), run_jobs)
2271
James E. Blairaf8b2082017-10-03 15:38:27 -07002272 @simple_layout('layouts/job-vars.yaml')
2273 def test_inherited_job_variables(self):
2274 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
2275 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
2276 self.waitUntilSettled()
2277 self.assertHistory([
2278 dict(name='parentjob', result='SUCCESS'),
2279 dict(name='child1', result='SUCCESS'),
2280 dict(name='child2', result='SUCCESS'),
2281 dict(name='child3', result='SUCCESS'),
2282 ], ordered=False)
2283 j = self.getJobFromHistory('parentjob')
2284 rp = set([p['name'] for p in j.parameters['projects']])
2285 self.assertEqual(j.parameters['vars']['override'], 0)
2286 self.assertEqual(j.parameters['vars']['child1override'], 0)
2287 self.assertEqual(j.parameters['vars']['parent'], 0)
2288 self.assertFalse('child1' in j.parameters['vars'])
2289 self.assertFalse('child2' in j.parameters['vars'])
2290 self.assertFalse('child3' in j.parameters['vars'])
2291 self.assertEqual(rp, set(['org/project', 'org/project0',
2292 'org/project0']))
2293 j = self.getJobFromHistory('child1')
2294 rp = set([p['name'] for p in j.parameters['projects']])
2295 self.assertEqual(j.parameters['vars']['override'], 1)
2296 self.assertEqual(j.parameters['vars']['child1override'], 1)
2297 self.assertEqual(j.parameters['vars']['parent'], 0)
2298 self.assertEqual(j.parameters['vars']['child1'], 1)
2299 self.assertFalse('child2' in j.parameters['vars'])
2300 self.assertFalse('child3' in j.parameters['vars'])
2301 self.assertEqual(rp, set(['org/project', 'org/project0',
2302 'org/project1']))
2303 j = self.getJobFromHistory('child2')
2304 rp = set([p['name'] for p in j.parameters['projects']])
2305 self.assertEqual(j.parameters['vars']['override'], 2)
2306 self.assertEqual(j.parameters['vars']['child1override'], 0)
2307 self.assertEqual(j.parameters['vars']['parent'], 0)
2308 self.assertFalse('child1' in j.parameters['vars'])
2309 self.assertEqual(j.parameters['vars']['child2'], 2)
2310 self.assertFalse('child3' in j.parameters['vars'])
2311 self.assertEqual(rp, set(['org/project', 'org/project0',
2312 'org/project2']))
2313 j = self.getJobFromHistory('child3')
2314 rp = set([p['name'] for p in j.parameters['projects']])
2315 self.assertEqual(j.parameters['vars']['override'], 3)
2316 self.assertEqual(j.parameters['vars']['child1override'], 0)
2317 self.assertEqual(j.parameters['vars']['parent'], 0)
2318 self.assertFalse('child1' in j.parameters['vars'])
2319 self.assertFalse('child2' in j.parameters['vars'])
2320 self.assertEqual(j.parameters['vars']['child3'], 3)
2321 self.assertEqual(rp, set(['org/project', 'org/project0',
2322 'org/project3']))
2323
James E. Blairc8a1e052014-02-25 09:29:26 -08002324 def test_queue_names(self):
2325 "Test shared change queue names"
Paul Belanger8e68fc22017-02-06 16:24:49 -05002326 tenant = self.sched.abide.tenants.get('tenant-one')
James E. Blair6053de42017-04-05 11:27:11 -07002327 (trusted, project1) = tenant.getProject('org/project1')
2328 (trusted, project2) = tenant.getProject('org/project2')
Paul Belanger8e68fc22017-02-06 16:24:49 -05002329 q1 = tenant.layout.pipelines['gate'].getQueue(project1)
2330 q2 = tenant.layout.pipelines['gate'].getQueue(project2)
2331 self.assertEqual(q1.name, 'integrated')
2332 self.assertEqual(q2.name, 'integrated')
James E. Blairc8a1e052014-02-25 09:29:26 -08002333
James E. Blair64ed6f22013-07-10 14:07:23 -07002334 def test_queue_precedence(self):
2335 "Test that queue precedence works"
2336
2337 self.gearman_server.hold_jobs_in_queue = True
Paul Belanger174a8272017-03-14 13:20:10 -04002338 self.executor_server.hold_jobs_in_build = True
James E. Blair64ed6f22013-07-10 14:07:23 -07002339 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
2340 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002341 A.addApproval('Code-Review', 2)
2342 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair64ed6f22013-07-10 14:07:23 -07002343
2344 self.waitUntilSettled()
2345 self.gearman_server.hold_jobs_in_queue = False
2346 self.gearman_server.release()
2347 self.waitUntilSettled()
2348
James E. Blair8de58bd2013-07-18 16:23:33 -07002349 # Run one build at a time to ensure non-race order:
James E. Blairb8c16472015-05-05 14:55:26 -07002350 self.orderedRelease()
Paul Belanger174a8272017-03-14 13:20:10 -04002351 self.executor_server.hold_jobs_in_build = False
James E. Blair8de58bd2013-07-18 16:23:33 -07002352 self.waitUntilSettled()
2353
James E. Blair64ed6f22013-07-10 14:07:23 -07002354 self.log.debug(self.history)
2355 self.assertEqual(self.history[0].pipeline, 'gate')
2356 self.assertEqual(self.history[1].pipeline, 'check')
2357 self.assertEqual(self.history[2].pipeline, 'gate')
2358 self.assertEqual(self.history[3].pipeline, 'gate')
2359 self.assertEqual(self.history[4].pipeline, 'check')
2360 self.assertEqual(self.history[5].pipeline, 'check')
2361
Clark Boylana5edbe42014-06-03 16:39:10 -07002362 def test_json_status(self):
James E. Blair1843a552013-07-03 14:19:52 -07002363 "Test that we can retrieve JSON status info"
Paul Belanger174a8272017-03-14 13:20:10 -04002364 self.executor_server.hold_jobs_in_build = True
James E. Blair1843a552013-07-03 14:19:52 -07002365 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002366 A.addApproval('Code-Review', 2)
2367 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair1843a552013-07-03 14:19:52 -07002368 self.waitUntilSettled()
2369
Paul Belanger174a8272017-03-14 13:20:10 -04002370 self.executor_server.release('project-merge')
James E. Blairb7273ef2016-04-19 08:58:51 -07002371 self.waitUntilSettled()
2372
James E. Blair1843a552013-07-03 14:19:52 -07002373 port = self.webapp.server.socket.getsockname()[1]
2374
Adam Gandelman77a12c72017-02-03 17:43:43 -08002375 req = urllib.request.Request(
2376 "http://localhost:%s/tenant-one/status" % port)
Morgan Fainberg293f7f82016-05-30 14:01:22 -07002377 f = urllib.request.urlopen(req)
Clark Boylanaa4f2e72014-06-03 21:22:40 -07002378 headers = f.info()
2379 self.assertIn('Content-Length', headers)
2380 self.assertIn('Content-Type', headers)
Sachi Kingdc963fc2016-03-23 16:00:33 +11002381 self.assertIsNotNone(re.match('^application/json(; charset=UTF-8)?$',
2382 headers['Content-Type']))
Timo Tijhof0ebd2932015-04-02 12:11:21 +01002383 self.assertIn('Access-Control-Allow-Origin', headers)
2384 self.assertIn('Cache-Control', headers)
Clark Boylanaa4f2e72014-06-03 21:22:40 -07002385 self.assertIn('Last-Modified', headers)
Timo Tijhof0ebd2932015-04-02 12:11:21 +01002386 self.assertIn('Expires', headers)
Clint Byrumf322fe22017-05-10 20:53:12 -07002387 data = f.read().decode('utf8')
James E. Blair1843a552013-07-03 14:19:52 -07002388
Paul Belanger174a8272017-03-14 13:20:10 -04002389 self.executor_server.hold_jobs_in_build = False
2390 self.executor_server.release()
James E. Blair1843a552013-07-03 14:19:52 -07002391 self.waitUntilSettled()
2392
2393 data = json.loads(data)
James E. Blairb7273ef2016-04-19 08:58:51 -07002394 status_jobs = []
James E. Blair1843a552013-07-03 14:19:52 -07002395 for p in data['pipelines']:
2396 for q in p['change_queues']:
James E. Blairbfb8e042014-12-30 17:01:44 -08002397 if p['name'] in ['gate', 'conflict']:
Clark Boylanaf2476f2014-01-23 14:47:36 -08002398 self.assertEqual(q['window'], 20)
2399 else:
2400 self.assertEqual(q['window'], 0)
James E. Blair1843a552013-07-03 14:19:52 -07002401 for head in q['heads']:
2402 for change in head:
Clark Boylanaf2476f2014-01-23 14:47:36 -08002403 self.assertTrue(change['active'])
Monty Taylor98f0f3e2013-07-06 16:02:31 -04002404 self.assertEqual(change['id'], '1,1')
James E. Blair1843a552013-07-03 14:19:52 -07002405 for job in change['jobs']:
James E. Blairb7273ef2016-04-19 08:58:51 -07002406 status_jobs.append(job)
2407 self.assertEqual('project-merge', status_jobs[0]['name'])
Monty Taylorde8242c2017-02-23 20:29:53 -06002408 # TODO(mordred) pull uuids from self.builds
Monty Taylor51139a02016-05-24 11:28:10 -05002409 self.assertEqual(
Tobias Henkelb4407fc2017-07-07 13:52:56 +02002410 'static/stream.html?uuid={uuid}&logfile=console.log'.format(
2411 uuid=status_jobs[0]['uuid']),
2412 status_jobs[0]['url'])
2413 self.assertEqual(
Monty Taylor51139a02016-05-24 11:28:10 -05002414 'finger://{hostname}/{uuid}'.format(
2415 hostname=self.executor_server.hostname,
2416 uuid=status_jobs[0]['uuid']),
Tobias Henkelb4407fc2017-07-07 13:52:56 +02002417 status_jobs[0]['finger_url'])
Monty Taylorde8242c2017-02-23 20:29:53 -06002418 # TOOD(mordred) configure a success-url on the base job
Monty Taylor51139a02016-05-24 11:28:10 -05002419 self.assertEqual(
2420 'finger://{hostname}/{uuid}'.format(
2421 hostname=self.executor_server.hostname,
2422 uuid=status_jobs[0]['uuid']),
2423 status_jobs[0]['report_url'])
James E. Blairb7273ef2016-04-19 08:58:51 -07002424 self.assertEqual('project-test1', status_jobs[1]['name'])
Monty Taylor51139a02016-05-24 11:28:10 -05002425 self.assertEqual(
Tobias Henkelb4407fc2017-07-07 13:52:56 +02002426 'static/stream.html?uuid={uuid}&logfile=console.log'.format(
2427 uuid=status_jobs[1]['uuid']),
2428 status_jobs[1]['url'])
2429 self.assertEqual(
Monty Taylor51139a02016-05-24 11:28:10 -05002430 'finger://{hostname}/{uuid}'.format(
2431 hostname=self.executor_server.hostname,
2432 uuid=status_jobs[1]['uuid']),
Tobias Henkelb4407fc2017-07-07 13:52:56 +02002433 status_jobs[1]['finger_url'])
Monty Taylor51139a02016-05-24 11:28:10 -05002434 self.assertEqual(
2435 'finger://{hostname}/{uuid}'.format(
2436 hostname=self.executor_server.hostname,
2437 uuid=status_jobs[1]['uuid']),
2438 status_jobs[1]['report_url'])
James E. Blairb7273ef2016-04-19 08:58:51 -07002439
2440 self.assertEqual('project-test2', status_jobs[2]['name'])
Monty Taylor51139a02016-05-24 11:28:10 -05002441 self.assertEqual(
Tobias Henkelb4407fc2017-07-07 13:52:56 +02002442 'static/stream.html?uuid={uuid}&logfile=console.log'.format(
2443 uuid=status_jobs[2]['uuid']),
2444 status_jobs[2]['url'])
2445 self.assertEqual(
Monty Taylor51139a02016-05-24 11:28:10 -05002446 'finger://{hostname}/{uuid}'.format(
2447 hostname=self.executor_server.hostname,
2448 uuid=status_jobs[2]['uuid']),
Tobias Henkelb4407fc2017-07-07 13:52:56 +02002449 status_jobs[2]['finger_url'])
Monty Taylor51139a02016-05-24 11:28:10 -05002450 self.assertEqual(
2451 'finger://{hostname}/{uuid}'.format(
2452 hostname=self.executor_server.hostname,
2453 uuid=status_jobs[2]['uuid']),
2454 status_jobs[2]['report_url'])
James E. Blair1843a552013-07-03 14:19:52 -07002455
Tobias Henkel65639f82017-07-10 10:25:42 +02002456 # check job dependencies
2457 self.assertIsNotNone(status_jobs[0]['dependencies'])
2458 self.assertIsNotNone(status_jobs[1]['dependencies'])
2459 self.assertIsNotNone(status_jobs[2]['dependencies'])
2460 self.assertEqual(len(status_jobs[0]['dependencies']), 0)
2461 self.assertEqual(len(status_jobs[1]['dependencies']), 1)
2462 self.assertEqual(len(status_jobs[2]['dependencies']), 1)
2463 self.assertIn('project-merge', status_jobs[1]['dependencies'])
2464 self.assertIn('project-merge', status_jobs[2]['dependencies'])
2465
James E. Blaircdccd972013-07-01 12:10:22 -07002466 def test_live_reconfiguration(self):
2467 "Test that live reconfiguration works"
Paul Belanger174a8272017-03-14 13:20:10 -04002468 self.executor_server.hold_jobs_in_build = True
James E. Blaircdccd972013-07-01 12:10:22 -07002469 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002470 A.addApproval('Code-Review', 2)
2471 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blaircdccd972013-07-01 12:10:22 -07002472 self.waitUntilSettled()
2473
2474 self.sched.reconfigure(self.config)
James E. Blairbd963632017-02-01 09:58:14 -08002475 self.waitUntilSettled()
James E. Blaircdccd972013-07-01 12:10:22 -07002476
Paul Belanger174a8272017-03-14 13:20:10 -04002477 self.executor_server.hold_jobs_in_build = False
2478 self.executor_server.release()
James E. Blaircdccd972013-07-01 12:10:22 -07002479 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -04002480 self.assertEqual(self.getJobFromHistory('project-merge').result,
2481 'SUCCESS')
2482 self.assertEqual(self.getJobFromHistory('project-test1').result,
2483 'SUCCESS')
2484 self.assertEqual(self.getJobFromHistory('project-test2').result,
2485 'SUCCESS')
2486 self.assertEqual(A.data['status'], 'MERGED')
2487 self.assertEqual(A.reported, 2)
James E. Blair287c06d2013-07-24 10:39:30 -07002488
James E. Blair027ba992017-09-20 13:48:32 -07002489 def test_live_reconfiguration_abort(self):
2490 # Raise an exception during reconfiguration and verify we
2491 # still function.
2492 self.executor_server.hold_jobs_in_build = True
2493 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
2494 A.addApproval('Code-Review', 2)
2495 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
2496 self.waitUntilSettled()
2497
2498 tenant = self.sched.abide.tenants.get('tenant-one')
2499 pipeline = tenant.layout.pipelines['gate']
2500 change = pipeline.getAllItems()[0].change
2501 # Set this to an invalid value to cause an exception during
2502 # reconfiguration.
2503 change.branch = None
2504
2505 self.sched.reconfigure(self.config)
2506 self.waitUntilSettled()
2507
2508 self.executor_server.hold_jobs_in_build = False
2509 self.executor_server.release()
2510
2511 self.waitUntilSettled()
2512 self.assertEqual(self.getJobFromHistory('project-merge').result,
2513 'ABORTED')
2514 self.assertEqual(A.data['status'], 'NEW')
2515 # The final report fails because of the invalid value set above.
2516 self.assertEqual(A.reported, 1)
2517
James E. Blair6bc782d2015-07-17 16:20:21 -07002518 def test_live_reconfiguration_merge_conflict(self):
2519 # A real-world bug: a change in a gate queue has a merge
2520 # conflict and a job is added to its project while it's
2521 # sitting in the queue. The job gets added to the change and
2522 # enqueued and the change gets stuck.
Paul Belanger174a8272017-03-14 13:20:10 -04002523 self.executor_server.hold_jobs_in_build = True
James E. Blair6bc782d2015-07-17 16:20:21 -07002524
2525 # This change is fine. It's here to stop the queue long
2526 # enough for the next change to be subject to the
2527 # reconfiguration, as well as to provide a conflict for the
2528 # next change. This change will succeed and merge.
2529 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Clint Byrum073c00b2017-05-01 13:41:23 -07002530 A.addPatchset({'conflict': 'A'})
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002531 A.addApproval('Code-Review', 2)
James E. Blair6bc782d2015-07-17 16:20:21 -07002532
2533 # This change will be in merge conflict. During the
2534 # reconfiguration, we will add a job. We want to make sure
2535 # that doesn't cause it to get stuck.
2536 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
Clint Byrum073c00b2017-05-01 13:41:23 -07002537 B.addPatchset({'conflict': 'B'})
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002538 B.addApproval('Code-Review', 2)
James E. Blair4eb21fa2015-07-27 14:56:47 -07002539
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002540 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
2541 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blair6bc782d2015-07-17 16:20:21 -07002542
2543 self.waitUntilSettled()
2544
2545 # No jobs have run yet
2546 self.assertEqual(A.data['status'], 'NEW')
2547 self.assertEqual(A.reported, 1)
2548 self.assertEqual(B.data['status'], 'NEW')
James E. Blair6bc782d2015-07-17 16:20:21 -07002549 self.assertEqual(len(self.history), 0)
2550
2551 # Add the "project-test3" job.
Clint Byrum073c00b2017-05-01 13:41:23 -07002552 self.commitConfigUpdate('common-config',
2553 'layouts/live-reconfiguration-add-job.yaml')
James E. Blair6bc782d2015-07-17 16:20:21 -07002554 self.sched.reconfigure(self.config)
2555 self.waitUntilSettled()
2556
Paul Belanger174a8272017-03-14 13:20:10 -04002557 self.executor_server.hold_jobs_in_build = False
2558 self.executor_server.release()
James E. Blair6bc782d2015-07-17 16:20:21 -07002559 self.waitUntilSettled()
2560
2561 self.assertEqual(A.data['status'], 'MERGED')
2562 self.assertEqual(A.reported, 2)
2563 self.assertEqual(B.data['status'], 'NEW')
Tobias Henkel9842bd72017-05-16 13:40:03 +02002564 self.assertIn('Merge Failed', B.messages[-1])
James E. Blair6bc782d2015-07-17 16:20:21 -07002565 self.assertEqual(self.getJobFromHistory('project-merge').result,
2566 'SUCCESS')
2567 self.assertEqual(self.getJobFromHistory('project-test1').result,
2568 'SUCCESS')
2569 self.assertEqual(self.getJobFromHistory('project-test2').result,
2570 'SUCCESS')
2571 self.assertEqual(self.getJobFromHistory('project-test3').result,
2572 'SUCCESS')
2573 self.assertEqual(len(self.history), 4)
2574
James E. Blair400e8fd2015-07-30 17:44:45 -07002575 def test_live_reconfiguration_failed_root(self):
James E. Blair6bc782d2015-07-17 16:20:21 -07002576 # An extrapolation of test_live_reconfiguration_merge_conflict
2577 # that tests a job added to a job tree with a failed root does
2578 # not run.
Paul Belanger174a8272017-03-14 13:20:10 -04002579 self.executor_server.hold_jobs_in_build = True
James E. Blair6bc782d2015-07-17 16:20:21 -07002580
2581 # This change is fine. It's here to stop the queue long
2582 # enough for the next change to be subject to the
2583 # reconfiguration. This change will succeed and merge.
2584 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Clint Byrum3147d652017-05-01 14:01:42 -07002585 A.addPatchset({'conflict': 'A'})
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002586 A.addApproval('Code-Review', 2)
2587 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair6bc782d2015-07-17 16:20:21 -07002588 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002589 self.executor_server.release('.*-merge')
James E. Blair6bc782d2015-07-17 16:20:21 -07002590 self.waitUntilSettled()
2591
2592 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
Paul Belanger174a8272017-03-14 13:20:10 -04002593 self.executor_server.failJob('project-merge', B)
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002594 B.addApproval('Code-Review', 2)
2595 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blair6bc782d2015-07-17 16:20:21 -07002596 self.waitUntilSettled()
2597
Paul Belanger174a8272017-03-14 13:20:10 -04002598 self.executor_server.release('.*-merge')
James E. Blair6bc782d2015-07-17 16:20:21 -07002599 self.waitUntilSettled()
2600
2601 # Both -merge jobs have run, but no others.
2602 self.assertEqual(A.data['status'], 'NEW')
2603 self.assertEqual(A.reported, 1)
2604 self.assertEqual(B.data['status'], 'NEW')
2605 self.assertEqual(B.reported, 1)
2606 self.assertEqual(self.history[0].result, 'SUCCESS')
2607 self.assertEqual(self.history[0].name, 'project-merge')
2608 self.assertEqual(self.history[1].result, 'FAILURE')
2609 self.assertEqual(self.history[1].name, 'project-merge')
2610 self.assertEqual(len(self.history), 2)
2611
2612 # Add the "project-test3" job.
Clint Byrum3147d652017-05-01 14:01:42 -07002613 self.commitConfigUpdate('common-config',
2614 'layouts/live-reconfiguration-add-job.yaml')
James E. Blair6bc782d2015-07-17 16:20:21 -07002615 self.sched.reconfigure(self.config)
2616 self.waitUntilSettled()
2617
Paul Belanger174a8272017-03-14 13:20:10 -04002618 self.executor_server.hold_jobs_in_build = False
2619 self.executor_server.release()
James E. Blair6bc782d2015-07-17 16:20:21 -07002620 self.waitUntilSettled()
2621
2622 self.assertEqual(A.data['status'], 'MERGED')
2623 self.assertEqual(A.reported, 2)
2624 self.assertEqual(B.data['status'], 'NEW')
2625 self.assertEqual(B.reported, 2)
2626 self.assertEqual(self.history[0].result, 'SUCCESS')
2627 self.assertEqual(self.history[0].name, 'project-merge')
2628 self.assertEqual(self.history[1].result, 'FAILURE')
2629 self.assertEqual(self.history[1].name, 'project-merge')
2630 self.assertEqual(self.history[2].result, 'SUCCESS')
2631 self.assertEqual(self.history[3].result, 'SUCCESS')
2632 self.assertEqual(self.history[4].result, 'SUCCESS')
2633 self.assertEqual(len(self.history), 5)
2634
James E. Blair400e8fd2015-07-30 17:44:45 -07002635 def test_live_reconfiguration_failed_job(self):
2636 # Test that a change with a removed failing job does not
2637 # disrupt reconfiguration. If a change has a failed job and
2638 # that job is removed during a reconfiguration, we observed a
2639 # bug where the code to re-set build statuses would run on
2640 # that build and raise an exception because the job no longer
2641 # existed.
Paul Belanger174a8272017-03-14 13:20:10 -04002642 self.executor_server.hold_jobs_in_build = True
James E. Blair400e8fd2015-07-30 17:44:45 -07002643
2644 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
2645
2646 # This change will fail and later be removed by the reconfiguration.
Paul Belanger174a8272017-03-14 13:20:10 -04002647 self.executor_server.failJob('project-test1', A)
James E. Blair400e8fd2015-07-30 17:44:45 -07002648
2649 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
2650 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002651 self.executor_server.release('.*-merge')
James E. Blair400e8fd2015-07-30 17:44:45 -07002652 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002653 self.executor_server.release('project-test1')
James E. Blair400e8fd2015-07-30 17:44:45 -07002654 self.waitUntilSettled()
2655
2656 self.assertEqual(A.data['status'], 'NEW')
2657 self.assertEqual(A.reported, 0)
2658
2659 self.assertEqual(self.getJobFromHistory('project-merge').result,
2660 'SUCCESS')
2661 self.assertEqual(self.getJobFromHistory('project-test1').result,
2662 'FAILURE')
2663 self.assertEqual(len(self.history), 2)
2664
2665 # Remove the test1 job.
Clint Byrum841a58c2017-05-01 14:17:21 -07002666 self.commitConfigUpdate('common-config',
2667 'layouts/live-reconfiguration-failed-job.yaml')
James E. Blair400e8fd2015-07-30 17:44:45 -07002668 self.sched.reconfigure(self.config)
2669 self.waitUntilSettled()
2670
Paul Belanger174a8272017-03-14 13:20:10 -04002671 self.executor_server.hold_jobs_in_build = False
2672 self.executor_server.release()
James E. Blair400e8fd2015-07-30 17:44:45 -07002673 self.waitUntilSettled()
2674
2675 self.assertEqual(self.getJobFromHistory('project-test2').result,
2676 'SUCCESS')
2677 self.assertEqual(self.getJobFromHistory('project-testfile').result,
2678 'SUCCESS')
2679 self.assertEqual(len(self.history), 4)
2680
2681 self.assertEqual(A.data['status'], 'NEW')
2682 self.assertEqual(A.reported, 1)
2683 self.assertIn('Build succeeded', A.messages[0])
2684 # Ensure the removed job was not included in the report.
2685 self.assertNotIn('project-test1', A.messages[0])
2686
James E. Blairfe707d12015-08-05 15:18:15 -07002687 def test_live_reconfiguration_shared_queue(self):
2688 # Test that a change with a failing job which was removed from
2689 # this project but otherwise still exists in the system does
2690 # not disrupt reconfiguration.
2691
Paul Belanger174a8272017-03-14 13:20:10 -04002692 self.executor_server.hold_jobs_in_build = True
James E. Blairfe707d12015-08-05 15:18:15 -07002693
2694 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
2695
Paul Belanger174a8272017-03-14 13:20:10 -04002696 self.executor_server.failJob('project1-project2-integration', A)
James E. Blairfe707d12015-08-05 15:18:15 -07002697
2698 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
2699 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002700 self.executor_server.release('.*-merge')
James E. Blairfe707d12015-08-05 15:18:15 -07002701 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002702 self.executor_server.release('project1-project2-integration')
James E. Blairfe707d12015-08-05 15:18:15 -07002703 self.waitUntilSettled()
2704
2705 self.assertEqual(A.data['status'], 'NEW')
2706 self.assertEqual(A.reported, 0)
2707
Clint Byrum69663d42017-05-01 14:51:04 -07002708 self.assertEqual(self.getJobFromHistory('project-merge').result,
James E. Blairfe707d12015-08-05 15:18:15 -07002709 'SUCCESS')
2710 self.assertEqual(self.getJobFromHistory(
2711 'project1-project2-integration').result, 'FAILURE')
2712 self.assertEqual(len(self.history), 2)
2713
2714 # Remove the integration job.
Clint Byrum69663d42017-05-01 14:51:04 -07002715 self.commitConfigUpdate(
2716 'common-config',
2717 'layouts/live-reconfiguration-shared-queue.yaml')
James E. Blairfe707d12015-08-05 15:18:15 -07002718 self.sched.reconfigure(self.config)
2719 self.waitUntilSettled()
2720
Paul Belanger174a8272017-03-14 13:20:10 -04002721 self.executor_server.hold_jobs_in_build = False
2722 self.executor_server.release()
James E. Blairfe707d12015-08-05 15:18:15 -07002723 self.waitUntilSettled()
2724
Clint Byrum69663d42017-05-01 14:51:04 -07002725 self.assertEqual(self.getJobFromHistory('project-merge').result,
James E. Blairfe707d12015-08-05 15:18:15 -07002726 'SUCCESS')
Clint Byrum69663d42017-05-01 14:51:04 -07002727 self.assertEqual(self.getJobFromHistory('project-test1').result,
James E. Blairfe707d12015-08-05 15:18:15 -07002728 'SUCCESS')
Clint Byrum69663d42017-05-01 14:51:04 -07002729 self.assertEqual(self.getJobFromHistory('project-test2').result,
James E. Blairfe707d12015-08-05 15:18:15 -07002730 'SUCCESS')
2731 self.assertEqual(self.getJobFromHistory(
2732 'project1-project2-integration').result, 'FAILURE')
2733 self.assertEqual(len(self.history), 4)
2734
2735 self.assertEqual(A.data['status'], 'NEW')
2736 self.assertEqual(A.reported, 1)
2737 self.assertIn('Build succeeded', A.messages[0])
2738 # Ensure the removed job was not included in the report.
2739 self.assertNotIn('project1-project2-integration', A.messages[0])
2740
Joshua Hesketh4bd7da32016-02-17 20:58:47 +11002741 def test_double_live_reconfiguration_shared_queue(self):
2742 # This was a real-world regression. A change is added to
2743 # gate; a reconfigure happens, a second change which depends
2744 # on the first is added, and a second reconfiguration happens.
2745 # Ensure that both changes merge.
2746
2747 # A failure may indicate incorrect caching or cleaning up of
2748 # references during a reconfiguration.
Paul Belanger174a8272017-03-14 13:20:10 -04002749 self.executor_server.hold_jobs_in_build = True
Joshua Hesketh4bd7da32016-02-17 20:58:47 +11002750
2751 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
2752 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
2753 B.setDependsOn(A, 1)
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002754 A.addApproval('Code-Review', 2)
2755 B.addApproval('Code-Review', 2)
Joshua Hesketh4bd7da32016-02-17 20:58:47 +11002756
2757 # Add the parent change.
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002758 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
Joshua Hesketh4bd7da32016-02-17 20:58:47 +11002759 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002760 self.executor_server.release('.*-merge')
Joshua Hesketh4bd7da32016-02-17 20:58:47 +11002761 self.waitUntilSettled()
2762
2763 # Reconfigure (with only one change in the pipeline).
2764 self.sched.reconfigure(self.config)
2765 self.waitUntilSettled()
2766
2767 # Add the child change.
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002768 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
Joshua Hesketh4bd7da32016-02-17 20:58:47 +11002769 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002770 self.executor_server.release('.*-merge')
Joshua Hesketh4bd7da32016-02-17 20:58:47 +11002771 self.waitUntilSettled()
2772
2773 # Reconfigure (with both in the pipeline).
2774 self.sched.reconfigure(self.config)
2775 self.waitUntilSettled()
2776
Paul Belanger174a8272017-03-14 13:20:10 -04002777 self.executor_server.hold_jobs_in_build = False
2778 self.executor_server.release()
Joshua Hesketh4bd7da32016-02-17 20:58:47 +11002779 self.waitUntilSettled()
2780
2781 self.assertEqual(len(self.history), 8)
2782
2783 self.assertEqual(A.data['status'], 'MERGED')
2784 self.assertEqual(A.reported, 2)
2785 self.assertEqual(B.data['status'], 'MERGED')
2786 self.assertEqual(B.reported, 2)
2787
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00002788 def test_live_reconfiguration_del_project(self):
2789 # Test project deletion from layout
2790 # while changes are enqueued
2791
Paul Belanger174a8272017-03-14 13:20:10 -04002792 self.executor_server.hold_jobs_in_build = True
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00002793 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
2794 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
2795 C = self.fake_gerrit.addFakeChange('org/project1', 'master', 'C')
2796
2797 # A Depends-On: B
2798 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
2799 A.subject, B.data['id'])
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002800 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00002801
2802 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
2803 self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
2804 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002805 self.executor_server.release('.*-merge')
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00002806 self.waitUntilSettled()
2807 self.assertEqual(len(self.builds), 5)
2808
2809 # This layout defines only org/project, not org/project1
James E. Blair31cb3e82017-04-20 13:08:56 -07002810 self.commitConfigUpdate(
2811 'common-config',
2812 'layouts/live-reconfiguration-del-project.yaml')
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00002813 self.sched.reconfigure(self.config)
2814 self.waitUntilSettled()
2815
2816 # Builds for C aborted, builds for A succeed,
2817 # and have change B applied ahead
Clint Byrum0149ada2017-04-04 12:22:01 -07002818 job_c = self.getJobFromHistory('project-test1')
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00002819 self.assertEqual(job_c.changes, '3,1')
2820 self.assertEqual(job_c.result, 'ABORTED')
2821
Paul Belanger174a8272017-03-14 13:20:10 -04002822 self.executor_server.hold_jobs_in_build = False
2823 self.executor_server.release()
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00002824 self.waitUntilSettled()
2825
Clint Byrum0149ada2017-04-04 12:22:01 -07002826 self.assertEqual(
2827 self.getJobFromHistory('project-test1', 'org/project').changes,
2828 '2,1 1,1')
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00002829
2830 self.assertEqual(A.data['status'], 'NEW')
2831 self.assertEqual(B.data['status'], 'NEW')
2832 self.assertEqual(C.data['status'], 'NEW')
2833 self.assertEqual(A.reported, 1)
2834 self.assertEqual(B.reported, 0)
2835 self.assertEqual(C.reported, 0)
2836
Clint Byrum0149ada2017-04-04 12:22:01 -07002837 tenant = self.sched.abide.tenants.get('tenant-one')
2838 self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00002839 self.assertIn('Build succeeded', A.messages[0])
2840
James E. Blair287c06d2013-07-24 10:39:30 -07002841 def test_delayed_repo_init(self):
James E. Blair287c06d2013-07-24 10:39:30 -07002842 self.init_repo("org/new-project")
Clint Byrum627ba362017-08-14 13:20:40 -07002843 files = {'README': ''}
2844 self.addCommitToRepo("org/new-project", 'Initial commit',
2845 files=files, tag='init')
2846 self.newTenantConfig('tenants/delayed-repo-init.yaml')
2847 self.commitConfigUpdate(
2848 'common-config',
2849 'layouts/delayed-repo-init.yaml')
2850 self.sched.reconfigure(self.config)
2851 self.waitUntilSettled()
2852
James E. Blair287c06d2013-07-24 10:39:30 -07002853 A = self.fake_gerrit.addFakeChange('org/new-project', 'master', 'A')
2854
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002855 A.addApproval('Code-Review', 2)
2856 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair287c06d2013-07-24 10:39:30 -07002857 self.waitUntilSettled()
2858 self.assertEqual(self.getJobFromHistory('project-merge').result,
2859 'SUCCESS')
2860 self.assertEqual(self.getJobFromHistory('project-test1').result,
2861 'SUCCESS')
2862 self.assertEqual(self.getJobFromHistory('project-test2').result,
2863 'SUCCESS')
2864 self.assertEqual(A.data['status'], 'MERGED')
2865 self.assertEqual(A.reported, 2)
James E. Blair63bb0ef2013-07-29 17:14:51 -07002866
James E. Blair9ea70072017-04-19 16:05:30 -07002867 @simple_layout('layouts/repo-deleted.yaml')
Clark Boylan6dbbc482013-10-18 10:57:31 -07002868 def test_repo_deleted(self):
Clark Boylan6dbbc482013-10-18 10:57:31 -07002869 self.init_repo("org/delete-project")
2870 A = self.fake_gerrit.addFakeChange('org/delete-project', 'master', 'A')
2871
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002872 A.addApproval('Code-Review', 2)
2873 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
Clark Boylan6dbbc482013-10-18 10:57:31 -07002874 self.waitUntilSettled()
2875 self.assertEqual(self.getJobFromHistory('project-merge').result,
2876 'SUCCESS')
2877 self.assertEqual(self.getJobFromHistory('project-test1').result,
2878 'SUCCESS')
2879 self.assertEqual(self.getJobFromHistory('project-test2').result,
2880 'SUCCESS')
2881 self.assertEqual(A.data['status'], 'MERGED')
2882 self.assertEqual(A.reported, 2)
2883
2884 # Delete org/new-project zuul repo. Should be recloned.
James E. Blair8c1be532017-02-07 14:04:12 -08002885 p = 'org/delete-project'
Monty Taylord642d852017-02-23 14:05:42 -05002886 if os.path.exists(os.path.join(self.merger_src_root, p)):
2887 shutil.rmtree(os.path.join(self.merger_src_root, p))
Paul Belanger174a8272017-03-14 13:20:10 -04002888 if os.path.exists(os.path.join(self.executor_src_root, p)):
2889 shutil.rmtree(os.path.join(self.executor_src_root, p))
Clark Boylan6dbbc482013-10-18 10:57:31 -07002890
2891 B = self.fake_gerrit.addFakeChange('org/delete-project', 'master', 'B')
2892
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002893 B.addApproval('Code-Review', 2)
2894 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
Clark Boylan6dbbc482013-10-18 10:57:31 -07002895 self.waitUntilSettled()
2896 self.assertEqual(self.getJobFromHistory('project-merge').result,
2897 'SUCCESS')
2898 self.assertEqual(self.getJobFromHistory('project-test1').result,
2899 'SUCCESS')
2900 self.assertEqual(self.getJobFromHistory('project-test2').result,
2901 'SUCCESS')
2902 self.assertEqual(B.data['status'], 'MERGED')
2903 self.assertEqual(B.reported, 2)
2904
James E. Blair892cca62017-08-09 11:36:58 -07002905 @simple_layout('layouts/untrusted-secrets.yaml')
2906 def test_untrusted_secrets(self):
2907 "Test untrusted secrets"
2908 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
2909 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
2910 self.waitUntilSettled()
2911
2912 self.assertHistory([])
2913 self.assertEqual(A.patchsets[0]['approvals'][0]['value'], "-1")
James E. Blair8eb564a2017-08-10 09:21:41 -07002914 self.assertIn('does not allow post-review job',
James E. Blair892cca62017-08-09 11:36:58 -07002915 A.messages[0])
2916
James E. Blaire68d43f2017-04-19 14:03:30 -07002917 @simple_layout('layouts/tags.yaml')
James E. Blair456f2fb2016-02-09 09:29:33 -08002918 def test_tags(self):
2919 "Test job tags"
James E. Blair456f2fb2016-02-09 09:29:33 -08002920 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
2921 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
2922 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
2923 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
2924 self.waitUntilSettled()
2925
James E. Blaire68d43f2017-04-19 14:03:30 -07002926 self.assertEqual(len(self.history), 2)
Cullen Taylor3aa6d472017-03-01 15:22:09 -06002927
2928 results = {self.getJobFromHistory('merge',
James E. Blair9d46f092017-07-20 16:06:20 -07002929 project='org/project1').uuid: ['extratag', 'merge'],
Cullen Taylor3aa6d472017-03-01 15:22:09 -06002930 self.getJobFromHistory('merge',
James E. Blair9d46f092017-07-20 16:06:20 -07002931 project='org/project2').uuid: ['merge']}
James E. Blair456f2fb2016-02-09 09:29:33 -08002932
2933 for build in self.history:
Cullen Taylor3aa6d472017-03-01 15:22:09 -06002934 self.assertEqual(results.get(build.uuid, ''),
James E. Blair9d46f092017-07-20 16:06:20 -07002935 build.parameters['zuul'].get('jobtags'))
James E. Blair456f2fb2016-02-09 09:29:33 -08002936
James E. Blair63bb0ef2013-07-29 17:14:51 -07002937 def test_timer(self):
2938 "Test that a periodic job is triggered"
James E. Blairdfdfcfc2017-04-20 10:19:20 -07002939 # This test can not use simple_layout because it must start
2940 # with a configuration which does not include a
2941 # timer-triggered job so that we have an opportunity to set
2942 # the hold flag before the first job.
James E. Blair21037782017-07-19 11:56:55 -07002943 self.create_branch('org/project', 'stable')
Paul Belanger174a8272017-03-14 13:20:10 -04002944 self.executor_server.hold_jobs_in_build = True
James E. Blairdfdfcfc2017-04-20 10:19:20 -07002945 self.commitConfigUpdate('common-config', 'layouts/timer.yaml')
James E. Blair63bb0ef2013-07-29 17:14:51 -07002946 self.sched.reconfigure(self.config)
James E. Blair63bb0ef2013-07-29 17:14:51 -07002947
Clark Boylan3ee090a2014-04-03 20:55:09 -07002948 # The pipeline triggers every second, so we should have seen
2949 # several by now.
2950 time.sleep(5)
James E. Blair63bb0ef2013-07-29 17:14:51 -07002951 self.waitUntilSettled()
Clark Boylan3ee090a2014-04-03 20:55:09 -07002952
2953 self.assertEqual(len(self.builds), 2)
2954
James E. Blair63bb0ef2013-07-29 17:14:51 -07002955 port = self.webapp.server.socket.getsockname()[1]
2956
Clint Byrum5e729e92017-02-08 06:34:57 -08002957 req = urllib.request.Request(
James E. Blairdfdfcfc2017-04-20 10:19:20 -07002958 "http://localhost:%s/tenant-one/status" % port)
Morgan Fainberg293f7f82016-05-30 14:01:22 -07002959 f = urllib.request.urlopen(req)
Clint Byrumf322fe22017-05-10 20:53:12 -07002960 data = f.read().decode('utf8')
James E. Blair63bb0ef2013-07-29 17:14:51 -07002961
Paul Belanger174a8272017-03-14 13:20:10 -04002962 self.executor_server.hold_jobs_in_build = False
Clark Boylan3ee090a2014-04-03 20:55:09 -07002963 # Stop queuing timer triggered jobs so that the assertions
2964 # below don't race against more jobs being queued.
James E. Blairdfdfcfc2017-04-20 10:19:20 -07002965 self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')
Clark Boylan3ee090a2014-04-03 20:55:09 -07002966 self.sched.reconfigure(self.config)
James E. Blair78ae4782017-08-02 14:19:10 -07002967 self.waitUntilSettled()
2968 # If APScheduler is in mid-event when we remove the job, we
2969 # can end up with one more event firing, so give it an extra
2970 # second to settle.
2971 time.sleep(1)
2972 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002973 self.executor_server.release()
James E. Blair63bb0ef2013-07-29 17:14:51 -07002974 self.waitUntilSettled()
2975
James E. Blair21037782017-07-19 11:56:55 -07002976 self.assertHistory([
2977 dict(name='project-bitrot', result='SUCCESS',
2978 ref='refs/heads/master'),
2979 dict(name='project-bitrot', result='SUCCESS',
2980 ref='refs/heads/stable'),
2981 ], ordered=False)
James E. Blair63bb0ef2013-07-29 17:14:51 -07002982
2983 data = json.loads(data)
2984 status_jobs = set()
2985 for p in data['pipelines']:
2986 for q in p['change_queues']:
2987 for head in q['heads']:
2988 for change in head:
James E. Blair63bb0ef2013-07-29 17:14:51 -07002989 for job in change['jobs']:
2990 status_jobs.add(job['name'])
James E. Blair21037782017-07-19 11:56:55 -07002991 self.assertIn('project-bitrot', status_jobs)
Joshua Hesketh5fea8672013-08-19 17:32:01 +10002992
James E. Blair4f6033c2014-03-27 15:49:09 -07002993 def test_idle(self):
2994 "Test that frequent periodic jobs work"
James E. Blairdfdfcfc2017-04-20 10:19:20 -07002995 # This test can not use simple_layout because it must start
2996 # with a configuration which does not include a
2997 # timer-triggered job so that we have an opportunity to set
2998 # the hold flag before the first job.
Paul Belanger174a8272017-03-14 13:20:10 -04002999 self.executor_server.hold_jobs_in_build = True
James E. Blair4f6033c2014-03-27 15:49:09 -07003000
Clark Boylan3ee090a2014-04-03 20:55:09 -07003001 for x in range(1, 3):
3002 # Test that timer triggers periodic jobs even across
3003 # layout config reloads.
3004 # Start timer trigger
James E. Blairdfdfcfc2017-04-20 10:19:20 -07003005 self.commitConfigUpdate('common-config',
3006 'layouts/idle.yaml')
Clark Boylan3ee090a2014-04-03 20:55:09 -07003007 self.sched.reconfigure(self.config)
James E. Blair995fc0f2016-02-04 16:48:31 -08003008 self.waitUntilSettled()
James E. Blair4f6033c2014-03-27 15:49:09 -07003009
Clark Boylan3ee090a2014-04-03 20:55:09 -07003010 # The pipeline triggers every second, so we should have seen
3011 # several by now.
3012 time.sleep(5)
Clark Boylan3ee090a2014-04-03 20:55:09 -07003013
3014 # Stop queuing timer triggered jobs so that the assertions
3015 # below don't race against more jobs being queued.
James E. Blairdfdfcfc2017-04-20 10:19:20 -07003016 self.commitConfigUpdate('common-config',
3017 'layouts/no-timer.yaml')
Clark Boylan3ee090a2014-04-03 20:55:09 -07003018 self.sched.reconfigure(self.config)
James E. Blair995fc0f2016-02-04 16:48:31 -08003019 self.waitUntilSettled()
James E. Blair78ae4782017-08-02 14:19:10 -07003020 # If APScheduler is in mid-event when we remove the job,
3021 # we can end up with one more event firing, so give it an
3022 # extra second to settle.
3023 time.sleep(1)
3024 self.waitUntilSettled()
James E. Blair21037782017-07-19 11:56:55 -07003025 self.assertEqual(len(self.builds), 1,
Clint Byrum58264dc2017-02-07 21:21:22 -08003026 'Timer builds iteration #%d' % x)
Paul Belanger174a8272017-03-14 13:20:10 -04003027 self.executor_server.release('.*')
Clark Boylan3ee090a2014-04-03 20:55:09 -07003028 self.waitUntilSettled()
3029 self.assertEqual(len(self.builds), 0)
James E. Blair21037782017-07-19 11:56:55 -07003030 self.assertEqual(len(self.history), x)
James E. Blair4f6033c2014-03-27 15:49:09 -07003031
James E. Blaircc324192017-04-19 14:36:08 -07003032 @simple_layout('layouts/smtp.yaml')
Joshua Hesketh5fea8672013-08-19 17:32:01 +10003033 def test_check_smtp_pool(self):
Joshua Hesketh5fea8672013-08-19 17:32:01 +10003034 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
3035 self.waitUntilSettled()
3036
3037 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
3038 self.waitUntilSettled()
3039
James E. Blairff80a2f2013-12-27 13:24:06 -08003040 self.assertEqual(len(self.smtp_messages), 2)
Joshua Hesketh5fea8672013-08-19 17:32:01 +10003041
3042 # A.messages only holds what FakeGerrit places in it. Thus we
3043 # work on the knowledge of what the first message should be as
3044 # it is only configured to go to SMTP.
3045
3046 self.assertEqual('zuul@example.com',
James E. Blairff80a2f2013-12-27 13:24:06 -08003047 self.smtp_messages[0]['from_email'])
Joshua Hesketh5fea8672013-08-19 17:32:01 +10003048 self.assertEqual(['you@example.com'],
James E. Blairff80a2f2013-12-27 13:24:06 -08003049 self.smtp_messages[0]['to_email'])
Joshua Hesketh5fea8672013-08-19 17:32:01 +10003050 self.assertEqual('Starting check jobs.',
James E. Blairff80a2f2013-12-27 13:24:06 -08003051 self.smtp_messages[0]['body'])
Joshua Hesketh5fea8672013-08-19 17:32:01 +10003052
3053 self.assertEqual('zuul_from@example.com',
James E. Blairff80a2f2013-12-27 13:24:06 -08003054 self.smtp_messages[1]['from_email'])
Joshua Hesketh5fea8672013-08-19 17:32:01 +10003055 self.assertEqual(['alternative_me@example.com'],
James E. Blairff80a2f2013-12-27 13:24:06 -08003056 self.smtp_messages[1]['to_email'])
Joshua Hesketh5fea8672013-08-19 17:32:01 +10003057 self.assertEqual(A.messages[0],
James E. Blairff80a2f2013-12-27 13:24:06 -08003058 self.smtp_messages[1]['body'])
James E. Blairad28e912013-11-27 10:43:22 -08003059
James E. Blaire5910202013-12-27 09:50:31 -08003060 def test_timer_smtp(self):
3061 "Test that a periodic job is triggered"
James E. Blairdfdfcfc2017-04-20 10:19:20 -07003062 # This test can not use simple_layout because it must start
3063 # with a configuration which does not include a
3064 # timer-triggered job so that we have an opportunity to set
3065 # the hold flag before the first job.
Paul Belanger174a8272017-03-14 13:20:10 -04003066 self.executor_server.hold_jobs_in_build = True
James E. Blairdfdfcfc2017-04-20 10:19:20 -07003067 self.commitConfigUpdate('common-config', 'layouts/timer-smtp.yaml')
James E. Blaire5910202013-12-27 09:50:31 -08003068 self.sched.reconfigure(self.config)
James E. Blaire5910202013-12-27 09:50:31 -08003069
Clark Boylan3ee090a2014-04-03 20:55:09 -07003070 # The pipeline triggers every second, so we should have seen
3071 # several by now.
3072 time.sleep(5)
James E. Blaire5910202013-12-27 09:50:31 -08003073 self.waitUntilSettled()
3074
Clark Boylan3ee090a2014-04-03 20:55:09 -07003075 self.assertEqual(len(self.builds), 2)
Paul Belanger174a8272017-03-14 13:20:10 -04003076 self.executor_server.release('.*')
Clark Boylan3ee090a2014-04-03 20:55:09 -07003077 self.waitUntilSettled()
3078 self.assertEqual(len(self.history), 2)
3079
James E. Blaire5910202013-12-27 09:50:31 -08003080 self.assertEqual(self.getJobFromHistory(
3081 'project-bitrot-stable-old').result, 'SUCCESS')
3082 self.assertEqual(self.getJobFromHistory(
3083 'project-bitrot-stable-older').result, 'SUCCESS')
3084
James E. Blairff80a2f2013-12-27 13:24:06 -08003085 self.assertEqual(len(self.smtp_messages), 1)
James E. Blaire5910202013-12-27 09:50:31 -08003086
3087 # A.messages only holds what FakeGerrit places in it. Thus we
3088 # work on the knowledge of what the first message should be as
3089 # it is only configured to go to SMTP.
3090
3091 self.assertEqual('zuul_from@example.com',
James E. Blairff80a2f2013-12-27 13:24:06 -08003092 self.smtp_messages[0]['from_email'])
James E. Blaire5910202013-12-27 09:50:31 -08003093 self.assertEqual(['alternative_me@example.com'],
James E. Blairff80a2f2013-12-27 13:24:06 -08003094 self.smtp_messages[0]['to_email'])
James E. Blaire5910202013-12-27 09:50:31 -08003095 self.assertIn('Subject: Periodic check for org/project succeeded',
James E. Blairff80a2f2013-12-27 13:24:06 -08003096 self.smtp_messages[0]['headers'])
James E. Blaire5910202013-12-27 09:50:31 -08003097
Clark Boylan3ee090a2014-04-03 20:55:09 -07003098 # Stop queuing timer triggered jobs and let any that may have
3099 # queued through so that end of test assertions pass.
James E. Blairdfdfcfc2017-04-20 10:19:20 -07003100 self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')
Clark Boylan3ee090a2014-04-03 20:55:09 -07003101 self.sched.reconfigure(self.config)
James E. Blairf8058972014-08-15 16:09:16 -07003102 self.waitUntilSettled()
James E. Blair78ae4782017-08-02 14:19:10 -07003103 # If APScheduler is in mid-event when we remove the job, we
3104 # can end up with one more event firing, so give it an extra
3105 # second to settle.
3106 time.sleep(1)
3107 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003108 self.executor_server.release('.*')
Clark Boylan3ee090a2014-04-03 20:55:09 -07003109 self.waitUntilSettled()
3110
Joshua Hesketh25695cb2017-03-06 12:50:04 +11003111 @skip("Disabled for early v3 development")
Evgeny Antyshevda90a502017-02-08 08:42:59 +00003112 def test_timer_sshkey(self):
3113 "Test that a periodic job can setup SSH key authentication"
3114 self.worker.hold_jobs_in_build = True
3115 self.config.set('zuul', 'layout_config',
3116 'tests/fixtures/layout-timer.yaml')
3117 self.sched.reconfigure(self.config)
3118 self.registerJobs()
3119
3120 # The pipeline triggers every second, so we should have seen
3121 # several by now.
3122 time.sleep(5)
3123 self.waitUntilSettled()
3124
3125 self.assertEqual(len(self.builds), 2)
3126
3127 ssh_wrapper = os.path.join(self.git_root, ".ssh_wrapper_gerrit")
3128 self.assertTrue(os.path.isfile(ssh_wrapper))
3129 with open(ssh_wrapper) as f:
3130 ssh_wrapper_content = f.read()
3131 self.assertIn("fake_id_rsa", ssh_wrapper_content)
3132 # In the unit tests Merger runs in the same process,
3133 # so we see its' environment variables
3134 self.assertEqual(os.environ['GIT_SSH'], ssh_wrapper)
3135
3136 self.worker.release('.*')
3137 self.waitUntilSettled()
3138 self.assertEqual(len(self.history), 2)
3139
3140 self.assertEqual(self.getJobFromHistory(
3141 'project-bitrot-stable-old').result, 'SUCCESS')
3142 self.assertEqual(self.getJobFromHistory(
3143 'project-bitrot-stable-older').result, 'SUCCESS')
3144
3145 # Stop queuing timer triggered jobs and let any that may have
3146 # queued through so that end of test assertions pass.
3147 self.config.set('zuul', 'layout_config',
3148 'tests/fixtures/layout-no-timer.yaml')
3149 self.sched.reconfigure(self.config)
3150 self.registerJobs()
3151 self.waitUntilSettled()
James E. Blair78ae4782017-08-02 14:19:10 -07003152 # If APScheduler is in mid-event when we remove the job, we
3153 # can end up with one more event firing, so give it an extra
3154 # second to settle.
3155 time.sleep(1)
3156 self.waitUntilSettled()
Evgeny Antyshevda90a502017-02-08 08:42:59 +00003157 self.worker.release('.*')
3158 self.waitUntilSettled()
3159
James E. Blair91e34592015-07-31 16:45:59 -07003160 def test_client_enqueue_change(self):
James E. Blairad28e912013-11-27 10:43:22 -08003161 "Test that the RPC client can enqueue a change"
3162 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003163 A.addApproval('Code-Review', 2)
3164 A.addApproval('Approved', 1)
James E. Blairad28e912013-11-27 10:43:22 -08003165
3166 client = zuul.rpcclient.RPCClient('127.0.0.1',
3167 self.gearman_server.port)
Clark Boylandf4eba22017-04-24 17:37:58 -07003168 self.addCleanup(client.shutdown)
Paul Belangerbaca3132016-11-04 12:49:54 -04003169 r = client.enqueue(tenant='tenant-one',
3170 pipeline='gate',
James E. Blairad28e912013-11-27 10:43:22 -08003171 project='org/project',
3172 trigger='gerrit',
James E. Blair36658cf2013-12-06 17:53:48 -08003173 change='1,1')
James E. Blairad28e912013-11-27 10:43:22 -08003174 self.waitUntilSettled()
3175 self.assertEqual(self.getJobFromHistory('project-merge').result,
3176 'SUCCESS')
3177 self.assertEqual(self.getJobFromHistory('project-test1').result,
3178 'SUCCESS')
3179 self.assertEqual(self.getJobFromHistory('project-test2').result,
3180 'SUCCESS')
3181 self.assertEqual(A.data['status'], 'MERGED')
3182 self.assertEqual(A.reported, 2)
3183 self.assertEqual(r, True)
3184
James E. Blair91e34592015-07-31 16:45:59 -07003185 def test_client_enqueue_ref(self):
3186 "Test that the RPC client can enqueue a ref"
James E. Blair289f5932017-07-27 15:02:29 -07003187 p = "review.example.com/org/project"
3188 upstream = self.getUpstreamRepos([p])
3189 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
3190 A.setMerged()
3191 A_commit = str(upstream[p].commit('master'))
3192 self.log.debug("A commit: %s" % A_commit)
James E. Blair91e34592015-07-31 16:45:59 -07003193
3194 client = zuul.rpcclient.RPCClient('127.0.0.1',
3195 self.gearman_server.port)
Clark Boylandf4eba22017-04-24 17:37:58 -07003196 self.addCleanup(client.shutdown)
James E. Blair91e34592015-07-31 16:45:59 -07003197 r = client.enqueue_ref(
Paul Belangerbaca3132016-11-04 12:49:54 -04003198 tenant='tenant-one',
James E. Blair91e34592015-07-31 16:45:59 -07003199 pipeline='post',
3200 project='org/project',
3201 trigger='gerrit',
3202 ref='master',
3203 oldrev='90f173846e3af9154517b88543ffbd1691f31366',
James E. Blair289f5932017-07-27 15:02:29 -07003204 newrev=A_commit)
James E. Blair91e34592015-07-31 16:45:59 -07003205 self.waitUntilSettled()
3206 job_names = [x.name for x in self.history]
3207 self.assertEqual(len(self.history), 1)
3208 self.assertIn('project-post', job_names)
3209 self.assertEqual(r, True)
3210
James E. Blairad28e912013-11-27 10:43:22 -08003211 def test_client_enqueue_negative(self):
3212 "Test that the RPC client returns errors"
3213 client = zuul.rpcclient.RPCClient('127.0.0.1',
3214 self.gearman_server.port)
Clark Boylandf4eba22017-04-24 17:37:58 -07003215 self.addCleanup(client.shutdown)
James E. Blairad28e912013-11-27 10:43:22 -08003216 with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
Paul Belangerbaca3132016-11-04 12:49:54 -04003217 "Invalid tenant"):
3218 r = client.enqueue(tenant='tenant-foo',
3219 pipeline='gate',
3220 project='org/project',
3221 trigger='gerrit',
3222 change='1,1')
Paul Belangerbaca3132016-11-04 12:49:54 -04003223 self.assertEqual(r, False)
3224
3225 with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
James E. Blairad28e912013-11-27 10:43:22 -08003226 "Invalid project"):
Paul Belangerbaca3132016-11-04 12:49:54 -04003227 r = client.enqueue(tenant='tenant-one',
3228 pipeline='gate',
James E. Blairad28e912013-11-27 10:43:22 -08003229 project='project-does-not-exist',
3230 trigger='gerrit',
James E. Blair36658cf2013-12-06 17:53:48 -08003231 change='1,1')
James E. Blairad28e912013-11-27 10:43:22 -08003232 self.assertEqual(r, False)
3233
3234 with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
3235 "Invalid pipeline"):
Paul Belangerbaca3132016-11-04 12:49:54 -04003236 r = client.enqueue(tenant='tenant-one',
3237 pipeline='pipeline-does-not-exist',
James E. Blairad28e912013-11-27 10:43:22 -08003238 project='org/project',
3239 trigger='gerrit',
James E. Blair36658cf2013-12-06 17:53:48 -08003240 change='1,1')
James E. Blairad28e912013-11-27 10:43:22 -08003241 self.assertEqual(r, False)
3242
3243 with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
3244 "Invalid trigger"):
Paul Belangerbaca3132016-11-04 12:49:54 -04003245 r = client.enqueue(tenant='tenant-one',
3246 pipeline='gate',
James E. Blairad28e912013-11-27 10:43:22 -08003247 project='org/project',
3248 trigger='trigger-does-not-exist',
James E. Blair36658cf2013-12-06 17:53:48 -08003249 change='1,1')
James E. Blairad28e912013-11-27 10:43:22 -08003250 self.assertEqual(r, False)
3251
3252 with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
3253 "Invalid change"):
Paul Belangerbaca3132016-11-04 12:49:54 -04003254 r = client.enqueue(tenant='tenant-one',
3255 pipeline='gate',
James E. Blairad28e912013-11-27 10:43:22 -08003256 project='org/project',
3257 trigger='gerrit',
James E. Blair36658cf2013-12-06 17:53:48 -08003258 change='1,1')
James E. Blairad28e912013-11-27 10:43:22 -08003259 self.assertEqual(r, False)
3260
3261 self.waitUntilSettled()
3262 self.assertEqual(len(self.history), 0)
3263 self.assertEqual(len(self.builds), 0)
James E. Blair36658cf2013-12-06 17:53:48 -08003264
3265 def test_client_promote(self):
3266 "Test that the RPC client can promote a change"
Paul Belanger174a8272017-03-14 13:20:10 -04003267 self.executor_server.hold_jobs_in_build = True
James E. Blair36658cf2013-12-06 17:53:48 -08003268 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
3269 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
3270 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003271 A.addApproval('Code-Review', 2)
3272 B.addApproval('Code-Review', 2)
3273 C.addApproval('Code-Review', 2)
James E. Blair36658cf2013-12-06 17:53:48 -08003274
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003275 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
3276 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
3277 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blair36658cf2013-12-06 17:53:48 -08003278
3279 self.waitUntilSettled()
3280
Paul Belangerbaca3132016-11-04 12:49:54 -04003281 tenant = self.sched.abide.tenants.get('tenant-one')
3282 items = tenant.layout.pipelines['gate'].getAllItems()
Sean Daguef39b9ca2014-01-10 21:34:35 -05003283 enqueue_times = {}
3284 for item in items:
3285 enqueue_times[str(item.change)] = item.enqueue_time
3286
James E. Blair36658cf2013-12-06 17:53:48 -08003287 client = zuul.rpcclient.RPCClient('127.0.0.1',
3288 self.gearman_server.port)
Clark Boylandf4eba22017-04-24 17:37:58 -07003289 self.addCleanup(client.shutdown)
Paul Belangerbaca3132016-11-04 12:49:54 -04003290 r = client.promote(tenant='tenant-one',
3291 pipeline='gate',
James E. Blair36658cf2013-12-06 17:53:48 -08003292 change_ids=['2,1', '3,1'])
3293
Sean Daguef39b9ca2014-01-10 21:34:35 -05003294 # ensure that enqueue times are durable
Paul Belangerbaca3132016-11-04 12:49:54 -04003295 items = tenant.layout.pipelines['gate'].getAllItems()
Sean Daguef39b9ca2014-01-10 21:34:35 -05003296 for item in items:
3297 self.assertEqual(
3298 enqueue_times[str(item.change)], item.enqueue_time)
3299
James E. Blair78acec92014-02-06 07:11:32 -08003300 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003301 self.executor_server.release('.*-merge')
James E. Blair36658cf2013-12-06 17:53:48 -08003302 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003303 self.executor_server.release('.*-merge')
James E. Blair36658cf2013-12-06 17:53:48 -08003304 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003305 self.executor_server.release('.*-merge')
James E. Blair36658cf2013-12-06 17:53:48 -08003306 self.waitUntilSettled()
3307
3308 self.assertEqual(len(self.builds), 6)
3309 self.assertEqual(self.builds[0].name, 'project-test1')
3310 self.assertEqual(self.builds[1].name, 'project-test2')
3311 self.assertEqual(self.builds[2].name, 'project-test1')
3312 self.assertEqual(self.builds[3].name, 'project-test2')
3313 self.assertEqual(self.builds[4].name, 'project-test1')
3314 self.assertEqual(self.builds[5].name, 'project-test2')
3315
Paul Belangerbaca3132016-11-04 12:49:54 -04003316 self.assertTrue(self.builds[0].hasChanges(B))
3317 self.assertFalse(self.builds[0].hasChanges(A))
3318 self.assertFalse(self.builds[0].hasChanges(C))
James E. Blair36658cf2013-12-06 17:53:48 -08003319
Paul Belangerbaca3132016-11-04 12:49:54 -04003320 self.assertTrue(self.builds[2].hasChanges(B))
3321 self.assertTrue(self.builds[2].hasChanges(C))
3322 self.assertFalse(self.builds[2].hasChanges(A))
James E. Blair36658cf2013-12-06 17:53:48 -08003323
Paul Belangerbaca3132016-11-04 12:49:54 -04003324 self.assertTrue(self.builds[4].hasChanges(B))
3325 self.assertTrue(self.builds[4].hasChanges(C))
3326 self.assertTrue(self.builds[4].hasChanges(A))
James E. Blair36658cf2013-12-06 17:53:48 -08003327
Paul Belanger174a8272017-03-14 13:20:10 -04003328 self.executor_server.release()
James E. Blair36658cf2013-12-06 17:53:48 -08003329 self.waitUntilSettled()
3330
3331 self.assertEqual(A.data['status'], 'MERGED')
3332 self.assertEqual(A.reported, 2)
3333 self.assertEqual(B.data['status'], 'MERGED')
3334 self.assertEqual(B.reported, 2)
3335 self.assertEqual(C.data['status'], 'MERGED')
3336 self.assertEqual(C.reported, 2)
3337
James E. Blair36658cf2013-12-06 17:53:48 -08003338 self.assertEqual(r, True)
3339
3340 def test_client_promote_dependent(self):
3341 "Test that the RPC client can promote a dependent change"
3342 # C (depends on B) -> B -> A ; then promote C to get:
3343 # A -> C (depends on B) -> B
Paul Belanger174a8272017-03-14 13:20:10 -04003344 self.executor_server.hold_jobs_in_build = True
James E. Blair36658cf2013-12-06 17:53:48 -08003345 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
3346 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
3347 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
3348
3349 C.setDependsOn(B, 1)
3350
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003351 A.addApproval('Code-Review', 2)
3352 B.addApproval('Code-Review', 2)
3353 C.addApproval('Code-Review', 2)
James E. Blair36658cf2013-12-06 17:53:48 -08003354
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003355 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
3356 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
3357 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blair36658cf2013-12-06 17:53:48 -08003358
3359 self.waitUntilSettled()
3360
3361 client = zuul.rpcclient.RPCClient('127.0.0.1',
3362 self.gearman_server.port)
Clark Boylandf4eba22017-04-24 17:37:58 -07003363 self.addCleanup(client.shutdown)
Paul Belangerbaca3132016-11-04 12:49:54 -04003364 r = client.promote(tenant='tenant-one',
3365 pipeline='gate',
James E. Blair36658cf2013-12-06 17:53:48 -08003366 change_ids=['3,1'])
3367
James E. Blair78acec92014-02-06 07:11:32 -08003368 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003369 self.executor_server.release('.*-merge')
James E. Blair36658cf2013-12-06 17:53:48 -08003370 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003371 self.executor_server.release('.*-merge')
James E. Blair36658cf2013-12-06 17:53:48 -08003372 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003373 self.executor_server.release('.*-merge')
James E. Blair36658cf2013-12-06 17:53:48 -08003374 self.waitUntilSettled()
3375
3376 self.assertEqual(len(self.builds), 6)
3377 self.assertEqual(self.builds[0].name, 'project-test1')
3378 self.assertEqual(self.builds[1].name, 'project-test2')
3379 self.assertEqual(self.builds[2].name, 'project-test1')
3380 self.assertEqual(self.builds[3].name, 'project-test2')
3381 self.assertEqual(self.builds[4].name, 'project-test1')
3382 self.assertEqual(self.builds[5].name, 'project-test2')
3383
Paul Belangerbaca3132016-11-04 12:49:54 -04003384 self.assertTrue(self.builds[0].hasChanges(B))
3385 self.assertFalse(self.builds[0].hasChanges(A))
3386 self.assertFalse(self.builds[0].hasChanges(C))
James E. Blair36658cf2013-12-06 17:53:48 -08003387
Paul Belangerbaca3132016-11-04 12:49:54 -04003388 self.assertTrue(self.builds[2].hasChanges(B))
3389 self.assertTrue(self.builds[2].hasChanges(C))
3390 self.assertFalse(self.builds[2].hasChanges(A))
James E. Blair36658cf2013-12-06 17:53:48 -08003391
Paul Belangerbaca3132016-11-04 12:49:54 -04003392 self.assertTrue(self.builds[4].hasChanges(B))
3393 self.assertTrue(self.builds[4].hasChanges(C))
3394 self.assertTrue(self.builds[4].hasChanges(A))
James E. Blair36658cf2013-12-06 17:53:48 -08003395
Paul Belanger174a8272017-03-14 13:20:10 -04003396 self.executor_server.release()
James E. Blair36658cf2013-12-06 17:53:48 -08003397 self.waitUntilSettled()
3398
3399 self.assertEqual(A.data['status'], 'MERGED')
3400 self.assertEqual(A.reported, 2)
3401 self.assertEqual(B.data['status'], 'MERGED')
3402 self.assertEqual(B.reported, 2)
3403 self.assertEqual(C.data['status'], 'MERGED')
3404 self.assertEqual(C.reported, 2)
3405
James E. Blair36658cf2013-12-06 17:53:48 -08003406 self.assertEqual(r, True)
3407
3408 def test_client_promote_negative(self):
3409 "Test that the RPC client returns errors for promotion"
Paul Belanger174a8272017-03-14 13:20:10 -04003410 self.executor_server.hold_jobs_in_build = True
James E. Blair36658cf2013-12-06 17:53:48 -08003411 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003412 A.addApproval('Code-Review', 2)
3413 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair36658cf2013-12-06 17:53:48 -08003414 self.waitUntilSettled()
3415
3416 client = zuul.rpcclient.RPCClient('127.0.0.1',
3417 self.gearman_server.port)
Clark Boylandf4eba22017-04-24 17:37:58 -07003418 self.addCleanup(client.shutdown)
James E. Blair36658cf2013-12-06 17:53:48 -08003419
3420 with testtools.ExpectedException(zuul.rpcclient.RPCFailure):
Paul Belangerbaca3132016-11-04 12:49:54 -04003421 r = client.promote(tenant='tenant-one',
3422 pipeline='nonexistent',
James E. Blair36658cf2013-12-06 17:53:48 -08003423 change_ids=['2,1', '3,1'])
James E. Blair36658cf2013-12-06 17:53:48 -08003424 self.assertEqual(r, False)
3425
3426 with testtools.ExpectedException(zuul.rpcclient.RPCFailure):
Paul Belangerbaca3132016-11-04 12:49:54 -04003427 r = client.promote(tenant='tenant-one',
3428 pipeline='gate',
James E. Blair36658cf2013-12-06 17:53:48 -08003429 change_ids=['4,1'])
James E. Blair36658cf2013-12-06 17:53:48 -08003430 self.assertEqual(r, False)
3431
Paul Belanger174a8272017-03-14 13:20:10 -04003432 self.executor_server.hold_jobs_in_build = False
3433 self.executor_server.release()
James E. Blair36658cf2013-12-06 17:53:48 -08003434 self.waitUntilSettled()
Clark Boylan7603a372014-01-21 11:43:20 -08003435
James E. Blaircc324192017-04-19 14:36:08 -07003436 @simple_layout('layouts/rate-limit.yaml')
Clark Boylan7603a372014-01-21 11:43:20 -08003437 def test_queue_rate_limiting(self):
3438 "Test that DependentPipelines are rate limited with dep across window"
Paul Belanger174a8272017-03-14 13:20:10 -04003439 self.executor_server.hold_jobs_in_build = True
Clark Boylan7603a372014-01-21 11:43:20 -08003440 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
3441 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
3442 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
3443
3444 C.setDependsOn(B, 1)
Paul Belanger174a8272017-03-14 13:20:10 -04003445 self.executor_server.failJob('project-test1', A)
Clark Boylan7603a372014-01-21 11:43:20 -08003446
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003447 A.addApproval('Code-Review', 2)
3448 B.addApproval('Code-Review', 2)
3449 C.addApproval('Code-Review', 2)
Clark Boylan7603a372014-01-21 11:43:20 -08003450
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003451 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
3452 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
3453 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
Clark Boylan7603a372014-01-21 11:43:20 -08003454 self.waitUntilSettled()
3455
3456 # Only A and B will have their merge jobs queued because
3457 # window is 2.
3458 self.assertEqual(len(self.builds), 2)
3459 self.assertEqual(self.builds[0].name, 'project-merge')
3460 self.assertEqual(self.builds[1].name, 'project-merge')
3461
James E. Blair732ee862017-04-21 07:49:26 -07003462 # Release the merge jobs one at a time.
3463 self.builds[0].release()
Clark Boylan7603a372014-01-21 11:43:20 -08003464 self.waitUntilSettled()
James E. Blair732ee862017-04-21 07:49:26 -07003465 self.builds[0].release()
Clark Boylan7603a372014-01-21 11:43:20 -08003466 self.waitUntilSettled()
3467
3468 # Only A and B will have their test jobs queued because
3469 # window is 2.
3470 self.assertEqual(len(self.builds), 4)
3471 self.assertEqual(self.builds[0].name, 'project-test1')
3472 self.assertEqual(self.builds[1].name, 'project-test2')
3473 self.assertEqual(self.builds[2].name, 'project-test1')
3474 self.assertEqual(self.builds[3].name, 'project-test2')
3475
Paul Belanger174a8272017-03-14 13:20:10 -04003476 self.executor_server.release('project-.*')
Clark Boylan7603a372014-01-21 11:43:20 -08003477 self.waitUntilSettled()
3478
James E. Blaircc324192017-04-19 14:36:08 -07003479 tenant = self.sched.abide.tenants.get('tenant-one')
Clint Byrumb01f1ca2017-04-02 10:19:52 -07003480 queue = tenant.layout.pipelines['gate'].queues[0]
Clark Boylan7603a372014-01-21 11:43:20 -08003481 # A failed so window is reduced by 1 to 1.
3482 self.assertEqual(queue.window, 1)
3483 self.assertEqual(queue.window_floor, 1)
3484 self.assertEqual(A.data['status'], 'NEW')
3485
3486 # Gate is reset and only B's merge job is queued because
3487 # window shrunk to 1.
3488 self.assertEqual(len(self.builds), 1)
3489 self.assertEqual(self.builds[0].name, 'project-merge')
3490
Paul Belanger174a8272017-03-14 13:20:10 -04003491 self.executor_server.release('.*-merge')
Clark Boylan7603a372014-01-21 11:43:20 -08003492 self.waitUntilSettled()
3493
3494 # Only B's test jobs are queued because window is still 1.
3495 self.assertEqual(len(self.builds), 2)
3496 self.assertEqual(self.builds[0].name, 'project-test1')
3497 self.assertEqual(self.builds[1].name, 'project-test2')
3498
Paul Belanger174a8272017-03-14 13:20:10 -04003499 self.executor_server.release('project-.*')
Clark Boylan7603a372014-01-21 11:43:20 -08003500 self.waitUntilSettled()
3501
3502 # B was successfully merged so window is increased to 2.
3503 self.assertEqual(queue.window, 2)
3504 self.assertEqual(queue.window_floor, 1)
3505 self.assertEqual(B.data['status'], 'MERGED')
3506
3507 # Only C is left and its merge job is queued.
3508 self.assertEqual(len(self.builds), 1)
3509 self.assertEqual(self.builds[0].name, 'project-merge')
3510
Paul Belanger174a8272017-03-14 13:20:10 -04003511 self.executor_server.release('.*-merge')
Clark Boylan7603a372014-01-21 11:43:20 -08003512 self.waitUntilSettled()
3513
3514 # After successful merge job the test jobs for C are queued.
3515 self.assertEqual(len(self.builds), 2)
3516 self.assertEqual(self.builds[0].name, 'project-test1')
3517 self.assertEqual(self.builds[1].name, 'project-test2')
3518
Paul Belanger174a8272017-03-14 13:20:10 -04003519 self.executor_server.release('project-.*')
Clark Boylan7603a372014-01-21 11:43:20 -08003520 self.waitUntilSettled()
3521
3522 # C successfully merged so window is bumped to 3.
3523 self.assertEqual(queue.window, 3)
3524 self.assertEqual(queue.window_floor, 1)
3525 self.assertEqual(C.data['status'], 'MERGED')
3526
James E. Blaircc324192017-04-19 14:36:08 -07003527 @simple_layout('layouts/rate-limit.yaml')
Clark Boylan7603a372014-01-21 11:43:20 -08003528 def test_queue_rate_limiting_dependent(self):
3529 "Test that DependentPipelines are rate limited with dep in window"
Paul Belanger174a8272017-03-14 13:20:10 -04003530 self.executor_server.hold_jobs_in_build = True
Clark Boylan7603a372014-01-21 11:43:20 -08003531 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
3532 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
3533 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
3534
3535 B.setDependsOn(A, 1)
3536
Paul Belanger174a8272017-03-14 13:20:10 -04003537 self.executor_server.failJob('project-test1', A)
Clark Boylan7603a372014-01-21 11:43:20 -08003538
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003539 A.addApproval('Code-Review', 2)
3540 B.addApproval('Code-Review', 2)
3541 C.addApproval('Code-Review', 2)
Clark Boylan7603a372014-01-21 11:43:20 -08003542
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003543 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
3544 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
3545 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
Clark Boylan7603a372014-01-21 11:43:20 -08003546 self.waitUntilSettled()
3547
3548 # Only A and B will have their merge jobs queued because
3549 # window is 2.
3550 self.assertEqual(len(self.builds), 2)
3551 self.assertEqual(self.builds[0].name, 'project-merge')
3552 self.assertEqual(self.builds[1].name, 'project-merge')
3553
James E. Blairf4a5f022017-04-18 14:01:10 -07003554 self.orderedRelease(2)
Clark Boylan7603a372014-01-21 11:43:20 -08003555
3556 # Only A and B will have their test jobs queued because
3557 # window is 2.
3558 self.assertEqual(len(self.builds), 4)
3559 self.assertEqual(self.builds[0].name, 'project-test1')
3560 self.assertEqual(self.builds[1].name, 'project-test2')
3561 self.assertEqual(self.builds[2].name, 'project-test1')
3562 self.assertEqual(self.builds[3].name, 'project-test2')
3563
Paul Belanger174a8272017-03-14 13:20:10 -04003564 self.executor_server.release('project-.*')
Clark Boylan7603a372014-01-21 11:43:20 -08003565 self.waitUntilSettled()
3566
James E. Blaircc324192017-04-19 14:36:08 -07003567 tenant = self.sched.abide.tenants.get('tenant-one')
Clint Byrum14ab6ca2017-04-03 10:15:10 -07003568 queue = tenant.layout.pipelines['gate'].queues[0]
Clark Boylan7603a372014-01-21 11:43:20 -08003569 # A failed so window is reduced by 1 to 1.
3570 self.assertEqual(queue.window, 1)
3571 self.assertEqual(queue.window_floor, 1)
3572 self.assertEqual(A.data['status'], 'NEW')
3573 self.assertEqual(B.data['status'], 'NEW')
3574
3575 # Gate is reset and only C's merge job is queued because
3576 # window shrunk to 1 and A and B were dequeued.
3577 self.assertEqual(len(self.builds), 1)
3578 self.assertEqual(self.builds[0].name, 'project-merge')
3579
James E. Blairf4a5f022017-04-18 14:01:10 -07003580 self.orderedRelease(1)
Clark Boylan7603a372014-01-21 11:43:20 -08003581
3582 # Only C's test jobs are queued because window is still 1.
3583 self.assertEqual(len(self.builds), 2)
James E. Blairdf25ddc2017-07-08 07:57:09 -07003584 builds = self.getSortedBuilds()
3585 self.assertEqual(builds[0].name, 'project-test1')
3586 self.assertEqual(builds[1].name, 'project-test2')
Clark Boylan7603a372014-01-21 11:43:20 -08003587
Paul Belanger174a8272017-03-14 13:20:10 -04003588 self.executor_server.release('project-.*')
Clark Boylan7603a372014-01-21 11:43:20 -08003589 self.waitUntilSettled()
3590
3591 # C was successfully merged so window is increased to 2.
3592 self.assertEqual(queue.window, 2)
3593 self.assertEqual(queue.window_floor, 1)
3594 self.assertEqual(C.data['status'], 'MERGED')
Joshua Heskethba8776a2014-01-12 14:35:40 +08003595
3596 def test_worker_update_metadata(self):
3597 "Test if a worker can send back metadata about itself"
Paul Belanger174a8272017-03-14 13:20:10 -04003598 self.executor_server.hold_jobs_in_build = True
Joshua Heskethba8776a2014-01-12 14:35:40 +08003599
3600 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003601 A.addApproval('Code-Review', 2)
3602 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
Joshua Heskethba8776a2014-01-12 14:35:40 +08003603 self.waitUntilSettled()
3604
Clint Byrumd4de79a2017-05-27 11:39:13 -07003605 self.assertEqual(len(self.executor_client.builds), 1)
Joshua Heskethba8776a2014-01-12 14:35:40 +08003606
3607 self.log.debug('Current builds:')
Clint Byrumd4de79a2017-05-27 11:39:13 -07003608 self.log.debug(self.executor_client.builds)
Joshua Heskethba8776a2014-01-12 14:35:40 +08003609
3610 start = time.time()
3611 while True:
3612 if time.time() - start > 10:
3613 raise Exception("Timeout waiting for gearman server to report "
3614 + "back to the client")
Clint Byrumd4de79a2017-05-27 11:39:13 -07003615 build = list(self.executor_client.builds.values())[0]
Monty Tayloreea6bb92017-06-14 11:56:54 -05003616 if build.worker.name == self.executor_server.hostname:
Joshua Heskethba8776a2014-01-12 14:35:40 +08003617 break
3618 else:
3619 time.sleep(0)
3620
3621 self.log.debug(build)
Monty Tayloreea6bb92017-06-14 11:56:54 -05003622 self.assertEqual(self.executor_server.hostname, build.worker.name)
Joshua Heskethba8776a2014-01-12 14:35:40 +08003623
Paul Belanger174a8272017-03-14 13:20:10 -04003624 self.executor_server.hold_jobs_in_build = False
3625 self.executor_server.release()
Joshua Heskethba8776a2014-01-12 14:35:40 +08003626 self.waitUntilSettled()
Joshua Hesketh3979e3e2014-03-04 11:21:10 +11003627
James E. Blaircc324192017-04-19 14:36:08 -07003628 @simple_layout('layouts/footer-message.yaml')
Joshua Hesketh3979e3e2014-03-04 11:21:10 +11003629 def test_footer_message(self):
3630 "Test a pipeline's footer message is correctly added to the report."
Joshua Hesketh3979e3e2014-03-04 11:21:10 +11003631 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003632 A.addApproval('Code-Review', 2)
Paul Belanger174a8272017-03-14 13:20:10 -04003633 self.executor_server.failJob('project-test1', A)
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003634 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
Joshua Hesketh3979e3e2014-03-04 11:21:10 +11003635 self.waitUntilSettled()
3636
3637 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003638 B.addApproval('Code-Review', 2)
3639 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
Joshua Hesketh3979e3e2014-03-04 11:21:10 +11003640 self.waitUntilSettled()
3641
3642 self.assertEqual(2, len(self.smtp_messages))
3643
Adam Gandelman82258fd2017-02-07 16:51:58 -08003644 failure_msg = """\
Joshua Hesketh3979e3e2014-03-04 11:21:10 +11003645Build failed. For information on how to proceed, see \
Adam Gandelman82258fd2017-02-07 16:51:58 -08003646http://wiki.example.org/Test_Failures"""
Joshua Hesketh3979e3e2014-03-04 11:21:10 +11003647
Adam Gandelman82258fd2017-02-07 16:51:58 -08003648 footer_msg = """\
Joshua Hesketh3979e3e2014-03-04 11:21:10 +11003649For CI problems and help debugging, contact ci@example.org"""
3650
Adam Gandelman82258fd2017-02-07 16:51:58 -08003651 self.assertTrue(self.smtp_messages[0]['body'].startswith(failure_msg))
3652 self.assertTrue(self.smtp_messages[0]['body'].endswith(footer_msg))
3653 self.assertFalse(self.smtp_messages[1]['body'].startswith(failure_msg))
3654 self.assertTrue(self.smtp_messages[1]['body'].endswith(footer_msg))
Joshua Heskethb7179772014-01-30 23:30:46 +11003655
Tobias Henkel9842bd72017-05-16 13:40:03 +02003656 @simple_layout('layouts/unmanaged-project.yaml')
3657 def test_unmanaged_project_start_message(self):
3658 "Test start reporting is not done for unmanaged projects."
3659 self.init_repo("org/project", tag='init')
3660 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
3661 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
3662 self.waitUntilSettled()
3663
3664 self.assertEqual(0, len(A.messages))
3665
Clint Byrumb9de8be2017-05-27 13:49:19 -07003666 @simple_layout('layouts/merge-failure.yaml')
Joshua Heskethb7179772014-01-30 23:30:46 +11003667 def test_merge_failure_reporters(self):
3668 """Check that the config is set up correctly"""
3669
Clint Byrumb9de8be2017-05-27 13:49:19 -07003670 tenant = self.sched.abide.tenants.get('tenant-one')
Joshua Heskethb7179772014-01-30 23:30:46 +11003671 self.assertEqual(
Jeremy Stanley1c2c3c22015-06-15 21:23:19 +00003672 "Merge Failed.\n\nThis change or one of its cross-repo "
3673 "dependencies was unable to be automatically merged with the "
3674 "current state of its repository. Please rebase the change and "
3675 "upload a new patchset.",
Clint Byrumb9de8be2017-05-27 13:49:19 -07003676 tenant.layout.pipelines['check'].merge_failure_message)
Joshua Heskethb7179772014-01-30 23:30:46 +11003677 self.assertEqual(
3678 "The merge failed! For more information...",
Clint Byrumb9de8be2017-05-27 13:49:19 -07003679 tenant.layout.pipelines['gate'].merge_failure_message)
Joshua Heskethb7179772014-01-30 23:30:46 +11003680
3681 self.assertEqual(
Clint Byrumb9de8be2017-05-27 13:49:19 -07003682 len(tenant.layout.pipelines['check'].merge_failure_actions), 1)
Joshua Heskethb7179772014-01-30 23:30:46 +11003683 self.assertEqual(
Clint Byrumb9de8be2017-05-27 13:49:19 -07003684 len(tenant.layout.pipelines['gate'].merge_failure_actions), 2)
Joshua Heskethb7179772014-01-30 23:30:46 +11003685
3686 self.assertTrue(isinstance(
Clint Byrumb9de8be2017-05-27 13:49:19 -07003687 tenant.layout.pipelines['check'].merge_failure_actions[0],
3688 gerritreporter.GerritReporter))
Joshua Heskethb7179772014-01-30 23:30:46 +11003689
3690 self.assertTrue(
3691 (
Clint Byrumb9de8be2017-05-27 13:49:19 -07003692 isinstance(tenant.layout.pipelines['gate'].
Joshua Heskethde958652015-11-10 19:19:50 +11003693 merge_failure_actions[0],
Clint Byrumb9de8be2017-05-27 13:49:19 -07003694 zuul.driver.smtp.smtpreporter.SMTPReporter) and
3695 isinstance(tenant.layout.pipelines['gate'].
Joshua Heskethde958652015-11-10 19:19:50 +11003696 merge_failure_actions[1],
Clint Byrumb9de8be2017-05-27 13:49:19 -07003697 gerritreporter.GerritReporter)
Joshua Heskethb7179772014-01-30 23:30:46 +11003698 ) or (
Clint Byrumb9de8be2017-05-27 13:49:19 -07003699 isinstance(tenant.layout.pipelines['gate'].
Joshua Heskethde958652015-11-10 19:19:50 +11003700 merge_failure_actions[0],
Clint Byrumb9de8be2017-05-27 13:49:19 -07003701 gerritreporter.GerritReporter) and
3702 isinstance(tenant.layout.pipelines['gate'].
Joshua Heskethde958652015-11-10 19:19:50 +11003703 merge_failure_actions[1],
Clint Byrumb9de8be2017-05-27 13:49:19 -07003704 zuul.driver.smtp.smtpreporter.SMTPReporter)
Joshua Heskethb7179772014-01-30 23:30:46 +11003705 )
3706 )
3707
James E. Blairec056492016-07-22 09:45:56 -07003708 @skip("Disabled for early v3 development")
Joshua Heskethb7179772014-01-30 23:30:46 +11003709 def test_merge_failure_reports(self):
3710 """Check that when a change fails to merge the correct message is sent
3711 to the correct reporter"""
James E. Blairf84026c2015-12-08 16:11:46 -08003712 self.updateConfigLayout(
3713 'tests/fixtures/layout-merge-failure.yaml')
Joshua Heskethb7179772014-01-30 23:30:46 +11003714 self.sched.reconfigure(self.config)
3715 self.registerJobs()
3716
3717 # Check a test failure isn't reported to SMTP
3718 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003719 A.addApproval('Code-Review', 2)
Paul Belanger174a8272017-03-14 13:20:10 -04003720 self.executor_server.failJob('project-test1', A)
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003721 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
Joshua Heskethb7179772014-01-30 23:30:46 +11003722 self.waitUntilSettled()
3723
3724 self.assertEqual(3, len(self.history)) # 3 jobs
3725 self.assertEqual(0, len(self.smtp_messages))
3726
3727 # Check a merge failure is reported to SMTP
3728 # B should be merged, but C will conflict with B
3729 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
3730 B.addPatchset(['conflict'])
3731 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
3732 C.addPatchset(['conflict'])
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003733 B.addApproval('Code-Review', 2)
3734 C.addApproval('Code-Review', 2)
3735 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
3736 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
Joshua Heskethb7179772014-01-30 23:30:46 +11003737 self.waitUntilSettled()
3738
3739 self.assertEqual(6, len(self.history)) # A and B jobs
3740 self.assertEqual(1, len(self.smtp_messages))
3741 self.assertEqual('The merge failed! For more information...',
3742 self.smtp_messages[0]['body'])
Joshua Hesketh36c3fa52014-01-22 11:40:52 +11003743
James E. Blairec056492016-07-22 09:45:56 -07003744 @skip("Disabled for early v3 development")
James E. Blairf760f0e2016-02-09 08:44:52 -08003745 def test_default_merge_failure_reports(self):
3746 """Check that the default merge failure reports are correct."""
3747
3748 # A should report success, B should report merge failure.
3749 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
3750 A.addPatchset(['conflict'])
3751 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
3752 B.addPatchset(['conflict'])
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003753 A.addApproval('Code-Review', 2)
3754 B.addApproval('Code-Review', 2)
3755 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
3756 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blairf760f0e2016-02-09 08:44:52 -08003757 self.waitUntilSettled()
3758
3759 self.assertEqual(3, len(self.history)) # A jobs
3760 self.assertEqual(A.reported, 2)
3761 self.assertEqual(B.reported, 2)
3762 self.assertEqual(A.data['status'], 'MERGED')
3763 self.assertEqual(B.data['status'], 'NEW')
3764 self.assertIn('Build succeeded', A.messages[1])
3765 self.assertIn('Merge Failed', B.messages[1])
3766 self.assertIn('automatically merged', B.messages[1])
3767 self.assertNotIn('logs.example.com', B.messages[1])
3768 self.assertNotIn('SKIPPED', B.messages[1])
3769
Joshua Hesketh85af4e92014-02-21 08:28:58 -08003770 def test_client_get_running_jobs(self):
3771 "Test that the RPC client can get a list of running jobs"
Paul Belanger174a8272017-03-14 13:20:10 -04003772 self.executor_server.hold_jobs_in_build = True
Joshua Hesketh85af4e92014-02-21 08:28:58 -08003773 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003774 A.addApproval('Code-Review', 2)
3775 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
Joshua Hesketh85af4e92014-02-21 08:28:58 -08003776 self.waitUntilSettled()
3777
3778 client = zuul.rpcclient.RPCClient('127.0.0.1',
3779 self.gearman_server.port)
Clark Boylandf4eba22017-04-24 17:37:58 -07003780 self.addCleanup(client.shutdown)
Joshua Hesketh85af4e92014-02-21 08:28:58 -08003781
3782 # Wait for gearman server to send the initial workData back to zuul
3783 start = time.time()
3784 while True:
3785 if time.time() - start > 10:
3786 raise Exception("Timeout waiting for gearman server to report "
3787 + "back to the client")
Clint Byrum1d0c7d12017-05-10 19:40:53 -07003788 build = list(self.executor_client.builds.values())[0]
Monty Tayloreea6bb92017-06-14 11:56:54 -05003789 if build.worker.name == self.executor_server.hostname:
Joshua Hesketh85af4e92014-02-21 08:28:58 -08003790 break
3791 else:
3792 time.sleep(0)
3793
3794 running_items = client.get_running_jobs()
3795
3796 self.assertEqual(1, len(running_items))
3797 running_item = running_items[0]
3798 self.assertEqual([], running_item['failing_reasons'])
3799 self.assertEqual([], running_item['items_behind'])
3800 self.assertEqual('https://hostname/1', running_item['url'])
Monty Taylor38b553a2017-06-05 13:06:10 -05003801 self.assertIsNone(running_item['item_ahead'])
Joshua Hesketh85af4e92014-02-21 08:28:58 -08003802 self.assertEqual('org/project', running_item['project'])
Monty Taylor38b553a2017-06-05 13:06:10 -05003803 self.assertIsNone(running_item['remaining_time'])
Joshua Hesketh85af4e92014-02-21 08:28:58 -08003804 self.assertEqual(True, running_item['active'])
3805 self.assertEqual('1,1', running_item['id'])
3806
3807 self.assertEqual(3, len(running_item['jobs']))
3808 for job in running_item['jobs']:
3809 if job['name'] == 'project-merge':
3810 self.assertEqual('project-merge', job['name'])
3811 self.assertEqual('gate', job['pipeline'])
3812 self.assertEqual(False, job['retry'])
Monty Taylor51139a02016-05-24 11:28:10 -05003813 self.assertEqual(
Tobias Henkelb4407fc2017-07-07 13:52:56 +02003814 'static/stream.html?uuid={uuid}&logfile=console.log'
3815 .format(uuid=job['uuid']), job['url'])
3816 self.assertEqual(
Monty Taylor51139a02016-05-24 11:28:10 -05003817 'finger://{hostname}/{uuid}'.format(
3818 hostname=self.executor_server.hostname,
3819 uuid=job['uuid']),
Tobias Henkelb4407fc2017-07-07 13:52:56 +02003820 job['finger_url'])
Clint Byrumd4de79a2017-05-27 11:39:13 -07003821 self.assertEqual(2, len(job['worker']))
Joshua Hesketh85af4e92014-02-21 08:28:58 -08003822 self.assertEqual(False, job['canceled'])
3823 self.assertEqual(True, job['voting'])
Monty Taylor38b553a2017-06-05 13:06:10 -05003824 self.assertIsNone(job['result'])
Joshua Hesketh85af4e92014-02-21 08:28:58 -08003825 self.assertEqual('gate', job['pipeline'])
3826 break
3827
Paul Belanger174a8272017-03-14 13:20:10 -04003828 self.executor_server.hold_jobs_in_build = False
3829 self.executor_server.release()
Joshua Hesketh85af4e92014-02-21 08:28:58 -08003830 self.waitUntilSettled()
3831
3832 running_items = client.get_running_jobs()
3833 self.assertEqual(0, len(running_items))
James E. Blairbadc1ad2014-04-28 13:55:14 -07003834
James E. Blairb70e55a2017-04-19 12:57:02 -07003835 @simple_layout('layouts/nonvoting-pipeline.yaml')
James E. Blairbadc1ad2014-04-28 13:55:14 -07003836 def test_nonvoting_pipeline(self):
3837 "Test that a nonvoting pipeline (experimental) can still report"
3838
Joshua Heskethcc017ea2014-04-30 19:55:25 +10003839 A = self.fake_gerrit.addFakeChange('org/experimental-project',
3840 'master', 'A')
James E. Blairbadc1ad2014-04-28 13:55:14 -07003841 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
3842 self.waitUntilSettled()
Paul Belanger15e3e202016-10-14 16:27:34 -04003843
3844 self.assertEqual(self.getJobFromHistory('project-merge').result,
3845 'SUCCESS')
Joshua Heskethcc017ea2014-04-30 19:55:25 +10003846 self.assertEqual(
3847 self.getJobFromHistory('experimental-project-test').result,
3848 'SUCCESS')
James E. Blairbadc1ad2014-04-28 13:55:14 -07003849 self.assertEqual(A.reported, 1)
James E. Blair5ee24252014-12-30 10:12:29 -08003850
3851 def test_crd_gate(self):
3852 "Test cross-repo dependencies"
3853 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
3854 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003855 A.addApproval('Code-Review', 2)
3856 B.addApproval('Code-Review', 2)
James E. Blair5ee24252014-12-30 10:12:29 -08003857
3858 AM2 = self.fake_gerrit.addFakeChange('org/project1', 'master', 'AM2')
3859 AM1 = self.fake_gerrit.addFakeChange('org/project1', 'master', 'AM1')
3860 AM2.setMerged()
3861 AM1.setMerged()
3862
3863 BM2 = self.fake_gerrit.addFakeChange('org/project2', 'master', 'BM2')
3864 BM1 = self.fake_gerrit.addFakeChange('org/project2', 'master', 'BM1')
3865 BM2.setMerged()
3866 BM1.setMerged()
3867
3868 # A -> AM1 -> AM2
3869 # B -> BM1 -> BM2
3870 # A Depends-On: B
3871 # M2 is here to make sure it is never queried. If it is, it
3872 # means zuul is walking down the entire history of merged
3873 # changes.
3874
3875 B.setDependsOn(BM1, 1)
3876 BM1.setDependsOn(BM2, 1)
3877
3878 A.setDependsOn(AM1, 1)
3879 AM1.setDependsOn(AM2, 1)
3880
3881 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
3882 A.subject, B.data['id'])
3883
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003884 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair5ee24252014-12-30 10:12:29 -08003885 self.waitUntilSettled()
3886
3887 self.assertEqual(A.data['status'], 'NEW')
3888 self.assertEqual(B.data['status'], 'NEW')
3889
Paul Belanger5dccbe72016-11-14 11:17:37 -05003890 for connection in self.connections.connections.values():
Joshua Hesketh4bd7da32016-02-17 20:58:47 +11003891 connection.maintainCache([])
James E. Blair5ee24252014-12-30 10:12:29 -08003892
Paul Belanger174a8272017-03-14 13:20:10 -04003893 self.executor_server.hold_jobs_in_build = True
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003894 B.addApproval('Approved', 1)
3895 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair5ee24252014-12-30 10:12:29 -08003896 self.waitUntilSettled()
3897
Paul Belanger174a8272017-03-14 13:20:10 -04003898 self.executor_server.release('.*-merge')
James E. Blair5ee24252014-12-30 10:12:29 -08003899 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003900 self.executor_server.release('.*-merge')
James E. Blair5ee24252014-12-30 10:12:29 -08003901 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003902 self.executor_server.hold_jobs_in_build = False
3903 self.executor_server.release()
James E. Blair5ee24252014-12-30 10:12:29 -08003904 self.waitUntilSettled()
3905
3906 self.assertEqual(AM2.queried, 0)
3907 self.assertEqual(BM2.queried, 0)
3908 self.assertEqual(A.data['status'], 'MERGED')
3909 self.assertEqual(B.data['status'], 'MERGED')
3910 self.assertEqual(A.reported, 2)
3911 self.assertEqual(B.reported, 2)
3912
Paul Belanger5dccbe72016-11-14 11:17:37 -05003913 changes = self.getJobFromHistory(
3914 'project-merge', 'org/project1').changes
3915 self.assertEqual(changes, '2,1 1,1')
James E. Blair8f78d882015-02-05 08:51:37 -08003916
3917 def test_crd_branch(self):
3918 "Test cross-repo dependencies in multiple branches"
Paul Belanger6379db12016-11-14 13:57:54 -05003919
3920 self.create_branch('org/project2', 'mp')
James E. Blair8f78d882015-02-05 08:51:37 -08003921 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
3922 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
K Jonathan Harkerd1d51d22017-08-23 11:57:40 -04003923 C1 = self.fake_gerrit.addFakeChange('org/project2', 'mp', 'C1')
3924 C2 = self.fake_gerrit.addFakeChange('org/project2', 'mp', 'C2',
3925 status='ABANDONED')
3926 C1.data['id'] = B.data['id']
3927 C2.data['id'] = B.data['id']
3928
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003929 A.addApproval('Code-Review', 2)
3930 B.addApproval('Code-Review', 2)
K Jonathan Harkerd1d51d22017-08-23 11:57:40 -04003931 C1.addApproval('Code-Review', 2)
James E. Blair8f78d882015-02-05 08:51:37 -08003932
K Jonathan Harkerd1d51d22017-08-23 11:57:40 -04003933 # A Depends-On: B+C1
James E. Blair8f78d882015-02-05 08:51:37 -08003934 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
3935 A.subject, B.data['id'])
3936
Paul Belanger174a8272017-03-14 13:20:10 -04003937 self.executor_server.hold_jobs_in_build = True
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003938 B.addApproval('Approved', 1)
K Jonathan Harkerd1d51d22017-08-23 11:57:40 -04003939 C1.addApproval('Approved', 1)
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003940 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair8f78d882015-02-05 08:51:37 -08003941 self.waitUntilSettled()
3942
Paul Belanger174a8272017-03-14 13:20:10 -04003943 self.executor_server.release('.*-merge')
James E. Blair8f78d882015-02-05 08:51:37 -08003944 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003945 self.executor_server.release('.*-merge')
James E. Blair8f78d882015-02-05 08:51:37 -08003946 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003947 self.executor_server.release('.*-merge')
James E. Blair8f78d882015-02-05 08:51:37 -08003948 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003949 self.executor_server.hold_jobs_in_build = False
3950 self.executor_server.release()
James E. Blair8f78d882015-02-05 08:51:37 -08003951 self.waitUntilSettled()
3952
3953 self.assertEqual(A.data['status'], 'MERGED')
3954 self.assertEqual(B.data['status'], 'MERGED')
K Jonathan Harkerd1d51d22017-08-23 11:57:40 -04003955 self.assertEqual(C1.data['status'], 'MERGED')
James E. Blair8f78d882015-02-05 08:51:37 -08003956 self.assertEqual(A.reported, 2)
3957 self.assertEqual(B.reported, 2)
K Jonathan Harkerd1d51d22017-08-23 11:57:40 -04003958 self.assertEqual(C1.reported, 2)
James E. Blair8f78d882015-02-05 08:51:37 -08003959
Paul Belanger6379db12016-11-14 13:57:54 -05003960 changes = self.getJobFromHistory(
3961 'project-merge', 'org/project1').changes
3962 self.assertEqual(changes, '2,1 3,1 1,1')
James E. Blair8f78d882015-02-05 08:51:37 -08003963
3964 def test_crd_multiline(self):
3965 "Test multiple depends-on lines in commit"
3966 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
3967 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
3968 C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003969 A.addApproval('Code-Review', 2)
3970 B.addApproval('Code-Review', 2)
3971 C.addApproval('Code-Review', 2)
James E. Blair8f78d882015-02-05 08:51:37 -08003972
3973 # A Depends-On: B+C
3974 A.data['commitMessage'] = '%s\n\nDepends-On: %s\nDepends-On: %s\n' % (
3975 A.subject, B.data['id'], C.data['id'])
3976
Paul Belanger174a8272017-03-14 13:20:10 -04003977 self.executor_server.hold_jobs_in_build = True
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003978 B.addApproval('Approved', 1)
3979 C.addApproval('Approved', 1)
3980 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair8f78d882015-02-05 08:51:37 -08003981 self.waitUntilSettled()
3982
Paul Belanger174a8272017-03-14 13:20:10 -04003983 self.executor_server.release('.*-merge')
James E. Blair8f78d882015-02-05 08:51:37 -08003984 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003985 self.executor_server.release('.*-merge')
James E. Blair8f78d882015-02-05 08:51:37 -08003986 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003987 self.executor_server.release('.*-merge')
James E. Blair8f78d882015-02-05 08:51:37 -08003988 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003989 self.executor_server.hold_jobs_in_build = False
3990 self.executor_server.release()
James E. Blair8f78d882015-02-05 08:51:37 -08003991 self.waitUntilSettled()
3992
3993 self.assertEqual(A.data['status'], 'MERGED')
3994 self.assertEqual(B.data['status'], 'MERGED')
3995 self.assertEqual(C.data['status'], 'MERGED')
3996 self.assertEqual(A.reported, 2)
3997 self.assertEqual(B.reported, 2)
3998 self.assertEqual(C.reported, 2)
3999
Paul Belanger5dccbe72016-11-14 11:17:37 -05004000 changes = self.getJobFromHistory(
4001 'project-merge', 'org/project1').changes
4002 self.assertEqual(changes, '2,1 3,1 1,1')
James E. Blair5ee24252014-12-30 10:12:29 -08004003
4004 def test_crd_unshared_gate(self):
4005 "Test cross-repo dependencies in unshared gate queues"
4006 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4007 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004008 A.addApproval('Code-Review', 2)
4009 B.addApproval('Code-Review', 2)
James E. Blair5ee24252014-12-30 10:12:29 -08004010
4011 # A Depends-On: B
4012 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4013 A.subject, B.data['id'])
4014
4015 # A and B do not share a queue, make sure that A is unable to
4016 # enqueue B (and therefore, A is unable to be enqueued).
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004017 B.addApproval('Approved', 1)
4018 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair5ee24252014-12-30 10:12:29 -08004019 self.waitUntilSettled()
4020
4021 self.assertEqual(A.data['status'], 'NEW')
4022 self.assertEqual(B.data['status'], 'NEW')
4023 self.assertEqual(A.reported, 0)
4024 self.assertEqual(B.reported, 0)
4025 self.assertEqual(len(self.history), 0)
4026
4027 # Enqueue and merge B alone.
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004028 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blair5ee24252014-12-30 10:12:29 -08004029 self.waitUntilSettled()
4030
4031 self.assertEqual(B.data['status'], 'MERGED')
4032 self.assertEqual(B.reported, 2)
4033
4034 # Now that B is merged, A should be able to be enqueued and
4035 # merged.
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004036 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair5ee24252014-12-30 10:12:29 -08004037 self.waitUntilSettled()
4038
4039 self.assertEqual(A.data['status'], 'MERGED')
4040 self.assertEqual(A.reported, 2)
4041
James E. Blair96698e22015-04-02 07:48:21 -07004042 def test_crd_gate_reverse(self):
4043 "Test reverse cross-repo dependencies"
4044 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4045 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004046 A.addApproval('Code-Review', 2)
4047 B.addApproval('Code-Review', 2)
James E. Blair96698e22015-04-02 07:48:21 -07004048
4049 # A Depends-On: B
4050
4051 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4052 A.subject, B.data['id'])
4053
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004054 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair96698e22015-04-02 07:48:21 -07004055 self.waitUntilSettled()
4056
4057 self.assertEqual(A.data['status'], 'NEW')
4058 self.assertEqual(B.data['status'], 'NEW')
4059
Paul Belanger174a8272017-03-14 13:20:10 -04004060 self.executor_server.hold_jobs_in_build = True
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004061 A.addApproval('Approved', 1)
4062 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blair96698e22015-04-02 07:48:21 -07004063 self.waitUntilSettled()
4064
Paul Belanger174a8272017-03-14 13:20:10 -04004065 self.executor_server.release('.*-merge')
James E. Blair96698e22015-04-02 07:48:21 -07004066 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04004067 self.executor_server.release('.*-merge')
James E. Blair96698e22015-04-02 07:48:21 -07004068 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04004069 self.executor_server.hold_jobs_in_build = False
4070 self.executor_server.release()
James E. Blair96698e22015-04-02 07:48:21 -07004071 self.waitUntilSettled()
4072
4073 self.assertEqual(A.data['status'], 'MERGED')
4074 self.assertEqual(B.data['status'], 'MERGED')
4075 self.assertEqual(A.reported, 2)
4076 self.assertEqual(B.reported, 2)
4077
Paul Belanger5dccbe72016-11-14 11:17:37 -05004078 changes = self.getJobFromHistory(
4079 'project-merge', 'org/project1').changes
4080 self.assertEqual(changes, '2,1 1,1')
James E. Blair96698e22015-04-02 07:48:21 -07004081
James E. Blair5ee24252014-12-30 10:12:29 -08004082 def test_crd_cycle(self):
4083 "Test cross-repo dependency cycles"
4084 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4085 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004086 A.addApproval('Code-Review', 2)
4087 B.addApproval('Code-Review', 2)
James E. Blair5ee24252014-12-30 10:12:29 -08004088
4089 # A -> B -> A (via commit-depends)
4090
4091 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4092 A.subject, B.data['id'])
4093 B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4094 B.subject, A.data['id'])
4095
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004096 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair5ee24252014-12-30 10:12:29 -08004097 self.waitUntilSettled()
4098
4099 self.assertEqual(A.reported, 0)
4100 self.assertEqual(B.reported, 0)
4101 self.assertEqual(A.data['status'], 'NEW')
4102 self.assertEqual(B.data['status'], 'NEW')
James E. Blairbfb8e042014-12-30 17:01:44 -08004103
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004104 def test_crd_gate_unknown(self):
4105 "Test unknown projects in dependent pipeline"
James E. Blairc4ba97a2017-04-19 16:26:24 -07004106 self.init_repo("org/unknown", tag='init')
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004107 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
4108 B = self.fake_gerrit.addFakeChange('org/unknown', 'master', 'B')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004109 A.addApproval('Code-Review', 2)
4110 B.addApproval('Code-Review', 2)
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004111
4112 # A Depends-On: B
4113 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4114 A.subject, B.data['id'])
4115
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004116 B.addApproval('Approved', 1)
4117 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004118 self.waitUntilSettled()
4119
4120 # Unknown projects cannot share a queue with any other
4121 # since they don't have common jobs with any other (they have no jobs).
4122 # Changes which depend on unknown project changes
4123 # should not be processed in dependent pipeline
4124 self.assertEqual(A.data['status'], 'NEW')
4125 self.assertEqual(B.data['status'], 'NEW')
4126 self.assertEqual(A.reported, 0)
4127 self.assertEqual(B.reported, 0)
4128 self.assertEqual(len(self.history), 0)
4129
James E. Blair7720b402017-06-07 08:51:49 -07004130 # Simulate change B being gated outside this layout Set the
4131 # change merged before submitting the event so that when the
4132 # event triggers a gerrit query to update the change, we get
4133 # the information that it was merged.
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004134 B.setMerged()
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004135 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004136 self.waitUntilSettled()
4137 self.assertEqual(len(self.history), 0)
4138
4139 # Now that B is merged, A should be able to be enqueued and
4140 # merged.
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004141 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004142 self.waitUntilSettled()
4143
4144 self.assertEqual(A.data['status'], 'MERGED')
4145 self.assertEqual(A.reported, 2)
4146 self.assertEqual(B.data['status'], 'MERGED')
4147 self.assertEqual(B.reported, 0)
4148
James E. Blairbfb8e042014-12-30 17:01:44 -08004149 def test_crd_check(self):
4150 "Test cross-repo dependencies in independent pipelines"
4151
Paul Belanger174a8272017-03-14 13:20:10 -04004152 self.executor_server.hold_jobs_in_build = True
James E. Blairbfb8e042014-12-30 17:01:44 -08004153 self.gearman_server.hold_jobs_in_queue = True
4154 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4155 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4156
4157 # A Depends-On: B
4158 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4159 A.subject, B.data['id'])
4160
4161 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4162 self.waitUntilSettled()
4163
James E. Blairbfb8e042014-12-30 17:01:44 -08004164 self.gearman_server.hold_jobs_in_queue = False
4165 self.gearman_server.release()
4166 self.waitUntilSettled()
4167
Paul Belanger174a8272017-03-14 13:20:10 -04004168 self.executor_server.release('.*-merge')
Paul Belangerb30342b2016-11-14 12:30:43 -05004169 self.waitUntilSettled()
4170
James E. Blair2a535672017-04-27 12:03:15 -07004171 self.assertTrue(self.builds[0].hasChanges(A, B))
James E. Blairbfb8e042014-12-30 17:01:44 -08004172
Paul Belanger174a8272017-03-14 13:20:10 -04004173 self.executor_server.hold_jobs_in_build = False
4174 self.executor_server.release()
Paul Belangerb30342b2016-11-14 12:30:43 -05004175 self.waitUntilSettled()
4176
James E. Blairbfb8e042014-12-30 17:01:44 -08004177 self.assertEqual(A.data['status'], 'NEW')
4178 self.assertEqual(B.data['status'], 'NEW')
4179 self.assertEqual(A.reported, 1)
4180 self.assertEqual(B.reported, 0)
4181
4182 self.assertEqual(self.history[0].changes, '2,1 1,1')
Paul Belangerb30342b2016-11-14 12:30:43 -05004183 tenant = self.sched.abide.tenants.get('tenant-one')
4184 self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
James E. Blair8f78d882015-02-05 08:51:37 -08004185
James E. Blairdbfe1cd2015-02-07 11:41:19 -08004186 def test_crd_check_git_depends(self):
4187 "Test single-repo dependencies in independent pipelines"
James E. Blairb8c16472015-05-05 14:55:26 -07004188 self.gearman_server.hold_jobs_in_build = True
James E. Blairdbfe1cd2015-02-07 11:41:19 -08004189 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4190 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
4191
4192 # Add two git-dependent changes and make sure they both report
4193 # success.
4194 B.setDependsOn(A, 1)
4195 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4196 self.waitUntilSettled()
4197 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4198 self.waitUntilSettled()
4199
James E. Blairb8c16472015-05-05 14:55:26 -07004200 self.orderedRelease()
4201 self.gearman_server.hold_jobs_in_build = False
James E. Blairdbfe1cd2015-02-07 11:41:19 -08004202 self.waitUntilSettled()
4203
4204 self.assertEqual(A.data['status'], 'NEW')
4205 self.assertEqual(B.data['status'], 'NEW')
4206 self.assertEqual(A.reported, 1)
4207 self.assertEqual(B.reported, 1)
4208
4209 self.assertEqual(self.history[0].changes, '1,1')
4210 self.assertEqual(self.history[-1].changes, '1,1 2,1')
Paul Belanger08136962016-11-03 17:57:38 -04004211 tenant = self.sched.abide.tenants.get('tenant-one')
4212 self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
James E. Blairdbfe1cd2015-02-07 11:41:19 -08004213
4214 self.assertIn('Build succeeded', A.messages[0])
4215 self.assertIn('Build succeeded', B.messages[0])
4216
4217 def test_crd_check_duplicate(self):
4218 "Test duplicate check in independent pipelines"
Paul Belanger174a8272017-03-14 13:20:10 -04004219 self.executor_server.hold_jobs_in_build = True
James E. Blairdbfe1cd2015-02-07 11:41:19 -08004220 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4221 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
Paul Belanger08136962016-11-03 17:57:38 -04004222 tenant = self.sched.abide.tenants.get('tenant-one')
4223 check_pipeline = tenant.layout.pipelines['check']
James E. Blairdbfe1cd2015-02-07 11:41:19 -08004224
4225 # Add two git-dependent changes...
4226 B.setDependsOn(A, 1)
4227 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4228 self.waitUntilSettled()
4229 self.assertEqual(len(check_pipeline.getAllItems()), 2)
4230
4231 # ...make sure the live one is not duplicated...
4232 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4233 self.waitUntilSettled()
4234 self.assertEqual(len(check_pipeline.getAllItems()), 2)
4235
4236 # ...but the non-live one is able to be.
4237 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4238 self.waitUntilSettled()
4239 self.assertEqual(len(check_pipeline.getAllItems()), 3)
4240
Clark Boylandd849822015-03-02 12:38:14 -08004241 # Release jobs in order to avoid races with change A jobs
4242 # finishing before change B jobs.
James E. Blaird7650852015-05-07 15:47:37 -07004243 self.orderedRelease()
Paul Belanger174a8272017-03-14 13:20:10 -04004244 self.executor_server.hold_jobs_in_build = False
4245 self.executor_server.release()
James E. Blairdbfe1cd2015-02-07 11:41:19 -08004246 self.waitUntilSettled()
4247
4248 self.assertEqual(A.data['status'], 'NEW')
4249 self.assertEqual(B.data['status'], 'NEW')
4250 self.assertEqual(A.reported, 1)
4251 self.assertEqual(B.reported, 1)
4252
4253 self.assertEqual(self.history[0].changes, '1,1 2,1')
4254 self.assertEqual(self.history[1].changes, '1,1')
Paul Belanger08136962016-11-03 17:57:38 -04004255 self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
James E. Blairdbfe1cd2015-02-07 11:41:19 -08004256
4257 self.assertIn('Build succeeded', A.messages[0])
4258 self.assertIn('Build succeeded', B.messages[0])
4259
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004260 def _test_crd_check_reconfiguration(self, project1, project2):
James E. Blair8f78d882015-02-05 08:51:37 -08004261 "Test cross-repo dependencies re-enqueued in independent pipelines"
4262
4263 self.gearman_server.hold_jobs_in_queue = True
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004264 A = self.fake_gerrit.addFakeChange(project1, 'master', 'A')
4265 B = self.fake_gerrit.addFakeChange(project2, 'master', 'B')
James E. Blair8f78d882015-02-05 08:51:37 -08004266
4267 # A Depends-On: B
4268 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4269 A.subject, B.data['id'])
4270
4271 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4272 self.waitUntilSettled()
4273
4274 self.sched.reconfigure(self.config)
4275
4276 # Make sure the items still share a change queue, and the
4277 # first one is not live.
Paul Belangerce4485f2016-11-11 17:01:56 -05004278 tenant = self.sched.abide.tenants.get('tenant-one')
4279 self.assertEqual(len(tenant.layout.pipelines['check'].queues), 1)
4280 queue = tenant.layout.pipelines['check'].queues[0]
James E. Blair8f78d882015-02-05 08:51:37 -08004281 first_item = queue.queue[0]
4282 for item in queue.queue:
4283 self.assertEqual(item.queue, first_item.queue)
4284 self.assertFalse(first_item.live)
4285 self.assertTrue(queue.queue[1].live)
4286
4287 self.gearman_server.hold_jobs_in_queue = False
4288 self.gearman_server.release()
4289 self.waitUntilSettled()
4290
4291 self.assertEqual(A.data['status'], 'NEW')
4292 self.assertEqual(B.data['status'], 'NEW')
4293 self.assertEqual(A.reported, 1)
4294 self.assertEqual(B.reported, 0)
4295
4296 self.assertEqual(self.history[0].changes, '2,1 1,1')
Paul Belangerce4485f2016-11-11 17:01:56 -05004297 self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
James E. Blair17dd6772015-02-09 14:45:18 -08004298
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004299 def test_crd_check_reconfiguration(self):
4300 self._test_crd_check_reconfiguration('org/project1', 'org/project2')
4301
4302 def test_crd_undefined_project(self):
4303 """Test that undefined projects in dependencies are handled for
4304 independent pipelines"""
4305 # It's a hack for fake gerrit,
4306 # as it implies repo creation upon the creation of any change
James E. Blairc4ba97a2017-04-19 16:26:24 -07004307 self.init_repo("org/unknown", tag='init')
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004308 self._test_crd_check_reconfiguration('org/project1', 'org/unknown')
4309
Jesse Keatingea68ab42017-04-19 15:03:30 -07004310 @simple_layout('layouts/ignore-dependencies.yaml')
James E. Blair17dd6772015-02-09 14:45:18 -08004311 def test_crd_check_ignore_dependencies(self):
4312 "Test cross-repo dependencies can be ignored"
James E. Blair17dd6772015-02-09 14:45:18 -08004313
4314 self.gearman_server.hold_jobs_in_queue = True
4315 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4316 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4317 C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
4318
4319 # A Depends-On: B
4320 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4321 A.subject, B.data['id'])
4322 # C git-depends on B
4323 C.setDependsOn(B, 1)
4324 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4325 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4326 self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
4327 self.waitUntilSettled()
4328
4329 # Make sure none of the items share a change queue, and all
4330 # are live.
Jesse Keatingea68ab42017-04-19 15:03:30 -07004331 tenant = self.sched.abide.tenants.get('tenant-one')
Clint Byrum12282ca2017-04-03 11:04:02 -07004332 check_pipeline = tenant.layout.pipelines['check']
James E. Blair17dd6772015-02-09 14:45:18 -08004333 self.assertEqual(len(check_pipeline.queues), 3)
4334 self.assertEqual(len(check_pipeline.getAllItems()), 3)
4335 for item in check_pipeline.getAllItems():
4336 self.assertTrue(item.live)
4337
4338 self.gearman_server.hold_jobs_in_queue = False
4339 self.gearman_server.release()
4340 self.waitUntilSettled()
4341
4342 self.assertEqual(A.data['status'], 'NEW')
4343 self.assertEqual(B.data['status'], 'NEW')
4344 self.assertEqual(C.data['status'], 'NEW')
4345 self.assertEqual(A.reported, 1)
4346 self.assertEqual(B.reported, 1)
4347 self.assertEqual(C.reported, 1)
4348
4349 # Each job should have tested exactly one change
4350 for job in self.history:
4351 self.assertEqual(len(job.changes.split()), 1)
James E. Blair96698e22015-04-02 07:48:21 -07004352
James E. Blair7cb84542017-04-19 13:35:05 -07004353 @simple_layout('layouts/three-projects.yaml')
James E. Blair96698e22015-04-02 07:48:21 -07004354 def test_crd_check_transitive(self):
4355 "Test transitive cross-repo dependencies"
4356 # Specifically, if A -> B -> C, and C gets a new patchset and
4357 # A gets a new patchset, ensure the test of A,2 includes B,1
4358 # and C,2 (not C,1 which would indicate stale data in the
4359 # cache for B).
4360 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4361 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4362 C = self.fake_gerrit.addFakeChange('org/project3', 'master', 'C')
4363
4364 # A Depends-On: B
4365 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4366 A.subject, B.data['id'])
4367
4368 # B Depends-On: C
4369 B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4370 B.subject, C.data['id'])
4371
4372 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4373 self.waitUntilSettled()
4374 self.assertEqual(self.history[-1].changes, '3,1 2,1 1,1')
4375
4376 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4377 self.waitUntilSettled()
4378 self.assertEqual(self.history[-1].changes, '3,1 2,1')
4379
4380 self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
4381 self.waitUntilSettled()
4382 self.assertEqual(self.history[-1].changes, '3,1')
4383
4384 C.addPatchset()
4385 self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(2))
4386 self.waitUntilSettled()
4387 self.assertEqual(self.history[-1].changes, '3,2')
4388
4389 A.addPatchset()
4390 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
4391 self.waitUntilSettled()
4392 self.assertEqual(self.history[-1].changes, '3,2 2,1 1,2')
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004393
James E. Blaira8b90b32016-08-24 15:18:50 -07004394 def test_crd_check_unknown(self):
4395 "Test unknown projects in independent pipeline"
James E. Blairc4ba97a2017-04-19 16:26:24 -07004396 self.init_repo("org/unknown", tag='init')
James E. Blaira8b90b32016-08-24 15:18:50 -07004397 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4398 B = self.fake_gerrit.addFakeChange('org/unknown', 'master', 'D')
4399 # A Depends-On: B
4400 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4401 A.subject, B.data['id'])
4402
4403 # Make sure zuul has seen an event on B.
4404 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4405 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4406 self.waitUntilSettled()
4407
4408 self.assertEqual(A.data['status'], 'NEW')
4409 self.assertEqual(A.reported, 1)
4410 self.assertEqual(B.data['status'], 'NEW')
4411 self.assertEqual(B.reported, 0)
4412
James E. Blair92464a22016-04-05 10:21:26 -07004413 def test_crd_cycle_join(self):
4414 "Test an updated change creates a cycle"
4415 A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
4416
4417 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4418 self.waitUntilSettled()
Tobias Henkel6b9390f2017-03-28 11:23:21 +02004419 self.assertEqual(A.reported, 1)
James E. Blair92464a22016-04-05 10:21:26 -07004420
4421 # Create B->A
4422 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
4423 B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4424 B.subject, A.data['id'])
4425 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4426 self.waitUntilSettled()
4427
Tobias Henkel6b9390f2017-03-28 11:23:21 +02004428 # Dep is there so zuul should have reported on B
4429 self.assertEqual(B.reported, 1)
4430
James E. Blair92464a22016-04-05 10:21:26 -07004431 # Update A to add A->B (a cycle).
4432 A.addPatchset()
4433 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4434 A.subject, B.data['id'])
Tobias Henkel6b9390f2017-03-28 11:23:21 +02004435 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
4436 self.waitUntilSettled()
James E. Blair92464a22016-04-05 10:21:26 -07004437
Tobias Henkel6b9390f2017-03-28 11:23:21 +02004438 # Dependency cycle injected so zuul should not have reported again on A
4439 self.assertEqual(A.reported, 1)
James E. Blair92464a22016-04-05 10:21:26 -07004440
4441 # Now if we update B to remove the depends-on, everything
4442 # should be okay. B; A->B
4443
4444 B.addPatchset()
4445 B.data['commitMessage'] = '%s\n' % (B.subject,)
Tobias Henkel6b9390f2017-03-28 11:23:21 +02004446 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
4447 self.waitUntilSettled()
Paul Belanger0e155e22016-11-14 14:12:23 -05004448
Tobias Henkel6b9390f2017-03-28 11:23:21 +02004449 # Cycle was removed so now zuul should have reported again on A
4450 self.assertEqual(A.reported, 2)
4451
4452 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
4453 self.waitUntilSettled()
4454 self.assertEqual(B.reported, 2)
James E. Blair92464a22016-04-05 10:21:26 -07004455
Jesse Keatingcc67b122017-04-19 14:54:57 -07004456 @simple_layout('layouts/disable_at.yaml')
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004457 def test_disable_at(self):
4458 "Test a pipeline will only report to the disabled trigger when failing"
4459
Jesse Keatingcc67b122017-04-19 14:54:57 -07004460 tenant = self.sched.abide.tenants.get('tenant-one')
Paul Belanger7dc76e82016-11-11 16:51:08 -05004461 self.assertEqual(3, tenant.layout.pipelines['check'].disable_at)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004462 self.assertEqual(
Paul Belanger7dc76e82016-11-11 16:51:08 -05004463 0, tenant.layout.pipelines['check']._consecutive_failures)
4464 self.assertFalse(tenant.layout.pipelines['check']._disabled)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004465
4466 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
4467 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
4468 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
4469 D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
4470 E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')
4471 F = self.fake_gerrit.addFakeChange('org/project', 'master', 'F')
4472 G = self.fake_gerrit.addFakeChange('org/project', 'master', 'G')
4473 H = self.fake_gerrit.addFakeChange('org/project', 'master', 'H')
4474 I = self.fake_gerrit.addFakeChange('org/project', 'master', 'I')
4475 J = self.fake_gerrit.addFakeChange('org/project', 'master', 'J')
4476 K = self.fake_gerrit.addFakeChange('org/project', 'master', 'K')
4477
Paul Belanger174a8272017-03-14 13:20:10 -04004478 self.executor_server.failJob('project-test1', A)
4479 self.executor_server.failJob('project-test1', B)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004480 # Let C pass, resetting the counter
Paul Belanger174a8272017-03-14 13:20:10 -04004481 self.executor_server.failJob('project-test1', D)
4482 self.executor_server.failJob('project-test1', E)
4483 self.executor_server.failJob('project-test1', F)
4484 self.executor_server.failJob('project-test1', G)
4485 self.executor_server.failJob('project-test1', H)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004486 # I also passes but should only report to the disabled reporters
Paul Belanger174a8272017-03-14 13:20:10 -04004487 self.executor_server.failJob('project-test1', J)
4488 self.executor_server.failJob('project-test1', K)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004489
4490 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4491 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4492 self.waitUntilSettled()
4493
4494 self.assertEqual(
Paul Belanger7dc76e82016-11-11 16:51:08 -05004495 2, tenant.layout.pipelines['check']._consecutive_failures)
4496 self.assertFalse(tenant.layout.pipelines['check']._disabled)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004497
4498 self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
4499 self.waitUntilSettled()
4500
4501 self.assertEqual(
Paul Belanger7dc76e82016-11-11 16:51:08 -05004502 0, tenant.layout.pipelines['check']._consecutive_failures)
4503 self.assertFalse(tenant.layout.pipelines['check']._disabled)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004504
4505 self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
4506 self.fake_gerrit.addEvent(E.getPatchsetCreatedEvent(1))
4507 self.fake_gerrit.addEvent(F.getPatchsetCreatedEvent(1))
4508 self.waitUntilSettled()
4509
4510 # We should be disabled now
4511 self.assertEqual(
Paul Belanger7dc76e82016-11-11 16:51:08 -05004512 3, tenant.layout.pipelines['check']._consecutive_failures)
4513 self.assertTrue(tenant.layout.pipelines['check']._disabled)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004514
4515 # We need to wait between each of these patches to make sure the
4516 # smtp messages come back in an expected order
4517 self.fake_gerrit.addEvent(G.getPatchsetCreatedEvent(1))
4518 self.waitUntilSettled()
4519 self.fake_gerrit.addEvent(H.getPatchsetCreatedEvent(1))
4520 self.waitUntilSettled()
4521 self.fake_gerrit.addEvent(I.getPatchsetCreatedEvent(1))
4522 self.waitUntilSettled()
4523
4524 # The first 6 (ABCDEF) jobs should have reported back to gerrt thus
4525 # leaving a message on each change
4526 self.assertEqual(1, len(A.messages))
4527 self.assertIn('Build failed.', A.messages[0])
4528 self.assertEqual(1, len(B.messages))
4529 self.assertIn('Build failed.', B.messages[0])
4530 self.assertEqual(1, len(C.messages))
4531 self.assertIn('Build succeeded.', C.messages[0])
4532 self.assertEqual(1, len(D.messages))
4533 self.assertIn('Build failed.', D.messages[0])
4534 self.assertEqual(1, len(E.messages))
4535 self.assertIn('Build failed.', E.messages[0])
4536 self.assertEqual(1, len(F.messages))
4537 self.assertIn('Build failed.', F.messages[0])
4538
4539 # The last 3 (GHI) would have only reported via smtp.
4540 self.assertEqual(3, len(self.smtp_messages))
4541 self.assertEqual(0, len(G.messages))
4542 self.assertIn('Build failed.', self.smtp_messages[0]['body'])
Paul Belanger7dc76e82016-11-11 16:51:08 -05004543 self.assertIn(
Monty Taylorde8242c2017-02-23 20:29:53 -06004544 'project-test1 finger://', self.smtp_messages[0]['body'])
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004545 self.assertEqual(0, len(H.messages))
4546 self.assertIn('Build failed.', self.smtp_messages[1]['body'])
Paul Belanger7dc76e82016-11-11 16:51:08 -05004547 self.assertIn(
Monty Taylorde8242c2017-02-23 20:29:53 -06004548 'project-test1 finger://', self.smtp_messages[1]['body'])
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004549 self.assertEqual(0, len(I.messages))
4550 self.assertIn('Build succeeded.', self.smtp_messages[2]['body'])
Paul Belanger7dc76e82016-11-11 16:51:08 -05004551 self.assertIn(
Monty Taylorde8242c2017-02-23 20:29:53 -06004552 'project-test1 finger://', self.smtp_messages[2]['body'])
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004553
4554 # Now reload the configuration (simulate a HUP) to check the pipeline
4555 # comes out of disabled
4556 self.sched.reconfigure(self.config)
4557
Jesse Keatingcc67b122017-04-19 14:54:57 -07004558 tenant = self.sched.abide.tenants.get('tenant-one')
Paul Belanger7dc76e82016-11-11 16:51:08 -05004559
4560 self.assertEqual(3, tenant.layout.pipelines['check'].disable_at)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004561 self.assertEqual(
Paul Belanger7dc76e82016-11-11 16:51:08 -05004562 0, tenant.layout.pipelines['check']._consecutive_failures)
4563 self.assertFalse(tenant.layout.pipelines['check']._disabled)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004564
4565 self.fake_gerrit.addEvent(J.getPatchsetCreatedEvent(1))
4566 self.fake_gerrit.addEvent(K.getPatchsetCreatedEvent(1))
4567 self.waitUntilSettled()
4568
4569 self.assertEqual(
Paul Belanger7dc76e82016-11-11 16:51:08 -05004570 2, tenant.layout.pipelines['check']._consecutive_failures)
4571 self.assertFalse(tenant.layout.pipelines['check']._disabled)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004572
4573 # J and K went back to gerrit
4574 self.assertEqual(1, len(J.messages))
4575 self.assertIn('Build failed.', J.messages[0])
4576 self.assertEqual(1, len(K.messages))
4577 self.assertIn('Build failed.', K.messages[0])
4578 # No more messages reported via smtp
4579 self.assertEqual(3, len(self.smtp_messages))
Joshua Heskethd6dbd682015-12-22 10:06:54 +11004580
James E. Blaircc020532017-04-19 13:04:44 -07004581 @simple_layout('layouts/one-job-project.yaml')
4582 def test_one_job_project(self):
4583 "Test that queueing works with one job"
4584 A = self.fake_gerrit.addFakeChange('org/one-job-project',
4585 'master', 'A')
4586 B = self.fake_gerrit.addFakeChange('org/one-job-project',
4587 'master', 'B')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004588 A.addApproval('Code-Review', 2)
4589 B.addApproval('Code-Review', 2)
4590 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
4591 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blaircc020532017-04-19 13:04:44 -07004592 self.waitUntilSettled()
4593
4594 self.assertEqual(A.data['status'], 'MERGED')
4595 self.assertEqual(A.reported, 2)
4596 self.assertEqual(B.data['status'], 'MERGED')
4597 self.assertEqual(B.reported, 2)
4598
Paul Belanger32023542017-10-06 14:48:50 -04004599 def test_job_aborted(self):
4600 "Test that if a execute server aborts a job, it is run again"
4601 self.executor_server.hold_jobs_in_build = True
4602 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
4603 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4604 self.waitUntilSettled()
4605
4606 self.executor_server.release('.*-merge')
4607 self.waitUntilSettled()
4608
4609 self.assertEqual(len(self.builds), 2)
Tobias Henkel26fa8a32017-10-07 07:08:36 +00004610
4611 # first abort
4612 self.builds[0].aborted = True
4613 self.executor_server.release('.*-test*')
4614 self.waitUntilSettled()
4615 self.assertEqual(len(self.builds), 1)
4616
4617 # second abort
4618 self.builds[0].aborted = True
4619 self.executor_server.release('.*-test*')
4620 self.waitUntilSettled()
4621 self.assertEqual(len(self.builds), 1)
4622
4623 # third abort
4624 self.builds[0].aborted = True
4625 self.executor_server.release('.*-test*')
4626 self.waitUntilSettled()
4627 self.assertEqual(len(self.builds), 1)
4628
4629 # fourth abort
Paul Belanger32023542017-10-06 14:48:50 -04004630 self.builds[0].aborted = True
4631 self.executor_server.release('.*-test*')
4632 self.waitUntilSettled()
4633 self.assertEqual(len(self.builds), 1)
4634
4635 self.executor_server.hold_jobs_in_build = False
4636 self.executor_server.release()
4637 self.waitUntilSettled()
4638
Tobias Henkel26fa8a32017-10-07 07:08:36 +00004639 self.assertEqual(len(self.history), 7)
4640 self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 4)
Paul Belanger32023542017-10-06 14:48:50 -04004641 self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 3)
4642
Paul Belanger71d98172016-11-08 10:56:31 -05004643 def test_rerun_on_abort(self):
Paul Belanger174a8272017-03-14 13:20:10 -04004644 "Test that if a execute server fails to run a job, it is run again"
Paul Belanger71d98172016-11-08 10:56:31 -05004645
Paul Belanger174a8272017-03-14 13:20:10 -04004646 self.executor_server.hold_jobs_in_build = True
Paul Belanger71d98172016-11-08 10:56:31 -05004647 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
4648 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4649 self.waitUntilSettled()
4650
Paul Belanger174a8272017-03-14 13:20:10 -04004651 self.executor_server.release('.*-merge')
Paul Belanger71d98172016-11-08 10:56:31 -05004652 self.waitUntilSettled()
4653
4654 self.assertEqual(len(self.builds), 2)
4655 self.builds[0].requeue = True
Paul Belanger174a8272017-03-14 13:20:10 -04004656 self.executor_server.release('.*-test*')
Paul Belanger71d98172016-11-08 10:56:31 -05004657 self.waitUntilSettled()
4658
Clint Byrum5bb5feb2016-12-08 12:39:05 -08004659 for x in range(3):
Clint Byrumdf0a55b2016-12-05 06:39:11 -08004660 self.assertEqual(len(self.builds), 1,
4661 'len of builds at x=%d is wrong' % x)
Paul Belanger71d98172016-11-08 10:56:31 -05004662 self.builds[0].requeue = True
Paul Belanger174a8272017-03-14 13:20:10 -04004663 self.executor_server.release('.*-test1')
Paul Belanger71d98172016-11-08 10:56:31 -05004664 self.waitUntilSettled()
4665
Paul Belanger174a8272017-03-14 13:20:10 -04004666 self.executor_server.hold_jobs_in_build = False
4667 self.executor_server.release()
Paul Belanger71d98172016-11-08 10:56:31 -05004668 self.waitUntilSettled()
Clint Byrum5bb5feb2016-12-08 12:39:05 -08004669 self.assertEqual(len(self.history), 6)
Paul Belanger71d98172016-11-08 10:56:31 -05004670 self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 2)
4671 self.assertEqual(A.reported, 1)
4672 self.assertIn('RETRY_LIMIT', A.messages[0])
Joshua Hesketh3f7def32016-11-21 17:36:44 +11004673
James E. Blair15be0e12017-01-03 13:45:20 -08004674 def test_zookeeper_disconnect(self):
Paul Belanger174a8272017-03-14 13:20:10 -04004675 "Test that jobs are executed after a zookeeper disconnect"
James E. Blair15be0e12017-01-03 13:45:20 -08004676
4677 self.fake_nodepool.paused = True
4678 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004679 A.addApproval('Code-Review', 2)
4680 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair15be0e12017-01-03 13:45:20 -08004681 self.waitUntilSettled()
4682
4683 self.zk.client.stop()
4684 self.zk.client.start()
4685 self.fake_nodepool.paused = False
4686 self.waitUntilSettled()
4687
4688 self.assertEqual(A.data['status'], 'MERGED')
4689 self.assertEqual(A.reported, 2)
4690
James E. Blair6ab79e02017-01-06 10:10:17 -08004691 def test_nodepool_failure(self):
4692 "Test that jobs are reported after a nodepool failure"
4693
4694 self.fake_nodepool.paused = True
4695 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004696 A.addApproval('Code-Review', 2)
4697 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair6ab79e02017-01-06 10:10:17 -08004698 self.waitUntilSettled()
4699
4700 req = self.fake_nodepool.getNodeRequests()[0]
4701 self.fake_nodepool.addFailRequest(req)
4702
4703 self.fake_nodepool.paused = False
4704 self.waitUntilSettled()
4705
4706 self.assertEqual(A.data['status'], 'NEW')
4707 self.assertEqual(A.reported, 2)
4708 self.assertIn('project-merge : NODE_FAILURE', A.messages[1])
4709 self.assertIn('project-test1 : SKIPPED', A.messages[1])
4710 self.assertIn('project-test2 : SKIPPED', A.messages[1])
4711
Monty Taylor6dc5bc12017-09-29 15:47:31 -05004712 def test_nodepool_priority(self):
4713 "Test that nodes are requested at the correct priority"
4714
4715 self.fake_nodepool.paused = True
4716
4717 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
4718 self.fake_gerrit.addEvent(A.getRefUpdatedEvent())
4719
4720 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
4721 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4722
4723 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
4724 C.addApproval('Code-Review', 2)
4725 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
4726
4727 self.waitUntilSettled()
4728
4729 reqs = self.fake_nodepool.getNodeRequests()
4730
4731 # The requests come back sorted by oid. Since we have three requests
4732 # for the three changes each with a different priority.
4733 # Also they get a serial number based on order they were received
4734 # so the number on the endof the oid should map to order submitted.
4735
4736 # * gate first - high priority - change C
4737 self.assertEqual(reqs[0]['_oid'], '100-0000000002')
4738 self.assertEqual(reqs[0]['node_types'], ['label1'])
4739 # * check second - normal priority - change B
4740 self.assertEqual(reqs[1]['_oid'], '200-0000000001')
4741 self.assertEqual(reqs[1]['node_types'], ['label1'])
4742 # * post third - low priority - change A
4743 # additionally, the post job defined uses an ubuntu-xenial node,
4744 # so we include that check just as an extra verification
4745 self.assertEqual(reqs[2]['_oid'], '300-0000000000')
4746 self.assertEqual(reqs[2]['node_types'], ['ubuntu-xenial'])
4747
4748 self.fake_nodepool.paused = False
4749 self.waitUntilSettled()
4750
James E. Blairfceaf412017-09-29 13:44:57 -07004751 @simple_layout('layouts/multiple-templates.yaml')
4752 def test_multiple_project_templates(self):
4753 # Test that applying multiple project templates to a project
4754 # doesn't alter them when used for a second project.
4755 A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
4756 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4757 self.waitUntilSettled()
4758
4759 build = self.getJobFromHistory('py27')
4760 self.assertEqual(build.parameters['zuul']['jobtags'], [])
4761
James E. Blair29c77002017-10-05 14:56:35 -07004762 def test_pending_merge_in_reconfig(self):
4763 # Test that if we are waiting for an outstanding merge on
4764 # reconfiguration that we continue to do so.
4765 self.gearman_server.hold_merge_jobs_in_queue = True
4766 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4767 A.setMerged()
4768 self.fake_gerrit.addEvent(A.getRefUpdatedEvent())
4769 self.waitUntilSettled()
4770 # Reconfigure while we still have an outstanding merge job
4771 self.sched.reconfigureTenant(self.sched.abide.tenants['tenant-one'],
4772 None)
4773 self.waitUntilSettled()
4774 # Verify the merge job is still running and that the item is
4775 # in the pipeline
4776 self.assertEqual(len(self.sched.merger.jobs), 1)
4777 tenant = self.sched.abide.tenants.get('tenant-one')
4778 pipeline = tenant.layout.pipelines['post']
4779 self.assertEqual(len(pipeline.getAllItems()), 1)
4780 self.gearman_server.hold_merge_jobs_in_queue = False
4781 self.gearman_server.release()
4782 self.waitUntilSettled()
4783
Arieb6f068c2016-10-09 13:11:06 +03004784
James E. Blaira002b032017-04-18 10:35:48 -07004785class TestExecutor(ZuulTestCase):
4786 tenant_config_file = 'config/single-tenant/main.yaml'
4787
4788 def assertFinalState(self):
4789 # In this test, we expect to shut down in a non-final state,
4790 # so skip these checks.
4791 pass
4792
4793 def assertCleanShutdown(self):
4794 self.log.debug("Assert clean shutdown")
4795
4796 # After shutdown, make sure no jobs are running
4797 self.assertEqual({}, self.executor_server.job_workers)
4798
4799 # Make sure that git.Repo objects have been garbage collected.
4800 repos = []
4801 gc.collect()
4802 for obj in gc.get_objects():
4803 if isinstance(obj, git.Repo):
4804 self.log.debug("Leaked git repo object: %s" % repr(obj))
4805 repos.append(obj)
4806 self.assertEqual(len(repos), 0)
4807
4808 def test_executor_shutdown(self):
4809 "Test that the executor can shut down with jobs running"
4810
4811 self.executor_server.hold_jobs_in_build = True
4812 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004813 A.addApproval('Code-Review', 2)
4814 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blaira002b032017-04-18 10:35:48 -07004815 self.waitUntilSettled()
4816
4817
Fredrik Medleyf8aec832015-09-28 13:40:20 +02004818class TestDependencyGraph(ZuulTestCase):
4819 tenant_config_file = 'config/dependency-graph/main.yaml'
4820
4821 def test_dependeny_graph_dispatch_jobs_once(self):
4822 "Test a job in a dependency graph is queued only once"
4823 # Job dependencies, starting with A
4824 # A
4825 # / \
4826 # B C
4827 # / \ / \
4828 # D F E
4829 # |
4830 # G
4831
4832 self.executor_server.hold_jobs_in_build = True
4833 change = self.fake_gerrit.addFakeChange(
4834 'org/project', 'master', 'change')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004835 change.addApproval('Code-Review', 2)
4836 self.fake_gerrit.addEvent(change.addApproval('Approved', 1))
Fredrik Medleyf8aec832015-09-28 13:40:20 +02004837
4838 self.waitUntilSettled()
4839 self.assertEqual([b.name for b in self.builds], ['A'])
4840
4841 self.executor_server.release('A')
4842 self.waitUntilSettled()
4843 self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C'])
4844
4845 self.executor_server.release('B')
4846 self.waitUntilSettled()
4847 self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D'])
4848
4849 self.executor_server.release('D')
4850 self.waitUntilSettled()
4851 self.assertEqual([b.name for b in self.builds], ['C'])
4852
4853 self.executor_server.release('C')
4854 self.waitUntilSettled()
4855 self.assertEqual(sorted(b.name for b in self.builds), ['E', 'F'])
4856
4857 self.executor_server.release('F')
4858 self.waitUntilSettled()
4859 self.assertEqual(sorted(b.name for b in self.builds), ['E', 'G'])
4860
4861 self.executor_server.release('G')
4862 self.waitUntilSettled()
4863 self.assertEqual([b.name for b in self.builds], ['E'])
4864
4865 self.executor_server.release('E')
4866 self.waitUntilSettled()
4867 self.assertEqual(len(self.builds), 0)
4868
4869 self.executor_server.hold_jobs_in_build = False
4870 self.executor_server.release()
4871 self.waitUntilSettled()
4872
4873 self.assertEqual(len(self.builds), 0)
4874 self.assertEqual(len(self.history), 7)
4875
4876 self.assertEqual(change.data['status'], 'MERGED')
4877 self.assertEqual(change.reported, 2)
4878
4879 def test_jobs_launched_only_if_all_dependencies_are_successful(self):
4880 "Test that a job waits till all dependencies are successful"
4881 # Job dependencies, starting with A
4882 # A
4883 # / \
4884 # B C*
4885 # / \ / \
4886 # D F E
4887 # |
4888 # G
4889
4890 self.executor_server.hold_jobs_in_build = True
4891 change = self.fake_gerrit.addFakeChange(
4892 'org/project', 'master', 'change')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004893 change.addApproval('Code-Review', 2)
Fredrik Medleyf8aec832015-09-28 13:40:20 +02004894
4895 self.executor_server.failJob('C', change)
4896
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004897 self.fake_gerrit.addEvent(change.addApproval('Approved', 1))
Fredrik Medleyf8aec832015-09-28 13:40:20 +02004898
4899 self.waitUntilSettled()
4900 self.assertEqual([b.name for b in self.builds], ['A'])
4901
4902 self.executor_server.release('A')
4903 self.waitUntilSettled()
4904 self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C'])
4905
4906 self.executor_server.release('B')
4907 self.waitUntilSettled()
4908 self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D'])
4909
4910 self.executor_server.release('D')
4911 self.waitUntilSettled()
4912 self.assertEqual([b.name for b in self.builds], ['C'])
4913
4914 self.executor_server.release('C')
4915 self.waitUntilSettled()
4916 self.assertEqual(len(self.builds), 0)
4917
4918 self.executor_server.hold_jobs_in_build = False
4919 self.executor_server.release()
4920 self.waitUntilSettled()
4921
4922 self.assertEqual(len(self.builds), 0)
4923 self.assertEqual(len(self.history), 4)
4924
4925 self.assertEqual(change.data['status'], 'NEW')
4926 self.assertEqual(change.reported, 2)
4927
4928
Arieb6f068c2016-10-09 13:11:06 +03004929class TestDuplicatePipeline(ZuulTestCase):
4930 tenant_config_file = 'config/duplicate-pipeline/main.yaml'
4931
4932 def test_duplicate_pipelines(self):
4933 "Test that a change matching multiple pipelines works"
4934
4935 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
4936 self.fake_gerrit.addEvent(A.getChangeRestoredEvent())
4937 self.waitUntilSettled()
4938
4939 self.assertHistory([
4940 dict(name='project-test1', result='SUCCESS', changes='1,1',
4941 pipeline='dup1'),
4942 dict(name='project-test1', result='SUCCESS', changes='1,1',
4943 pipeline='dup2'),
Paul Belanger73a7d8e2016-11-09 11:12:42 -05004944 ], ordered=False)
Arieb6f068c2016-10-09 13:11:06 +03004945
4946 self.assertEqual(len(A.messages), 2)
4947
Paul Belangera46a3742016-11-09 14:23:03 -05004948 if 'dup1' in A.messages[0]:
4949 self.assertIn('dup1', A.messages[0])
4950 self.assertNotIn('dup2', A.messages[0])
4951 self.assertIn('project-test1', A.messages[0])
4952 self.assertIn('dup2', A.messages[1])
4953 self.assertNotIn('dup1', A.messages[1])
4954 self.assertIn('project-test1', A.messages[1])
4955 else:
4956 self.assertIn('dup1', A.messages[1])
4957 self.assertNotIn('dup2', A.messages[1])
4958 self.assertIn('project-test1', A.messages[1])
4959 self.assertIn('dup2', A.messages[0])
4960 self.assertNotIn('dup1', A.messages[0])
4961 self.assertIn('project-test1', A.messages[0])
Paul Belangerfac69ba2016-11-03 09:03:13 -04004962
4963
Paul Belanger86085b32016-11-03 12:48:57 -04004964class TestSchedulerTemplatedProject(ZuulTestCase):
4965 tenant_config_file = 'config/templated-project/main.yaml'
4966
Paul Belanger174a8272017-03-14 13:20:10 -04004967 def test_job_from_templates_executed(self):
4968 "Test whether a job generated via a template can be executed"
Paul Belanger86085b32016-11-03 12:48:57 -04004969
4970 A = self.fake_gerrit.addFakeChange(
4971 'org/templated-project', 'master', 'A')
4972 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4973 self.waitUntilSettled()
4974
4975 self.assertEqual(self.getJobFromHistory('project-test1').result,
4976 'SUCCESS')
4977 self.assertEqual(self.getJobFromHistory('project-test2').result,
4978 'SUCCESS')
Paul Belanger3adf72f2016-11-03 14:57:31 -04004979
4980 def test_layered_templates(self):
Paul Belanger174a8272017-03-14 13:20:10 -04004981 "Test whether a job generated via a template can be executed"
Paul Belanger3adf72f2016-11-03 14:57:31 -04004982
4983 A = self.fake_gerrit.addFakeChange(
4984 'org/layered-project', 'master', 'A')
4985 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4986 self.waitUntilSettled()
4987
4988 self.assertEqual(self.getJobFromHistory('project-test1').result,
4989 'SUCCESS')
4990 self.assertEqual(self.getJobFromHistory('project-test2').result,
4991 'SUCCESS')
4992 self.assertEqual(self.getJobFromHistory('layered-project-test3'
4993 ).result, 'SUCCESS')
4994 self.assertEqual(self.getJobFromHistory('layered-project-test4'
4995 ).result, 'SUCCESS')
4996 self.assertEqual(self.getJobFromHistory('layered-project-foo-test5'
4997 ).result, 'SUCCESS')
4998 self.assertEqual(self.getJobFromHistory('project-test6').result,
4999 'SUCCESS')
Adam Gandelman94a60062016-11-21 16:43:14 -08005000
James E. Blaire74f5712017-09-29 15:14:31 -07005001 def test_unimplied_branch_matchers(self):
5002 # This tests that there are no implied branch matchers added
5003 # by project templates.
5004 self.create_branch('org/layered-project', 'stable')
5005
5006 A = self.fake_gerrit.addFakeChange(
5007 'org/layered-project', 'stable', 'A')
5008
5009 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5010 self.waitUntilSettled()
5011
5012 self.assertEqual(self.getJobFromHistory('project-test1').result,
5013 'SUCCESS')
5014 print(self.getJobFromHistory('project-test1').
5015 parameters['zuul']['_inheritance_path'])
5016
5017 def test_implied_branch_matchers(self):
5018 # This tests that there is an implied branch matcher when a
5019 # template is used on an in-repo project pipeline definition.
5020 self.create_branch('untrusted-config', 'stable')
5021 self.fake_gerrit.addEvent(
5022 self.fake_gerrit.getFakeBranchCreatedEvent(
5023 'untrusted-config', 'stable'))
5024 self.waitUntilSettled()
5025
5026 A = self.fake_gerrit.addFakeChange(
5027 'untrusted-config', 'stable', 'A')
5028
5029 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5030 self.waitUntilSettled()
5031
5032 self.assertEqual(self.getJobFromHistory('project-test1').result,
5033 'SUCCESS')
5034 print(self.getJobFromHistory('project-test1').
5035 parameters['zuul']['_inheritance_path'])
5036
Adam Gandelman94a60062016-11-21 16:43:14 -08005037
5038class TestSchedulerSuccessURL(ZuulTestCase):
5039 tenant_config_file = 'config/success-url/main.yaml'
5040
5041 def test_success_url(self):
5042 "Ensure bad build params are ignored"
5043 self.sched.reconfigure(self.config)
5044 self.init_repo('org/docs')
5045
5046 A = self.fake_gerrit.addFakeChange('org/docs', 'master', 'A')
5047 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5048 self.waitUntilSettled()
5049
5050 # Both builds ran: docs-draft-test + docs-draft-test2
5051 self.assertEqual(len(self.history), 2)
5052
5053 # Grab build id
James E. Blaire7576802016-12-21 16:15:00 -08005054 for build in self.history:
5055 if build.name == 'docs-draft-test':
5056 uuid = build.uuid[:7]
Monty Taylorde8242c2017-02-23 20:29:53 -06005057 elif build.name == 'docs-draft-test2':
5058 uuid_test2 = build.uuid
Adam Gandelman94a60062016-11-21 16:43:14 -08005059
5060 # Two msgs: 'Starting...' + results
5061 self.assertEqual(len(self.smtp_messages), 2)
5062 body = self.smtp_messages[1]['body'].splitlines()
5063 self.assertEqual('Build succeeded.', body[0])
5064
5065 self.assertIn(
5066 '- docs-draft-test http://docs-draft.example.org/1/1/1/check/'
5067 'docs-draft-test/%s/publish-docs/' % uuid,
5068 body[2])
5069
Paul Belanger174a8272017-03-14 13:20:10 -04005070 # NOTE: This default URL is currently hard-coded in executor/server.py
Adam Gandelman94a60062016-11-21 16:43:14 -08005071 self.assertIn(
Monty Taylor51139a02016-05-24 11:28:10 -05005072 '- docs-draft-test2 finger://{hostname}/{uuid}'.format(
5073 hostname=self.executor_server.hostname,
Monty Taylorde8242c2017-02-23 20:29:53 -06005074 uuid=uuid_test2),
Adam Gandelman94a60062016-11-21 16:43:14 -08005075 body[3])
Adam Gandelman8bd57102016-12-02 12:58:42 -08005076
5077
Adam Gandelman4da00f62016-12-09 15:47:33 -08005078class TestSchedulerMerges(ZuulTestCase):
5079 tenant_config_file = 'config/merges/main.yaml'
Adam Gandelman8bd57102016-12-02 12:58:42 -08005080
5081 def _test_project_merge_mode(self, mode):
Paul Belanger174a8272017-03-14 13:20:10 -04005082 self.executor_server.keep_jobdir = False
Adam Gandelman8bd57102016-12-02 12:58:42 -08005083 project = 'org/project-%s' % mode
Paul Belanger174a8272017-03-14 13:20:10 -04005084 self.executor_server.hold_jobs_in_build = True
Adam Gandelman8bd57102016-12-02 12:58:42 -08005085 A = self.fake_gerrit.addFakeChange(project, 'master', 'A')
5086 B = self.fake_gerrit.addFakeChange(project, 'master', 'B')
5087 C = self.fake_gerrit.addFakeChange(project, 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02005088 A.addApproval('Code-Review', 2)
5089 B.addApproval('Code-Review', 2)
5090 C.addApproval('Code-Review', 2)
5091 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
5092 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
5093 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
Adam Gandelman8bd57102016-12-02 12:58:42 -08005094 self.waitUntilSettled()
5095
5096 build = self.builds[-1]
James E. Blair2a535672017-04-27 12:03:15 -07005097 path = os.path.join(build.jobdir.src_root, 'review.example.com',
5098 project)
Adam Gandelman8bd57102016-12-02 12:58:42 -08005099 repo = git.Repo(path)
James E. Blair247cab72017-07-20 16:52:36 -07005100 repo_messages = [c.message.strip() for c in repo.iter_commits()]
Adam Gandelman8bd57102016-12-02 12:58:42 -08005101 repo_messages.reverse()
5102
Paul Belanger174a8272017-03-14 13:20:10 -04005103 self.executor_server.hold_jobs_in_build = False
5104 self.executor_server.release()
Adam Gandelman8bd57102016-12-02 12:58:42 -08005105 self.waitUntilSettled()
5106
5107 return repo_messages
5108
5109 def _test_merge(self, mode):
5110 us_path = os.path.join(
5111 self.upstream_root, 'org/project-%s' % mode)
5112 expected_messages = [
5113 'initial commit',
5114 'add content from fixture',
5115 # the intermediate commits order is nondeterministic
5116 "Merge commit 'refs/changes/1/2/1' of %s into HEAD" % us_path,
5117 "Merge commit 'refs/changes/1/3/1' of %s into HEAD" % us_path,
5118 ]
5119 result = self._test_project_merge_mode(mode)
5120 self.assertEqual(result[:2], expected_messages[:2])
5121 self.assertEqual(result[-2:], expected_messages[-2:])
5122
5123 def test_project_merge_mode_merge(self):
5124 self._test_merge('merge')
5125
5126 def test_project_merge_mode_merge_resolve(self):
5127 self._test_merge('merge-resolve')
5128
5129 def test_project_merge_mode_cherrypick(self):
5130 expected_messages = [
5131 'initial commit',
5132 'add content from fixture',
5133 'A-1',
5134 'B-1',
5135 'C-1']
5136 result = self._test_project_merge_mode('cherry-pick')
5137 self.assertEqual(result, expected_messages)
Adam Gandelman4da00f62016-12-09 15:47:33 -08005138
5139 def test_merge_branch(self):
5140 "Test that the right commits are on alternate branches"
5141 self.create_branch('org/project-merge-branches', 'mp')
5142
Paul Belanger174a8272017-03-14 13:20:10 -04005143 self.executor_server.hold_jobs_in_build = True
Adam Gandelman4da00f62016-12-09 15:47:33 -08005144 A = self.fake_gerrit.addFakeChange(
5145 'org/project-merge-branches', 'mp', 'A')
5146 B = self.fake_gerrit.addFakeChange(
5147 'org/project-merge-branches', 'mp', 'B')
5148 C = self.fake_gerrit.addFakeChange(
5149 'org/project-merge-branches', 'mp', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02005150 A.addApproval('Code-Review', 2)
5151 B.addApproval('Code-Review', 2)
5152 C.addApproval('Code-Review', 2)
5153 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
5154 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
5155 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
Adam Gandelman4da00f62016-12-09 15:47:33 -08005156 self.waitUntilSettled()
5157
Paul Belanger174a8272017-03-14 13:20:10 -04005158 self.executor_server.release('.*-merge')
Adam Gandelman4da00f62016-12-09 15:47:33 -08005159 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04005160 self.executor_server.release('.*-merge')
Adam Gandelman4da00f62016-12-09 15:47:33 -08005161 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04005162 self.executor_server.release('.*-merge')
Adam Gandelman4da00f62016-12-09 15:47:33 -08005163 self.waitUntilSettled()
5164
5165 build = self.builds[-1]
James E. Blair3b222492017-07-21 15:17:37 -07005166 self.assertEqual(build.parameters['zuul']['branch'], 'mp')
James E. Blair2a535672017-04-27 12:03:15 -07005167 path = os.path.join(build.jobdir.src_root, 'review.example.com',
5168 'org/project-merge-branches')
Adam Gandelman4da00f62016-12-09 15:47:33 -08005169 repo = git.Repo(path)
5170
James E. Blair247cab72017-07-20 16:52:36 -07005171 repo_messages = [c.message.strip() for c in repo.iter_commits()]
Adam Gandelman4da00f62016-12-09 15:47:33 -08005172 repo_messages.reverse()
5173 correct_messages = [
5174 'initial commit',
5175 'add content from fixture',
5176 'mp commit',
5177 'A-1', 'B-1', 'C-1']
5178 self.assertEqual(repo_messages, correct_messages)
5179
Paul Belanger174a8272017-03-14 13:20:10 -04005180 self.executor_server.hold_jobs_in_build = False
5181 self.executor_server.release()
Adam Gandelman4da00f62016-12-09 15:47:33 -08005182 self.waitUntilSettled()
5183
5184 def test_merge_multi_branch(self):
5185 "Test that dependent changes on multiple branches are merged"
5186 self.create_branch('org/project-merge-branches', 'mp')
5187
Paul Belanger174a8272017-03-14 13:20:10 -04005188 self.executor_server.hold_jobs_in_build = True
Adam Gandelman4da00f62016-12-09 15:47:33 -08005189 A = self.fake_gerrit.addFakeChange(
5190 'org/project-merge-branches', 'master', 'A')
5191 B = self.fake_gerrit.addFakeChange(
5192 'org/project-merge-branches', 'mp', 'B')
5193 C = self.fake_gerrit.addFakeChange(
5194 'org/project-merge-branches', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02005195 A.addApproval('Code-Review', 2)
5196 B.addApproval('Code-Review', 2)
5197 C.addApproval('Code-Review', 2)
5198 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
5199 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
5200 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
Adam Gandelman4da00f62016-12-09 15:47:33 -08005201 self.waitUntilSettled()
5202
5203 job_A = None
5204 for job in self.builds:
5205 if 'project-merge' in job.name:
5206 job_A = job
Adam Gandelman4da00f62016-12-09 15:47:33 -08005207
James E. Blair2a535672017-04-27 12:03:15 -07005208 path = os.path.join(job_A.jobdir.src_root, 'review.example.com',
5209 'org/project-merge-branches')
Adam Gandelman4da00f62016-12-09 15:47:33 -08005210 repo = git.Repo(path)
5211 repo_messages = [c.message.strip()
James E. Blair247cab72017-07-20 16:52:36 -07005212 for c in repo.iter_commits()]
Adam Gandelman4da00f62016-12-09 15:47:33 -08005213 repo_messages.reverse()
5214 correct_messages = [
5215 'initial commit', 'add content from fixture', 'A-1']
5216 self.assertEqual(repo_messages, correct_messages)
5217
Paul Belanger174a8272017-03-14 13:20:10 -04005218 self.executor_server.release('.*-merge')
Adam Gandelman4da00f62016-12-09 15:47:33 -08005219 self.waitUntilSettled()
5220
5221 job_B = None
5222 for job in self.builds:
5223 if 'project-merge' in job.name:
5224 job_B = job
Adam Gandelman4da00f62016-12-09 15:47:33 -08005225
James E. Blair2a535672017-04-27 12:03:15 -07005226 path = os.path.join(job_B.jobdir.src_root, 'review.example.com',
5227 'org/project-merge-branches')
Adam Gandelman4da00f62016-12-09 15:47:33 -08005228 repo = git.Repo(path)
James E. Blair247cab72017-07-20 16:52:36 -07005229 repo_messages = [c.message.strip() for c in repo.iter_commits()]
Adam Gandelman4da00f62016-12-09 15:47:33 -08005230 repo_messages.reverse()
5231 correct_messages = [
5232 'initial commit', 'add content from fixture', 'mp commit', 'B-1']
5233 self.assertEqual(repo_messages, correct_messages)
5234
Paul Belanger174a8272017-03-14 13:20:10 -04005235 self.executor_server.release('.*-merge')
Adam Gandelman4da00f62016-12-09 15:47:33 -08005236 self.waitUntilSettled()
5237
5238 job_C = None
5239 for job in self.builds:
5240 if 'project-merge' in job.name:
5241 job_C = job
James E. Blairb1345832017-07-21 14:51:35 -07005242
James E. Blair2a535672017-04-27 12:03:15 -07005243 path = os.path.join(job_C.jobdir.src_root, 'review.example.com',
5244 'org/project-merge-branches')
Adam Gandelman4da00f62016-12-09 15:47:33 -08005245 repo = git.Repo(path)
James E. Blair247cab72017-07-20 16:52:36 -07005246 repo_messages = [c.message.strip() for c in repo.iter_commits()]
Adam Gandelman4da00f62016-12-09 15:47:33 -08005247
5248 repo_messages.reverse()
5249 correct_messages = [
5250 'initial commit', 'add content from fixture',
5251 'A-1', 'C-1']
5252 # Ensure the right commits are in the history for this ref
5253 self.assertEqual(repo_messages, correct_messages)
5254
Paul Belanger174a8272017-03-14 13:20:10 -04005255 self.executor_server.hold_jobs_in_build = False
5256 self.executor_server.release()
Adam Gandelman4da00f62016-12-09 15:47:33 -08005257 self.waitUntilSettled()
Tobias Henkel9a0e1942017-03-20 16:16:02 +01005258
5259
James E. Blair9ea0d0b2017-04-20 09:27:15 -07005260class TestSemaphore(ZuulTestCase):
5261 tenant_config_file = 'config/semaphore/main.yaml'
5262
5263 def test_semaphore_one(self):
5264 "Test semaphores with max=1 (mutex)"
5265 tenant = self.sched.abide.tenants.get('tenant-one')
5266
5267 self.executor_server.hold_jobs_in_build = True
5268
5269 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
5270 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
5271 self.assertFalse('test-semaphore' in
5272 tenant.semaphore_handler.semaphores)
5273
5274 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5275 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
5276 self.waitUntilSettled()
5277
5278 self.assertEqual(len(self.builds), 3)
5279 self.assertEqual(self.builds[0].name, 'project-test1')
5280 self.assertEqual(self.builds[1].name, 'semaphore-one-test1')
5281 self.assertEqual(self.builds[2].name, 'project-test1')
5282
5283 self.executor_server.release('semaphore-one-test1')
5284 self.waitUntilSettled()
5285
5286 self.assertEqual(len(self.builds), 3)
5287 self.assertEqual(self.builds[0].name, 'project-test1')
5288 self.assertEqual(self.builds[1].name, 'project-test1')
5289 self.assertEqual(self.builds[2].name, 'semaphore-one-test2')
5290 self.assertTrue('test-semaphore' in
5291 tenant.semaphore_handler.semaphores)
5292
5293 self.executor_server.release('semaphore-one-test2')
5294 self.waitUntilSettled()
5295
5296 self.assertEqual(len(self.builds), 3)
5297 self.assertEqual(self.builds[0].name, 'project-test1')
5298 self.assertEqual(self.builds[1].name, 'project-test1')
5299 self.assertEqual(self.builds[2].name, 'semaphore-one-test1')
5300 self.assertTrue('test-semaphore' in
5301 tenant.semaphore_handler.semaphores)
5302
5303 self.executor_server.release('semaphore-one-test1')
5304 self.waitUntilSettled()
5305
5306 self.assertEqual(len(self.builds), 3)
5307 self.assertEqual(self.builds[0].name, 'project-test1')
5308 self.assertEqual(self.builds[1].name, 'project-test1')
5309 self.assertEqual(self.builds[2].name, 'semaphore-one-test2')
5310 self.assertTrue('test-semaphore' in
5311 tenant.semaphore_handler.semaphores)
5312
5313 self.executor_server.release('semaphore-one-test2')
5314 self.waitUntilSettled()
5315
5316 self.assertEqual(len(self.builds), 2)
5317 self.assertEqual(self.builds[0].name, 'project-test1')
5318 self.assertEqual(self.builds[1].name, 'project-test1')
5319 self.assertFalse('test-semaphore' in
5320 tenant.semaphore_handler.semaphores)
5321
5322 self.executor_server.hold_jobs_in_build = False
5323 self.executor_server.release()
5324
5325 self.waitUntilSettled()
5326 self.assertEqual(len(self.builds), 0)
5327
5328 self.assertEqual(A.reported, 1)
5329 self.assertEqual(B.reported, 1)
5330 self.assertFalse('test-semaphore' in
5331 tenant.semaphore_handler.semaphores)
5332
5333 def test_semaphore_two(self):
5334 "Test semaphores with max>1"
5335 tenant = self.sched.abide.tenants.get('tenant-one')
5336
5337 self.executor_server.hold_jobs_in_build = True
5338 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
5339 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
5340 self.assertFalse('test-semaphore-two' in
5341 tenant.semaphore_handler.semaphores)
5342
5343 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5344 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
5345 self.waitUntilSettled()
5346
5347 self.assertEqual(len(self.builds), 4)
5348 self.assertEqual(self.builds[0].name, 'project-test1')
5349 self.assertEqual(self.builds[1].name, 'semaphore-two-test1')
5350 self.assertEqual(self.builds[2].name, 'semaphore-two-test2')
5351 self.assertEqual(self.builds[3].name, 'project-test1')
5352 self.assertTrue('test-semaphore-two' in
5353 tenant.semaphore_handler.semaphores)
5354 self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
5355 'test-semaphore-two', [])), 2)
5356
5357 self.executor_server.release('semaphore-two-test1')
5358 self.waitUntilSettled()
5359
5360 self.assertEqual(len(self.builds), 4)
5361 self.assertEqual(self.builds[0].name, 'project-test1')
5362 self.assertEqual(self.builds[1].name, 'semaphore-two-test2')
5363 self.assertEqual(self.builds[2].name, 'project-test1')
5364 self.assertEqual(self.builds[3].name, 'semaphore-two-test1')
5365 self.assertTrue('test-semaphore-two' in
5366 tenant.semaphore_handler.semaphores)
5367 self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
5368 'test-semaphore-two', [])), 2)
5369
5370 self.executor_server.release('semaphore-two-test2')
5371 self.waitUntilSettled()
5372
5373 self.assertEqual(len(self.builds), 4)
5374 self.assertEqual(self.builds[0].name, 'project-test1')
5375 self.assertEqual(self.builds[1].name, 'project-test1')
5376 self.assertEqual(self.builds[2].name, 'semaphore-two-test1')
5377 self.assertEqual(self.builds[3].name, 'semaphore-two-test2')
5378 self.assertTrue('test-semaphore-two' in
5379 tenant.semaphore_handler.semaphores)
5380 self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
5381 'test-semaphore-two', [])), 2)
5382
5383 self.executor_server.release('semaphore-two-test1')
5384 self.waitUntilSettled()
5385
5386 self.assertEqual(len(self.builds), 3)
5387 self.assertEqual(self.builds[0].name, 'project-test1')
5388 self.assertEqual(self.builds[1].name, 'project-test1')
5389 self.assertEqual(self.builds[2].name, 'semaphore-two-test2')
5390 self.assertTrue('test-semaphore-two' in
5391 tenant.semaphore_handler.semaphores)
5392 self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
5393 'test-semaphore-two', [])), 1)
5394
5395 self.executor_server.release('semaphore-two-test2')
5396 self.waitUntilSettled()
5397
5398 self.assertEqual(len(self.builds), 2)
5399 self.assertEqual(self.builds[0].name, 'project-test1')
5400 self.assertEqual(self.builds[1].name, 'project-test1')
5401 self.assertFalse('test-semaphore-two' in
5402 tenant.semaphore_handler.semaphores)
5403
5404 self.executor_server.hold_jobs_in_build = False
5405 self.executor_server.release()
5406
5407 self.waitUntilSettled()
5408 self.assertEqual(len(self.builds), 0)
5409
5410 self.assertEqual(A.reported, 1)
5411 self.assertEqual(B.reported, 1)
5412
5413 def test_semaphore_abandon(self):
5414 "Test abandon with job semaphores"
5415 self.executor_server.hold_jobs_in_build = True
5416 tenant = self.sched.abide.tenants.get('tenant-one')
5417 check_pipeline = tenant.layout.pipelines['check']
5418
5419 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
5420 self.assertFalse('test-semaphore' in
5421 tenant.semaphore_handler.semaphores)
5422
5423 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5424 self.waitUntilSettled()
5425
5426 self.assertTrue('test-semaphore' in
5427 tenant.semaphore_handler.semaphores)
5428
5429 self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
5430 self.waitUntilSettled()
5431
5432 # The check pipeline should be empty
5433 items = check_pipeline.getAllItems()
5434 self.assertEqual(len(items), 0)
5435
5436 # The semaphore should be released
5437 self.assertFalse('test-semaphore' in
5438 tenant.semaphore_handler.semaphores)
5439
5440 self.executor_server.hold_jobs_in_build = False
5441 self.executor_server.release()
5442 self.waitUntilSettled()
5443
Tobias Henkel0f714002017-06-30 23:30:52 +02005444 def test_semaphore_new_patchset(self):
5445 "Test new patchset with job semaphores"
5446 self.executor_server.hold_jobs_in_build = True
5447 tenant = self.sched.abide.tenants.get('tenant-one')
5448 check_pipeline = tenant.layout.pipelines['check']
5449
5450 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
5451 self.assertFalse('test-semaphore' in
5452 tenant.semaphore_handler.semaphores)
5453
5454 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5455 self.waitUntilSettled()
5456
5457 self.assertTrue('test-semaphore' in
5458 tenant.semaphore_handler.semaphores)
5459 semaphore = tenant.semaphore_handler.semaphores['test-semaphore']
5460 self.assertEqual(len(semaphore), 1)
5461
5462 A.addPatchset()
5463 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
5464 self.waitUntilSettled()
5465
5466 self.assertTrue('test-semaphore' in
5467 tenant.semaphore_handler.semaphores)
5468 semaphore = tenant.semaphore_handler.semaphores['test-semaphore']
5469 self.assertEqual(len(semaphore), 1)
5470
5471 items = check_pipeline.getAllItems()
5472 self.assertEqual(items[0].change.number, '1')
5473 self.assertEqual(items[0].change.patchset, '2')
5474 self.assertTrue(items[0].live)
5475
5476 self.executor_server.hold_jobs_in_build = False
5477 self.executor_server.release()
5478 self.waitUntilSettled()
5479
5480 # The semaphore should be released
5481 self.assertFalse('test-semaphore' in
5482 tenant.semaphore_handler.semaphores)
5483
James E. Blair9ea0d0b2017-04-20 09:27:15 -07005484 def test_semaphore_reconfigure(self):
5485 "Test reconfigure with job semaphores"
5486 self.executor_server.hold_jobs_in_build = True
5487 tenant = self.sched.abide.tenants.get('tenant-one')
5488 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
5489 self.assertFalse('test-semaphore' in
5490 tenant.semaphore_handler.semaphores)
5491
5492 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5493 self.waitUntilSettled()
5494
5495 self.assertTrue('test-semaphore' in
5496 tenant.semaphore_handler.semaphores)
5497
5498 # reconfigure without layout change
5499 self.sched.reconfigure(self.config)
5500 self.waitUntilSettled()
5501 tenant = self.sched.abide.tenants.get('tenant-one')
5502
5503 # semaphore still must be held
5504 self.assertTrue('test-semaphore' in
5505 tenant.semaphore_handler.semaphores)
5506
5507 self.commitConfigUpdate(
5508 'common-config',
5509 'config/semaphore/zuul-reconfiguration.yaml')
5510 self.sched.reconfigure(self.config)
5511 self.waitUntilSettled()
5512 tenant = self.sched.abide.tenants.get('tenant-one')
5513
5514 self.executor_server.release('project-test1')
5515 self.waitUntilSettled()
5516
5517 # There should be no builds anymore
5518 self.assertEqual(len(self.builds), 0)
5519
5520 # The semaphore should be released
5521 self.assertFalse('test-semaphore' in
5522 tenant.semaphore_handler.semaphores)
5523
5524
Tobias Henkel9a0e1942017-03-20 16:16:02 +01005525class TestSemaphoreMultiTenant(ZuulTestCase):
5526 tenant_config_file = 'config/multi-tenant-semaphore/main.yaml'
5527
5528 def test_semaphore_tenant_isolation(self):
5529 "Test semaphores in multiple tenants"
5530
5531 self.waitUntilSettled()
5532 tenant_one = self.sched.abide.tenants.get('tenant-one')
5533 tenant_two = self.sched.abide.tenants.get('tenant-two')
5534
5535 self.executor_server.hold_jobs_in_build = True
5536 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
5537 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
5538 C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
5539 D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')
5540 E = self.fake_gerrit.addFakeChange('org/project2', 'master', 'E')
5541 self.assertFalse('test-semaphore' in
5542 tenant_one.semaphore_handler.semaphores)
5543 self.assertFalse('test-semaphore' in
5544 tenant_two.semaphore_handler.semaphores)
5545
5546 # add patches to project1 of tenant-one
5547 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5548 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
5549 self.waitUntilSettled()
5550
5551 # one build of project1-test1 must run
5552 # semaphore of tenant-one must be acquired once
5553 # semaphore of tenant-two must not be acquired
5554 self.assertEqual(len(self.builds), 1)
5555 self.assertEqual(self.builds[0].name, 'project1-test1')
5556 self.assertTrue('test-semaphore' in
5557 tenant_one.semaphore_handler.semaphores)
5558 self.assertEqual(len(tenant_one.semaphore_handler.semaphores.get(
5559 'test-semaphore', [])), 1)
5560 self.assertFalse('test-semaphore' in
5561 tenant_two.semaphore_handler.semaphores)
5562
5563 # add patches to project2 of tenant-two
5564 self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
5565 self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
5566 self.fake_gerrit.addEvent(E.getPatchsetCreatedEvent(1))
5567 self.waitUntilSettled()
5568
5569 # one build of project1-test1 must run
5570 # two builds of project2-test1 must run
5571 # semaphore of tenant-one must be acquired once
5572 # semaphore of tenant-two must be acquired twice
5573 self.assertEqual(len(self.builds), 3)
5574 self.assertEqual(self.builds[0].name, 'project1-test1')
5575 self.assertEqual(self.builds[1].name, 'project2-test1')
5576 self.assertEqual(self.builds[2].name, 'project2-test1')
5577 self.assertTrue('test-semaphore' in
5578 tenant_one.semaphore_handler.semaphores)
5579 self.assertEqual(len(tenant_one.semaphore_handler.semaphores.get(
5580 'test-semaphore', [])), 1)
5581 self.assertTrue('test-semaphore' in
5582 tenant_two.semaphore_handler.semaphores)
5583 self.assertEqual(len(tenant_two.semaphore_handler.semaphores.get(
5584 'test-semaphore', [])), 2)
5585
5586 self.executor_server.release('project1-test1')
5587 self.waitUntilSettled()
5588
5589 # one build of project1-test1 must run
5590 # two builds of project2-test1 must run
5591 # semaphore of tenant-one must be acquired once
5592 # semaphore of tenant-two must be acquired twice
5593 self.assertEqual(len(self.builds), 3)
5594 self.assertEqual(self.builds[0].name, 'project2-test1')
5595 self.assertEqual(self.builds[1].name, 'project2-test1')
5596 self.assertEqual(self.builds[2].name, 'project1-test1')
5597 self.assertTrue('test-semaphore' in
5598 tenant_one.semaphore_handler.semaphores)
5599 self.assertEqual(len(tenant_one.semaphore_handler.semaphores.get(
5600 'test-semaphore', [])), 1)
5601 self.assertTrue('test-semaphore' in
5602 tenant_two.semaphore_handler.semaphores)
5603 self.assertEqual(len(tenant_two.semaphore_handler.semaphores.get(
5604 'test-semaphore', [])), 2)
5605
5606 self.executor_server.release('project2-test1')
5607 self.waitUntilSettled()
5608
5609 # one build of project1-test1 must run
5610 # one build of project2-test1 must run
5611 # semaphore of tenant-one must be acquired once
5612 # semaphore of tenant-two must be acquired once
5613 self.assertEqual(len(self.builds), 2)
5614 self.assertTrue('test-semaphore' in
5615 tenant_one.semaphore_handler.semaphores)
5616 self.assertEqual(len(tenant_one.semaphore_handler.semaphores.get(
5617 'test-semaphore', [])), 1)
5618 self.assertTrue('test-semaphore' in
5619 tenant_two.semaphore_handler.semaphores)
5620 self.assertEqual(len(tenant_two.semaphore_handler.semaphores.get(
5621 'test-semaphore', [])), 1)
5622
5623 self.executor_server.hold_jobs_in_build = False
5624 self.executor_server.release()
5625
5626 self.waitUntilSettled()
5627
5628 # no build must run
5629 # semaphore of tenant-one must not be acquired
5630 # semaphore of tenant-two must not be acquired
5631 self.assertEqual(len(self.builds), 0)
5632 self.assertFalse('test-semaphore' in
5633 tenant_one.semaphore_handler.semaphores)
5634 self.assertFalse('test-semaphore' in
5635 tenant_two.semaphore_handler.semaphores)
5636
5637 self.assertEqual(A.reported, 1)
5638 self.assertEqual(B.reported, 1)
5639
5640
5641class TestSemaphoreInRepo(ZuulTestCase):
Tobias Henkelabf973e2017-07-28 10:07:34 +02005642 config_file = 'zuul-connections-gerrit-and-github.conf'
Tobias Henkel9a0e1942017-03-20 16:16:02 +01005643 tenant_config_file = 'config/in-repo/main.yaml'
5644
5645 def test_semaphore_in_repo(self):
5646 "Test semaphores in repo config"
5647
5648 # This tests dynamic semaphore handling in project repos. The semaphore
5649 # max value should not be evaluated dynamically but must be updated
5650 # after the change lands.
5651
5652 self.waitUntilSettled()
5653 tenant = self.sched.abide.tenants.get('tenant-one')
5654
5655 in_repo_conf = textwrap.dedent(
5656 """
5657 - job:
Tobias Henkelf02cf512017-07-21 22:55:34 +02005658 name: project-test1
5659
5660 - job:
Tobias Henkel9a0e1942017-03-20 16:16:02 +01005661 name: project-test2
5662 semaphore: test-semaphore
5663
5664 - project:
5665 name: org/project
5666 tenant-one-gate:
5667 jobs:
5668 - project-test2
5669
5670 # the max value in dynamic layout must be ignored
5671 - semaphore:
5672 name: test-semaphore
5673 max: 2
5674 """)
5675
5676 in_repo_playbook = textwrap.dedent(
5677 """
5678 - hosts: all
5679 tasks: []
5680 """)
5681
5682 file_dict = {'.zuul.yaml': in_repo_conf,
5683 'playbooks/project-test2.yaml': in_repo_playbook}
5684 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
5685 files=file_dict)
5686 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
5687 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
5688 B.setDependsOn(A, 1)
5689 C.setDependsOn(A, 1)
5690
5691 self.executor_server.hold_jobs_in_build = True
5692
Tobias Henkelbf24fd12017-07-27 06:13:07 +02005693 A.addApproval('Code-Review', 2)
5694 B.addApproval('Code-Review', 2)
5695 C.addApproval('Code-Review', 2)
5696 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
5697 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
5698 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
Tobias Henkel9a0e1942017-03-20 16:16:02 +01005699 self.waitUntilSettled()
5700
5701 # check that the layout in a queue item still has max value of 1
5702 # for test-semaphore
5703 pipeline = tenant.layout.pipelines.get('tenant-one-gate')
5704 queue = None
5705 for queue_candidate in pipeline.queues:
5706 if queue_candidate.name == 'org/project':
5707 queue = queue_candidate
5708 break
5709 queue_item = queue.queue[0]
James E. Blair29a24fd2017-10-02 15:04:56 -07005710 item_dynamic_layout = queue_item.layout
Tobias Henkel9a0e1942017-03-20 16:16:02 +01005711 dynamic_test_semaphore = \
5712 item_dynamic_layout.semaphores.get('test-semaphore')
5713 self.assertEqual(dynamic_test_semaphore.max, 1)
5714
5715 # one build must be in queue, one semaphores acquired
5716 self.assertEqual(len(self.builds), 1)
5717 self.assertEqual(self.builds[0].name, 'project-test2')
5718 self.assertTrue('test-semaphore' in
5719 tenant.semaphore_handler.semaphores)
5720 self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
5721 'test-semaphore', [])), 1)
5722
5723 self.executor_server.release('project-test2')
5724 self.waitUntilSettled()
5725
5726 # change A must be merged
5727 self.assertEqual(A.data['status'], 'MERGED')
5728 self.assertEqual(A.reported, 2)
5729
5730 # send change-merged event as the gerrit mock doesn't send it
5731 self.fake_gerrit.addEvent(A.getChangeMergedEvent())
5732 self.waitUntilSettled()
5733
5734 # now that change A was merged, the new semaphore max must be effective
5735 tenant = self.sched.abide.tenants.get('tenant-one')
5736 self.assertEqual(tenant.layout.semaphores.get('test-semaphore').max, 2)
5737
5738 # two builds must be in queue, two semaphores acquired
5739 self.assertEqual(len(self.builds), 2)
5740 self.assertEqual(self.builds[0].name, 'project-test2')
5741 self.assertEqual(self.builds[1].name, 'project-test2')
5742 self.assertTrue('test-semaphore' in
5743 tenant.semaphore_handler.semaphores)
5744 self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
5745 'test-semaphore', [])), 2)
5746
5747 self.executor_server.release('project-test2')
5748 self.waitUntilSettled()
5749
5750 self.assertEqual(len(self.builds), 0)
5751 self.assertFalse('test-semaphore' in
5752 tenant.semaphore_handler.semaphores)
5753
5754 self.executor_server.hold_jobs_in_build = False
5755 self.executor_server.release()
5756
5757 self.waitUntilSettled()
5758 self.assertEqual(len(self.builds), 0)
5759
5760 self.assertEqual(A.reported, 2)
5761 self.assertEqual(B.reported, 2)
5762 self.assertEqual(C.reported, 2)
David Shrewsburyf6dc1762017-10-02 13:34:37 -04005763
5764
5765class TestSchedulerBranchMatcher(ZuulTestCase):
5766
5767 @simple_layout('layouts/matcher-test.yaml')
5768 def test_job_branch_ignored(self):
5769 '''
5770 Test that branch matching logic works.
5771
5772 The 'ignore-branch' job has a branch matcher that is supposed to
5773 match every branch except for the 'featureA' branch, so it should
5774 not be run on a change to that branch.
5775 '''
5776 self.create_branch('org/project', 'featureA')
5777 A = self.fake_gerrit.addFakeChange('org/project', 'featureA', 'A')
5778 A.addApproval('Code-Review', 2)
5779 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
5780 self.waitUntilSettled()
5781 self.printHistory()
5782 self.assertEqual(self.getJobFromHistory('project-test1').result,
5783 'SUCCESS')
5784 self.assertJobNotInHistory('ignore-branch')
5785 self.assertEqual(A.data['status'], 'MERGED')
5786 self.assertEqual(A.reported, 2,
5787 "A should report start and success")
5788 self.assertIn('gate', A.messages[1],
5789 "A should transit gate")