blob: cad557e69518347b27315e39dfd710369ac46076 [file] [log] [blame]
James E. Blairb0fcae42012-07-17 11:12:10 -07001#!/usr/bin/env python
2
3# Copyright 2012 Hewlett-Packard Development Company, L.P.
4#
5# Licensed under the Apache License, Version 2.0 (the "License"); you may
6# not use this file except in compliance with the License. You may obtain
7# a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14# License for the specific language governing permissions and limitations
15# under the License.
16
James E. Blaira002b032017-04-18 10:35:48 -070017import gc
James E. Blairb0fcae42012-07-17 11:12:10 -070018import json
Tobias Henkel9a0e1942017-03-20 16:16:02 +010019import textwrap
20
Monty Taylorbc758832013-06-17 17:22:42 -040021import os
James E. Blairb0fcae42012-07-17 11:12:10 -070022import re
James E. Blair4886cc12012-07-18 15:39:41 -070023import shutil
Monty Taylorbc758832013-06-17 17:22:42 -040024import time
Clint Byrum03454a52017-05-26 17:14:02 -070025from unittest import skip
Monty Taylorbc758832013-06-17 17:22:42 -040026
James E. Blair4886cc12012-07-18 15:39:41 -070027import git
Monty Taylorbc758832013-06-17 17:22:42 -040028import testtools
Monty Taylorb934c1a2017-06-16 19:31:47 -050029import urllib
James E. Blairb0fcae42012-07-17 11:12:10 -070030
Maru Newby3fe5f852015-01-13 04:22:14 +000031import zuul.change_matcher
Clint Byrumb9de8be2017-05-27 13:49:19 -070032from zuul.driver.gerrit import gerritreporter
James E. Blairb0fcae42012-07-17 11:12:10 -070033import zuul.scheduler
James E. Blairad28e912013-11-27 10:43:22 -080034import zuul.rpcclient
Paul Belanger9bba4902016-11-02 16:07:33 -040035import zuul.model
James E. Blairb0fcae42012-07-17 11:12:10 -070036
Maru Newby3fe5f852015-01-13 04:22:14 +000037from tests.base import (
Paul Belanger0a21f0a2017-06-13 13:14:42 -040038 SSLZuulTestCase,
Maru Newby3fe5f852015-01-13 04:22:14 +000039 ZuulTestCase,
40 repack_repo,
James E. Blair06cc3922017-04-19 10:08:10 -070041 simple_layout,
Maru Newby3fe5f852015-01-13 04:22:14 +000042)
James E. Blairb0fcae42012-07-17 11:12:10 -070043
James E. Blairb0fcae42012-07-17 11:12:10 -070044
Paul Belanger0a21f0a2017-06-13 13:14:42 -040045class TestSchedulerSSL(SSLZuulTestCase):
46 tenant_config_file = 'config/single-tenant/main.yaml'
47
48 def test_jobs_executed(self):
49 "Test that jobs are executed and a change is merged"
Paul Belanger0a21f0a2017-06-13 13:14:42 -040050 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +020051 A.addApproval('Code-Review', 2)
52 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
Paul Belanger0a21f0a2017-06-13 13:14:42 -040053 self.waitUntilSettled()
54 self.assertEqual(self.getJobFromHistory('project-merge').result,
55 'SUCCESS')
56 self.assertEqual(self.getJobFromHistory('project-test1').result,
57 'SUCCESS')
58 self.assertEqual(self.getJobFromHistory('project-test2').result,
59 'SUCCESS')
60 self.assertEqual(A.data['status'], 'MERGED')
61 self.assertEqual(A.reported, 2)
62 self.assertEqual(self.getJobFromHistory('project-test1').node,
63 'label1')
James E. Blairfcbf95d2017-07-25 16:38:09 -070064 self.assertEqual(self.getJobFromHistory('project-test2').node,
65 'label1')
Paul Belanger0a21f0a2017-06-13 13:14:42 -040066
67
Clark Boylanb640e052014-04-03 16:41:46 -070068class TestScheduler(ZuulTestCase):
James E. Blair552b54f2016-07-22 13:55:32 -070069 tenant_config_file = 'config/single-tenant/main.yaml'
Antoine Mussobd86a312014-01-08 14:51:33 +010070
Paul Belanger174a8272017-03-14 13:20:10 -040071 def test_jobs_executed(self):
72 "Test that jobs are executed and a change is merged"
James E. Blair1f4c2bb2013-04-26 08:40:46 -070073
James E. Blairb0fcae42012-07-17 11:12:10 -070074 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +020075 A.addApproval('Code-Review', 2)
76 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blairb0fcae42012-07-17 11:12:10 -070077 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -040078 self.assertEqual(self.getJobFromHistory('project-merge').result,
79 'SUCCESS')
80 self.assertEqual(self.getJobFromHistory('project-test1').result,
81 'SUCCESS')
82 self.assertEqual(self.getJobFromHistory('project-test2').result,
83 'SUCCESS')
84 self.assertEqual(A.data['status'], 'MERGED')
85 self.assertEqual(A.reported, 2)
James E. Blair34776ee2016-08-25 13:53:54 -070086 self.assertEqual(self.getJobFromHistory('project-test1').node,
James E. Blair16d96a02017-06-08 11:32:56 -070087 'label1')
James E. Blairfcbf95d2017-07-25 16:38:09 -070088 self.assertEqual(self.getJobFromHistory('project-test2').node,
89 'label1')
James E. Blairb0fcae42012-07-17 11:12:10 -070090
James E. Blair80ac1582017-10-09 07:02:40 -070091 for stat in self.statsd.stats:
92 k, v = stat.decode('utf-8').split(':')
93 self.log.debug('stat %s:%s', k, v)
James E. Blair552b54f2016-07-22 13:55:32 -070094 # TODOv3(jeblair): we may want to report stats by tenant (also?).
Monty Taylorf1385042017-05-27 12:25:05 -050095 # Per-driver
96 self.assertReportedStat('zuul.event.gerrit.comment-added', value='1|c')
97 # Per-driver per-connection
98 self.assertReportedStat('zuul.event.gerrit.gerrit.comment-added',
99 value='1|c')
James E. Blair412e5582013-04-22 15:50:12 -0700100 self.assertReportedStat(
James E. Blair80ac1582017-10-09 07:02:40 -0700101 'zuul.tenant.tenant-one.pipeline.gate.current_changes',
102 value='1|g')
James E. Blair412e5582013-04-22 15:50:12 -0700103 self.assertReportedStat(
James E. Blair80ac1582017-10-09 07:02:40 -0700104 'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.'
105 'org_project.master.job.project-merge.SUCCESS', kind='ms')
106 self.assertReportedStat(
107 'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.'
108 'org_project.master.job.project-merge.SUCCESS', value='1|c')
109 self.assertReportedStat(
110 'zuul.tenant.tenant-one.pipeline.gate.resident_time', kind='ms')
111 self.assertReportedStat(
112 'zuul.tenant.tenant-one.pipeline.gate.total_changes', value='1|c')
113 self.assertReportedStat(
114 'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.'
115 'org_project.master.resident_time', kind='ms')
116 self.assertReportedStat(
117 'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.'
118 'org_project.master.total_changes', value='1|c')
James E. Blairfaf81982017-10-10 15:42:26 -0700119 exec_key = 'zuul.executor.%s' % self.executor_server.hostname
120 self.assertReportedStat(exec_key + '.builds', value='1|c')
James E. Blair4f1731b2017-10-10 18:11:42 -0700121 self.assertReportedStat('zuul.nodepool.requested', value='1|c')
122 self.assertReportedStat('zuul.nodepool.requested.label.label1',
123 value='1|c')
124 self.assertReportedStat('zuul.nodepool.fulfilled.label.label1',
125 value='1|c')
126 self.assertReportedStat('zuul.nodepool.requested.size.1', value='1|c')
127 self.assertReportedStat('zuul.nodepool.fulfilled.size.1', value='1|c')
128 self.assertReportedStat('zuul.nodepool.current_requests', value='1|g')
James E. Blairbdd50e62017-10-21 08:18:55 -0700129 self.assertReportedStat('zuul.executors.online', value='1|g')
130 self.assertReportedStat('zuul.executors.accepting', value='1|g')
131 self.assertReportedStat('zuul.mergers.online', value='1|g')
James E. Blair412e5582013-04-22 15:50:12 -0700132
James E. Blair2ef29e92017-07-21 15:25:05 -0700133 for build in self.history:
134 self.assertTrue(build.parameters['zuul']['voting'])
James E. Blair5821bd92015-09-16 08:48:15 -0700135
James E. Blair3cb10702013-08-24 08:56:03 -0700136 def test_initial_pipeline_gauges(self):
137 "Test that each pipeline reported its length on start"
James E. Blair57bf14d2017-10-21 09:09:48 -0700138 self.assertReportedStat('zuul.tenant.tenant-one.pipeline.gate.'
139 'current_changes',
James E. Blair7da51b72016-07-25 15:29:43 -0700140 value='0|g')
James E. Blair57bf14d2017-10-21 09:09:48 -0700141 self.assertReportedStat('zuul.tenant.tenant-one.pipeline.check.'
142 'current_changes',
James E. Blair7da51b72016-07-25 15:29:43 -0700143 value='0|g')
James E. Blair3cb10702013-08-24 08:56:03 -0700144
James E. Blair34776ee2016-08-25 13:53:54 -0700145 def test_job_branch(self):
146 "Test the correct variant of a job runs on a branch"
147 self.create_branch('org/project', 'stable')
148 A = self.fake_gerrit.addFakeChange('org/project', 'stable', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200149 A.addApproval('Code-Review', 2)
150 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair34776ee2016-08-25 13:53:54 -0700151 self.waitUntilSettled()
152 self.assertEqual(self.getJobFromHistory('project-test1').result,
153 'SUCCESS')
154 self.assertEqual(self.getJobFromHistory('project-test2').result,
155 'SUCCESS')
156 self.assertEqual(A.data['status'], 'MERGED')
157 self.assertEqual(A.reported, 2,
158 "A should report start and success")
159 self.assertIn('gate', A.messages[1],
160 "A should transit gate")
161 self.assertEqual(self.getJobFromHistory('project-test1').node,
James E. Blair16d96a02017-06-08 11:32:56 -0700162 'label2')
James E. Blair34776ee2016-08-25 13:53:54 -0700163
James E. Blairb0fcae42012-07-17 11:12:10 -0700164 def test_parallel_changes(self):
165 "Test that changes are tested in parallel and merged in series"
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700166
Paul Belanger174a8272017-03-14 13:20:10 -0400167 self.executor_server.hold_jobs_in_build = True
James E. Blairb0fcae42012-07-17 11:12:10 -0700168 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
169 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
170 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200171 A.addApproval('Code-Review', 2)
172 B.addApproval('Code-Review', 2)
173 C.addApproval('Code-Review', 2)
James E. Blairb0fcae42012-07-17 11:12:10 -0700174
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200175 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
176 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
177 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blairb0fcae42012-07-17 11:12:10 -0700178
179 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400180 self.assertEqual(len(self.builds), 1)
181 self.assertEqual(self.builds[0].name, 'project-merge')
James E. Blair791b5392016-08-03 11:25:56 -0700182 self.assertTrue(self.builds[0].hasChanges(A))
James E. Blairb0fcae42012-07-17 11:12:10 -0700183
Paul Belanger174a8272017-03-14 13:20:10 -0400184 self.executor_server.release('.*-merge')
James E. Blairb0fcae42012-07-17 11:12:10 -0700185 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400186 self.assertEqual(len(self.builds), 3)
187 self.assertEqual(self.builds[0].name, 'project-test1')
James E. Blair791b5392016-08-03 11:25:56 -0700188 self.assertTrue(self.builds[0].hasChanges(A))
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400189 self.assertEqual(self.builds[1].name, 'project-test2')
James E. Blair791b5392016-08-03 11:25:56 -0700190 self.assertTrue(self.builds[1].hasChanges(A))
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400191 self.assertEqual(self.builds[2].name, 'project-merge')
James E. Blair791b5392016-08-03 11:25:56 -0700192 self.assertTrue(self.builds[2].hasChanges(A, B))
James E. Blairb0fcae42012-07-17 11:12:10 -0700193
Paul Belanger174a8272017-03-14 13:20:10 -0400194 self.executor_server.release('.*-merge')
James E. Blairb0fcae42012-07-17 11:12:10 -0700195 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400196 self.assertEqual(len(self.builds), 5)
197 self.assertEqual(self.builds[0].name, 'project-test1')
James E. Blair791b5392016-08-03 11:25:56 -0700198 self.assertTrue(self.builds[0].hasChanges(A))
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400199 self.assertEqual(self.builds[1].name, 'project-test2')
James E. Blair791b5392016-08-03 11:25:56 -0700200 self.assertTrue(self.builds[1].hasChanges(A))
James E. Blairb0fcae42012-07-17 11:12:10 -0700201
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400202 self.assertEqual(self.builds[2].name, 'project-test1')
James E. Blair791b5392016-08-03 11:25:56 -0700203 self.assertTrue(self.builds[2].hasChanges(A, B))
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400204 self.assertEqual(self.builds[3].name, 'project-test2')
James E. Blair791b5392016-08-03 11:25:56 -0700205 self.assertTrue(self.builds[3].hasChanges(A, B))
James E. Blairb0fcae42012-07-17 11:12:10 -0700206
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400207 self.assertEqual(self.builds[4].name, 'project-merge')
James E. Blair791b5392016-08-03 11:25:56 -0700208 self.assertTrue(self.builds[4].hasChanges(A, B, C))
James E. Blairb0fcae42012-07-17 11:12:10 -0700209
Paul Belanger174a8272017-03-14 13:20:10 -0400210 self.executor_server.release('.*-merge')
James E. Blairb0fcae42012-07-17 11:12:10 -0700211 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400212 self.assertEqual(len(self.builds), 6)
213 self.assertEqual(self.builds[0].name, 'project-test1')
James E. Blair791b5392016-08-03 11:25:56 -0700214 self.assertTrue(self.builds[0].hasChanges(A))
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400215 self.assertEqual(self.builds[1].name, 'project-test2')
James E. Blair791b5392016-08-03 11:25:56 -0700216 self.assertTrue(self.builds[1].hasChanges(A))
James E. Blairb0fcae42012-07-17 11:12:10 -0700217
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400218 self.assertEqual(self.builds[2].name, 'project-test1')
James E. Blair791b5392016-08-03 11:25:56 -0700219 self.assertTrue(self.builds[2].hasChanges(A, B))
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400220 self.assertEqual(self.builds[3].name, 'project-test2')
James E. Blair791b5392016-08-03 11:25:56 -0700221 self.assertTrue(self.builds[3].hasChanges(A, B))
James E. Blairb0fcae42012-07-17 11:12:10 -0700222
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400223 self.assertEqual(self.builds[4].name, 'project-test1')
James E. Blair791b5392016-08-03 11:25:56 -0700224 self.assertTrue(self.builds[4].hasChanges(A, B, C))
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400225 self.assertEqual(self.builds[5].name, 'project-test2')
James E. Blair791b5392016-08-03 11:25:56 -0700226 self.assertTrue(self.builds[5].hasChanges(A, B, C))
James E. Blairb0fcae42012-07-17 11:12:10 -0700227
Paul Belanger174a8272017-03-14 13:20:10 -0400228 self.executor_server.hold_jobs_in_build = False
229 self.executor_server.release()
James E. Blairb0fcae42012-07-17 11:12:10 -0700230 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400231 self.assertEqual(len(self.builds), 0)
James E. Blairb0fcae42012-07-17 11:12:10 -0700232
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400233 self.assertEqual(len(self.history), 9)
234 self.assertEqual(A.data['status'], 'MERGED')
235 self.assertEqual(B.data['status'], 'MERGED')
236 self.assertEqual(C.data['status'], 'MERGED')
237 self.assertEqual(A.reported, 2)
238 self.assertEqual(B.reported, 2)
239 self.assertEqual(C.reported, 2)
James E. Blairb02a3bb2012-07-30 17:49:55 -0700240
241 def test_failed_changes(self):
242 "Test that a change behind a failed change is retested"
Paul Belanger174a8272017-03-14 13:20:10 -0400243 self.executor_server.hold_jobs_in_build = True
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700244
James E. Blairb02a3bb2012-07-30 17:49:55 -0700245 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
246 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200247 A.addApproval('Code-Review', 2)
248 B.addApproval('Code-Review', 2)
James E. Blairb02a3bb2012-07-30 17:49:55 -0700249
Paul Belanger174a8272017-03-14 13:20:10 -0400250 self.executor_server.failJob('project-test1', A)
James E. Blairb02a3bb2012-07-30 17:49:55 -0700251
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200252 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
253 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blairb02a3bb2012-07-30 17:49:55 -0700254 self.waitUntilSettled()
James E. Blair2b2a8ab2016-08-11 14:39:11 -0700255 self.assertBuilds([dict(name='project-merge', changes='1,1')])
James E. Blaire2819012013-06-28 17:17:26 -0400256
Paul Belanger174a8272017-03-14 13:20:10 -0400257 self.executor_server.release('.*-merge')
James E. Blaire2819012013-06-28 17:17:26 -0400258 self.waitUntilSettled()
James E. Blair2b2a8ab2016-08-11 14:39:11 -0700259 # A/project-merge is complete
260 self.assertBuilds([
261 dict(name='project-test1', changes='1,1'),
262 dict(name='project-test2', changes='1,1'),
263 dict(name='project-merge', changes='1,1 2,1'),
264 ])
James E. Blaire2819012013-06-28 17:17:26 -0400265
Paul Belanger174a8272017-03-14 13:20:10 -0400266 self.executor_server.release('.*-merge')
James E. Blaire2819012013-06-28 17:17:26 -0400267 self.waitUntilSettled()
James E. Blair2b2a8ab2016-08-11 14:39:11 -0700268 # A/project-merge is complete
269 # B/project-merge is complete
270 self.assertBuilds([
271 dict(name='project-test1', changes='1,1'),
272 dict(name='project-test2', changes='1,1'),
273 dict(name='project-test1', changes='1,1 2,1'),
274 dict(name='project-test2', changes='1,1 2,1'),
275 ])
276
277 # Release project-test1 for A which will fail. This will
Paul Belanger174a8272017-03-14 13:20:10 -0400278 # abort both running B jobs and reexecute project-merge for B.
James E. Blair2b2a8ab2016-08-11 14:39:11 -0700279 self.builds[0].release()
280 self.waitUntilSettled()
281
282 self.orderedRelease()
283 self.assertHistory([
284 dict(name='project-merge', result='SUCCESS', changes='1,1'),
285 dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'),
286 dict(name='project-test1', result='FAILURE', changes='1,1'),
287 dict(name='project-test1', result='ABORTED', changes='1,1 2,1'),
288 dict(name='project-test2', result='ABORTED', changes='1,1 2,1'),
289 dict(name='project-test2', result='SUCCESS', changes='1,1'),
290 dict(name='project-merge', result='SUCCESS', changes='2,1'),
291 dict(name='project-test1', result='SUCCESS', changes='2,1'),
292 dict(name='project-test2', result='SUCCESS', changes='2,1'),
James E. Blairb536ecc2016-08-31 10:11:42 -0700293 ], ordered=False)
James E. Blair2b2a8ab2016-08-11 14:39:11 -0700294
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400295 self.assertEqual(A.data['status'], 'NEW')
296 self.assertEqual(B.data['status'], 'MERGED')
297 self.assertEqual(A.reported, 2)
298 self.assertEqual(B.reported, 2)
James E. Blairb02a3bb2012-07-30 17:49:55 -0700299
300 def test_independent_queues(self):
301 "Test that changes end up in the right queues"
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700302
Paul Belanger174a8272017-03-14 13:20:10 -0400303 self.executor_server.hold_jobs_in_build = True
Zhongyue Luo5d556072012-09-21 02:00:47 +0900304 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
James E. Blairb02a3bb2012-07-30 17:49:55 -0700305 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
306 C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200307 A.addApproval('Code-Review', 2)
308 B.addApproval('Code-Review', 2)
309 C.addApproval('Code-Review', 2)
James E. Blairb02a3bb2012-07-30 17:49:55 -0700310
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200311 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blairf59f3cf2017-02-19 14:50:26 -0800312 self.waitUntilSettled()
James E. Blairb02a3bb2012-07-30 17:49:55 -0700313
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200314 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blairf59f3cf2017-02-19 14:50:26 -0800315 self.waitUntilSettled()
316
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200317 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blairb02a3bb2012-07-30 17:49:55 -0700318 self.waitUntilSettled()
319
320 # There should be one merge job at the head of each queue running
James E. Blair5f0d49e2016-08-19 09:36:50 -0700321 self.assertBuilds([
322 dict(name='project-merge', changes='1,1'),
323 dict(name='project-merge', changes='2,1'),
324 ])
James E. Blairb02a3bb2012-07-30 17:49:55 -0700325
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700326 # Release the current merge builds
James E. Blair5f0d49e2016-08-19 09:36:50 -0700327 self.builds[0].release()
328 self.waitUntilSettled()
329 self.builds[0].release()
James E. Blairb02a3bb2012-07-30 17:49:55 -0700330 self.waitUntilSettled()
331 # Release the merge job for project2 which is behind project1
Paul Belanger174a8272017-03-14 13:20:10 -0400332 self.executor_server.release('.*-merge')
James E. Blairb02a3bb2012-07-30 17:49:55 -0700333 self.waitUntilSettled()
334
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700335 # All the test builds should be running:
James E. Blair5f0d49e2016-08-19 09:36:50 -0700336 self.assertBuilds([
337 dict(name='project-test1', changes='1,1'),
338 dict(name='project-test2', changes='1,1'),
339 dict(name='project-test1', changes='2,1'),
340 dict(name='project-test2', changes='2,1'),
Paul Belanger58bf6912016-11-11 19:36:01 -0500341 dict(name='project1-project2-integration', changes='2,1'),
James E. Blair5f0d49e2016-08-19 09:36:50 -0700342 dict(name='project-test1', changes='2,1 3,1'),
343 dict(name='project-test2', changes='2,1 3,1'),
Clint Byrumcd9393f2016-12-21 12:21:07 -0800344 dict(name='project1-project2-integration', changes='2,1 3,1'),
James E. Blair5f0d49e2016-08-19 09:36:50 -0700345 ])
James E. Blairb02a3bb2012-07-30 17:49:55 -0700346
James E. Blair5f0d49e2016-08-19 09:36:50 -0700347 self.orderedRelease()
348 self.assertHistory([
349 dict(name='project-merge', result='SUCCESS', changes='1,1'),
350 dict(name='project-merge', result='SUCCESS', changes='2,1'),
351 dict(name='project-merge', result='SUCCESS', changes='2,1 3,1'),
352 dict(name='project-test1', result='SUCCESS', changes='1,1'),
353 dict(name='project-test2', result='SUCCESS', changes='1,1'),
354 dict(name='project-test1', result='SUCCESS', changes='2,1'),
355 dict(name='project-test2', result='SUCCESS', changes='2,1'),
Paul Belanger58bf6912016-11-11 19:36:01 -0500356 dict(
357 name='project1-project2-integration',
358 result='SUCCESS',
359 changes='2,1'),
James E. Blair5f0d49e2016-08-19 09:36:50 -0700360 dict(name='project-test1', result='SUCCESS', changes='2,1 3,1'),
361 dict(name='project-test2', result='SUCCESS', changes='2,1 3,1'),
Clint Byrumcd9393f2016-12-21 12:21:07 -0800362 dict(name='project1-project2-integration',
363 result='SUCCESS',
364 changes='2,1 3,1'),
James E. Blair5f0d49e2016-08-19 09:36:50 -0700365 ])
James E. Blairb02a3bb2012-07-30 17:49:55 -0700366
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400367 self.assertEqual(A.data['status'], 'MERGED')
368 self.assertEqual(B.data['status'], 'MERGED')
369 self.assertEqual(C.data['status'], 'MERGED')
370 self.assertEqual(A.reported, 2)
371 self.assertEqual(B.reported, 2)
372 self.assertEqual(C.reported, 2)
James E. Blaird466dc42012-07-31 10:42:56 -0700373
374 def test_failed_change_at_head(self):
375 "Test that if a change at the head fails, jobs behind it are canceled"
James E. Blaird466dc42012-07-31 10:42:56 -0700376
Paul Belanger174a8272017-03-14 13:20:10 -0400377 self.executor_server.hold_jobs_in_build = True
James E. Blaird466dc42012-07-31 10:42:56 -0700378 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
379 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
380 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200381 A.addApproval('Code-Review', 2)
382 B.addApproval('Code-Review', 2)
383 C.addApproval('Code-Review', 2)
James E. Blaird466dc42012-07-31 10:42:56 -0700384
Paul Belanger174a8272017-03-14 13:20:10 -0400385 self.executor_server.failJob('project-test1', A)
James E. Blaird466dc42012-07-31 10:42:56 -0700386
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200387 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
388 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
389 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blaird466dc42012-07-31 10:42:56 -0700390
391 self.waitUntilSettled()
James E. Blaird466dc42012-07-31 10:42:56 -0700392
James E. Blairb536ecc2016-08-31 10:11:42 -0700393 self.assertBuilds([
394 dict(name='project-merge', changes='1,1'),
395 ])
James E. Blaird466dc42012-07-31 10:42:56 -0700396
Paul Belanger174a8272017-03-14 13:20:10 -0400397 self.executor_server.release('.*-merge')
James E. Blaird466dc42012-07-31 10:42:56 -0700398 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -0400399 self.executor_server.release('.*-merge')
James E. Blaird466dc42012-07-31 10:42:56 -0700400 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -0400401 self.executor_server.release('.*-merge')
James E. Blaird466dc42012-07-31 10:42:56 -0700402 self.waitUntilSettled()
403
James E. Blairb536ecc2016-08-31 10:11:42 -0700404 self.assertBuilds([
405 dict(name='project-test1', changes='1,1'),
406 dict(name='project-test2', changes='1,1'),
407 dict(name='project-test1', changes='1,1 2,1'),
408 dict(name='project-test2', changes='1,1 2,1'),
409 dict(name='project-test1', changes='1,1 2,1 3,1'),
410 dict(name='project-test2', changes='1,1 2,1 3,1'),
411 ])
James E. Blaird466dc42012-07-31 10:42:56 -0700412
Monty Taylor6bef8ef2013-06-02 08:17:12 -0400413 self.release(self.builds[0])
James E. Blaird466dc42012-07-31 10:42:56 -0700414 self.waitUntilSettled()
415
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400416 # project-test2, project-merge for B
James E. Blairb536ecc2016-08-31 10:11:42 -0700417 self.assertBuilds([
418 dict(name='project-test2', changes='1,1'),
419 dict(name='project-merge', changes='2,1'),
420 ])
421 # Unordered history comparison because the aborts can finish
422 # in any order.
423 self.assertHistory([
424 dict(name='project-merge', result='SUCCESS',
425 changes='1,1'),
426 dict(name='project-merge', result='SUCCESS',
427 changes='1,1 2,1'),
428 dict(name='project-merge', result='SUCCESS',
429 changes='1,1 2,1 3,1'),
430 dict(name='project-test1', result='FAILURE',
431 changes='1,1'),
432 dict(name='project-test1', result='ABORTED',
433 changes='1,1 2,1'),
434 dict(name='project-test2', result='ABORTED',
435 changes='1,1 2,1'),
436 dict(name='project-test1', result='ABORTED',
437 changes='1,1 2,1 3,1'),
438 dict(name='project-test2', result='ABORTED',
439 changes='1,1 2,1 3,1'),
440 ], ordered=False)
James E. Blaird466dc42012-07-31 10:42:56 -0700441
Paul Belanger174a8272017-03-14 13:20:10 -0400442 self.executor_server.release('.*-merge')
James E. Blaird466dc42012-07-31 10:42:56 -0700443 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -0400444 self.executor_server.release('.*-merge')
James E. Blairb536ecc2016-08-31 10:11:42 -0700445 self.waitUntilSettled()
446 self.orderedRelease()
James E. Blaird466dc42012-07-31 10:42:56 -0700447
James E. Blairb536ecc2016-08-31 10:11:42 -0700448 self.assertBuilds([])
449 self.assertHistory([
450 dict(name='project-merge', result='SUCCESS',
451 changes='1,1'),
452 dict(name='project-merge', result='SUCCESS',
453 changes='1,1 2,1'),
454 dict(name='project-merge', result='SUCCESS',
455 changes='1,1 2,1 3,1'),
456 dict(name='project-test1', result='FAILURE',
457 changes='1,1'),
458 dict(name='project-test1', result='ABORTED',
459 changes='1,1 2,1'),
460 dict(name='project-test2', result='ABORTED',
461 changes='1,1 2,1'),
462 dict(name='project-test1', result='ABORTED',
463 changes='1,1 2,1 3,1'),
464 dict(name='project-test2', result='ABORTED',
465 changes='1,1 2,1 3,1'),
466 dict(name='project-merge', result='SUCCESS',
467 changes='2,1'),
468 dict(name='project-merge', result='SUCCESS',
469 changes='2,1 3,1'),
470 dict(name='project-test2', result='SUCCESS',
471 changes='1,1'),
472 dict(name='project-test1', result='SUCCESS',
473 changes='2,1'),
474 dict(name='project-test2', result='SUCCESS',
475 changes='2,1'),
476 dict(name='project-test1', result='SUCCESS',
477 changes='2,1 3,1'),
478 dict(name='project-test2', result='SUCCESS',
479 changes='2,1 3,1'),
480 ], ordered=False)
481
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400482 self.assertEqual(A.data['status'], 'NEW')
483 self.assertEqual(B.data['status'], 'MERGED')
484 self.assertEqual(C.data['status'], 'MERGED')
485 self.assertEqual(A.reported, 2)
486 self.assertEqual(B.reported, 2)
487 self.assertEqual(C.reported, 2)
James E. Blaird466dc42012-07-31 10:42:56 -0700488
James E. Blair0aac4872013-08-23 14:02:38 -0700489 def test_failed_change_in_middle(self):
490 "Test a failed change in the middle of the queue"
491
Paul Belanger174a8272017-03-14 13:20:10 -0400492 self.executor_server.hold_jobs_in_build = True
James E. Blair0aac4872013-08-23 14:02:38 -0700493 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
494 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
495 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200496 A.addApproval('Code-Review', 2)
497 B.addApproval('Code-Review', 2)
498 C.addApproval('Code-Review', 2)
James E. Blair0aac4872013-08-23 14:02:38 -0700499
Paul Belanger174a8272017-03-14 13:20:10 -0400500 self.executor_server.failJob('project-test1', B)
James E. Blair0aac4872013-08-23 14:02:38 -0700501
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200502 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
503 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
504 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blair0aac4872013-08-23 14:02:38 -0700505
506 self.waitUntilSettled()
507
Paul Belanger174a8272017-03-14 13:20:10 -0400508 self.executor_server.release('.*-merge')
James E. Blair0aac4872013-08-23 14:02:38 -0700509 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -0400510 self.executor_server.release('.*-merge')
James E. Blair0aac4872013-08-23 14:02:38 -0700511 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -0400512 self.executor_server.release('.*-merge')
James E. Blair0aac4872013-08-23 14:02:38 -0700513 self.waitUntilSettled()
514
515 self.assertEqual(len(self.builds), 6)
516 self.assertEqual(self.builds[0].name, 'project-test1')
517 self.assertEqual(self.builds[1].name, 'project-test2')
518 self.assertEqual(self.builds[2].name, 'project-test1')
519 self.assertEqual(self.builds[3].name, 'project-test2')
520 self.assertEqual(self.builds[4].name, 'project-test1')
521 self.assertEqual(self.builds[5].name, 'project-test2')
522
523 self.release(self.builds[2])
524 self.waitUntilSettled()
525
James E. Blair972e3c72013-08-29 12:04:55 -0700526 # project-test1 and project-test2 for A
527 # project-test2 for B
528 # project-merge for C (without B)
529 self.assertEqual(len(self.builds), 4)
James E. Blair0aac4872013-08-23 14:02:38 -0700530 self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 2)
531
Paul Belanger174a8272017-03-14 13:20:10 -0400532 self.executor_server.release('.*-merge')
James E. Blair972e3c72013-08-29 12:04:55 -0700533 self.waitUntilSettled()
534
535 # project-test1 and project-test2 for A
536 # project-test2 for B
537 # project-test1 and project-test2 for C
538 self.assertEqual(len(self.builds), 5)
539
Paul Belanger2e2a0492016-10-30 18:33:37 -0400540 tenant = self.sched.abide.tenants.get('tenant-one')
541 items = tenant.layout.pipelines['gate'].getAllItems()
James E. Blair0aac4872013-08-23 14:02:38 -0700542 builds = items[0].current_build_set.getBuilds()
543 self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1)
544 self.assertEqual(self.countJobResults(builds, None), 2)
545 builds = items[1].current_build_set.getBuilds()
546 self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1)
547 self.assertEqual(self.countJobResults(builds, 'FAILURE'), 1)
548 self.assertEqual(self.countJobResults(builds, None), 1)
549 builds = items[2].current_build_set.getBuilds()
550 self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1)
James E. Blair972e3c72013-08-29 12:04:55 -0700551 self.assertEqual(self.countJobResults(builds, None), 2)
James E. Blair0aac4872013-08-23 14:02:38 -0700552
Paul Belanger174a8272017-03-14 13:20:10 -0400553 self.executor_server.hold_jobs_in_build = False
554 self.executor_server.release()
James E. Blair0aac4872013-08-23 14:02:38 -0700555 self.waitUntilSettled()
556
557 self.assertEqual(len(self.builds), 0)
558 self.assertEqual(len(self.history), 12)
559 self.assertEqual(A.data['status'], 'MERGED')
560 self.assertEqual(B.data['status'], 'NEW')
561 self.assertEqual(C.data['status'], 'MERGED')
562 self.assertEqual(A.reported, 2)
563 self.assertEqual(B.reported, 2)
564 self.assertEqual(C.reported, 2)
565
James E. Blaird466dc42012-07-31 10:42:56 -0700566 def test_failed_change_at_head_with_queue(self):
567 "Test that if a change at the head fails, queued jobs are canceled"
James E. Blaird466dc42012-07-31 10:42:56 -0700568
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700569 self.gearman_server.hold_jobs_in_queue = True
James E. Blaird466dc42012-07-31 10:42:56 -0700570 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
571 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
572 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200573 A.addApproval('Code-Review', 2)
574 B.addApproval('Code-Review', 2)
575 C.addApproval('Code-Review', 2)
James E. Blaird466dc42012-07-31 10:42:56 -0700576
Paul Belanger174a8272017-03-14 13:20:10 -0400577 self.executor_server.failJob('project-test1', A)
James E. Blaird466dc42012-07-31 10:42:56 -0700578
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200579 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
580 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
581 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blaird466dc42012-07-31 10:42:56 -0700582
583 self.waitUntilSettled()
James E. Blair701c5b42013-06-06 09:34:59 -0700584 queue = self.gearman_server.getQueue()
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400585 self.assertEqual(len(self.builds), 0)
586 self.assertEqual(len(queue), 1)
Clint Byrumf322fe22017-05-10 20:53:12 -0700587 self.assertEqual(queue[0].name, b'executor:execute')
588 job_args = json.loads(queue[0].arguments.decode('utf8'))
Clint Byrum8b2bc0c2016-11-16 16:28:50 -0800589 self.assertEqual(job_args['job'], 'project-merge')
590 self.assertEqual(job_args['items'][0]['number'], '%d' % A.number)
James E. Blaird466dc42012-07-31 10:42:56 -0700591
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700592 self.gearman_server.release('.*-merge')
James E. Blaird466dc42012-07-31 10:42:56 -0700593 self.waitUntilSettled()
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700594 self.gearman_server.release('.*-merge')
James E. Blaird466dc42012-07-31 10:42:56 -0700595 self.waitUntilSettled()
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700596 self.gearman_server.release('.*-merge')
James E. Blaird466dc42012-07-31 10:42:56 -0700597 self.waitUntilSettled()
James E. Blair701c5b42013-06-06 09:34:59 -0700598 queue = self.gearman_server.getQueue()
James E. Blaird466dc42012-07-31 10:42:56 -0700599
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400600 self.assertEqual(len(self.builds), 0)
601 self.assertEqual(len(queue), 6)
Clint Byrum8b2bc0c2016-11-16 16:28:50 -0800602
603 self.assertEqual(
Clint Byrumf322fe22017-05-10 20:53:12 -0700604 json.loads(queue[0].arguments.decode('utf8'))['job'],
605 'project-test1')
Clint Byrum8b2bc0c2016-11-16 16:28:50 -0800606 self.assertEqual(
Clint Byrumf322fe22017-05-10 20:53:12 -0700607 json.loads(queue[1].arguments.decode('utf8'))['job'],
608 'project-test2')
Clint Byrum8b2bc0c2016-11-16 16:28:50 -0800609 self.assertEqual(
Clint Byrumf322fe22017-05-10 20:53:12 -0700610 json.loads(queue[2].arguments.decode('utf8'))['job'],
611 'project-test1')
Clint Byrum8b2bc0c2016-11-16 16:28:50 -0800612 self.assertEqual(
Clint Byrumf322fe22017-05-10 20:53:12 -0700613 json.loads(queue[3].arguments.decode('utf8'))['job'],
614 'project-test2')
Clint Byrum8b2bc0c2016-11-16 16:28:50 -0800615 self.assertEqual(
Clint Byrumf322fe22017-05-10 20:53:12 -0700616 json.loads(queue[4].arguments.decode('utf8'))['job'],
617 'project-test1')
Clint Byrum8b2bc0c2016-11-16 16:28:50 -0800618 self.assertEqual(
Clint Byrumf322fe22017-05-10 20:53:12 -0700619 json.loads(queue[5].arguments.decode('utf8'))['job'],
620 'project-test2')
James E. Blaird466dc42012-07-31 10:42:56 -0700621
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700622 self.release(queue[0])
James E. Blaird466dc42012-07-31 10:42:56 -0700623 self.waitUntilSettled()
624
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400625 self.assertEqual(len(self.builds), 0)
James E. Blair701c5b42013-06-06 09:34:59 -0700626 queue = self.gearman_server.getQueue()
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400627 self.assertEqual(len(queue), 2) # project-test2, project-merge for B
628 self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 0)
James E. Blaird466dc42012-07-31 10:42:56 -0700629
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700630 self.gearman_server.hold_jobs_in_queue = False
631 self.gearman_server.release()
James E. Blaird466dc42012-07-31 10:42:56 -0700632 self.waitUntilSettled()
633
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400634 self.assertEqual(len(self.builds), 0)
635 self.assertEqual(len(self.history), 11)
636 self.assertEqual(A.data['status'], 'NEW')
637 self.assertEqual(B.data['status'], 'MERGED')
638 self.assertEqual(C.data['status'], 'MERGED')
639 self.assertEqual(A.reported, 2)
640 self.assertEqual(B.reported, 2)
641 self.assertEqual(C.reported, 2)
James E. Blair8c803f82012-07-31 16:25:42 -0700642
James E. Blairce8a2132016-05-19 15:21:52 -0700643 def _test_time_database(self, iteration):
Paul Belanger174a8272017-03-14 13:20:10 -0400644 self.executor_server.hold_jobs_in_build = True
James E. Blairce8a2132016-05-19 15:21:52 -0700645 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200646 A.addApproval('Code-Review', 2)
647 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blairce8a2132016-05-19 15:21:52 -0700648 self.waitUntilSettled()
649 time.sleep(2)
650
James E. Blairae0f23c2017-09-13 10:55:15 -0600651 data = json.loads(self.sched.formatStatusJSON('tenant-one'))
James E. Blairce8a2132016-05-19 15:21:52 -0700652 found_job = None
653 for pipeline in data['pipelines']:
654 if pipeline['name'] != 'gate':
655 continue
656 for queue in pipeline['change_queues']:
657 for head in queue['heads']:
658 for item in head:
659 for job in item['jobs']:
660 if job['name'] == 'project-merge':
661 found_job = job
662 break
663
664 self.assertIsNotNone(found_job)
665 if iteration == 1:
666 self.assertIsNotNone(found_job['estimated_time'])
667 self.assertIsNone(found_job['remaining_time'])
668 else:
669 self.assertIsNotNone(found_job['estimated_time'])
670 self.assertTrue(found_job['estimated_time'] >= 2)
671 self.assertIsNotNone(found_job['remaining_time'])
672
Paul Belanger174a8272017-03-14 13:20:10 -0400673 self.executor_server.hold_jobs_in_build = False
674 self.executor_server.release()
James E. Blairce8a2132016-05-19 15:21:52 -0700675 self.waitUntilSettled()
676
677 def test_time_database(self):
678 "Test the time database"
679
680 self._test_time_database(1)
681 self._test_time_database(2)
682
James E. Blairfef71632013-09-23 11:15:47 -0700683 def test_two_failed_changes_at_head(self):
684 "Test that changes are reparented correctly if 2 fail at head"
685
Paul Belanger174a8272017-03-14 13:20:10 -0400686 self.executor_server.hold_jobs_in_build = True
James E. Blairfef71632013-09-23 11:15:47 -0700687 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
688 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
689 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200690 A.addApproval('Code-Review', 2)
691 B.addApproval('Code-Review', 2)
692 C.addApproval('Code-Review', 2)
James E. Blairfef71632013-09-23 11:15:47 -0700693
Paul Belanger174a8272017-03-14 13:20:10 -0400694 self.executor_server.failJob('project-test1', A)
695 self.executor_server.failJob('project-test1', B)
James E. Blairfef71632013-09-23 11:15:47 -0700696
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200697 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
698 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
699 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blairfef71632013-09-23 11:15:47 -0700700 self.waitUntilSettled()
701
Paul Belanger174a8272017-03-14 13:20:10 -0400702 self.executor_server.release('.*-merge')
James E. Blairfef71632013-09-23 11:15:47 -0700703 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -0400704 self.executor_server.release('.*-merge')
James E. Blairfef71632013-09-23 11:15:47 -0700705 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -0400706 self.executor_server.release('.*-merge')
James E. Blairfef71632013-09-23 11:15:47 -0700707 self.waitUntilSettled()
708
709 self.assertEqual(len(self.builds), 6)
710 self.assertEqual(self.builds[0].name, 'project-test1')
711 self.assertEqual(self.builds[1].name, 'project-test2')
712 self.assertEqual(self.builds[2].name, 'project-test1')
713 self.assertEqual(self.builds[3].name, 'project-test2')
714 self.assertEqual(self.builds[4].name, 'project-test1')
715 self.assertEqual(self.builds[5].name, 'project-test2')
716
Paul Belanger71dd2fc2016-11-04 19:05:55 -0400717 self.assertTrue(self.builds[0].hasChanges(A))
718 self.assertTrue(self.builds[2].hasChanges(A))
719 self.assertTrue(self.builds[2].hasChanges(B))
720 self.assertTrue(self.builds[4].hasChanges(A))
721 self.assertTrue(self.builds[4].hasChanges(B))
722 self.assertTrue(self.builds[4].hasChanges(C))
James E. Blairfef71632013-09-23 11:15:47 -0700723
724 # Fail change B first
725 self.release(self.builds[2])
726 self.waitUntilSettled()
727
728 # restart of C after B failure
Paul Belanger174a8272017-03-14 13:20:10 -0400729 self.executor_server.release('.*-merge')
James E. Blairfef71632013-09-23 11:15:47 -0700730 self.waitUntilSettled()
731
732 self.assertEqual(len(self.builds), 5)
733 self.assertEqual(self.builds[0].name, 'project-test1')
734 self.assertEqual(self.builds[1].name, 'project-test2')
735 self.assertEqual(self.builds[2].name, 'project-test2')
736 self.assertEqual(self.builds[3].name, 'project-test1')
737 self.assertEqual(self.builds[4].name, 'project-test2')
738
Paul Belanger71dd2fc2016-11-04 19:05:55 -0400739 self.assertTrue(self.builds[1].hasChanges(A))
740 self.assertTrue(self.builds[2].hasChanges(A))
741 self.assertTrue(self.builds[2].hasChanges(B))
742 self.assertTrue(self.builds[4].hasChanges(A))
743 self.assertFalse(self.builds[4].hasChanges(B))
744 self.assertTrue(self.builds[4].hasChanges(C))
James E. Blairfef71632013-09-23 11:15:47 -0700745
746 # Finish running all passing jobs for change A
747 self.release(self.builds[1])
748 self.waitUntilSettled()
749 # Fail and report change A
750 self.release(self.builds[0])
751 self.waitUntilSettled()
752
753 # restart of B,C after A failure
Paul Belanger174a8272017-03-14 13:20:10 -0400754 self.executor_server.release('.*-merge')
James E. Blairfef71632013-09-23 11:15:47 -0700755 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -0400756 self.executor_server.release('.*-merge')
James E. Blairfef71632013-09-23 11:15:47 -0700757 self.waitUntilSettled()
758
759 self.assertEqual(len(self.builds), 4)
760 self.assertEqual(self.builds[0].name, 'project-test1') # B
761 self.assertEqual(self.builds[1].name, 'project-test2') # B
762 self.assertEqual(self.builds[2].name, 'project-test1') # C
763 self.assertEqual(self.builds[3].name, 'project-test2') # C
764
Paul Belanger71dd2fc2016-11-04 19:05:55 -0400765 self.assertFalse(self.builds[1].hasChanges(A))
766 self.assertTrue(self.builds[1].hasChanges(B))
767 self.assertFalse(self.builds[1].hasChanges(C))
James E. Blairfef71632013-09-23 11:15:47 -0700768
Paul Belanger71dd2fc2016-11-04 19:05:55 -0400769 self.assertFalse(self.builds[2].hasChanges(A))
James E. Blairfef71632013-09-23 11:15:47 -0700770 # After A failed and B and C restarted, B should be back in
771 # C's tests because it has not failed yet.
Paul Belanger71dd2fc2016-11-04 19:05:55 -0400772 self.assertTrue(self.builds[2].hasChanges(B))
773 self.assertTrue(self.builds[2].hasChanges(C))
James E. Blairfef71632013-09-23 11:15:47 -0700774
Paul Belanger174a8272017-03-14 13:20:10 -0400775 self.executor_server.hold_jobs_in_build = False
776 self.executor_server.release()
James E. Blairfef71632013-09-23 11:15:47 -0700777 self.waitUntilSettled()
778
779 self.assertEqual(len(self.builds), 0)
780 self.assertEqual(len(self.history), 21)
781 self.assertEqual(A.data['status'], 'NEW')
782 self.assertEqual(B.data['status'], 'NEW')
783 self.assertEqual(C.data['status'], 'MERGED')
784 self.assertEqual(A.reported, 2)
785 self.assertEqual(B.reported, 2)
786 self.assertEqual(C.reported, 2)
787
James E. Blair8c803f82012-07-31 16:25:42 -0700788 def test_patch_order(self):
789 "Test that dependent patches are tested in the right order"
790 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
791 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
792 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200793 A.addApproval('Code-Review', 2)
794 B.addApproval('Code-Review', 2)
795 C.addApproval('Code-Review', 2)
James E. Blair8c803f82012-07-31 16:25:42 -0700796
797 M2 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M2')
798 M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1')
799 M2.setMerged()
800 M1.setMerged()
801
802 # C -> B -> A -> M1 -> M2
803 # M2 is here to make sure it is never queried. If it is, it
804 # means zuul is walking down the entire history of merged
805 # changes.
806
807 C.setDependsOn(B, 1)
808 B.setDependsOn(A, 1)
809 A.setDependsOn(M1, 1)
810 M1.setDependsOn(M2, 1)
811
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200812 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blair8c803f82012-07-31 16:25:42 -0700813
814 self.waitUntilSettled()
815
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400816 self.assertEqual(A.data['status'], 'NEW')
817 self.assertEqual(B.data['status'], 'NEW')
818 self.assertEqual(C.data['status'], 'NEW')
James E. Blair8c803f82012-07-31 16:25:42 -0700819
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200820 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
821 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair8c803f82012-07-31 16:25:42 -0700822
823 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -0400824 self.assertEqual(M2.queried, 0)
825 self.assertEqual(A.data['status'], 'MERGED')
826 self.assertEqual(B.data['status'], 'MERGED')
827 self.assertEqual(C.data['status'], 'MERGED')
828 self.assertEqual(A.reported, 2)
829 self.assertEqual(B.reported, 2)
830 self.assertEqual(C.reported, 2)
James E. Blair8c803f82012-07-31 16:25:42 -0700831
James E. Blair063672f2015-01-29 13:09:12 -0800832 def test_needed_changes_enqueue(self):
833 "Test that a needed change is enqueued ahead"
834 # A Given a git tree like this, if we enqueue
835 # / \ change C, we should walk up and down the tree
836 # B G and enqueue changes in the order ABCDEFG.
837 # /|\ This is also the order that you would get if
838 # *C E F you enqueued changes in the order ABCDEFG, so
839 # / the ordering is stable across re-enqueue events.
840 # D
841
842 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
843 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
844 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
845 D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
846 E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')
847 F = self.fake_gerrit.addFakeChange('org/project', 'master', 'F')
848 G = self.fake_gerrit.addFakeChange('org/project', 'master', 'G')
849 B.setDependsOn(A, 1)
850 C.setDependsOn(B, 1)
851 D.setDependsOn(C, 1)
852 E.setDependsOn(B, 1)
853 F.setDependsOn(B, 1)
854 G.setDependsOn(A, 1)
855
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200856 A.addApproval('Code-Review', 2)
857 B.addApproval('Code-Review', 2)
858 C.addApproval('Code-Review', 2)
859 D.addApproval('Code-Review', 2)
860 E.addApproval('Code-Review', 2)
861 F.addApproval('Code-Review', 2)
862 G.addApproval('Code-Review', 2)
863 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blair063672f2015-01-29 13:09:12 -0800864
865 self.waitUntilSettled()
866
867 self.assertEqual(A.data['status'], 'NEW')
868 self.assertEqual(B.data['status'], 'NEW')
869 self.assertEqual(C.data['status'], 'NEW')
870 self.assertEqual(D.data['status'], 'NEW')
871 self.assertEqual(E.data['status'], 'NEW')
872 self.assertEqual(F.data['status'], 'NEW')
873 self.assertEqual(G.data['status'], 'NEW')
874
875 # We're about to add approvals to changes without adding the
876 # triggering events to Zuul, so that we can be sure that it is
877 # enqueing the changes based on dependencies, not because of
878 # triggering events. Since it will have the changes cached
879 # already (without approvals), we need to clear the cache
880 # first.
Paul Belanger593480a2016-11-02 19:06:28 -0400881 for connection in self.connections.connections.values():
James E. Blairb0a95ab2017-10-18 09:39:18 -0700882 connection.maintainCache([])
James E. Blair063672f2015-01-29 13:09:12 -0800883
Paul Belanger174a8272017-03-14 13:20:10 -0400884 self.executor_server.hold_jobs_in_build = True
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200885 A.addApproval('Approved', 1)
886 B.addApproval('Approved', 1)
887 D.addApproval('Approved', 1)
888 E.addApproval('Approved', 1)
889 F.addApproval('Approved', 1)
890 G.addApproval('Approved', 1)
891 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blair063672f2015-01-29 13:09:12 -0800892
893 for x in range(8):
Paul Belanger174a8272017-03-14 13:20:10 -0400894 self.executor_server.release('.*-merge')
James E. Blair063672f2015-01-29 13:09:12 -0800895 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -0400896 self.executor_server.hold_jobs_in_build = False
897 self.executor_server.release()
James E. Blair063672f2015-01-29 13:09:12 -0800898 self.waitUntilSettled()
899
900 self.assertEqual(A.data['status'], 'MERGED')
901 self.assertEqual(B.data['status'], 'MERGED')
902 self.assertEqual(C.data['status'], 'MERGED')
903 self.assertEqual(D.data['status'], 'MERGED')
904 self.assertEqual(E.data['status'], 'MERGED')
905 self.assertEqual(F.data['status'], 'MERGED')
906 self.assertEqual(G.data['status'], 'MERGED')
907 self.assertEqual(A.reported, 2)
908 self.assertEqual(B.reported, 2)
909 self.assertEqual(C.reported, 2)
910 self.assertEqual(D.reported, 2)
911 self.assertEqual(E.reported, 2)
912 self.assertEqual(F.reported, 2)
913 self.assertEqual(G.reported, 2)
914 self.assertEqual(self.history[6].changes,
915 '1,1 2,1 3,1 4,1 5,1 6,1 7,1')
916
Joshua Hesketh850ccb62014-11-27 11:31:02 +1100917 def test_source_cache(self):
918 "Test that the source cache operates correctly"
Paul Belanger174a8272017-03-14 13:20:10 -0400919 self.executor_server.hold_jobs_in_build = True
James E. Blair0e933c52013-07-11 10:18:52 -0700920
921 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
922 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
923 X = self.fake_gerrit.addFakeChange('org/project', 'master', 'X')
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200924 A.addApproval('Code-Review', 2)
925 B.addApproval('Code-Review', 2)
James E. Blair0e933c52013-07-11 10:18:52 -0700926
927 M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1')
928 M1.setMerged()
929
930 B.setDependsOn(A, 1)
931 A.setDependsOn(M1, 1)
932
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200933 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair0e933c52013-07-11 10:18:52 -0700934 self.fake_gerrit.addEvent(X.getPatchsetCreatedEvent(1))
935
936 self.waitUntilSettled()
937
938 for build in self.builds:
James E. Blaire675d682017-07-21 15:29:35 -0700939 if build.pipeline == 'check':
James E. Blair0e933c52013-07-11 10:18:52 -0700940 build.release()
941 self.waitUntilSettled()
942 for build in self.builds:
James E. Blaire675d682017-07-21 15:29:35 -0700943 if build.pipeline == 'check':
James E. Blair0e933c52013-07-11 10:18:52 -0700944 build.release()
945 self.waitUntilSettled()
946
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200947 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blair0e933c52013-07-11 10:18:52 -0700948 self.waitUntilSettled()
949
Joshua Hesketh352264b2015-08-11 23:42:08 +1000950 self.log.debug("len %s" % self.fake_gerrit._change_cache.keys())
James E. Blair0e933c52013-07-11 10:18:52 -0700951 # there should still be changes in the cache
James E. Blairb0a95ab2017-10-18 09:39:18 -0700952 self.assertNotEqual(len(self.fake_gerrit._change_cache.keys()), 0)
James E. Blair0e933c52013-07-11 10:18:52 -0700953
Paul Belanger174a8272017-03-14 13:20:10 -0400954 self.executor_server.hold_jobs_in_build = False
955 self.executor_server.release()
James E. Blair0e933c52013-07-11 10:18:52 -0700956 self.waitUntilSettled()
957
958 self.assertEqual(A.data['status'], 'MERGED')
959 self.assertEqual(B.data['status'], 'MERGED')
960 self.assertEqual(A.queried, 2) # Initial and isMerged
961 self.assertEqual(B.queried, 3) # Initial A, refresh from B, isMerged
962
James E. Blair8c803f82012-07-31 16:25:42 -0700963 def test_can_merge(self):
James E. Blair4886cc12012-07-18 15:39:41 -0700964 "Test whether a change is ready to merge"
James E. Blair8c803f82012-07-31 16:25:42 -0700965 # TODO: move to test_gerrit (this is a unit test!)
966 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Paul Belanger9bba4902016-11-02 16:07:33 -0400967 tenant = self.sched.abide.tenants.get('tenant-one')
James E. Blair6053de42017-04-05 11:27:11 -0700968 (trusted, project) = tenant.getProject('org/project')
969 source = project.source
Paul Belanger9bba4902016-11-02 16:07:33 -0400970
971 # TODO(pabelanger): As we add more source / trigger APIs we should make
972 # it easier for users to create events for testing.
973 event = zuul.model.TriggerEvent()
974 event.trigger_name = 'gerrit'
975 event.change_number = '1'
976 event.patch_number = '2'
977
978 a = source.getChange(event)
979 mgr = tenant.layout.pipelines['gate'].manager
James E. Blairc0dedf82014-08-06 09:37:52 -0700980 self.assertFalse(source.canMerge(a, mgr.getSubmitAllowNeeds()))
James E. Blair8c803f82012-07-31 16:25:42 -0700981
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200982 A.addApproval('Code-Review', 2)
Paul Belanger9bba4902016-11-02 16:07:33 -0400983 a = source.getChange(event, refresh=True)
James E. Blairc0dedf82014-08-06 09:37:52 -0700984 self.assertFalse(source.canMerge(a, mgr.getSubmitAllowNeeds()))
James E. Blair8c803f82012-07-31 16:25:42 -0700985
Tobias Henkelbf24fd12017-07-27 06:13:07 +0200986 A.addApproval('Approved', 1)
Paul Belanger9bba4902016-11-02 16:07:33 -0400987 a = source.getChange(event, refresh=True)
James E. Blairc0dedf82014-08-06 09:37:52 -0700988 self.assertTrue(source.canMerge(a, mgr.getSubmitAllowNeeds()))
James E. Blair4886cc12012-07-18 15:39:41 -0700989
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -0700990 def test_project_merge_conflict(self):
991 "Test that gate merge conflicts are handled properly"
James E. Blair1f4c2bb2013-04-26 08:40:46 -0700992
993 self.gearman_server.hold_jobs_in_queue = True
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -0700994 A = self.fake_gerrit.addFakeChange('org/project',
995 'master', 'A',
996 files={'conflict': 'foo'})
997 B = self.fake_gerrit.addFakeChange('org/project',
998 'master', 'B',
999 files={'conflict': 'bar'})
1000 C = self.fake_gerrit.addFakeChange('org/project',
James E. Blair6736beb2013-07-11 15:18:15 -07001001 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001002 A.addApproval('Code-Review', 2)
1003 B.addApproval('Code-Review', 2)
1004 C.addApproval('Code-Review', 2)
1005 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
1006 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
1007 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blair973721f2012-08-15 10:19:43 -07001008 self.waitUntilSettled()
1009
James E. Blair6736beb2013-07-11 15:18:15 -07001010 self.assertEqual(A.reported, 1)
James E. Blair6736beb2013-07-11 15:18:15 -07001011 self.assertEqual(C.reported, 1)
1012
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -07001013 self.gearman_server.release('project-merge')
James E. Blair973721f2012-08-15 10:19:43 -07001014 self.waitUntilSettled()
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -07001015 self.gearman_server.release('project-merge')
James E. Blair973721f2012-08-15 10:19:43 -07001016 self.waitUntilSettled()
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -07001017 self.gearman_server.release('project-merge')
James E. Blair973721f2012-08-15 10:19:43 -07001018 self.waitUntilSettled()
James E. Blair972e3c72013-08-29 12:04:55 -07001019
James E. Blair1f4c2bb2013-04-26 08:40:46 -07001020 self.gearman_server.hold_jobs_in_queue = False
1021 self.gearman_server.release()
James E. Blair973721f2012-08-15 10:19:43 -07001022 self.waitUntilSettled()
1023
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001024 self.assertEqual(A.data['status'], 'MERGED')
1025 self.assertEqual(B.data['status'], 'NEW')
1026 self.assertEqual(C.data['status'], 'MERGED')
1027 self.assertEqual(A.reported, 2)
Tobias Henkel9842bd72017-05-16 13:40:03 +02001028 self.assertIn('Merge Failed', B.messages[-1])
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001029 self.assertEqual(C.reported, 2)
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -07001030
1031 self.assertHistory([
1032 dict(name='project-merge', result='SUCCESS', changes='1,1'),
1033 dict(name='project-test1', result='SUCCESS', changes='1,1'),
1034 dict(name='project-test2', result='SUCCESS', changes='1,1'),
1035 dict(name='project-merge', result='SUCCESS', changes='1,1 3,1'),
1036 dict(name='project-test1', result='SUCCESS', changes='1,1 3,1'),
1037 dict(name='project-test2', result='SUCCESS', changes='1,1 3,1'),
1038 ], ordered=False)
1039
1040 def test_delayed_merge_conflict(self):
1041 "Test that delayed check merge conflicts are handled properly"
1042
1043 # Hold jobs in the gearman queue so that we can test whether
James E. Blair1960d682017-04-28 15:44:14 -07001044 # the executor sucesfully merges a change based on an old
1045 # repo state (frozen by the scheduler) which would otherwise
1046 # conflict.
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -07001047 self.gearman_server.hold_jobs_in_queue = True
1048 A = self.fake_gerrit.addFakeChange('org/project',
1049 'master', 'A',
1050 files={'conflict': 'foo'})
1051 B = self.fake_gerrit.addFakeChange('org/project',
1052 'master', 'B',
1053 files={'conflict': 'bar'})
1054 C = self.fake_gerrit.addFakeChange('org/project',
1055 'master', 'C')
1056 C.setDependsOn(B, 1)
1057
1058 # A enters the gate queue; B and C enter the check queue
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001059 A.addApproval('Code-Review', 2)
1060 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -07001061 self.waitUntilSettled()
1062 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
James E. Blair8d144dc2017-05-05 10:13:45 -07001063 self.waitUntilSettled()
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -07001064 self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
1065 self.waitUntilSettled()
1066
1067 self.assertEqual(A.reported, 1)
1068 self.assertEqual(B.reported, 0) # Check does not report start
1069 self.assertEqual(C.reported, 0) # Check does not report start
1070
1071 # A merges while B and C are queued in check
1072 # Release A project-merge
1073 queue = self.gearman_server.getQueue()
1074 self.release(queue[0])
1075 self.waitUntilSettled()
1076
1077 # Release A project-test*
1078 # gate has higher precedence, so A's test jobs are added in
1079 # front of the merge jobs for B and C
1080 queue = self.gearman_server.getQueue()
1081 self.release(queue[0])
1082 self.release(queue[1])
1083 self.waitUntilSettled()
1084
1085 self.assertEqual(A.data['status'], 'MERGED')
1086 self.assertEqual(B.data['status'], 'NEW')
1087 self.assertEqual(C.data['status'], 'NEW')
1088 self.assertEqual(A.reported, 2)
1089 self.assertEqual(B.reported, 0)
1090 self.assertEqual(C.reported, 0)
1091 self.assertHistory([
1092 dict(name='project-merge', result='SUCCESS', changes='1,1'),
1093 dict(name='project-test1', result='SUCCESS', changes='1,1'),
1094 dict(name='project-test2', result='SUCCESS', changes='1,1'),
1095 ], ordered=False)
1096
1097 # B and C report merge conflicts
1098 # Release B project-merge
1099 queue = self.gearman_server.getQueue()
1100 self.release(queue[0])
1101 self.waitUntilSettled()
1102
1103 # Release C
1104 self.gearman_server.hold_jobs_in_queue = False
1105 self.gearman_server.release()
1106 self.waitUntilSettled()
1107
1108 self.assertEqual(A.data['status'], 'MERGED')
1109 self.assertEqual(B.data['status'], 'NEW')
1110 self.assertEqual(C.data['status'], 'NEW')
1111 self.assertEqual(A.reported, 2)
1112 self.assertEqual(B.reported, 1)
1113 self.assertEqual(C.reported, 1)
1114
1115 self.assertHistory([
1116 dict(name='project-merge', result='SUCCESS', changes='1,1'),
1117 dict(name='project-test1', result='SUCCESS', changes='1,1'),
1118 dict(name='project-test2', result='SUCCESS', changes='1,1'),
James E. Blair1960d682017-04-28 15:44:14 -07001119 dict(name='project-merge', result='SUCCESS', changes='2,1'),
1120 dict(name='project-test1', result='SUCCESS', changes='2,1'),
1121 dict(name='project-test2', result='SUCCESS', changes='2,1'),
1122 dict(name='project-merge', result='SUCCESS', changes='2,1 3,1'),
1123 dict(name='project-test1', result='SUCCESS', changes='2,1 3,1'),
1124 dict(name='project-test2', result='SUCCESS', changes='2,1 3,1'),
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -07001125 ], ordered=False)
James E. Blair6736beb2013-07-11 15:18:15 -07001126
James E. Blairdaabed22012-08-15 15:38:57 -07001127 def test_post(self):
1128 "Test that post jobs run"
James E. Blair289f5932017-07-27 15:02:29 -07001129 p = "review.example.com/org/project"
1130 upstream = self.getUpstreamRepos([p])
1131 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1132 A.setMerged()
1133 A_commit = str(upstream[p].commit('master'))
1134 self.log.debug("A commit: %s" % A_commit)
James E. Blair1f4c2bb2013-04-26 08:40:46 -07001135
Zhongyue Luo5d556072012-09-21 02:00:47 +09001136 e = {
1137 "type": "ref-updated",
1138 "submitter": {
1139 "name": "User Name",
1140 },
1141 "refUpdate": {
1142 "oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
James E. Blair289f5932017-07-27 15:02:29 -07001143 "newRev": A_commit,
Zhongyue Luo5d556072012-09-21 02:00:47 +09001144 "refName": "master",
1145 "project": "org/project",
1146 }
1147 }
James E. Blairdaabed22012-08-15 15:38:57 -07001148 self.fake_gerrit.addEvent(e)
1149 self.waitUntilSettled()
1150
Monty Taylor6bef8ef2013-06-02 08:17:12 -04001151 job_names = [x.name for x in self.history]
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001152 self.assertEqual(len(self.history), 1)
1153 self.assertIn('project-post', job_names)
James E. Blairc6294a52012-08-17 10:19:48 -07001154
K Jonathan Harkerf95e7232015-04-29 13:33:16 -07001155 def test_post_ignore_deletes(self):
1156 "Test that deleting refs does not trigger post jobs"
1157
1158 e = {
1159 "type": "ref-updated",
1160 "submitter": {
1161 "name": "User Name",
1162 },
1163 "refUpdate": {
1164 "oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
1165 "newRev": "0000000000000000000000000000000000000000",
1166 "refName": "master",
1167 "project": "org/project",
1168 }
1169 }
1170 self.fake_gerrit.addEvent(e)
1171 self.waitUntilSettled()
1172
1173 job_names = [x.name for x in self.history]
1174 self.assertEqual(len(self.history), 0)
1175 self.assertNotIn('project-post', job_names)
1176
James E. Blaircc324192017-04-19 14:36:08 -07001177 @simple_layout('layouts/dont-ignore-ref-deletes.yaml')
K Jonathan Harkerf95e7232015-04-29 13:33:16 -07001178 def test_post_ignore_deletes_negative(self):
1179 "Test that deleting refs does trigger post jobs"
K Jonathan Harkerf95e7232015-04-29 13:33:16 -07001180 e = {
1181 "type": "ref-updated",
1182 "submitter": {
1183 "name": "User Name",
1184 },
1185 "refUpdate": {
1186 "oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
1187 "newRev": "0000000000000000000000000000000000000000",
James E. Blair289f5932017-07-27 15:02:29 -07001188 "refName": "testbranch",
K Jonathan Harkerf95e7232015-04-29 13:33:16 -07001189 "project": "org/project",
1190 }
1191 }
1192 self.fake_gerrit.addEvent(e)
1193 self.waitUntilSettled()
1194
1195 job_names = [x.name for x in self.history]
1196 self.assertEqual(len(self.history), 1)
1197 self.assertIn('project-post', job_names)
1198
James E. Blairec056492016-07-22 09:45:56 -07001199 @skip("Disabled for early v3 development")
James E. Blairc6294a52012-08-17 10:19:48 -07001200 def test_build_configuration_branch_interaction(self):
1201 "Test that switching between branches works"
1202 self.test_build_configuration()
1203 self.test_build_configuration_branch()
1204 # C has been merged, undo that
Monty Taylorbc758832013-06-17 17:22:42 -04001205 path = os.path.join(self.upstream_root, "org/project")
James E. Blairc6294a52012-08-17 10:19:48 -07001206 repo = git.Repo(path)
1207 repo.heads.master.commit = repo.commit('init')
1208 self.test_build_configuration()
1209
James E. Blairdb93b302017-07-19 15:33:11 -07001210 def test_dependent_changes_rebase(self):
1211 # Test that no errors occur when we walk a dependency tree
1212 # with an unused leaf node due to a rebase.
1213 # Start by constructing: C -> B -> A
1214 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1215 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
1216 B.setDependsOn(A, 1)
1217
1218 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
1219 C.setDependsOn(B, 1)
1220
1221 # Then rebase to form: D -> C -> A
1222 C.addPatchset() # C,2
1223 C.setDependsOn(A, 1)
1224
1225 D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
1226 D.setDependsOn(C, 2)
1227
1228 # Walk the entire tree
1229 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
1230 self.waitUntilSettled()
1231 self.assertEqual(len(self.history), 3)
1232
1233 # Verify that walking just part of the tree still works
1234 self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
1235 self.waitUntilSettled()
1236 self.assertEqual(len(self.history), 6)
1237
James E. Blaircaec0c52012-08-22 14:52:22 -07001238 def test_dependent_changes_dequeue(self):
1239 "Test that dependent patches are not needlessly tested"
James E. Blair1f4c2bb2013-04-26 08:40:46 -07001240
James E. Blaircaec0c52012-08-22 14:52:22 -07001241 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1242 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
1243 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001244 A.addApproval('Code-Review', 2)
1245 B.addApproval('Code-Review', 2)
1246 C.addApproval('Code-Review', 2)
James E. Blaircaec0c52012-08-22 14:52:22 -07001247
1248 M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1')
1249 M1.setMerged()
1250
1251 # C -> B -> A -> M1
1252
1253 C.setDependsOn(B, 1)
1254 B.setDependsOn(A, 1)
1255 A.setDependsOn(M1, 1)
1256
Paul Belanger174a8272017-03-14 13:20:10 -04001257 self.executor_server.failJob('project-merge', A)
James E. Blaircaec0c52012-08-22 14:52:22 -07001258
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001259 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
1260 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
1261 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blaircaec0c52012-08-22 14:52:22 -07001262
1263 self.waitUntilSettled()
1264
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001265 self.assertEqual(A.data['status'], 'NEW')
1266 self.assertEqual(A.reported, 2)
1267 self.assertEqual(B.data['status'], 'NEW')
1268 self.assertEqual(B.reported, 2)
1269 self.assertEqual(C.data['status'], 'NEW')
James E. Blair0d3e83b2017-06-05 13:51:57 -07001270 self.assertIn('This change depends on a change that failed to merge.',
1271 C.messages[-1])
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001272 self.assertEqual(len(self.history), 1)
James E. Blairec590122012-08-22 15:19:31 -07001273
James E. Blair972e3c72013-08-29 12:04:55 -07001274 def test_failing_dependent_changes(self):
1275 "Test that failing dependent patches are taken out of stream"
Paul Belanger174a8272017-03-14 13:20:10 -04001276 self.executor_server.hold_jobs_in_build = True
James E. Blair972e3c72013-08-29 12:04:55 -07001277 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1278 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
1279 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
1280 D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
1281 E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001282 A.addApproval('Code-Review', 2)
1283 B.addApproval('Code-Review', 2)
1284 C.addApproval('Code-Review', 2)
1285 D.addApproval('Code-Review', 2)
1286 E.addApproval('Code-Review', 2)
James E. Blair972e3c72013-08-29 12:04:55 -07001287
1288 # E, D -> C -> B, A
1289
1290 D.setDependsOn(C, 1)
1291 C.setDependsOn(B, 1)
1292
Paul Belanger174a8272017-03-14 13:20:10 -04001293 self.executor_server.failJob('project-test1', B)
James E. Blair972e3c72013-08-29 12:04:55 -07001294
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001295 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
1296 self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
1297 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
1298 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
1299 self.fake_gerrit.addEvent(E.addApproval('Approved', 1))
James E. Blair972e3c72013-08-29 12:04:55 -07001300
1301 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001302 self.executor_server.release('.*-merge')
James E. Blair972e3c72013-08-29 12:04:55 -07001303 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001304 self.executor_server.release('.*-merge')
James E. Blair972e3c72013-08-29 12:04:55 -07001305 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001306 self.executor_server.release('.*-merge')
James E. Blair972e3c72013-08-29 12:04:55 -07001307 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001308 self.executor_server.release('.*-merge')
James E. Blair972e3c72013-08-29 12:04:55 -07001309 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001310 self.executor_server.release('.*-merge')
James E. Blair972e3c72013-08-29 12:04:55 -07001311 self.waitUntilSettled()
1312
Paul Belanger174a8272017-03-14 13:20:10 -04001313 self.executor_server.hold_jobs_in_build = False
James E. Blair972e3c72013-08-29 12:04:55 -07001314 for build in self.builds:
James E. Blaire3db2952017-07-21 15:03:36 -07001315 if build.parameters['zuul']['change'] != '1':
James E. Blair972e3c72013-08-29 12:04:55 -07001316 build.release()
1317 self.waitUntilSettled()
1318
Paul Belanger174a8272017-03-14 13:20:10 -04001319 self.executor_server.release()
James E. Blair972e3c72013-08-29 12:04:55 -07001320 self.waitUntilSettled()
1321
1322 self.assertEqual(A.data['status'], 'MERGED')
1323 self.assertEqual(A.reported, 2)
Jeremy Stanley10837132014-08-02 16:10:56 +00001324 self.assertIn('Build succeeded', A.messages[1])
James E. Blair972e3c72013-08-29 12:04:55 -07001325 self.assertEqual(B.data['status'], 'NEW')
1326 self.assertEqual(B.reported, 2)
Jeremy Stanley10837132014-08-02 16:10:56 +00001327 self.assertIn('Build failed', B.messages[1])
James E. Blair972e3c72013-08-29 12:04:55 -07001328 self.assertEqual(C.data['status'], 'NEW')
1329 self.assertEqual(C.reported, 2)
Jeremy Stanley10837132014-08-02 16:10:56 +00001330 self.assertIn('depends on a change', C.messages[1])
James E. Blair972e3c72013-08-29 12:04:55 -07001331 self.assertEqual(D.data['status'], 'NEW')
1332 self.assertEqual(D.reported, 2)
Jeremy Stanley10837132014-08-02 16:10:56 +00001333 self.assertIn('depends on a change', D.messages[1])
James E. Blair972e3c72013-08-29 12:04:55 -07001334 self.assertEqual(E.data['status'], 'MERGED')
1335 self.assertEqual(E.reported, 2)
Jeremy Stanley10837132014-08-02 16:10:56 +00001336 self.assertIn('Build succeeded', E.messages[1])
James E. Blair972e3c72013-08-29 12:04:55 -07001337 self.assertEqual(len(self.history), 18)
1338
James E. Blairec590122012-08-22 15:19:31 -07001339 def test_head_is_dequeued_once(self):
James E. Blair2fa50962013-01-30 21:50:41 -08001340 "Test that if a change at the head fails it is dequeued only once"
James E. Blairec590122012-08-22 15:19:31 -07001341 # If it's dequeued more than once, we should see extra
1342 # aborted jobs.
James E. Blairec590122012-08-22 15:19:31 -07001343
Paul Belanger174a8272017-03-14 13:20:10 -04001344 self.executor_server.hold_jobs_in_build = True
James E. Blair6df67132017-04-19 13:53:20 -07001345 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1346 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
1347 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001348 A.addApproval('Code-Review', 2)
1349 B.addApproval('Code-Review', 2)
1350 C.addApproval('Code-Review', 2)
James E. Blairec590122012-08-22 15:19:31 -07001351
Paul Belanger174a8272017-03-14 13:20:10 -04001352 self.executor_server.failJob('project-test1', A)
1353 self.executor_server.failJob('project-test2', A)
James E. Blairec590122012-08-22 15:19:31 -07001354
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001355 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
1356 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
1357 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blairec590122012-08-22 15:19:31 -07001358
1359 self.waitUntilSettled()
James E. Blairec590122012-08-22 15:19:31 -07001360
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001361 self.assertEqual(len(self.builds), 1)
Paul Belanger58bf6912016-11-11 19:36:01 -05001362 self.assertEqual(self.builds[0].name, 'project-merge')
1363 self.assertTrue(self.builds[0].hasChanges(A))
James E. Blairec590122012-08-22 15:19:31 -07001364
Paul Belanger174a8272017-03-14 13:20:10 -04001365 self.executor_server.release('.*-merge')
James E. Blairec590122012-08-22 15:19:31 -07001366 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001367 self.executor_server.release('.*-merge')
James E. Blairec590122012-08-22 15:19:31 -07001368 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001369 self.executor_server.release('.*-merge')
James E. Blairec590122012-08-22 15:19:31 -07001370 self.waitUntilSettled()
1371
James E. Blair6df67132017-04-19 13:53:20 -07001372 self.assertEqual(len(self.builds), 6)
Paul Belanger58bf6912016-11-11 19:36:01 -05001373 self.assertEqual(self.builds[0].name, 'project-test1')
1374 self.assertEqual(self.builds[1].name, 'project-test2')
James E. Blair6df67132017-04-19 13:53:20 -07001375 self.assertEqual(self.builds[2].name, 'project-test1')
1376 self.assertEqual(self.builds[3].name, 'project-test2')
1377 self.assertEqual(self.builds[4].name, 'project-test1')
1378 self.assertEqual(self.builds[5].name, 'project-test2')
James E. Blairec590122012-08-22 15:19:31 -07001379
Monty Taylor6bef8ef2013-06-02 08:17:12 -04001380 self.release(self.builds[0])
James E. Blairec590122012-08-22 15:19:31 -07001381 self.waitUntilSettled()
1382
James E. Blair6df67132017-04-19 13:53:20 -07001383 self.assertEqual(len(self.builds), 2) # test2, merge for B
1384 self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 4)
James E. Blairec590122012-08-22 15:19:31 -07001385
Paul Belanger174a8272017-03-14 13:20:10 -04001386 self.executor_server.hold_jobs_in_build = False
1387 self.executor_server.release()
James E. Blairec590122012-08-22 15:19:31 -07001388 self.waitUntilSettled()
1389
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001390 self.assertEqual(len(self.builds), 0)
James E. Blair6df67132017-04-19 13:53:20 -07001391 self.assertEqual(len(self.history), 15)
James E. Blaircaec0c52012-08-22 14:52:22 -07001392
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001393 self.assertEqual(A.data['status'], 'NEW')
1394 self.assertEqual(B.data['status'], 'MERGED')
1395 self.assertEqual(C.data['status'], 'MERGED')
1396 self.assertEqual(A.reported, 2)
1397 self.assertEqual(B.reported, 2)
1398 self.assertEqual(C.reported, 2)
James E. Blair4ec821f2012-08-23 15:28:28 -07001399
James E. Blaircc020532017-04-19 13:04:44 -07001400 @simple_layout('layouts/nonvoting-job.yaml')
James E. Blair4ec821f2012-08-23 15:28:28 -07001401 def test_nonvoting_job(self):
1402 "Test that non-voting jobs don't vote."
James E. Blair1f4c2bb2013-04-26 08:40:46 -07001403
James E. Blair4ec821f2012-08-23 15:28:28 -07001404 A = self.fake_gerrit.addFakeChange('org/nonvoting-project',
1405 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001406 A.addApproval('Code-Review', 2)
Paul Belanger174a8272017-03-14 13:20:10 -04001407 self.executor_server.failJob('nonvoting-project-test2', A)
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001408 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair4ec821f2012-08-23 15:28:28 -07001409
1410 self.waitUntilSettled()
James E. Blair4ec821f2012-08-23 15:28:28 -07001411
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001412 self.assertEqual(A.data['status'], 'MERGED')
1413 self.assertEqual(A.reported, 2)
1414 self.assertEqual(
1415 self.getJobFromHistory('nonvoting-project-merge').result,
1416 'SUCCESS')
1417 self.assertEqual(
1418 self.getJobFromHistory('nonvoting-project-test1').result,
1419 'SUCCESS')
1420 self.assertEqual(
1421 self.getJobFromHistory('nonvoting-project-test2').result,
1422 'FAILURE')
James E. Blaire0487072012-08-29 17:38:31 -07001423
James E. Blair2ef29e92017-07-21 15:25:05 -07001424 self.assertTrue(self.getJobFromHistory('nonvoting-project-merge').
1425 parameters['zuul']['voting'])
1426 self.assertTrue(self.getJobFromHistory('nonvoting-project-test1').
1427 parameters['zuul']['voting'])
1428 self.assertFalse(self.getJobFromHistory('nonvoting-project-test2').
1429 parameters['zuul']['voting'])
James E. Blair5821bd92015-09-16 08:48:15 -07001430
James E. Blaire0487072012-08-29 17:38:31 -07001431 def test_check_queue_success(self):
1432 "Test successful check queue jobs."
James E. Blair1f4c2bb2013-04-26 08:40:46 -07001433
James E. Blaire0487072012-08-29 17:38:31 -07001434 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1435 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
1436
1437 self.waitUntilSettled()
James E. Blaire0487072012-08-29 17:38:31 -07001438
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001439 self.assertEqual(A.data['status'], 'NEW')
1440 self.assertEqual(A.reported, 1)
1441 self.assertEqual(self.getJobFromHistory('project-merge').result,
1442 'SUCCESS')
1443 self.assertEqual(self.getJobFromHistory('project-test1').result,
1444 'SUCCESS')
1445 self.assertEqual(self.getJobFromHistory('project-test2').result,
1446 'SUCCESS')
James E. Blaire0487072012-08-29 17:38:31 -07001447
1448 def test_check_queue_failure(self):
1449 "Test failed check queue jobs."
James E. Blair1f4c2bb2013-04-26 08:40:46 -07001450
James E. Blaire0487072012-08-29 17:38:31 -07001451 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Paul Belanger174a8272017-03-14 13:20:10 -04001452 self.executor_server.failJob('project-test2', A)
James E. Blaire0487072012-08-29 17:38:31 -07001453 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
1454
1455 self.waitUntilSettled()
James E. Blaire0487072012-08-29 17:38:31 -07001456
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001457 self.assertEqual(A.data['status'], 'NEW')
1458 self.assertEqual(A.reported, 1)
1459 self.assertEqual(self.getJobFromHistory('project-merge').result,
James E. Blair78e31b32013-07-09 09:11:34 -07001460 'SUCCESS')
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001461 self.assertEqual(self.getJobFromHistory('project-test1').result,
1462 'SUCCESS')
1463 self.assertEqual(self.getJobFromHistory('project-test2').result,
1464 'FAILURE')
James E. Blair127bc182012-08-28 15:55:15 -07001465
David Shrewsburyffab07a2017-07-24 12:45:07 -04001466 @simple_layout('layouts/autohold.yaml')
1467 def test_autohold(self):
David Shrewsburyffab07a2017-07-24 12:45:07 -04001468 client = zuul.rpcclient.RPCClient('127.0.0.1',
1469 self.gearman_server.port)
1470 self.addCleanup(client.shutdown)
David Shrewsbury36b2adf2017-07-31 15:40:13 -04001471 r = client.autohold('tenant-one', 'org/project', 'project-test2',
1472 "reason text", 1)
David Shrewsburyffab07a2017-07-24 12:45:07 -04001473 self.assertTrue(r)
1474
Clark Boylanaeb5a122017-10-20 14:00:11 -07001475 # First check that successful jobs do not autohold
1476 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
David Shrewsburyffab07a2017-07-24 12:45:07 -04001477 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
1478
1479 self.waitUntilSettled()
1480
1481 self.assertEqual(A.data['status'], 'NEW')
1482 self.assertEqual(A.reported, 1)
Clark Boylanaeb5a122017-10-20 14:00:11 -07001483 # project-test2
1484 self.assertEqual(self.history[0].result, 'SUCCESS')
1485
1486 # Check nodepool for a held node
1487 held_node = None
1488 for node in self.fake_nodepool.getNodes():
1489 if node['state'] == zuul.model.STATE_HOLD:
1490 held_node = node
1491 break
1492 self.assertIsNone(held_node)
1493
1494 # Now test that failed jobs are autoheld
1495 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
1496 self.executor_server.failJob('project-test2', B)
1497 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
1498
1499 self.waitUntilSettled()
1500
1501 self.assertEqual(B.data['status'], 'NEW')
1502 self.assertEqual(B.reported, 1)
1503 # project-test2
1504 self.assertEqual(self.history[1].result, 'FAILURE')
David Shrewsburyffab07a2017-07-24 12:45:07 -04001505
1506 # Check nodepool for a held node
1507 held_node = None
1508 for node in self.fake_nodepool.getNodes():
1509 if node['state'] == zuul.model.STATE_HOLD:
1510 held_node = node
1511 break
1512 self.assertIsNotNone(held_node)
1513
1514 # Validate node has recorded the failed job
1515 self.assertEqual(
1516 held_node['hold_job'],
1517 " ".join(['tenant-one',
1518 'review.example.com/org/project',
1519 'project-test2'])
1520 )
David Shrewsburyf9af9df2017-08-01 15:19:26 -04001521 self.assertEqual(held_node['comment'], "reason text")
David Shrewsburyffab07a2017-07-24 12:45:07 -04001522
1523 # Another failed change should not hold any more nodes
Clark Boylanaeb5a122017-10-20 14:00:11 -07001524 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
1525 self.executor_server.failJob('project-test2', C)
1526 self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
David Shrewsburyffab07a2017-07-24 12:45:07 -04001527 self.waitUntilSettled()
Clark Boylanaeb5a122017-10-20 14:00:11 -07001528 self.assertEqual(C.data['status'], 'NEW')
1529 self.assertEqual(C.reported, 1)
1530 # project-test2
1531 self.assertEqual(self.history[2].result, 'FAILURE')
David Shrewsburyffab07a2017-07-24 12:45:07 -04001532
1533 held_nodes = 0
1534 for node in self.fake_nodepool.getNodes():
1535 if node['state'] == zuul.model.STATE_HOLD:
1536 held_nodes += 1
1537 self.assertEqual(held_nodes, 1)
1538
David Shrewsburyc4676132017-10-16 12:41:57 -04001539 @simple_layout('layouts/autohold.yaml')
Clark Boylanaeb5a122017-10-20 14:00:11 -07001540 def test_autohold_ignores_aborted_jobs(self):
1541 client = zuul.rpcclient.RPCClient('127.0.0.1',
1542 self.gearman_server.port)
1543 self.addCleanup(client.shutdown)
1544 r = client.autohold('tenant-one', 'org/project', 'project-test2',
1545 "reason text", 1)
1546 self.assertTrue(r)
1547
1548 self.executor_server.hold_jobs_in_build = True
1549
1550 # Create a change that will have its job aborted
1551 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1552 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
1553 self.waitUntilSettled()
1554
1555 # Creating new patchset on change A will abort A,1's job because
1556 # a new patchset arrived replacing A,1 with A,2.
1557 A.addPatchset()
1558 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
1559
1560 self.waitUntilSettled()
1561 self.executor_server.hold_jobs_in_build = False
1562 self.executor_server.release()
1563 self.waitUntilSettled()
1564
1565 self.assertEqual(A.data['status'], 'NEW')
1566 # Note only the successful job for A,2 will report as we don't
1567 # report aborted builds for old patchsets.
1568 self.assertEqual(A.reported, 1)
1569 # A,1 project-test2
1570 self.assertEqual(self.history[0].result, 'ABORTED')
1571 # A,2 project-test2
1572 self.assertEqual(self.history[1].result, 'SUCCESS')
1573
1574 # Check nodepool for a held node
1575 held_node = None
1576 for node in self.fake_nodepool.getNodes():
1577 if node['state'] == zuul.model.STATE_HOLD:
1578 held_node = node
1579 break
1580 self.assertIsNone(held_node)
1581
1582 @simple_layout('layouts/autohold.yaml')
David Shrewsburyc4676132017-10-16 12:41:57 -04001583 def test_autohold_list(self):
1584 client = zuul.rpcclient.RPCClient('127.0.0.1',
1585 self.gearman_server.port)
1586 self.addCleanup(client.shutdown)
1587
1588 r = client.autohold('tenant-one', 'org/project', 'project-test2',
1589 "reason text", 1)
1590 self.assertTrue(r)
1591
1592 autohold_requests = client.autohold_list()
1593 self.assertNotEqual({}, autohold_requests)
1594 self.assertEqual(1, len(autohold_requests.keys()))
1595
1596 # The single dict key should be a CSV string value
1597 key = list(autohold_requests.keys())[0]
1598 tenant, project, job = key.split(',')
1599
1600 self.assertEqual('tenant-one', tenant)
1601 self.assertIn('org/project', project)
1602 self.assertEqual('project-test2', job)
1603
1604 # Note: the value is converted from set to list by json.
1605 self.assertEqual([1, "reason text"], autohold_requests[key])
1606
James E. Blair7cb84542017-04-19 13:35:05 -07001607 @simple_layout('layouts/three-projects.yaml')
James E. Blair127bc182012-08-28 15:55:15 -07001608 def test_dependent_behind_dequeue(self):
Clint Byruma9626572017-02-22 14:04:00 -05001609 # This particular test does a large amount of merges and needs a little
1610 # more time to complete
James E. Blair267e5162017-04-07 10:08:20 -07001611 self.wait_timeout = 120
James E. Blair127bc182012-08-28 15:55:15 -07001612 "test that dependent changes behind dequeued changes work"
1613 # This complicated test is a reproduction of a real life bug
1614 self.sched.reconfigure(self.config)
James E. Blair127bc182012-08-28 15:55:15 -07001615
Paul Belanger174a8272017-03-14 13:20:10 -04001616 self.executor_server.hold_jobs_in_build = True
James E. Blair127bc182012-08-28 15:55:15 -07001617 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
1618 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
1619 C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
1620 D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')
1621 E = self.fake_gerrit.addFakeChange('org/project2', 'master', 'E')
1622 F = self.fake_gerrit.addFakeChange('org/project3', 'master', 'F')
1623 D.setDependsOn(C, 1)
1624 E.setDependsOn(D, 1)
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001625 A.addApproval('Code-Review', 2)
1626 B.addApproval('Code-Review', 2)
1627 C.addApproval('Code-Review', 2)
1628 D.addApproval('Code-Review', 2)
1629 E.addApproval('Code-Review', 2)
1630 F.addApproval('Code-Review', 2)
James E. Blair127bc182012-08-28 15:55:15 -07001631
1632 A.fail_merge = True
James E. Blair127bc182012-08-28 15:55:15 -07001633
1634 # Change object re-use in the gerrit trigger is hidden if
1635 # changes are added in quick succession; waiting makes it more
1636 # like real life.
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001637 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair127bc182012-08-28 15:55:15 -07001638 self.waitUntilSettled()
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001639 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blair127bc182012-08-28 15:55:15 -07001640 self.waitUntilSettled()
1641
Paul Belanger174a8272017-03-14 13:20:10 -04001642 self.executor_server.release('.*-merge')
James E. Blair127bc182012-08-28 15:55:15 -07001643 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001644 self.executor_server.release('.*-merge')
James E. Blair127bc182012-08-28 15:55:15 -07001645 self.waitUntilSettled()
1646
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001647 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blair127bc182012-08-28 15:55:15 -07001648 self.waitUntilSettled()
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001649 self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
James E. Blair127bc182012-08-28 15:55:15 -07001650 self.waitUntilSettled()
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001651 self.fake_gerrit.addEvent(E.addApproval('Approved', 1))
James E. Blair127bc182012-08-28 15:55:15 -07001652 self.waitUntilSettled()
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001653 self.fake_gerrit.addEvent(F.addApproval('Approved', 1))
James E. Blair127bc182012-08-28 15:55:15 -07001654 self.waitUntilSettled()
1655
Paul Belanger174a8272017-03-14 13:20:10 -04001656 self.executor_server.release('.*-merge')
James E. Blair127bc182012-08-28 15:55:15 -07001657 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001658 self.executor_server.release('.*-merge')
James E. Blair127bc182012-08-28 15:55:15 -07001659 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001660 self.executor_server.release('.*-merge')
James E. Blair127bc182012-08-28 15:55:15 -07001661 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001662 self.executor_server.release('.*-merge')
James E. Blair127bc182012-08-28 15:55:15 -07001663 self.waitUntilSettled()
1664
1665 # all jobs running
James E. Blaire955e062012-10-08 09:49:03 -07001666
1667 # Grab pointers to the jobs we want to release before
1668 # releasing any, because list indexes may change as
1669 # the jobs complete.
Monty Taylor6bef8ef2013-06-02 08:17:12 -04001670 a, b, c = self.builds[:3]
James E. Blaire955e062012-10-08 09:49:03 -07001671 a.release()
1672 b.release()
1673 c.release()
James E. Blair127bc182012-08-28 15:55:15 -07001674 self.waitUntilSettled()
1675
Paul Belanger174a8272017-03-14 13:20:10 -04001676 self.executor_server.hold_jobs_in_build = False
1677 self.executor_server.release()
James E. Blair127bc182012-08-28 15:55:15 -07001678 self.waitUntilSettled()
1679
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001680 self.assertEqual(A.data['status'], 'NEW')
1681 self.assertEqual(B.data['status'], 'MERGED')
1682 self.assertEqual(C.data['status'], 'MERGED')
1683 self.assertEqual(D.data['status'], 'MERGED')
1684 self.assertEqual(E.data['status'], 'MERGED')
1685 self.assertEqual(F.data['status'], 'MERGED')
James E. Blair127bc182012-08-28 15:55:15 -07001686
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001687 self.assertEqual(A.reported, 2)
1688 self.assertEqual(B.reported, 2)
1689 self.assertEqual(C.reported, 2)
1690 self.assertEqual(D.reported, 2)
1691 self.assertEqual(E.reported, 2)
1692 self.assertEqual(F.reported, 2)
James E. Blair127bc182012-08-28 15:55:15 -07001693
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001694 self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 15)
1695 self.assertEqual(len(self.history), 44)
James E. Blair05fed602012-09-07 12:45:24 -07001696
1697 def test_merger_repack(self):
1698 "Test that the merger works after a repack"
James E. Blair1f4c2bb2013-04-26 08:40:46 -07001699
James E. Blair05fed602012-09-07 12:45:24 -07001700 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001701 A.addApproval('Code-Review', 2)
1702 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair05fed602012-09-07 12:45:24 -07001703 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001704 self.assertEqual(self.getJobFromHistory('project-merge').result,
1705 'SUCCESS')
1706 self.assertEqual(self.getJobFromHistory('project-test1').result,
1707 'SUCCESS')
1708 self.assertEqual(self.getJobFromHistory('project-test2').result,
1709 'SUCCESS')
1710 self.assertEqual(A.data['status'], 'MERGED')
1711 self.assertEqual(A.reported, 2)
James E. Blair05fed602012-09-07 12:45:24 -07001712 self.assertEmptyQueues()
Paul Belanger5bfe9e72016-11-04 19:20:04 -04001713 self.build_history = []
James E. Blair05fed602012-09-07 12:45:24 -07001714
James E. Blair2a535672017-04-27 12:03:15 -07001715 path = os.path.join(self.merger_src_root, "review.example.com",
1716 "org/project")
James E. Blair8c1be532017-02-07 14:04:12 -08001717 if os.path.exists(path):
1718 repack_repo(path)
James E. Blair2a535672017-04-27 12:03:15 -07001719 path = os.path.join(self.executor_src_root, "review.example.com",
1720 "org/project")
James E. Blair8c1be532017-02-07 14:04:12 -08001721 if os.path.exists(path):
1722 repack_repo(path)
James E. Blair05fed602012-09-07 12:45:24 -07001723
1724 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001725 A.addApproval('Code-Review', 2)
1726 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair05fed602012-09-07 12:45:24 -07001727 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001728 self.assertEqual(self.getJobFromHistory('project-merge').result,
1729 'SUCCESS')
1730 self.assertEqual(self.getJobFromHistory('project-test1').result,
1731 'SUCCESS')
1732 self.assertEqual(self.getJobFromHistory('project-test2').result,
1733 'SUCCESS')
1734 self.assertEqual(A.data['status'], 'MERGED')
1735 self.assertEqual(A.reported, 2)
James E. Blair7ee88a22012-09-12 18:59:31 +02001736
James E. Blair4886f282012-11-15 09:27:33 -08001737 def test_merger_repack_large_change(self):
1738 "Test that the merger works with large changes after a repack"
Paul Belanger174a8272017-03-14 13:20:10 -04001739 # https://bugs.executepad.net/zuul/+bug/1078946
James E. Blairac2c3242014-01-24 13:38:51 -08001740 # This test assumes the repo is already cloned; make sure it is
Paul Belanger503bded2016-11-11 19:43:38 -05001741 tenant = self.sched.abide.tenants.get('tenant-one')
James E. Blair6df67132017-04-19 13:53:20 -07001742 trusted, project = tenant.getProject('org/project')
James E. Blair0ffa0102017-03-30 13:11:33 -07001743 url = self.fake_gerrit.getGitUrl(project)
James E. Blair8d144dc2017-05-05 10:13:45 -07001744 self.executor_server.merger._addProject('review.example.com',
James E. Blair197e8202017-06-09 12:54:28 -07001745 'org/project', url, None)
James E. Blair6df67132017-04-19 13:53:20 -07001746 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
James E. Blair4886f282012-11-15 09:27:33 -08001747 A.addPatchset(large=True)
James E. Blair2a535672017-04-27 12:03:15 -07001748 # TODOv3(jeblair): add hostname to upstream root
1749 path = os.path.join(self.upstream_root, 'org/project')
James E. Blair8c1be532017-02-07 14:04:12 -08001750 repack_repo(path)
James E. Blair2a535672017-04-27 12:03:15 -07001751 path = os.path.join(self.merger_src_root, 'review.example.com',
1752 'org/project')
James E. Blair8c1be532017-02-07 14:04:12 -08001753 if os.path.exists(path):
1754 repack_repo(path)
James E. Blair2a535672017-04-27 12:03:15 -07001755 path = os.path.join(self.executor_src_root, 'review.example.com',
1756 'org/project')
James E. Blair8c1be532017-02-07 14:04:12 -08001757 if os.path.exists(path):
1758 repack_repo(path)
James E. Blair4886f282012-11-15 09:27:33 -08001759
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001760 A.addApproval('Code-Review', 2)
1761 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair4886f282012-11-15 09:27:33 -08001762 self.waitUntilSettled()
Paul Belanger503bded2016-11-11 19:43:38 -05001763 self.assertEqual(self.getJobFromHistory('project-merge').result,
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001764 'SUCCESS')
Paul Belanger503bded2016-11-11 19:43:38 -05001765 self.assertEqual(self.getJobFromHistory('project-test1').result,
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001766 'SUCCESS')
Paul Belanger503bded2016-11-11 19:43:38 -05001767 self.assertEqual(self.getJobFromHistory('project-test2').result,
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001768 'SUCCESS')
1769 self.assertEqual(A.data['status'], 'MERGED')
1770 self.assertEqual(A.reported, 2)
James E. Blair4886f282012-11-15 09:27:33 -08001771
James E. Blair2fa50962013-01-30 21:50:41 -08001772 def test_new_patchset_dequeues_old(self):
1773 "Test that a new patchset causes the old to be dequeued"
1774 # D -> C (depends on B) -> B (depends on A) -> A -> M
Paul Belanger174a8272017-03-14 13:20:10 -04001775 self.executor_server.hold_jobs_in_build = True
James E. Blair2fa50962013-01-30 21:50:41 -08001776 M = self.fake_gerrit.addFakeChange('org/project', 'master', 'M')
1777 M.setMerged()
1778
1779 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1780 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
1781 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
1782 D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001783 A.addApproval('Code-Review', 2)
1784 B.addApproval('Code-Review', 2)
1785 C.addApproval('Code-Review', 2)
1786 D.addApproval('Code-Review', 2)
James E. Blair2fa50962013-01-30 21:50:41 -08001787
1788 C.setDependsOn(B, 1)
1789 B.setDependsOn(A, 1)
1790 A.setDependsOn(M, 1)
1791
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001792 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
1793 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
1794 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
1795 self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
James E. Blair2fa50962013-01-30 21:50:41 -08001796 self.waitUntilSettled()
1797
1798 B.addPatchset()
1799 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
1800 self.waitUntilSettled()
1801
Paul Belanger174a8272017-03-14 13:20:10 -04001802 self.executor_server.hold_jobs_in_build = False
1803 self.executor_server.release()
James E. Blair2fa50962013-01-30 21:50:41 -08001804 self.waitUntilSettled()
1805
Monty Taylor98f0f3e2013-07-06 16:02:31 -04001806 self.assertEqual(A.data['status'], 'MERGED')
1807 self.assertEqual(A.reported, 2)
1808 self.assertEqual(B.data['status'], 'NEW')
1809 self.assertEqual(B.reported, 2)
1810 self.assertEqual(C.data['status'], 'NEW')
1811 self.assertEqual(C.reported, 2)
1812 self.assertEqual(D.data['status'], 'MERGED')
1813 self.assertEqual(D.reported, 2)
1814 self.assertEqual(len(self.history), 9) # 3 each for A, B, D.
James E. Blair2fa50962013-01-30 21:50:41 -08001815
James E. Blairba437362015-02-07 11:41:52 -08001816 def test_new_patchset_check(self):
1817 "Test a new patchset in check"
Antoine Mussobd86a312014-01-08 14:51:33 +01001818
Paul Belanger174a8272017-03-14 13:20:10 -04001819 self.executor_server.hold_jobs_in_build = True
Antoine Mussobd86a312014-01-08 14:51:33 +01001820
1821 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
James E. Blairba437362015-02-07 11:41:52 -08001822 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
Paul Belanger18e5f632016-11-03 15:24:16 -04001823 tenant = self.sched.abide.tenants.get('tenant-one')
1824 check_pipeline = tenant.layout.pipelines['check']
James E. Blairba437362015-02-07 11:41:52 -08001825
1826 # Add two git-dependent changes
1827 B.setDependsOn(A, 1)
1828 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
1829 self.waitUntilSettled()
Antoine Mussobd86a312014-01-08 14:51:33 +01001830 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
1831 self.waitUntilSettled()
James E. Blairba437362015-02-07 11:41:52 -08001832
1833 # A live item, and a non-live/live pair
1834 items = check_pipeline.getAllItems()
1835 self.assertEqual(len(items), 3)
1836
1837 self.assertEqual(items[0].change.number, '1')
1838 self.assertEqual(items[0].change.patchset, '1')
1839 self.assertFalse(items[0].live)
1840
1841 self.assertEqual(items[1].change.number, '2')
1842 self.assertEqual(items[1].change.patchset, '1')
1843 self.assertTrue(items[1].live)
1844
1845 self.assertEqual(items[2].change.number, '1')
1846 self.assertEqual(items[2].change.patchset, '1')
1847 self.assertTrue(items[2].live)
1848
1849 # Add a new patchset to A
1850 A.addPatchset()
1851 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
1852 self.waitUntilSettled()
1853
1854 # The live copy of A,1 should be gone, but the non-live and B
1855 # should continue, and we should have a new A,2
1856 items = check_pipeline.getAllItems()
1857 self.assertEqual(len(items), 3)
1858
1859 self.assertEqual(items[0].change.number, '1')
1860 self.assertEqual(items[0].change.patchset, '1')
1861 self.assertFalse(items[0].live)
1862
1863 self.assertEqual(items[1].change.number, '2')
1864 self.assertEqual(items[1].change.patchset, '1')
1865 self.assertTrue(items[1].live)
1866
1867 self.assertEqual(items[2].change.number, '1')
1868 self.assertEqual(items[2].change.patchset, '2')
1869 self.assertTrue(items[2].live)
1870
1871 # Add a new patchset to B
1872 B.addPatchset()
1873 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
1874 self.waitUntilSettled()
1875
1876 # The live copy of B,1 should be gone, and it's non-live copy of A,1
1877 # but we should have a new B,2 (still based on A,1)
1878 items = check_pipeline.getAllItems()
1879 self.assertEqual(len(items), 3)
1880
1881 self.assertEqual(items[0].change.number, '1')
1882 self.assertEqual(items[0].change.patchset, '2')
1883 self.assertTrue(items[0].live)
1884
1885 self.assertEqual(items[1].change.number, '1')
1886 self.assertEqual(items[1].change.patchset, '1')
1887 self.assertFalse(items[1].live)
1888
1889 self.assertEqual(items[2].change.number, '2')
1890 self.assertEqual(items[2].change.patchset, '2')
1891 self.assertTrue(items[2].live)
1892
1893 self.builds[0].release()
1894 self.waitUntilSettled()
1895 self.builds[0].release()
1896 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04001897 self.executor_server.hold_jobs_in_build = False
1898 self.executor_server.release()
James E. Blairba437362015-02-07 11:41:52 -08001899 self.waitUntilSettled()
1900
1901 self.assertEqual(A.reported, 1)
1902 self.assertEqual(B.reported, 1)
1903 self.assertEqual(self.history[0].result, 'ABORTED')
1904 self.assertEqual(self.history[0].changes, '1,1')
1905 self.assertEqual(self.history[1].result, 'ABORTED')
1906 self.assertEqual(self.history[1].changes, '1,1 2,1')
1907 self.assertEqual(self.history[2].result, 'SUCCESS')
1908 self.assertEqual(self.history[2].changes, '1,2')
1909 self.assertEqual(self.history[3].result, 'SUCCESS')
1910 self.assertEqual(self.history[3].changes, '1,1 2,2')
1911
1912 def test_abandoned_gate(self):
1913 "Test that an abandoned change is dequeued from gate"
1914
Paul Belanger174a8272017-03-14 13:20:10 -04001915 self.executor_server.hold_jobs_in_build = True
James E. Blairba437362015-02-07 11:41:52 -08001916
1917 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02001918 A.addApproval('Code-Review', 2)
1919 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blairba437362015-02-07 11:41:52 -08001920 self.waitUntilSettled()
Antoine Mussobd86a312014-01-08 14:51:33 +01001921 self.assertEqual(len(self.builds), 1, "One job being built (on hold)")
1922 self.assertEqual(self.builds[0].name, 'project-merge')
1923
1924 self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
1925 self.waitUntilSettled()
1926
Paul Belanger174a8272017-03-14 13:20:10 -04001927 self.executor_server.release('.*-merge')
Antoine Mussobd86a312014-01-08 14:51:33 +01001928 self.waitUntilSettled()
1929
Arie5b4048c2016-10-05 16:12:06 +03001930 self.assertBuilds([])
1931 self.assertHistory([
1932 dict(name='project-merge', result='ABORTED', changes='1,1')],
1933 ordered=False)
James E. Blairba437362015-02-07 11:41:52 -08001934 self.assertEqual(A.reported, 1,
1935 "Abandoned gate change should report only start")
1936
1937 def test_abandoned_check(self):
1938 "Test that an abandoned change is dequeued from check"
1939
Paul Belanger174a8272017-03-14 13:20:10 -04001940 self.executor_server.hold_jobs_in_build = True
James E. Blairba437362015-02-07 11:41:52 -08001941
1942 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
1943 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
Paul Belanger18e5f632016-11-03 15:24:16 -04001944 tenant = self.sched.abide.tenants.get('tenant-one')
1945 check_pipeline = tenant.layout.pipelines['check']
James E. Blairba437362015-02-07 11:41:52 -08001946
1947 # Add two git-dependent changes
1948 B.setDependsOn(A, 1)
1949 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
1950 self.waitUntilSettled()
1951 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
1952 self.waitUntilSettled()
1953 # A live item, and a non-live/live pair
1954 items = check_pipeline.getAllItems()
1955 self.assertEqual(len(items), 3)
1956
1957 self.assertEqual(items[0].change.number, '1')
1958 self.assertFalse(items[0].live)
1959
1960 self.assertEqual(items[1].change.number, '2')
1961 self.assertTrue(items[1].live)
1962
1963 self.assertEqual(items[2].change.number, '1')
1964 self.assertTrue(items[2].live)
1965
1966 # Abandon A
1967 self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
1968 self.waitUntilSettled()
1969
1970 # The live copy of A should be gone, but the non-live and B
1971 # should continue
1972 items = check_pipeline.getAllItems()
1973 self.assertEqual(len(items), 2)
1974
1975 self.assertEqual(items[0].change.number, '1')
1976 self.assertFalse(items[0].live)
1977
1978 self.assertEqual(items[1].change.number, '2')
1979 self.assertTrue(items[1].live)
1980
Paul Belanger174a8272017-03-14 13:20:10 -04001981 self.executor_server.hold_jobs_in_build = False
1982 self.executor_server.release()
James E. Blairba437362015-02-07 11:41:52 -08001983 self.waitUntilSettled()
1984
1985 self.assertEqual(len(self.history), 4)
1986 self.assertEqual(self.history[0].result, 'ABORTED',
Antoine Mussobd86a312014-01-08 14:51:33 +01001987 'Build should have been aborted')
1988 self.assertEqual(A.reported, 0, "Abandoned change should not report")
James E. Blairba437362015-02-07 11:41:52 -08001989 self.assertEqual(B.reported, 1, "Change should report")
Antoine Mussobd86a312014-01-08 14:51:33 +01001990
Steve Varnau7b78b312015-04-03 14:49:46 -07001991 def test_abandoned_not_timer(self):
1992 "Test that an abandoned change does not cancel timer jobs"
James E. Blairdfdfcfc2017-04-20 10:19:20 -07001993 # This test can not use simple_layout because it must start
1994 # with a configuration which does not include a
1995 # timer-triggered job so that we have an opportunity to set
1996 # the hold flag before the first job.
Paul Belanger174a8272017-03-14 13:20:10 -04001997 self.executor_server.hold_jobs_in_build = True
Steve Varnau7b78b312015-04-03 14:49:46 -07001998 # Start timer trigger - also org/project
James E. Blairdfdfcfc2017-04-20 10:19:20 -07001999 self.commitConfigUpdate('common-config', 'layouts/idle.yaml')
Steve Varnau7b78b312015-04-03 14:49:46 -07002000 self.sched.reconfigure(self.config)
Steve Varnau7b78b312015-04-03 14:49:46 -07002001 # The pipeline triggers every second, so we should have seen
2002 # several by now.
2003 time.sleep(5)
2004 self.waitUntilSettled()
2005 # Stop queuing timer triggered jobs so that the assertions
2006 # below don't race against more jobs being queued.
Clint Byrumead6c562017-02-01 16:34:04 -08002007 # Must be in same repo, so overwrite config with another one
James E. Blairdfdfcfc2017-04-20 10:19:20 -07002008 self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')
Steve Varnau7b78b312015-04-03 14:49:46 -07002009 self.sched.reconfigure(self.config)
James E. Blair0b137b42017-07-27 08:51:25 -07002010 self.waitUntilSettled()
James E. Blair78ae4782017-08-02 14:19:10 -07002011 # If APScheduler is in mid-event when we remove the job, we
2012 # can end up with one more event firing, so give it an extra
2013 # second to settle.
2014 time.sleep(1)
2015 self.waitUntilSettled()
James E. Blairdfdfcfc2017-04-20 10:19:20 -07002016
James E. Blair21037782017-07-19 11:56:55 -07002017 self.assertEqual(len(self.builds), 1, "One timer job")
Steve Varnau7b78b312015-04-03 14:49:46 -07002018
2019 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
2020 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
2021 self.waitUntilSettled()
James E. Blair21037782017-07-19 11:56:55 -07002022 self.assertEqual(len(self.builds), 2, "One change plus one timer job")
Steve Varnau7b78b312015-04-03 14:49:46 -07002023
2024 self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
2025 self.waitUntilSettled()
2026
James E. Blair21037782017-07-19 11:56:55 -07002027 self.assertEqual(len(self.builds), 1, "One timer job remains")
Steve Varnau7b78b312015-04-03 14:49:46 -07002028
Paul Belanger174a8272017-03-14 13:20:10 -04002029 self.executor_server.release()
Steve Varnau7b78b312015-04-03 14:49:46 -07002030 self.waitUntilSettled()
2031
James E. Blair2fa50962013-01-30 21:50:41 -08002032 def test_new_patchset_dequeues_old_on_head(self):
2033 "Test that a new patchset causes the old to be dequeued (at head)"
2034 # D -> C (depends on B) -> B (depends on A) -> A -> M
Paul Belanger174a8272017-03-14 13:20:10 -04002035 self.executor_server.hold_jobs_in_build = True
James E. Blair2fa50962013-01-30 21:50:41 -08002036 M = self.fake_gerrit.addFakeChange('org/project', 'master', 'M')
2037 M.setMerged()
2038 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
2039 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
2040 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
2041 D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002042 A.addApproval('Code-Review', 2)
2043 B.addApproval('Code-Review', 2)
2044 C.addApproval('Code-Review', 2)
2045 D.addApproval('Code-Review', 2)
James E. Blair2fa50962013-01-30 21:50:41 -08002046
2047 C.setDependsOn(B, 1)
2048 B.setDependsOn(A, 1)
2049 A.setDependsOn(M, 1)
2050
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002051 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
2052 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
2053 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
2054 self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
James E. Blair2fa50962013-01-30 21:50:41 -08002055 self.waitUntilSettled()
2056
2057 A.addPatchset()
2058 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
2059 self.waitUntilSettled()
2060
Paul Belanger174a8272017-03-14 13:20:10 -04002061 self.executor_server.hold_jobs_in_build = False
2062 self.executor_server.release()
James E. Blair2fa50962013-01-30 21:50:41 -08002063 self.waitUntilSettled()
2064
Monty Taylor98f0f3e2013-07-06 16:02:31 -04002065 self.assertEqual(A.data['status'], 'NEW')
2066 self.assertEqual(A.reported, 2)
2067 self.assertEqual(B.data['status'], 'NEW')
2068 self.assertEqual(B.reported, 2)
2069 self.assertEqual(C.data['status'], 'NEW')
2070 self.assertEqual(C.reported, 2)
2071 self.assertEqual(D.data['status'], 'MERGED')
2072 self.assertEqual(D.reported, 2)
2073 self.assertEqual(len(self.history), 7)
James E. Blair2fa50962013-01-30 21:50:41 -08002074
2075 def test_new_patchset_dequeues_old_without_dependents(self):
2076 "Test that a new patchset causes only the old to be dequeued"
Paul Belanger174a8272017-03-14 13:20:10 -04002077 self.executor_server.hold_jobs_in_build = True
James E. Blair2fa50962013-01-30 21:50:41 -08002078 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
2079 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
2080 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002081 A.addApproval('Code-Review', 2)
2082 B.addApproval('Code-Review', 2)
2083 C.addApproval('Code-Review', 2)
James E. Blair2fa50962013-01-30 21:50:41 -08002084
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002085 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
2086 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
2087 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair2fa50962013-01-30 21:50:41 -08002088 self.waitUntilSettled()
2089
2090 B.addPatchset()
2091 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
2092 self.waitUntilSettled()
2093
Paul Belanger174a8272017-03-14 13:20:10 -04002094 self.executor_server.hold_jobs_in_build = False
2095 self.executor_server.release()
James E. Blair2fa50962013-01-30 21:50:41 -08002096 self.waitUntilSettled()
2097
Monty Taylor98f0f3e2013-07-06 16:02:31 -04002098 self.assertEqual(A.data['status'], 'MERGED')
2099 self.assertEqual(A.reported, 2)
2100 self.assertEqual(B.data['status'], 'NEW')
2101 self.assertEqual(B.reported, 2)
2102 self.assertEqual(C.data['status'], 'MERGED')
2103 self.assertEqual(C.reported, 2)
2104 self.assertEqual(len(self.history), 9)
James E. Blair2fa50962013-01-30 21:50:41 -08002105
2106 def test_new_patchset_dequeues_old_independent_queue(self):
2107 "Test that a new patchset causes the old to be dequeued (independent)"
Paul Belanger174a8272017-03-14 13:20:10 -04002108 self.executor_server.hold_jobs_in_build = True
James E. Blair2fa50962013-01-30 21:50:41 -08002109 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
2110 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
2111 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
2112 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
2113 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
2114 self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
2115 self.waitUntilSettled()
2116
2117 B.addPatchset()
2118 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
2119 self.waitUntilSettled()
2120
Paul Belanger174a8272017-03-14 13:20:10 -04002121 self.executor_server.hold_jobs_in_build = False
2122 self.executor_server.release()
James E. Blair2fa50962013-01-30 21:50:41 -08002123 self.waitUntilSettled()
2124
Monty Taylor98f0f3e2013-07-06 16:02:31 -04002125 self.assertEqual(A.data['status'], 'NEW')
2126 self.assertEqual(A.reported, 1)
2127 self.assertEqual(B.data['status'], 'NEW')
2128 self.assertEqual(B.reported, 1)
2129 self.assertEqual(C.data['status'], 'NEW')
2130 self.assertEqual(C.reported, 1)
2131 self.assertEqual(len(self.history), 10)
2132 self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 1)
James E. Blair7d0dedc2013-02-21 17:26:09 -08002133
James E. Blaircc020532017-04-19 13:04:44 -07002134 @simple_layout('layouts/noop-job.yaml')
James E. Blair18c64442014-03-18 10:14:45 -07002135 def test_noop_job(self):
2136 "Test that the internal noop job works"
2137 A = self.fake_gerrit.addFakeChange('org/noop-project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002138 A.addApproval('Code-Review', 2)
2139 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair18c64442014-03-18 10:14:45 -07002140 self.waitUntilSettled()
2141
2142 self.assertEqual(len(self.gearman_server.getQueue()), 0)
2143 self.assertTrue(self.sched._areAllBuildsComplete())
2144 self.assertEqual(len(self.history), 0)
2145 self.assertEqual(A.data['status'], 'MERGED')
2146 self.assertEqual(A.reported, 2)
2147
James E. Blair06cc3922017-04-19 10:08:10 -07002148 @simple_layout('layouts/no-jobs-project.yaml')
Evgeny Antyshevd6e546c2015-06-11 15:13:57 +00002149 def test_no_job_project(self):
2150 "Test that reports with no jobs don't get sent"
2151 A = self.fake_gerrit.addFakeChange('org/no-jobs-project',
2152 'master', 'A')
2153 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
2154 self.waitUntilSettled()
2155
2156 # Change wasn't reported to
2157 self.assertEqual(A.reported, False)
2158
2159 # Check queue is empty afterwards
Clint Byrum883c30e2016-12-13 11:54:33 -08002160 tenant = self.sched.abide.tenants.get('tenant-one')
2161 check_pipeline = tenant.layout.pipelines['check']
Evgeny Antyshevd6e546c2015-06-11 15:13:57 +00002162 items = check_pipeline.getAllItems()
2163 self.assertEqual(len(items), 0)
2164
2165 self.assertEqual(len(self.history), 0)
2166
James E. Blair7d0dedc2013-02-21 17:26:09 -08002167 def test_zuul_refs(self):
2168 "Test that zuul refs exist and have the right changes"
Paul Belanger174a8272017-03-14 13:20:10 -04002169 self.executor_server.hold_jobs_in_build = True
James E. Blair7d0dedc2013-02-21 17:26:09 -08002170 M1 = self.fake_gerrit.addFakeChange('org/project1', 'master', 'M1')
2171 M1.setMerged()
2172 M2 = self.fake_gerrit.addFakeChange('org/project2', 'master', 'M2')
2173 M2.setMerged()
2174
2175 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
2176 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
2177 C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
2178 D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002179 A.addApproval('Code-Review', 2)
2180 B.addApproval('Code-Review', 2)
2181 C.addApproval('Code-Review', 2)
2182 D.addApproval('Code-Review', 2)
2183 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
2184 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
2185 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
2186 self.fake_gerrit.addEvent(D.addApproval('Approved', 1))
James E. Blair7d0dedc2013-02-21 17:26:09 -08002187
2188 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002189 self.executor_server.release('.*-merge')
James E. Blair7d0dedc2013-02-21 17:26:09 -08002190 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002191 self.executor_server.release('.*-merge')
James E. Blair7d0dedc2013-02-21 17:26:09 -08002192 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002193 self.executor_server.release('.*-merge')
James E. Blair7d0dedc2013-02-21 17:26:09 -08002194 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002195 self.executor_server.release('.*-merge')
James E. Blair7d0dedc2013-02-21 17:26:09 -08002196 self.waitUntilSettled()
2197
Clint Byrum3343e3e2016-11-15 16:05:03 -08002198 a_build = b_build = c_build = d_build = None
Monty Taylor6bef8ef2013-06-02 08:17:12 -04002199 for x in self.builds:
James E. Blaire3db2952017-07-21 15:03:36 -07002200 if x.parameters['zuul']['change'] == '3':
Clint Byrum3343e3e2016-11-15 16:05:03 -08002201 a_build = x
James E. Blaire3db2952017-07-21 15:03:36 -07002202 elif x.parameters['zuul']['change'] == '4':
Clint Byrum3343e3e2016-11-15 16:05:03 -08002203 b_build = x
James E. Blaire3db2952017-07-21 15:03:36 -07002204 elif x.parameters['zuul']['change'] == '5':
Clint Byrum3343e3e2016-11-15 16:05:03 -08002205 c_build = x
James E. Blaire3db2952017-07-21 15:03:36 -07002206 elif x.parameters['zuul']['change'] == '6':
Clint Byrum3343e3e2016-11-15 16:05:03 -08002207 d_build = x
2208 if a_build and b_build and c_build and d_build:
2209 break
James E. Blair7d0dedc2013-02-21 17:26:09 -08002210
Clint Byrum3343e3e2016-11-15 16:05:03 -08002211 # should have a, not b, and should not be in project2
2212 self.assertTrue(a_build.hasChanges(A))
2213 self.assertFalse(a_build.hasChanges(B, M2))
James E. Blair7d0dedc2013-02-21 17:26:09 -08002214
Clint Byrum3343e3e2016-11-15 16:05:03 -08002215 # should have a and b, and should not be in project2
2216 self.assertTrue(b_build.hasChanges(A, B))
2217 self.assertFalse(b_build.hasChanges(M2))
James E. Blair7d0dedc2013-02-21 17:26:09 -08002218
Clint Byrum3343e3e2016-11-15 16:05:03 -08002219 # should have a and b in 1, c in 2
2220 self.assertTrue(c_build.hasChanges(A, B, C))
2221 self.assertFalse(c_build.hasChanges(D))
James E. Blair7d0dedc2013-02-21 17:26:09 -08002222
Clint Byrum3343e3e2016-11-15 16:05:03 -08002223 # should have a and b in 1, c and d in 2
2224 self.assertTrue(d_build.hasChanges(A, B, C, D))
James E. Blair7d0dedc2013-02-21 17:26:09 -08002225
Paul Belanger174a8272017-03-14 13:20:10 -04002226 self.executor_server.hold_jobs_in_build = False
2227 self.executor_server.release()
James E. Blair7d0dedc2013-02-21 17:26:09 -08002228 self.waitUntilSettled()
2229
Monty Taylor98f0f3e2013-07-06 16:02:31 -04002230 self.assertEqual(A.data['status'], 'MERGED')
2231 self.assertEqual(A.reported, 2)
2232 self.assertEqual(B.data['status'], 'MERGED')
2233 self.assertEqual(B.reported, 2)
2234 self.assertEqual(C.data['status'], 'MERGED')
2235 self.assertEqual(C.reported, 2)
2236 self.assertEqual(D.data['status'], 'MERGED')
2237 self.assertEqual(D.reported, 2)
James E. Blair70c71582013-03-06 08:50:50 -08002238
James E. Blair4a28a882013-08-23 15:17:33 -07002239 def test_rerun_on_error(self):
2240 "Test that if a worker fails to run a job, it is run again"
Paul Belanger174a8272017-03-14 13:20:10 -04002241 self.executor_server.hold_jobs_in_build = True
James E. Blair4a28a882013-08-23 15:17:33 -07002242 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002243 A.addApproval('Code-Review', 2)
2244 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair4a28a882013-08-23 15:17:33 -07002245 self.waitUntilSettled()
2246
James E. Blair412fba82017-01-26 15:00:50 -08002247 self.builds[0].requeue = True
Paul Belanger174a8272017-03-14 13:20:10 -04002248 self.executor_server.hold_jobs_in_build = False
2249 self.executor_server.release()
James E. Blair4a28a882013-08-23 15:17:33 -07002250 self.waitUntilSettled()
James E. Blair412fba82017-01-26 15:00:50 -08002251 self.assertEqual(self.countJobResults(self.history, None), 1)
James E. Blair4a28a882013-08-23 15:17:33 -07002252 self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 3)
2253
James E. Blair412e5582013-04-22 15:50:12 -07002254 def test_statsd(self):
2255 "Test each of the statsd methods used in the scheduler"
James E. Blairded241e2017-10-10 13:22:40 -07002256 statsd = self.sched.statsd
James E. Blair412e5582013-04-22 15:50:12 -07002257 statsd.incr('test-incr')
2258 statsd.timing('test-timing', 3)
Alex Gaynor813d39b2014-05-17 16:17:16 -07002259 statsd.gauge('test-gauge', 12)
James E. Blair412e5582013-04-22 15:50:12 -07002260 self.assertReportedStat('test-incr', '1|c')
2261 self.assertReportedStat('test-timing', '3|ms')
Alex Gaynor813d39b2014-05-17 16:17:16 -07002262 self.assertReportedStat('test-gauge', '12|g')
James E. Blair412e5582013-04-22 15:50:12 -07002263
James E. Blairdad52252014-02-07 16:59:17 -08002264 def test_stuck_job_cleanup(self):
2265 "Test that pending jobs are cleaned up if removed from layout"
Clint Byrum678e2c32017-03-16 16:27:21 -07002266
2267 # We want to hold the project-merge job that the fake change enqueues
James E. Blairdad52252014-02-07 16:59:17 -08002268 self.gearman_server.hold_jobs_in_queue = True
2269 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002270 A.addApproval('Code-Review', 2)
2271 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blairdad52252014-02-07 16:59:17 -08002272 self.waitUntilSettled()
Clint Byrum678e2c32017-03-16 16:27:21 -07002273 # The assertion is that we have one job in the queue, project-merge
James E. Blairdad52252014-02-07 16:59:17 -08002274 self.assertEqual(len(self.gearman_server.getQueue()), 1)
2275
James E. Blair31cb3e82017-04-20 13:08:56 -07002276 self.commitConfigUpdate('common-config', 'layouts/no-jobs.yaml')
James E. Blairdad52252014-02-07 16:59:17 -08002277 self.sched.reconfigure(self.config)
2278 self.waitUntilSettled()
2279
James E. Blair18c64442014-03-18 10:14:45 -07002280 self.gearman_server.release('gate-noop')
James E. Blairdad52252014-02-07 16:59:17 -08002281 self.waitUntilSettled()
Clint Byrum678e2c32017-03-16 16:27:21 -07002282 # asserting that project-merge is removed from queue
James E. Blairdad52252014-02-07 16:59:17 -08002283 self.assertEqual(len(self.gearman_server.getQueue()), 0)
2284 self.assertTrue(self.sched._areAllBuildsComplete())
2285
2286 self.assertEqual(len(self.history), 1)
James E. Blair18c64442014-03-18 10:14:45 -07002287 self.assertEqual(self.history[0].name, 'gate-noop')
James E. Blairdad52252014-02-07 16:59:17 -08002288 self.assertEqual(self.history[0].result, 'SUCCESS')
2289
James E. Blair879dafb2015-07-17 14:04:49 -07002290 def test_file_head(self):
2291 # This is a regression test for an observed bug. A change
2292 # with a file named "HEAD" in the root directory of the repo
2293 # was processed by a merger. It then was unable to reset the
2294 # repo because of:
2295 # GitCommandError: 'git reset --hard HEAD' returned
2296 # with exit code 128
2297 # stderr: 'fatal: ambiguous argument 'HEAD': both revision
2298 # and filename
2299 # Use '--' to separate filenames from revisions'
2300
2301 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Paul Belangera91095f2016-11-04 19:28:48 -04002302 A.addPatchset({'HEAD': ''})
James E. Blair879dafb2015-07-17 14:04:49 -07002303 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
2304
2305 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
2306 self.waitUntilSettled()
2307
2308 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
2309 self.waitUntilSettled()
2310
2311 self.assertIn('Build succeeded', A.messages[0])
2312 self.assertIn('Build succeeded', B.messages[0])
2313
James E. Blair70c71582013-03-06 08:50:50 -08002314 def test_file_jobs(self):
2315 "Test that file jobs run only when appropriate"
2316 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Adam Gandelman5661bda2017-02-03 15:58:03 -08002317 A.addPatchset({'pip-requires': 'foo'})
James E. Blair70c71582013-03-06 08:50:50 -08002318 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002319 A.addApproval('Code-Review', 2)
2320 B.addApproval('Code-Review', 2)
2321 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
2322 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blair70c71582013-03-06 08:50:50 -08002323 self.waitUntilSettled()
2324
Monty Taylor6bef8ef2013-06-02 08:17:12 -04002325 testfile_jobs = [x for x in self.history
James E. Blair70c71582013-03-06 08:50:50 -08002326 if x.name == 'project-testfile']
2327
Monty Taylor98f0f3e2013-07-06 16:02:31 -04002328 self.assertEqual(len(testfile_jobs), 1)
2329 self.assertEqual(testfile_jobs[0].changes, '1,2')
2330 self.assertEqual(A.data['status'], 'MERGED')
2331 self.assertEqual(A.reported, 2)
2332 self.assertEqual(B.data['status'], 'MERGED')
2333 self.assertEqual(B.reported, 2)
James E. Blair3c5e5b52013-04-26 11:17:03 -07002334
Clint Byrum3ee39f32016-11-17 23:45:07 -08002335 def _test_irrelevant_files_jobs(self, should_skip):
2336 "Test that jobs with irrelevant-files filter run only when appropriate"
Clint Byrum3ee39f32016-11-17 23:45:07 -08002337 if should_skip:
2338 files = {'ignoreme': 'ignored\n'}
2339 else:
2340 files = {'respectme': 'please!\n'}
Maru Newby3fe5f852015-01-13 04:22:14 +00002341
2342 change = self.fake_gerrit.addFakeChange('org/project',
Clint Byrum3ee39f32016-11-17 23:45:07 -08002343 'master',
2344 'test irrelevant-files',
2345 files=files)
Maru Newby3fe5f852015-01-13 04:22:14 +00002346 self.fake_gerrit.addEvent(change.getPatchsetCreatedEvent(1))
2347 self.waitUntilSettled()
2348
2349 tested_change_ids = [x.changes[0] for x in self.history
Clint Byrum3ee39f32016-11-17 23:45:07 -08002350 if x.name == 'project-test-irrelevant-files']
Maru Newby3fe5f852015-01-13 04:22:14 +00002351
2352 if should_skip:
2353 self.assertEqual([], tested_change_ids)
2354 else:
2355 self.assertIn(change.data['number'], tested_change_ids)
2356
James E. Blaircc324192017-04-19 14:36:08 -07002357 @simple_layout('layouts/irrelevant-files.yaml')
Clint Byrum3ee39f32016-11-17 23:45:07 -08002358 def test_irrelevant_files_match_skips_job(self):
2359 self._test_irrelevant_files_jobs(should_skip=True)
Maru Newby3fe5f852015-01-13 04:22:14 +00002360
James E. Blaircc324192017-04-19 14:36:08 -07002361 @simple_layout('layouts/irrelevant-files.yaml')
Clint Byrum3ee39f32016-11-17 23:45:07 -08002362 def test_irrelevant_files_no_match_runs_job(self):
2363 self._test_irrelevant_files_jobs(should_skip=False)
Maru Newby3fe5f852015-01-13 04:22:14 +00002364
James E. Blaircc324192017-04-19 14:36:08 -07002365 @simple_layout('layouts/inheritance.yaml')
Clint Byrum85493602016-11-18 11:59:47 -08002366 def test_inherited_jobs_keep_matchers(self):
Clint Byrum85493602016-11-18 11:59:47 -08002367 files = {'ignoreme': 'ignored\n'}
2368
2369 change = self.fake_gerrit.addFakeChange('org/project',
2370 'master',
2371 'test irrelevant-files',
2372 files=files)
2373 self.fake_gerrit.addEvent(change.getPatchsetCreatedEvent(1))
2374 self.waitUntilSettled()
2375
2376 run_jobs = set([build.name for build in self.history])
2377
2378 self.assertEqual(set(['project-test-nomatch-starts-empty',
2379 'project-test-nomatch-starts-full']), run_jobs)
2380
James E. Blairaf8b2082017-10-03 15:38:27 -07002381 @simple_layout('layouts/job-vars.yaml')
2382 def test_inherited_job_variables(self):
2383 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
2384 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
2385 self.waitUntilSettled()
2386 self.assertHistory([
2387 dict(name='parentjob', result='SUCCESS'),
2388 dict(name='child1', result='SUCCESS'),
2389 dict(name='child2', result='SUCCESS'),
2390 dict(name='child3', result='SUCCESS'),
2391 ], ordered=False)
2392 j = self.getJobFromHistory('parentjob')
2393 rp = set([p['name'] for p in j.parameters['projects']])
2394 self.assertEqual(j.parameters['vars']['override'], 0)
2395 self.assertEqual(j.parameters['vars']['child1override'], 0)
2396 self.assertEqual(j.parameters['vars']['parent'], 0)
2397 self.assertFalse('child1' in j.parameters['vars'])
2398 self.assertFalse('child2' in j.parameters['vars'])
2399 self.assertFalse('child3' in j.parameters['vars'])
2400 self.assertEqual(rp, set(['org/project', 'org/project0',
2401 'org/project0']))
2402 j = self.getJobFromHistory('child1')
2403 rp = set([p['name'] for p in j.parameters['projects']])
2404 self.assertEqual(j.parameters['vars']['override'], 1)
2405 self.assertEqual(j.parameters['vars']['child1override'], 1)
2406 self.assertEqual(j.parameters['vars']['parent'], 0)
2407 self.assertEqual(j.parameters['vars']['child1'], 1)
2408 self.assertFalse('child2' in j.parameters['vars'])
2409 self.assertFalse('child3' in j.parameters['vars'])
2410 self.assertEqual(rp, set(['org/project', 'org/project0',
2411 'org/project1']))
2412 j = self.getJobFromHistory('child2')
2413 rp = set([p['name'] for p in j.parameters['projects']])
2414 self.assertEqual(j.parameters['vars']['override'], 2)
2415 self.assertEqual(j.parameters['vars']['child1override'], 0)
2416 self.assertEqual(j.parameters['vars']['parent'], 0)
2417 self.assertFalse('child1' in j.parameters['vars'])
2418 self.assertEqual(j.parameters['vars']['child2'], 2)
2419 self.assertFalse('child3' in j.parameters['vars'])
2420 self.assertEqual(rp, set(['org/project', 'org/project0',
2421 'org/project2']))
2422 j = self.getJobFromHistory('child3')
2423 rp = set([p['name'] for p in j.parameters['projects']])
2424 self.assertEqual(j.parameters['vars']['override'], 3)
2425 self.assertEqual(j.parameters['vars']['child1override'], 0)
2426 self.assertEqual(j.parameters['vars']['parent'], 0)
2427 self.assertFalse('child1' in j.parameters['vars'])
2428 self.assertFalse('child2' in j.parameters['vars'])
2429 self.assertEqual(j.parameters['vars']['child3'], 3)
2430 self.assertEqual(rp, set(['org/project', 'org/project0',
2431 'org/project3']))
2432
James E. Blairb09a0c52017-10-04 07:35:14 -07002433 @simple_layout('layouts/job-variants.yaml')
2434 def test_job_branch_variants(self):
2435 self.create_branch('org/project', 'stable/diablo')
2436 self.create_branch('org/project', 'stable/essex')
2437 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
2438 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
2439 self.waitUntilSettled()
2440
2441 B = self.fake_gerrit.addFakeChange('org/project', 'stable/diablo', 'B')
2442 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
2443 self.waitUntilSettled()
2444
2445 C = self.fake_gerrit.addFakeChange('org/project', 'stable/essex', 'C')
2446 self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
2447 self.waitUntilSettled()
2448 self.assertHistory([
2449 dict(name='python27', result='SUCCESS'),
2450 dict(name='python27', result='SUCCESS'),
2451 dict(name='python27', result='SUCCESS'),
2452 ])
2453
2454 p = self.history[0].parameters
2455 self.assertEqual(p['timeout'], 40)
2456 self.assertEqual(len(p['nodes']), 1)
2457 self.assertEqual(p['nodes'][0]['label'], 'new')
2458 self.assertEqual([x['path'] for x in p['pre_playbooks']],
2459 ['base-pre', 'py27-pre'])
2460 self.assertEqual([x['path'] for x in p['post_playbooks']],
2461 ['py27-post-a', 'py27-post-b', 'base-post'])
2462 self.assertEqual([x['path'] for x in p['playbooks']],
James E. Blair2f589fe2017-10-26 12:57:41 -07002463 ['playbooks/python27.yaml'])
James E. Blairb09a0c52017-10-04 07:35:14 -07002464
2465 p = self.history[1].parameters
2466 self.assertEqual(p['timeout'], 50)
2467 self.assertEqual(len(p['nodes']), 1)
2468 self.assertEqual(p['nodes'][0]['label'], 'old')
2469 self.assertEqual([x['path'] for x in p['pre_playbooks']],
2470 ['base-pre', 'py27-pre', 'py27-diablo-pre'])
2471 self.assertEqual([x['path'] for x in p['post_playbooks']],
2472 ['py27-diablo-post', 'py27-post-a', 'py27-post-b',
2473 'base-post'])
2474 self.assertEqual([x['path'] for x in p['playbooks']],
2475 ['py27-diablo'])
2476
2477 p = self.history[2].parameters
2478 self.assertEqual(p['timeout'], 40)
2479 self.assertEqual(len(p['nodes']), 1)
2480 self.assertEqual(p['nodes'][0]['label'], 'new')
2481 self.assertEqual([x['path'] for x in p['pre_playbooks']],
2482 ['base-pre', 'py27-pre', 'py27-essex-pre'])
2483 self.assertEqual([x['path'] for x in p['post_playbooks']],
2484 ['py27-essex-post', 'py27-post-a', 'py27-post-b',
2485 'base-post'])
2486 self.assertEqual([x['path'] for x in p['playbooks']],
James E. Blair2f589fe2017-10-26 12:57:41 -07002487 ['playbooks/python27.yaml'])
James E. Blairb09a0c52017-10-04 07:35:14 -07002488
James E. Blair00292672017-10-26 15:29:39 -07002489 @simple_layout("layouts/no-run.yaml")
2490 def test_job_without_run(self):
2491 "Test that a job without a run playbook errors"
2492 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
2493 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
2494 self.waitUntilSettled()
2495 self.assertIn('Job base does not specify a run playbook',
2496 A.messages[-1])
2497
James E. Blairc8a1e052014-02-25 09:29:26 -08002498 def test_queue_names(self):
2499 "Test shared change queue names"
Paul Belanger8e68fc22017-02-06 16:24:49 -05002500 tenant = self.sched.abide.tenants.get('tenant-one')
James E. Blair6053de42017-04-05 11:27:11 -07002501 (trusted, project1) = tenant.getProject('org/project1')
2502 (trusted, project2) = tenant.getProject('org/project2')
Paul Belanger8e68fc22017-02-06 16:24:49 -05002503 q1 = tenant.layout.pipelines['gate'].getQueue(project1)
2504 q2 = tenant.layout.pipelines['gate'].getQueue(project2)
2505 self.assertEqual(q1.name, 'integrated')
2506 self.assertEqual(q2.name, 'integrated')
James E. Blairc8a1e052014-02-25 09:29:26 -08002507
James E. Blair64ed6f22013-07-10 14:07:23 -07002508 def test_queue_precedence(self):
2509 "Test that queue precedence works"
2510
2511 self.gearman_server.hold_jobs_in_queue = True
Paul Belanger174a8272017-03-14 13:20:10 -04002512 self.executor_server.hold_jobs_in_build = True
James E. Blair64ed6f22013-07-10 14:07:23 -07002513 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
2514 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002515 A.addApproval('Code-Review', 2)
2516 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair64ed6f22013-07-10 14:07:23 -07002517
2518 self.waitUntilSettled()
2519 self.gearman_server.hold_jobs_in_queue = False
2520 self.gearman_server.release()
2521 self.waitUntilSettled()
2522
James E. Blair8de58bd2013-07-18 16:23:33 -07002523 # Run one build at a time to ensure non-race order:
James E. Blairb8c16472015-05-05 14:55:26 -07002524 self.orderedRelease()
Paul Belanger174a8272017-03-14 13:20:10 -04002525 self.executor_server.hold_jobs_in_build = False
James E. Blair8de58bd2013-07-18 16:23:33 -07002526 self.waitUntilSettled()
2527
James E. Blair64ed6f22013-07-10 14:07:23 -07002528 self.log.debug(self.history)
2529 self.assertEqual(self.history[0].pipeline, 'gate')
2530 self.assertEqual(self.history[1].pipeline, 'check')
2531 self.assertEqual(self.history[2].pipeline, 'gate')
2532 self.assertEqual(self.history[3].pipeline, 'gate')
2533 self.assertEqual(self.history[4].pipeline, 'check')
2534 self.assertEqual(self.history[5].pipeline, 'check')
2535
Clark Boylana5edbe42014-06-03 16:39:10 -07002536 def test_json_status(self):
James E. Blair1843a552013-07-03 14:19:52 -07002537 "Test that we can retrieve JSON status info"
Paul Belanger174a8272017-03-14 13:20:10 -04002538 self.executor_server.hold_jobs_in_build = True
James E. Blair1843a552013-07-03 14:19:52 -07002539 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002540 A.addApproval('Code-Review', 2)
2541 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair1843a552013-07-03 14:19:52 -07002542 self.waitUntilSettled()
2543
Paul Belanger174a8272017-03-14 13:20:10 -04002544 self.executor_server.release('project-merge')
James E. Blairb7273ef2016-04-19 08:58:51 -07002545 self.waitUntilSettled()
2546
James E. Blair1843a552013-07-03 14:19:52 -07002547 port = self.webapp.server.socket.getsockname()[1]
2548
Adam Gandelman77a12c72017-02-03 17:43:43 -08002549 req = urllib.request.Request(
2550 "http://localhost:%s/tenant-one/status" % port)
Morgan Fainberg293f7f82016-05-30 14:01:22 -07002551 f = urllib.request.urlopen(req)
Clark Boylanaa4f2e72014-06-03 21:22:40 -07002552 headers = f.info()
2553 self.assertIn('Content-Length', headers)
2554 self.assertIn('Content-Type', headers)
Sachi Kingdc963fc2016-03-23 16:00:33 +11002555 self.assertIsNotNone(re.match('^application/json(; charset=UTF-8)?$',
2556 headers['Content-Type']))
Timo Tijhof0ebd2932015-04-02 12:11:21 +01002557 self.assertIn('Access-Control-Allow-Origin', headers)
2558 self.assertIn('Cache-Control', headers)
Clark Boylanaa4f2e72014-06-03 21:22:40 -07002559 self.assertIn('Last-Modified', headers)
Timo Tijhof0ebd2932015-04-02 12:11:21 +01002560 self.assertIn('Expires', headers)
Clint Byrumf322fe22017-05-10 20:53:12 -07002561 data = f.read().decode('utf8')
James E. Blair1843a552013-07-03 14:19:52 -07002562
Paul Belanger174a8272017-03-14 13:20:10 -04002563 self.executor_server.hold_jobs_in_build = False
2564 self.executor_server.release()
James E. Blair1843a552013-07-03 14:19:52 -07002565 self.waitUntilSettled()
2566
2567 data = json.loads(data)
James E. Blairb7273ef2016-04-19 08:58:51 -07002568 status_jobs = []
James E. Blair1843a552013-07-03 14:19:52 -07002569 for p in data['pipelines']:
2570 for q in p['change_queues']:
James E. Blairbfb8e042014-12-30 17:01:44 -08002571 if p['name'] in ['gate', 'conflict']:
Clark Boylanaf2476f2014-01-23 14:47:36 -08002572 self.assertEqual(q['window'], 20)
2573 else:
2574 self.assertEqual(q['window'], 0)
James E. Blair1843a552013-07-03 14:19:52 -07002575 for head in q['heads']:
2576 for change in head:
Clark Boylanaf2476f2014-01-23 14:47:36 -08002577 self.assertTrue(change['active'])
Monty Taylor98f0f3e2013-07-06 16:02:31 -04002578 self.assertEqual(change['id'], '1,1')
James E. Blair1843a552013-07-03 14:19:52 -07002579 for job in change['jobs']:
James E. Blairb7273ef2016-04-19 08:58:51 -07002580 status_jobs.append(job)
2581 self.assertEqual('project-merge', status_jobs[0]['name'])
Monty Taylorde8242c2017-02-23 20:29:53 -06002582 # TODO(mordred) pull uuids from self.builds
Monty Taylor51139a02016-05-24 11:28:10 -05002583 self.assertEqual(
Tobias Henkelb4407fc2017-07-07 13:52:56 +02002584 'static/stream.html?uuid={uuid}&logfile=console.log'.format(
2585 uuid=status_jobs[0]['uuid']),
2586 status_jobs[0]['url'])
2587 self.assertEqual(
Monty Taylor51139a02016-05-24 11:28:10 -05002588 'finger://{hostname}/{uuid}'.format(
2589 hostname=self.executor_server.hostname,
2590 uuid=status_jobs[0]['uuid']),
Tobias Henkelb4407fc2017-07-07 13:52:56 +02002591 status_jobs[0]['finger_url'])
Monty Taylorde8242c2017-02-23 20:29:53 -06002592 # TOOD(mordred) configure a success-url on the base job
Monty Taylor51139a02016-05-24 11:28:10 -05002593 self.assertEqual(
2594 'finger://{hostname}/{uuid}'.format(
2595 hostname=self.executor_server.hostname,
2596 uuid=status_jobs[0]['uuid']),
2597 status_jobs[0]['report_url'])
James E. Blairb7273ef2016-04-19 08:58:51 -07002598 self.assertEqual('project-test1', status_jobs[1]['name'])
Monty Taylor51139a02016-05-24 11:28:10 -05002599 self.assertEqual(
Tobias Henkelb4407fc2017-07-07 13:52:56 +02002600 'static/stream.html?uuid={uuid}&logfile=console.log'.format(
2601 uuid=status_jobs[1]['uuid']),
2602 status_jobs[1]['url'])
2603 self.assertEqual(
Monty Taylor51139a02016-05-24 11:28:10 -05002604 'finger://{hostname}/{uuid}'.format(
2605 hostname=self.executor_server.hostname,
2606 uuid=status_jobs[1]['uuid']),
Tobias Henkelb4407fc2017-07-07 13:52:56 +02002607 status_jobs[1]['finger_url'])
Monty Taylor51139a02016-05-24 11:28:10 -05002608 self.assertEqual(
2609 'finger://{hostname}/{uuid}'.format(
2610 hostname=self.executor_server.hostname,
2611 uuid=status_jobs[1]['uuid']),
2612 status_jobs[1]['report_url'])
James E. Blairb7273ef2016-04-19 08:58:51 -07002613
2614 self.assertEqual('project-test2', status_jobs[2]['name'])
Monty Taylor51139a02016-05-24 11:28:10 -05002615 self.assertEqual(
Tobias Henkelb4407fc2017-07-07 13:52:56 +02002616 'static/stream.html?uuid={uuid}&logfile=console.log'.format(
2617 uuid=status_jobs[2]['uuid']),
2618 status_jobs[2]['url'])
2619 self.assertEqual(
Monty Taylor51139a02016-05-24 11:28:10 -05002620 'finger://{hostname}/{uuid}'.format(
2621 hostname=self.executor_server.hostname,
2622 uuid=status_jobs[2]['uuid']),
Tobias Henkelb4407fc2017-07-07 13:52:56 +02002623 status_jobs[2]['finger_url'])
Monty Taylor51139a02016-05-24 11:28:10 -05002624 self.assertEqual(
2625 'finger://{hostname}/{uuid}'.format(
2626 hostname=self.executor_server.hostname,
2627 uuid=status_jobs[2]['uuid']),
2628 status_jobs[2]['report_url'])
James E. Blair1843a552013-07-03 14:19:52 -07002629
Tobias Henkel65639f82017-07-10 10:25:42 +02002630 # check job dependencies
2631 self.assertIsNotNone(status_jobs[0]['dependencies'])
2632 self.assertIsNotNone(status_jobs[1]['dependencies'])
2633 self.assertIsNotNone(status_jobs[2]['dependencies'])
2634 self.assertEqual(len(status_jobs[0]['dependencies']), 0)
2635 self.assertEqual(len(status_jobs[1]['dependencies']), 1)
2636 self.assertEqual(len(status_jobs[2]['dependencies']), 1)
2637 self.assertIn('project-merge', status_jobs[1]['dependencies'])
2638 self.assertIn('project-merge', status_jobs[2]['dependencies'])
2639
James E. Blair419a8672017-10-18 14:48:25 -07002640 def test_reconfigure_merge(self):
2641 """Test that two reconfigure events are merged"""
2642
2643 tenant = self.sched.abide.tenants['tenant-one']
2644 (trusted, project) = tenant.getProject('org/project')
2645
2646 self.sched.run_handler_lock.acquire()
2647 self.assertEqual(self.sched.management_event_queue.qsize(), 0)
2648
2649 self.sched.reconfigureTenant(tenant, project)
2650 self.assertEqual(self.sched.management_event_queue.qsize(), 1)
2651
2652 self.sched.reconfigureTenant(tenant, project)
2653 # The second event should have been combined with the first
2654 # so we should still only have one entry.
2655 self.assertEqual(self.sched.management_event_queue.qsize(), 1)
2656
2657 self.sched.run_handler_lock.release()
2658 self.waitUntilSettled()
2659
2660 self.assertEqual(self.sched.management_event_queue.qsize(), 0)
2661
James E. Blaircdccd972013-07-01 12:10:22 -07002662 def test_live_reconfiguration(self):
2663 "Test that live reconfiguration works"
Paul Belanger174a8272017-03-14 13:20:10 -04002664 self.executor_server.hold_jobs_in_build = True
James E. Blaircdccd972013-07-01 12:10:22 -07002665 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002666 A.addApproval('Code-Review', 2)
2667 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blaircdccd972013-07-01 12:10:22 -07002668 self.waitUntilSettled()
2669
2670 self.sched.reconfigure(self.config)
James E. Blairbd963632017-02-01 09:58:14 -08002671 self.waitUntilSettled()
James E. Blaircdccd972013-07-01 12:10:22 -07002672
Paul Belanger174a8272017-03-14 13:20:10 -04002673 self.executor_server.hold_jobs_in_build = False
2674 self.executor_server.release()
James E. Blaircdccd972013-07-01 12:10:22 -07002675 self.waitUntilSettled()
Monty Taylor98f0f3e2013-07-06 16:02:31 -04002676 self.assertEqual(self.getJobFromHistory('project-merge').result,
2677 'SUCCESS')
2678 self.assertEqual(self.getJobFromHistory('project-test1').result,
2679 'SUCCESS')
2680 self.assertEqual(self.getJobFromHistory('project-test2').result,
2681 'SUCCESS')
2682 self.assertEqual(A.data['status'], 'MERGED')
2683 self.assertEqual(A.reported, 2)
James E. Blair287c06d2013-07-24 10:39:30 -07002684
James E. Blair027ba992017-09-20 13:48:32 -07002685 def test_live_reconfiguration_abort(self):
2686 # Raise an exception during reconfiguration and verify we
2687 # still function.
2688 self.executor_server.hold_jobs_in_build = True
2689 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
2690 A.addApproval('Code-Review', 2)
2691 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
2692 self.waitUntilSettled()
2693
2694 tenant = self.sched.abide.tenants.get('tenant-one')
2695 pipeline = tenant.layout.pipelines['gate']
2696 change = pipeline.getAllItems()[0].change
2697 # Set this to an invalid value to cause an exception during
2698 # reconfiguration.
2699 change.branch = None
2700
2701 self.sched.reconfigure(self.config)
2702 self.waitUntilSettled()
2703
2704 self.executor_server.hold_jobs_in_build = False
2705 self.executor_server.release()
2706
2707 self.waitUntilSettled()
2708 self.assertEqual(self.getJobFromHistory('project-merge').result,
2709 'ABORTED')
2710 self.assertEqual(A.data['status'], 'NEW')
2711 # The final report fails because of the invalid value set above.
2712 self.assertEqual(A.reported, 1)
2713
James E. Blair6bc782d2015-07-17 16:20:21 -07002714 def test_live_reconfiguration_merge_conflict(self):
2715 # A real-world bug: a change in a gate queue has a merge
2716 # conflict and a job is added to its project while it's
2717 # sitting in the queue. The job gets added to the change and
2718 # enqueued and the change gets stuck.
Paul Belanger174a8272017-03-14 13:20:10 -04002719 self.executor_server.hold_jobs_in_build = True
James E. Blair6bc782d2015-07-17 16:20:21 -07002720
2721 # This change is fine. It's here to stop the queue long
2722 # enough for the next change to be subject to the
2723 # reconfiguration, as well as to provide a conflict for the
2724 # next change. This change will succeed and merge.
2725 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Clint Byrum073c00b2017-05-01 13:41:23 -07002726 A.addPatchset({'conflict': 'A'})
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002727 A.addApproval('Code-Review', 2)
James E. Blair6bc782d2015-07-17 16:20:21 -07002728
2729 # This change will be in merge conflict. During the
2730 # reconfiguration, we will add a job. We want to make sure
2731 # that doesn't cause it to get stuck.
2732 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
Clint Byrum073c00b2017-05-01 13:41:23 -07002733 B.addPatchset({'conflict': 'B'})
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002734 B.addApproval('Code-Review', 2)
James E. Blair4eb21fa2015-07-27 14:56:47 -07002735
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002736 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
2737 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blair6bc782d2015-07-17 16:20:21 -07002738
2739 self.waitUntilSettled()
2740
2741 # No jobs have run yet
2742 self.assertEqual(A.data['status'], 'NEW')
2743 self.assertEqual(A.reported, 1)
2744 self.assertEqual(B.data['status'], 'NEW')
James E. Blair6bc782d2015-07-17 16:20:21 -07002745 self.assertEqual(len(self.history), 0)
2746
2747 # Add the "project-test3" job.
Clint Byrum073c00b2017-05-01 13:41:23 -07002748 self.commitConfigUpdate('common-config',
2749 'layouts/live-reconfiguration-add-job.yaml')
James E. Blair6bc782d2015-07-17 16:20:21 -07002750 self.sched.reconfigure(self.config)
2751 self.waitUntilSettled()
2752
Paul Belanger174a8272017-03-14 13:20:10 -04002753 self.executor_server.hold_jobs_in_build = False
2754 self.executor_server.release()
James E. Blair6bc782d2015-07-17 16:20:21 -07002755 self.waitUntilSettled()
2756
2757 self.assertEqual(A.data['status'], 'MERGED')
2758 self.assertEqual(A.reported, 2)
2759 self.assertEqual(B.data['status'], 'NEW')
Tobias Henkel9842bd72017-05-16 13:40:03 +02002760 self.assertIn('Merge Failed', B.messages[-1])
James E. Blair6bc782d2015-07-17 16:20:21 -07002761 self.assertEqual(self.getJobFromHistory('project-merge').result,
2762 'SUCCESS')
2763 self.assertEqual(self.getJobFromHistory('project-test1').result,
2764 'SUCCESS')
2765 self.assertEqual(self.getJobFromHistory('project-test2').result,
2766 'SUCCESS')
2767 self.assertEqual(self.getJobFromHistory('project-test3').result,
2768 'SUCCESS')
2769 self.assertEqual(len(self.history), 4)
2770
James E. Blair400e8fd2015-07-30 17:44:45 -07002771 def test_live_reconfiguration_failed_root(self):
James E. Blair6bc782d2015-07-17 16:20:21 -07002772 # An extrapolation of test_live_reconfiguration_merge_conflict
2773 # that tests a job added to a job tree with a failed root does
2774 # not run.
Paul Belanger174a8272017-03-14 13:20:10 -04002775 self.executor_server.hold_jobs_in_build = True
James E. Blair6bc782d2015-07-17 16:20:21 -07002776
2777 # This change is fine. It's here to stop the queue long
2778 # enough for the next change to be subject to the
2779 # reconfiguration. This change will succeed and merge.
2780 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Clint Byrum3147d652017-05-01 14:01:42 -07002781 A.addPatchset({'conflict': 'A'})
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002782 A.addApproval('Code-Review', 2)
2783 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair6bc782d2015-07-17 16:20:21 -07002784 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002785 self.executor_server.release('.*-merge')
James E. Blair6bc782d2015-07-17 16:20:21 -07002786 self.waitUntilSettled()
2787
2788 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
Paul Belanger174a8272017-03-14 13:20:10 -04002789 self.executor_server.failJob('project-merge', B)
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002790 B.addApproval('Code-Review', 2)
2791 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blair6bc782d2015-07-17 16:20:21 -07002792 self.waitUntilSettled()
2793
Paul Belanger174a8272017-03-14 13:20:10 -04002794 self.executor_server.release('.*-merge')
James E. Blair6bc782d2015-07-17 16:20:21 -07002795 self.waitUntilSettled()
2796
2797 # Both -merge jobs have run, but no others.
2798 self.assertEqual(A.data['status'], 'NEW')
2799 self.assertEqual(A.reported, 1)
2800 self.assertEqual(B.data['status'], 'NEW')
2801 self.assertEqual(B.reported, 1)
2802 self.assertEqual(self.history[0].result, 'SUCCESS')
2803 self.assertEqual(self.history[0].name, 'project-merge')
2804 self.assertEqual(self.history[1].result, 'FAILURE')
2805 self.assertEqual(self.history[1].name, 'project-merge')
2806 self.assertEqual(len(self.history), 2)
2807
2808 # Add the "project-test3" job.
Clint Byrum3147d652017-05-01 14:01:42 -07002809 self.commitConfigUpdate('common-config',
2810 'layouts/live-reconfiguration-add-job.yaml')
James E. Blair6bc782d2015-07-17 16:20:21 -07002811 self.sched.reconfigure(self.config)
2812 self.waitUntilSettled()
2813
Paul Belanger174a8272017-03-14 13:20:10 -04002814 self.executor_server.hold_jobs_in_build = False
2815 self.executor_server.release()
James E. Blair6bc782d2015-07-17 16:20:21 -07002816 self.waitUntilSettled()
2817
2818 self.assertEqual(A.data['status'], 'MERGED')
2819 self.assertEqual(A.reported, 2)
2820 self.assertEqual(B.data['status'], 'NEW')
2821 self.assertEqual(B.reported, 2)
2822 self.assertEqual(self.history[0].result, 'SUCCESS')
2823 self.assertEqual(self.history[0].name, 'project-merge')
2824 self.assertEqual(self.history[1].result, 'FAILURE')
2825 self.assertEqual(self.history[1].name, 'project-merge')
2826 self.assertEqual(self.history[2].result, 'SUCCESS')
2827 self.assertEqual(self.history[3].result, 'SUCCESS')
2828 self.assertEqual(self.history[4].result, 'SUCCESS')
2829 self.assertEqual(len(self.history), 5)
2830
James E. Blair400e8fd2015-07-30 17:44:45 -07002831 def test_live_reconfiguration_failed_job(self):
2832 # Test that a change with a removed failing job does not
2833 # disrupt reconfiguration. If a change has a failed job and
2834 # that job is removed during a reconfiguration, we observed a
2835 # bug where the code to re-set build statuses would run on
2836 # that build and raise an exception because the job no longer
2837 # existed.
Paul Belanger174a8272017-03-14 13:20:10 -04002838 self.executor_server.hold_jobs_in_build = True
James E. Blair400e8fd2015-07-30 17:44:45 -07002839
2840 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
2841
2842 # This change will fail and later be removed by the reconfiguration.
Paul Belanger174a8272017-03-14 13:20:10 -04002843 self.executor_server.failJob('project-test1', A)
James E. Blair400e8fd2015-07-30 17:44:45 -07002844
2845 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
2846 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002847 self.executor_server.release('.*-merge')
James E. Blair400e8fd2015-07-30 17:44:45 -07002848 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002849 self.executor_server.release('project-test1')
James E. Blair400e8fd2015-07-30 17:44:45 -07002850 self.waitUntilSettled()
2851
2852 self.assertEqual(A.data['status'], 'NEW')
2853 self.assertEqual(A.reported, 0)
2854
2855 self.assertEqual(self.getJobFromHistory('project-merge').result,
2856 'SUCCESS')
2857 self.assertEqual(self.getJobFromHistory('project-test1').result,
2858 'FAILURE')
2859 self.assertEqual(len(self.history), 2)
2860
2861 # Remove the test1 job.
Clint Byrum841a58c2017-05-01 14:17:21 -07002862 self.commitConfigUpdate('common-config',
2863 'layouts/live-reconfiguration-failed-job.yaml')
James E. Blair400e8fd2015-07-30 17:44:45 -07002864 self.sched.reconfigure(self.config)
2865 self.waitUntilSettled()
2866
Paul Belanger174a8272017-03-14 13:20:10 -04002867 self.executor_server.hold_jobs_in_build = False
2868 self.executor_server.release()
James E. Blair400e8fd2015-07-30 17:44:45 -07002869 self.waitUntilSettled()
2870
2871 self.assertEqual(self.getJobFromHistory('project-test2').result,
2872 'SUCCESS')
2873 self.assertEqual(self.getJobFromHistory('project-testfile').result,
2874 'SUCCESS')
2875 self.assertEqual(len(self.history), 4)
2876
2877 self.assertEqual(A.data['status'], 'NEW')
2878 self.assertEqual(A.reported, 1)
2879 self.assertIn('Build succeeded', A.messages[0])
2880 # Ensure the removed job was not included in the report.
2881 self.assertNotIn('project-test1', A.messages[0])
2882
James E. Blairfe707d12015-08-05 15:18:15 -07002883 def test_live_reconfiguration_shared_queue(self):
2884 # Test that a change with a failing job which was removed from
2885 # this project but otherwise still exists in the system does
2886 # not disrupt reconfiguration.
2887
Paul Belanger174a8272017-03-14 13:20:10 -04002888 self.executor_server.hold_jobs_in_build = True
James E. Blairfe707d12015-08-05 15:18:15 -07002889
2890 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
2891
Paul Belanger174a8272017-03-14 13:20:10 -04002892 self.executor_server.failJob('project1-project2-integration', A)
James E. Blairfe707d12015-08-05 15:18:15 -07002893
2894 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
2895 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002896 self.executor_server.release('.*-merge')
James E. Blairfe707d12015-08-05 15:18:15 -07002897 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002898 self.executor_server.release('project1-project2-integration')
James E. Blairfe707d12015-08-05 15:18:15 -07002899 self.waitUntilSettled()
2900
2901 self.assertEqual(A.data['status'], 'NEW')
2902 self.assertEqual(A.reported, 0)
2903
Clint Byrum69663d42017-05-01 14:51:04 -07002904 self.assertEqual(self.getJobFromHistory('project-merge').result,
James E. Blairfe707d12015-08-05 15:18:15 -07002905 'SUCCESS')
2906 self.assertEqual(self.getJobFromHistory(
2907 'project1-project2-integration').result, 'FAILURE')
2908 self.assertEqual(len(self.history), 2)
2909
2910 # Remove the integration job.
Clint Byrum69663d42017-05-01 14:51:04 -07002911 self.commitConfigUpdate(
2912 'common-config',
2913 'layouts/live-reconfiguration-shared-queue.yaml')
James E. Blairfe707d12015-08-05 15:18:15 -07002914 self.sched.reconfigure(self.config)
2915 self.waitUntilSettled()
2916
Paul Belanger174a8272017-03-14 13:20:10 -04002917 self.executor_server.hold_jobs_in_build = False
2918 self.executor_server.release()
James E. Blairfe707d12015-08-05 15:18:15 -07002919 self.waitUntilSettled()
2920
Clint Byrum69663d42017-05-01 14:51:04 -07002921 self.assertEqual(self.getJobFromHistory('project-merge').result,
James E. Blairfe707d12015-08-05 15:18:15 -07002922 'SUCCESS')
Clint Byrum69663d42017-05-01 14:51:04 -07002923 self.assertEqual(self.getJobFromHistory('project-test1').result,
James E. Blairfe707d12015-08-05 15:18:15 -07002924 'SUCCESS')
Clint Byrum69663d42017-05-01 14:51:04 -07002925 self.assertEqual(self.getJobFromHistory('project-test2').result,
James E. Blairfe707d12015-08-05 15:18:15 -07002926 'SUCCESS')
2927 self.assertEqual(self.getJobFromHistory(
2928 'project1-project2-integration').result, 'FAILURE')
2929 self.assertEqual(len(self.history), 4)
2930
2931 self.assertEqual(A.data['status'], 'NEW')
2932 self.assertEqual(A.reported, 1)
2933 self.assertIn('Build succeeded', A.messages[0])
2934 # Ensure the removed job was not included in the report.
2935 self.assertNotIn('project1-project2-integration', A.messages[0])
2936
Joshua Hesketh4bd7da32016-02-17 20:58:47 +11002937 def test_double_live_reconfiguration_shared_queue(self):
2938 # This was a real-world regression. A change is added to
2939 # gate; a reconfigure happens, a second change which depends
2940 # on the first is added, and a second reconfiguration happens.
2941 # Ensure that both changes merge.
2942
2943 # A failure may indicate incorrect caching or cleaning up of
2944 # references during a reconfiguration.
Paul Belanger174a8272017-03-14 13:20:10 -04002945 self.executor_server.hold_jobs_in_build = True
Joshua Hesketh4bd7da32016-02-17 20:58:47 +11002946
2947 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
2948 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
2949 B.setDependsOn(A, 1)
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002950 A.addApproval('Code-Review', 2)
2951 B.addApproval('Code-Review', 2)
Joshua Hesketh4bd7da32016-02-17 20:58:47 +11002952
2953 # Add the parent change.
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002954 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
Joshua Hesketh4bd7da32016-02-17 20:58:47 +11002955 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002956 self.executor_server.release('.*-merge')
Joshua Hesketh4bd7da32016-02-17 20:58:47 +11002957 self.waitUntilSettled()
2958
2959 # Reconfigure (with only one change in the pipeline).
2960 self.sched.reconfigure(self.config)
2961 self.waitUntilSettled()
2962
2963 # Add the child change.
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002964 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
Joshua Hesketh4bd7da32016-02-17 20:58:47 +11002965 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04002966 self.executor_server.release('.*-merge')
Joshua Hesketh4bd7da32016-02-17 20:58:47 +11002967 self.waitUntilSettled()
2968
2969 # Reconfigure (with both in the pipeline).
2970 self.sched.reconfigure(self.config)
2971 self.waitUntilSettled()
2972
Paul Belanger174a8272017-03-14 13:20:10 -04002973 self.executor_server.hold_jobs_in_build = False
2974 self.executor_server.release()
Joshua Hesketh4bd7da32016-02-17 20:58:47 +11002975 self.waitUntilSettled()
2976
2977 self.assertEqual(len(self.history), 8)
2978
2979 self.assertEqual(A.data['status'], 'MERGED')
2980 self.assertEqual(A.reported, 2)
2981 self.assertEqual(B.data['status'], 'MERGED')
2982 self.assertEqual(B.reported, 2)
2983
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00002984 def test_live_reconfiguration_del_project(self):
2985 # Test project deletion from layout
2986 # while changes are enqueued
2987
Paul Belanger174a8272017-03-14 13:20:10 -04002988 self.executor_server.hold_jobs_in_build = True
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00002989 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
2990 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
2991 C = self.fake_gerrit.addFakeChange('org/project1', 'master', 'C')
2992
2993 # A Depends-On: B
2994 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
2995 A.subject, B.data['id'])
Tobias Henkelbf24fd12017-07-27 06:13:07 +02002996 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00002997
2998 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
2999 self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
3000 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003001 self.executor_server.release('.*-merge')
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00003002 self.waitUntilSettled()
3003 self.assertEqual(len(self.builds), 5)
3004
3005 # This layout defines only org/project, not org/project1
James E. Blair31cb3e82017-04-20 13:08:56 -07003006 self.commitConfigUpdate(
3007 'common-config',
3008 'layouts/live-reconfiguration-del-project.yaml')
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00003009 self.sched.reconfigure(self.config)
3010 self.waitUntilSettled()
3011
3012 # Builds for C aborted, builds for A succeed,
3013 # and have change B applied ahead
Clint Byrum0149ada2017-04-04 12:22:01 -07003014 job_c = self.getJobFromHistory('project-test1')
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00003015 self.assertEqual(job_c.changes, '3,1')
3016 self.assertEqual(job_c.result, 'ABORTED')
3017
Paul Belanger174a8272017-03-14 13:20:10 -04003018 self.executor_server.hold_jobs_in_build = False
3019 self.executor_server.release()
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00003020 self.waitUntilSettled()
3021
Clint Byrum0149ada2017-04-04 12:22:01 -07003022 self.assertEqual(
3023 self.getJobFromHistory('project-test1', 'org/project').changes,
3024 '2,1 1,1')
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00003025
3026 self.assertEqual(A.data['status'], 'NEW')
3027 self.assertEqual(B.data['status'], 'NEW')
3028 self.assertEqual(C.data['status'], 'NEW')
3029 self.assertEqual(A.reported, 1)
3030 self.assertEqual(B.reported, 0)
3031 self.assertEqual(C.reported, 0)
3032
Clint Byrum0149ada2017-04-04 12:22:01 -07003033 tenant = self.sched.abide.tenants.get('tenant-one')
3034 self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00003035 self.assertIn('Build succeeded', A.messages[0])
3036
James E. Blair872738f2017-10-31 16:40:36 -07003037 @simple_layout("layouts/reconfigure-failed-head.yaml")
3038 def test_live_reconfiguration_failed_change_at_head(self):
3039 # Test that if we reconfigure with a failed change at head,
3040 # that the change behind it isn't reparented onto it.
3041
3042 self.executor_server.hold_jobs_in_build = True
3043 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
3044 A.addApproval('Code-Review', 2)
3045 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
3046 B.addApproval('Code-Review', 2)
3047
3048 self.executor_server.failJob('job1', A)
3049
3050 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
3051 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
3052
3053 self.waitUntilSettled()
3054
3055 self.assertBuilds([
3056 dict(name='job1', changes='1,1'),
3057 dict(name='job2', changes='1,1'),
3058 dict(name='job1', changes='1,1 2,1'),
3059 dict(name='job2', changes='1,1 2,1'),
3060 ])
3061
3062 self.release(self.builds[0])
3063 self.waitUntilSettled()
3064
3065 self.assertBuilds([
3066 dict(name='job2', changes='1,1'),
3067 dict(name='job1', changes='2,1'),
3068 dict(name='job2', changes='2,1'),
3069 ])
3070
3071 # Unordered history comparison because the aborts can finish
3072 # in any order.
3073 self.assertHistory([
3074 dict(name='job1', result='FAILURE', changes='1,1'),
3075 dict(name='job1', result='ABORTED', changes='1,1 2,1'),
3076 dict(name='job2', result='ABORTED', changes='1,1 2,1'),
3077 ], ordered=False)
3078
3079 self.sched.reconfigure(self.config)
3080 self.waitUntilSettled()
3081
3082 self.executor_server.hold_jobs_in_build = False
3083 self.executor_server.release()
3084 self.waitUntilSettled()
3085
3086 self.assertBuilds([])
3087
3088 self.assertHistory([
3089 dict(name='job1', result='FAILURE', changes='1,1'),
3090 dict(name='job1', result='ABORTED', changes='1,1 2,1'),
3091 dict(name='job2', result='ABORTED', changes='1,1 2,1'),
3092 dict(name='job2', result='SUCCESS', changes='1,1'),
3093 dict(name='job1', result='SUCCESS', changes='2,1'),
3094 dict(name='job2', result='SUCCESS', changes='2,1'),
3095 ], ordered=False)
3096 self.assertEqual(A.data['status'], 'NEW')
3097 self.assertEqual(B.data['status'], 'MERGED')
3098 self.assertEqual(A.reported, 2)
3099 self.assertEqual(B.reported, 2)
3100
James E. Blair287c06d2013-07-24 10:39:30 -07003101 def test_delayed_repo_init(self):
James E. Blair287c06d2013-07-24 10:39:30 -07003102 self.init_repo("org/new-project")
Clint Byrum627ba362017-08-14 13:20:40 -07003103 files = {'README': ''}
3104 self.addCommitToRepo("org/new-project", 'Initial commit',
3105 files=files, tag='init')
3106 self.newTenantConfig('tenants/delayed-repo-init.yaml')
3107 self.commitConfigUpdate(
3108 'common-config',
3109 'layouts/delayed-repo-init.yaml')
3110 self.sched.reconfigure(self.config)
3111 self.waitUntilSettled()
3112
James E. Blair287c06d2013-07-24 10:39:30 -07003113 A = self.fake_gerrit.addFakeChange('org/new-project', 'master', 'A')
3114
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003115 A.addApproval('Code-Review', 2)
3116 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair287c06d2013-07-24 10:39:30 -07003117 self.waitUntilSettled()
3118 self.assertEqual(self.getJobFromHistory('project-merge').result,
3119 'SUCCESS')
3120 self.assertEqual(self.getJobFromHistory('project-test1').result,
3121 'SUCCESS')
3122 self.assertEqual(self.getJobFromHistory('project-test2').result,
3123 'SUCCESS')
3124 self.assertEqual(A.data['status'], 'MERGED')
3125 self.assertEqual(A.reported, 2)
James E. Blair63bb0ef2013-07-29 17:14:51 -07003126
James E. Blair9ea70072017-04-19 16:05:30 -07003127 @simple_layout('layouts/repo-deleted.yaml')
Clark Boylan6dbbc482013-10-18 10:57:31 -07003128 def test_repo_deleted(self):
Clark Boylan6dbbc482013-10-18 10:57:31 -07003129 self.init_repo("org/delete-project")
3130 A = self.fake_gerrit.addFakeChange('org/delete-project', 'master', 'A')
3131
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003132 A.addApproval('Code-Review', 2)
3133 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
Clark Boylan6dbbc482013-10-18 10:57:31 -07003134 self.waitUntilSettled()
3135 self.assertEqual(self.getJobFromHistory('project-merge').result,
3136 'SUCCESS')
3137 self.assertEqual(self.getJobFromHistory('project-test1').result,
3138 'SUCCESS')
3139 self.assertEqual(self.getJobFromHistory('project-test2').result,
3140 'SUCCESS')
3141 self.assertEqual(A.data['status'], 'MERGED')
3142 self.assertEqual(A.reported, 2)
3143
3144 # Delete org/new-project zuul repo. Should be recloned.
James E. Blair8c1be532017-02-07 14:04:12 -08003145 p = 'org/delete-project'
Monty Taylord642d852017-02-23 14:05:42 -05003146 if os.path.exists(os.path.join(self.merger_src_root, p)):
3147 shutil.rmtree(os.path.join(self.merger_src_root, p))
Paul Belanger174a8272017-03-14 13:20:10 -04003148 if os.path.exists(os.path.join(self.executor_src_root, p)):
3149 shutil.rmtree(os.path.join(self.executor_src_root, p))
Clark Boylan6dbbc482013-10-18 10:57:31 -07003150
3151 B = self.fake_gerrit.addFakeChange('org/delete-project', 'master', 'B')
3152
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003153 B.addApproval('Code-Review', 2)
3154 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
Clark Boylan6dbbc482013-10-18 10:57:31 -07003155 self.waitUntilSettled()
3156 self.assertEqual(self.getJobFromHistory('project-merge').result,
3157 'SUCCESS')
3158 self.assertEqual(self.getJobFromHistory('project-test1').result,
3159 'SUCCESS')
3160 self.assertEqual(self.getJobFromHistory('project-test2').result,
3161 'SUCCESS')
3162 self.assertEqual(B.data['status'], 'MERGED')
3163 self.assertEqual(B.reported, 2)
3164
James E. Blair892cca62017-08-09 11:36:58 -07003165 @simple_layout('layouts/untrusted-secrets.yaml')
3166 def test_untrusted_secrets(self):
3167 "Test untrusted secrets"
3168 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
3169 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
3170 self.waitUntilSettled()
3171
3172 self.assertHistory([])
3173 self.assertEqual(A.patchsets[0]['approvals'][0]['value'], "-1")
James E. Blair8eb564a2017-08-10 09:21:41 -07003174 self.assertIn('does not allow post-review job',
James E. Blair892cca62017-08-09 11:36:58 -07003175 A.messages[0])
3176
James E. Blaire68d43f2017-04-19 14:03:30 -07003177 @simple_layout('layouts/tags.yaml')
James E. Blair456f2fb2016-02-09 09:29:33 -08003178 def test_tags(self):
3179 "Test job tags"
James E. Blair456f2fb2016-02-09 09:29:33 -08003180 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
3181 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
3182 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
3183 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
3184 self.waitUntilSettled()
3185
James E. Blaire68d43f2017-04-19 14:03:30 -07003186 self.assertEqual(len(self.history), 2)
Cullen Taylor3aa6d472017-03-01 15:22:09 -06003187
3188 results = {self.getJobFromHistory('merge',
James E. Blair9d46f092017-07-20 16:06:20 -07003189 project='org/project1').uuid: ['extratag', 'merge'],
Cullen Taylor3aa6d472017-03-01 15:22:09 -06003190 self.getJobFromHistory('merge',
James E. Blair9d46f092017-07-20 16:06:20 -07003191 project='org/project2').uuid: ['merge']}
James E. Blair456f2fb2016-02-09 09:29:33 -08003192
3193 for build in self.history:
Cullen Taylor3aa6d472017-03-01 15:22:09 -06003194 self.assertEqual(results.get(build.uuid, ''),
James E. Blair9d46f092017-07-20 16:06:20 -07003195 build.parameters['zuul'].get('jobtags'))
James E. Blair456f2fb2016-02-09 09:29:33 -08003196
James E. Blair63bb0ef2013-07-29 17:14:51 -07003197 def test_timer(self):
3198 "Test that a periodic job is triggered"
James E. Blairdfdfcfc2017-04-20 10:19:20 -07003199 # This test can not use simple_layout because it must start
3200 # with a configuration which does not include a
3201 # timer-triggered job so that we have an opportunity to set
3202 # the hold flag before the first job.
James E. Blair21037782017-07-19 11:56:55 -07003203 self.create_branch('org/project', 'stable')
Paul Belanger174a8272017-03-14 13:20:10 -04003204 self.executor_server.hold_jobs_in_build = True
James E. Blairdfdfcfc2017-04-20 10:19:20 -07003205 self.commitConfigUpdate('common-config', 'layouts/timer.yaml')
James E. Blair63bb0ef2013-07-29 17:14:51 -07003206 self.sched.reconfigure(self.config)
James E. Blair63bb0ef2013-07-29 17:14:51 -07003207
Clark Boylan3ee090a2014-04-03 20:55:09 -07003208 # The pipeline triggers every second, so we should have seen
3209 # several by now.
3210 time.sleep(5)
James E. Blair63bb0ef2013-07-29 17:14:51 -07003211 self.waitUntilSettled()
Clark Boylan3ee090a2014-04-03 20:55:09 -07003212
3213 self.assertEqual(len(self.builds), 2)
3214
James E. Blair63bb0ef2013-07-29 17:14:51 -07003215 port = self.webapp.server.socket.getsockname()[1]
3216
Clint Byrum5e729e92017-02-08 06:34:57 -08003217 req = urllib.request.Request(
James E. Blairdfdfcfc2017-04-20 10:19:20 -07003218 "http://localhost:%s/tenant-one/status" % port)
Morgan Fainberg293f7f82016-05-30 14:01:22 -07003219 f = urllib.request.urlopen(req)
Clint Byrumf322fe22017-05-10 20:53:12 -07003220 data = f.read().decode('utf8')
James E. Blair63bb0ef2013-07-29 17:14:51 -07003221
Paul Belanger174a8272017-03-14 13:20:10 -04003222 self.executor_server.hold_jobs_in_build = False
Clark Boylan3ee090a2014-04-03 20:55:09 -07003223 # Stop queuing timer triggered jobs so that the assertions
3224 # below don't race against more jobs being queued.
James E. Blairdfdfcfc2017-04-20 10:19:20 -07003225 self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')
Clark Boylan3ee090a2014-04-03 20:55:09 -07003226 self.sched.reconfigure(self.config)
James E. Blair78ae4782017-08-02 14:19:10 -07003227 self.waitUntilSettled()
3228 # If APScheduler is in mid-event when we remove the job, we
3229 # can end up with one more event firing, so give it an extra
3230 # second to settle.
3231 time.sleep(1)
3232 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003233 self.executor_server.release()
James E. Blair63bb0ef2013-07-29 17:14:51 -07003234 self.waitUntilSettled()
3235
James E. Blair21037782017-07-19 11:56:55 -07003236 self.assertHistory([
3237 dict(name='project-bitrot', result='SUCCESS',
3238 ref='refs/heads/master'),
3239 dict(name='project-bitrot', result='SUCCESS',
3240 ref='refs/heads/stable'),
3241 ], ordered=False)
James E. Blair63bb0ef2013-07-29 17:14:51 -07003242
3243 data = json.loads(data)
3244 status_jobs = set()
3245 for p in data['pipelines']:
3246 for q in p['change_queues']:
3247 for head in q['heads']:
3248 for change in head:
James E. Blair63bb0ef2013-07-29 17:14:51 -07003249 for job in change['jobs']:
3250 status_jobs.add(job['name'])
James E. Blair21037782017-07-19 11:56:55 -07003251 self.assertIn('project-bitrot', status_jobs)
Joshua Hesketh5fea8672013-08-19 17:32:01 +10003252
James E. Blair4f6033c2014-03-27 15:49:09 -07003253 def test_idle(self):
3254 "Test that frequent periodic jobs work"
James E. Blairdfdfcfc2017-04-20 10:19:20 -07003255 # This test can not use simple_layout because it must start
3256 # with a configuration which does not include a
3257 # timer-triggered job so that we have an opportunity to set
3258 # the hold flag before the first job.
Paul Belanger174a8272017-03-14 13:20:10 -04003259 self.executor_server.hold_jobs_in_build = True
James E. Blair4f6033c2014-03-27 15:49:09 -07003260
Clark Boylan3ee090a2014-04-03 20:55:09 -07003261 for x in range(1, 3):
3262 # Test that timer triggers periodic jobs even across
3263 # layout config reloads.
3264 # Start timer trigger
James E. Blairdfdfcfc2017-04-20 10:19:20 -07003265 self.commitConfigUpdate('common-config',
3266 'layouts/idle.yaml')
Clark Boylan3ee090a2014-04-03 20:55:09 -07003267 self.sched.reconfigure(self.config)
James E. Blair995fc0f2016-02-04 16:48:31 -08003268 self.waitUntilSettled()
James E. Blair4f6033c2014-03-27 15:49:09 -07003269
Clark Boylan3ee090a2014-04-03 20:55:09 -07003270 # The pipeline triggers every second, so we should have seen
3271 # several by now.
3272 time.sleep(5)
Clark Boylan3ee090a2014-04-03 20:55:09 -07003273
3274 # Stop queuing timer triggered jobs so that the assertions
3275 # below don't race against more jobs being queued.
James E. Blairdfdfcfc2017-04-20 10:19:20 -07003276 self.commitConfigUpdate('common-config',
3277 'layouts/no-timer.yaml')
Clark Boylan3ee090a2014-04-03 20:55:09 -07003278 self.sched.reconfigure(self.config)
James E. Blair995fc0f2016-02-04 16:48:31 -08003279 self.waitUntilSettled()
James E. Blair78ae4782017-08-02 14:19:10 -07003280 # If APScheduler is in mid-event when we remove the job,
3281 # we can end up with one more event firing, so give it an
3282 # extra second to settle.
3283 time.sleep(1)
3284 self.waitUntilSettled()
James E. Blair21037782017-07-19 11:56:55 -07003285 self.assertEqual(len(self.builds), 1,
Clint Byrum58264dc2017-02-07 21:21:22 -08003286 'Timer builds iteration #%d' % x)
Paul Belanger174a8272017-03-14 13:20:10 -04003287 self.executor_server.release('.*')
Clark Boylan3ee090a2014-04-03 20:55:09 -07003288 self.waitUntilSettled()
3289 self.assertEqual(len(self.builds), 0)
James E. Blair21037782017-07-19 11:56:55 -07003290 self.assertEqual(len(self.history), x)
James E. Blair4f6033c2014-03-27 15:49:09 -07003291
James E. Blaircc324192017-04-19 14:36:08 -07003292 @simple_layout('layouts/smtp.yaml')
Joshua Hesketh5fea8672013-08-19 17:32:01 +10003293 def test_check_smtp_pool(self):
Joshua Hesketh5fea8672013-08-19 17:32:01 +10003294 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
3295 self.waitUntilSettled()
3296
3297 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
3298 self.waitUntilSettled()
3299
James E. Blairff80a2f2013-12-27 13:24:06 -08003300 self.assertEqual(len(self.smtp_messages), 2)
Joshua Hesketh5fea8672013-08-19 17:32:01 +10003301
3302 # A.messages only holds what FakeGerrit places in it. Thus we
3303 # work on the knowledge of what the first message should be as
3304 # it is only configured to go to SMTP.
3305
3306 self.assertEqual('zuul@example.com',
James E. Blairff80a2f2013-12-27 13:24:06 -08003307 self.smtp_messages[0]['from_email'])
Joshua Hesketh5fea8672013-08-19 17:32:01 +10003308 self.assertEqual(['you@example.com'],
James E. Blairff80a2f2013-12-27 13:24:06 -08003309 self.smtp_messages[0]['to_email'])
Joshua Hesketh5fea8672013-08-19 17:32:01 +10003310 self.assertEqual('Starting check jobs.',
James E. Blairff80a2f2013-12-27 13:24:06 -08003311 self.smtp_messages[0]['body'])
Joshua Hesketh5fea8672013-08-19 17:32:01 +10003312
3313 self.assertEqual('zuul_from@example.com',
James E. Blairff80a2f2013-12-27 13:24:06 -08003314 self.smtp_messages[1]['from_email'])
Joshua Hesketh5fea8672013-08-19 17:32:01 +10003315 self.assertEqual(['alternative_me@example.com'],
James E. Blairff80a2f2013-12-27 13:24:06 -08003316 self.smtp_messages[1]['to_email'])
Joshua Hesketh5fea8672013-08-19 17:32:01 +10003317 self.assertEqual(A.messages[0],
James E. Blairff80a2f2013-12-27 13:24:06 -08003318 self.smtp_messages[1]['body'])
James E. Blairad28e912013-11-27 10:43:22 -08003319
James E. Blaire5910202013-12-27 09:50:31 -08003320 def test_timer_smtp(self):
3321 "Test that a periodic job is triggered"
James E. Blairdfdfcfc2017-04-20 10:19:20 -07003322 # This test can not use simple_layout because it must start
3323 # with a configuration which does not include a
3324 # timer-triggered job so that we have an opportunity to set
3325 # the hold flag before the first job.
Paul Belanger174a8272017-03-14 13:20:10 -04003326 self.executor_server.hold_jobs_in_build = True
James E. Blairdfdfcfc2017-04-20 10:19:20 -07003327 self.commitConfigUpdate('common-config', 'layouts/timer-smtp.yaml')
James E. Blaire5910202013-12-27 09:50:31 -08003328 self.sched.reconfigure(self.config)
James E. Blaire5910202013-12-27 09:50:31 -08003329
Clark Boylan3ee090a2014-04-03 20:55:09 -07003330 # The pipeline triggers every second, so we should have seen
3331 # several by now.
3332 time.sleep(5)
James E. Blaire5910202013-12-27 09:50:31 -08003333 self.waitUntilSettled()
3334
Clark Boylan3ee090a2014-04-03 20:55:09 -07003335 self.assertEqual(len(self.builds), 2)
Paul Belanger174a8272017-03-14 13:20:10 -04003336 self.executor_server.release('.*')
Clark Boylan3ee090a2014-04-03 20:55:09 -07003337 self.waitUntilSettled()
3338 self.assertEqual(len(self.history), 2)
3339
James E. Blaire5910202013-12-27 09:50:31 -08003340 self.assertEqual(self.getJobFromHistory(
3341 'project-bitrot-stable-old').result, 'SUCCESS')
3342 self.assertEqual(self.getJobFromHistory(
3343 'project-bitrot-stable-older').result, 'SUCCESS')
3344
James E. Blairff80a2f2013-12-27 13:24:06 -08003345 self.assertEqual(len(self.smtp_messages), 1)
James E. Blaire5910202013-12-27 09:50:31 -08003346
3347 # A.messages only holds what FakeGerrit places in it. Thus we
3348 # work on the knowledge of what the first message should be as
3349 # it is only configured to go to SMTP.
3350
3351 self.assertEqual('zuul_from@example.com',
James E. Blairff80a2f2013-12-27 13:24:06 -08003352 self.smtp_messages[0]['from_email'])
James E. Blaire5910202013-12-27 09:50:31 -08003353 self.assertEqual(['alternative_me@example.com'],
James E. Blairff80a2f2013-12-27 13:24:06 -08003354 self.smtp_messages[0]['to_email'])
James E. Blaire5910202013-12-27 09:50:31 -08003355 self.assertIn('Subject: Periodic check for org/project succeeded',
James E. Blairff80a2f2013-12-27 13:24:06 -08003356 self.smtp_messages[0]['headers'])
James E. Blaire5910202013-12-27 09:50:31 -08003357
Clark Boylan3ee090a2014-04-03 20:55:09 -07003358 # Stop queuing timer triggered jobs and let any that may have
3359 # queued through so that end of test assertions pass.
James E. Blairdfdfcfc2017-04-20 10:19:20 -07003360 self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml')
Clark Boylan3ee090a2014-04-03 20:55:09 -07003361 self.sched.reconfigure(self.config)
James E. Blairf8058972014-08-15 16:09:16 -07003362 self.waitUntilSettled()
James E. Blair78ae4782017-08-02 14:19:10 -07003363 # If APScheduler is in mid-event when we remove the job, we
3364 # can end up with one more event firing, so give it an extra
3365 # second to settle.
3366 time.sleep(1)
3367 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003368 self.executor_server.release('.*')
Clark Boylan3ee090a2014-04-03 20:55:09 -07003369 self.waitUntilSettled()
3370
Joshua Hesketh25695cb2017-03-06 12:50:04 +11003371 @skip("Disabled for early v3 development")
Evgeny Antyshevda90a502017-02-08 08:42:59 +00003372 def test_timer_sshkey(self):
3373 "Test that a periodic job can setup SSH key authentication"
3374 self.worker.hold_jobs_in_build = True
3375 self.config.set('zuul', 'layout_config',
3376 'tests/fixtures/layout-timer.yaml')
3377 self.sched.reconfigure(self.config)
3378 self.registerJobs()
3379
3380 # The pipeline triggers every second, so we should have seen
3381 # several by now.
3382 time.sleep(5)
3383 self.waitUntilSettled()
3384
3385 self.assertEqual(len(self.builds), 2)
3386
3387 ssh_wrapper = os.path.join(self.git_root, ".ssh_wrapper_gerrit")
3388 self.assertTrue(os.path.isfile(ssh_wrapper))
3389 with open(ssh_wrapper) as f:
3390 ssh_wrapper_content = f.read()
3391 self.assertIn("fake_id_rsa", ssh_wrapper_content)
3392 # In the unit tests Merger runs in the same process,
3393 # so we see its' environment variables
3394 self.assertEqual(os.environ['GIT_SSH'], ssh_wrapper)
3395
3396 self.worker.release('.*')
3397 self.waitUntilSettled()
3398 self.assertEqual(len(self.history), 2)
3399
3400 self.assertEqual(self.getJobFromHistory(
3401 'project-bitrot-stable-old').result, 'SUCCESS')
3402 self.assertEqual(self.getJobFromHistory(
3403 'project-bitrot-stable-older').result, 'SUCCESS')
3404
3405 # Stop queuing timer triggered jobs and let any that may have
3406 # queued through so that end of test assertions pass.
3407 self.config.set('zuul', 'layout_config',
3408 'tests/fixtures/layout-no-timer.yaml')
3409 self.sched.reconfigure(self.config)
3410 self.registerJobs()
3411 self.waitUntilSettled()
James E. Blair78ae4782017-08-02 14:19:10 -07003412 # If APScheduler is in mid-event when we remove the job, we
3413 # can end up with one more event firing, so give it an extra
3414 # second to settle.
3415 time.sleep(1)
3416 self.waitUntilSettled()
Evgeny Antyshevda90a502017-02-08 08:42:59 +00003417 self.worker.release('.*')
3418 self.waitUntilSettled()
3419
James E. Blair91e34592015-07-31 16:45:59 -07003420 def test_client_enqueue_change(self):
James E. Blairad28e912013-11-27 10:43:22 -08003421 "Test that the RPC client can enqueue a change"
3422 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003423 A.addApproval('Code-Review', 2)
3424 A.addApproval('Approved', 1)
James E. Blairad28e912013-11-27 10:43:22 -08003425
3426 client = zuul.rpcclient.RPCClient('127.0.0.1',
3427 self.gearman_server.port)
Clark Boylandf4eba22017-04-24 17:37:58 -07003428 self.addCleanup(client.shutdown)
Paul Belangerbaca3132016-11-04 12:49:54 -04003429 r = client.enqueue(tenant='tenant-one',
3430 pipeline='gate',
James E. Blairad28e912013-11-27 10:43:22 -08003431 project='org/project',
3432 trigger='gerrit',
James E. Blair36658cf2013-12-06 17:53:48 -08003433 change='1,1')
James E. Blairad28e912013-11-27 10:43:22 -08003434 self.waitUntilSettled()
3435 self.assertEqual(self.getJobFromHistory('project-merge').result,
3436 'SUCCESS')
3437 self.assertEqual(self.getJobFromHistory('project-test1').result,
3438 'SUCCESS')
3439 self.assertEqual(self.getJobFromHistory('project-test2').result,
3440 'SUCCESS')
3441 self.assertEqual(A.data['status'], 'MERGED')
3442 self.assertEqual(A.reported, 2)
3443 self.assertEqual(r, True)
3444
James E. Blair91e34592015-07-31 16:45:59 -07003445 def test_client_enqueue_ref(self):
3446 "Test that the RPC client can enqueue a ref"
James E. Blair289f5932017-07-27 15:02:29 -07003447 p = "review.example.com/org/project"
3448 upstream = self.getUpstreamRepos([p])
3449 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
3450 A.setMerged()
3451 A_commit = str(upstream[p].commit('master'))
3452 self.log.debug("A commit: %s" % A_commit)
James E. Blair91e34592015-07-31 16:45:59 -07003453
3454 client = zuul.rpcclient.RPCClient('127.0.0.1',
3455 self.gearman_server.port)
Clark Boylandf4eba22017-04-24 17:37:58 -07003456 self.addCleanup(client.shutdown)
James E. Blair91e34592015-07-31 16:45:59 -07003457 r = client.enqueue_ref(
Paul Belangerbaca3132016-11-04 12:49:54 -04003458 tenant='tenant-one',
James E. Blair91e34592015-07-31 16:45:59 -07003459 pipeline='post',
3460 project='org/project',
3461 trigger='gerrit',
3462 ref='master',
3463 oldrev='90f173846e3af9154517b88543ffbd1691f31366',
James E. Blair289f5932017-07-27 15:02:29 -07003464 newrev=A_commit)
James E. Blair91e34592015-07-31 16:45:59 -07003465 self.waitUntilSettled()
3466 job_names = [x.name for x in self.history]
3467 self.assertEqual(len(self.history), 1)
3468 self.assertIn('project-post', job_names)
3469 self.assertEqual(r, True)
3470
James E. Blairad28e912013-11-27 10:43:22 -08003471 def test_client_enqueue_negative(self):
3472 "Test that the RPC client returns errors"
3473 client = zuul.rpcclient.RPCClient('127.0.0.1',
3474 self.gearman_server.port)
Clark Boylandf4eba22017-04-24 17:37:58 -07003475 self.addCleanup(client.shutdown)
James E. Blairad28e912013-11-27 10:43:22 -08003476 with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
Paul Belangerbaca3132016-11-04 12:49:54 -04003477 "Invalid tenant"):
3478 r = client.enqueue(tenant='tenant-foo',
3479 pipeline='gate',
3480 project='org/project',
3481 trigger='gerrit',
3482 change='1,1')
Paul Belangerbaca3132016-11-04 12:49:54 -04003483 self.assertEqual(r, False)
3484
3485 with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
James E. Blairad28e912013-11-27 10:43:22 -08003486 "Invalid project"):
Paul Belangerbaca3132016-11-04 12:49:54 -04003487 r = client.enqueue(tenant='tenant-one',
3488 pipeline='gate',
James E. Blairad28e912013-11-27 10:43:22 -08003489 project='project-does-not-exist',
3490 trigger='gerrit',
James E. Blair36658cf2013-12-06 17:53:48 -08003491 change='1,1')
James E. Blairad28e912013-11-27 10:43:22 -08003492 self.assertEqual(r, False)
3493
3494 with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
3495 "Invalid pipeline"):
Paul Belangerbaca3132016-11-04 12:49:54 -04003496 r = client.enqueue(tenant='tenant-one',
3497 pipeline='pipeline-does-not-exist',
James E. Blairad28e912013-11-27 10:43:22 -08003498 project='org/project',
3499 trigger='gerrit',
James E. Blair36658cf2013-12-06 17:53:48 -08003500 change='1,1')
James E. Blairad28e912013-11-27 10:43:22 -08003501 self.assertEqual(r, False)
3502
3503 with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
3504 "Invalid trigger"):
Paul Belangerbaca3132016-11-04 12:49:54 -04003505 r = client.enqueue(tenant='tenant-one',
3506 pipeline='gate',
James E. Blairad28e912013-11-27 10:43:22 -08003507 project='org/project',
3508 trigger='trigger-does-not-exist',
James E. Blair36658cf2013-12-06 17:53:48 -08003509 change='1,1')
James E. Blairad28e912013-11-27 10:43:22 -08003510 self.assertEqual(r, False)
3511
3512 with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
3513 "Invalid change"):
Paul Belangerbaca3132016-11-04 12:49:54 -04003514 r = client.enqueue(tenant='tenant-one',
3515 pipeline='gate',
James E. Blairad28e912013-11-27 10:43:22 -08003516 project='org/project',
3517 trigger='gerrit',
James E. Blair36658cf2013-12-06 17:53:48 -08003518 change='1,1')
James E. Blairad28e912013-11-27 10:43:22 -08003519 self.assertEqual(r, False)
3520
3521 self.waitUntilSettled()
3522 self.assertEqual(len(self.history), 0)
3523 self.assertEqual(len(self.builds), 0)
James E. Blair36658cf2013-12-06 17:53:48 -08003524
3525 def test_client_promote(self):
3526 "Test that the RPC client can promote a change"
Paul Belanger174a8272017-03-14 13:20:10 -04003527 self.executor_server.hold_jobs_in_build = True
James E. Blair36658cf2013-12-06 17:53:48 -08003528 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
3529 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
3530 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003531 A.addApproval('Code-Review', 2)
3532 B.addApproval('Code-Review', 2)
3533 C.addApproval('Code-Review', 2)
James E. Blair36658cf2013-12-06 17:53:48 -08003534
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003535 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
3536 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
3537 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blair36658cf2013-12-06 17:53:48 -08003538
3539 self.waitUntilSettled()
3540
Paul Belangerbaca3132016-11-04 12:49:54 -04003541 tenant = self.sched.abide.tenants.get('tenant-one')
3542 items = tenant.layout.pipelines['gate'].getAllItems()
Sean Daguef39b9ca2014-01-10 21:34:35 -05003543 enqueue_times = {}
3544 for item in items:
3545 enqueue_times[str(item.change)] = item.enqueue_time
3546
James E. Blair36658cf2013-12-06 17:53:48 -08003547 client = zuul.rpcclient.RPCClient('127.0.0.1',
3548 self.gearman_server.port)
Clark Boylandf4eba22017-04-24 17:37:58 -07003549 self.addCleanup(client.shutdown)
Paul Belangerbaca3132016-11-04 12:49:54 -04003550 r = client.promote(tenant='tenant-one',
3551 pipeline='gate',
James E. Blair36658cf2013-12-06 17:53:48 -08003552 change_ids=['2,1', '3,1'])
3553
Sean Daguef39b9ca2014-01-10 21:34:35 -05003554 # ensure that enqueue times are durable
Paul Belangerbaca3132016-11-04 12:49:54 -04003555 items = tenant.layout.pipelines['gate'].getAllItems()
Sean Daguef39b9ca2014-01-10 21:34:35 -05003556 for item in items:
3557 self.assertEqual(
3558 enqueue_times[str(item.change)], item.enqueue_time)
3559
James E. Blair78acec92014-02-06 07:11:32 -08003560 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003561 self.executor_server.release('.*-merge')
James E. Blair36658cf2013-12-06 17:53:48 -08003562 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003563 self.executor_server.release('.*-merge')
James E. Blair36658cf2013-12-06 17:53:48 -08003564 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003565 self.executor_server.release('.*-merge')
James E. Blair36658cf2013-12-06 17:53:48 -08003566 self.waitUntilSettled()
3567
3568 self.assertEqual(len(self.builds), 6)
3569 self.assertEqual(self.builds[0].name, 'project-test1')
3570 self.assertEqual(self.builds[1].name, 'project-test2')
3571 self.assertEqual(self.builds[2].name, 'project-test1')
3572 self.assertEqual(self.builds[3].name, 'project-test2')
3573 self.assertEqual(self.builds[4].name, 'project-test1')
3574 self.assertEqual(self.builds[5].name, 'project-test2')
3575
Paul Belangerbaca3132016-11-04 12:49:54 -04003576 self.assertTrue(self.builds[0].hasChanges(B))
3577 self.assertFalse(self.builds[0].hasChanges(A))
3578 self.assertFalse(self.builds[0].hasChanges(C))
James E. Blair36658cf2013-12-06 17:53:48 -08003579
Paul Belangerbaca3132016-11-04 12:49:54 -04003580 self.assertTrue(self.builds[2].hasChanges(B))
3581 self.assertTrue(self.builds[2].hasChanges(C))
3582 self.assertFalse(self.builds[2].hasChanges(A))
James E. Blair36658cf2013-12-06 17:53:48 -08003583
Paul Belangerbaca3132016-11-04 12:49:54 -04003584 self.assertTrue(self.builds[4].hasChanges(B))
3585 self.assertTrue(self.builds[4].hasChanges(C))
3586 self.assertTrue(self.builds[4].hasChanges(A))
James E. Blair36658cf2013-12-06 17:53:48 -08003587
Paul Belanger174a8272017-03-14 13:20:10 -04003588 self.executor_server.release()
James E. Blair36658cf2013-12-06 17:53:48 -08003589 self.waitUntilSettled()
3590
3591 self.assertEqual(A.data['status'], 'MERGED')
3592 self.assertEqual(A.reported, 2)
3593 self.assertEqual(B.data['status'], 'MERGED')
3594 self.assertEqual(B.reported, 2)
3595 self.assertEqual(C.data['status'], 'MERGED')
3596 self.assertEqual(C.reported, 2)
3597
James E. Blair36658cf2013-12-06 17:53:48 -08003598 self.assertEqual(r, True)
3599
3600 def test_client_promote_dependent(self):
3601 "Test that the RPC client can promote a dependent change"
3602 # C (depends on B) -> B -> A ; then promote C to get:
3603 # A -> C (depends on B) -> B
Paul Belanger174a8272017-03-14 13:20:10 -04003604 self.executor_server.hold_jobs_in_build = True
James E. Blair36658cf2013-12-06 17:53:48 -08003605 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
3606 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
3607 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
3608
3609 C.setDependsOn(B, 1)
3610
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003611 A.addApproval('Code-Review', 2)
3612 B.addApproval('Code-Review', 2)
3613 C.addApproval('Code-Review', 2)
James E. Blair36658cf2013-12-06 17:53:48 -08003614
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003615 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
3616 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
3617 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
James E. Blair36658cf2013-12-06 17:53:48 -08003618
3619 self.waitUntilSettled()
3620
3621 client = zuul.rpcclient.RPCClient('127.0.0.1',
3622 self.gearman_server.port)
Clark Boylandf4eba22017-04-24 17:37:58 -07003623 self.addCleanup(client.shutdown)
Paul Belangerbaca3132016-11-04 12:49:54 -04003624 r = client.promote(tenant='tenant-one',
3625 pipeline='gate',
James E. Blair36658cf2013-12-06 17:53:48 -08003626 change_ids=['3,1'])
3627
James E. Blair78acec92014-02-06 07:11:32 -08003628 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003629 self.executor_server.release('.*-merge')
James E. Blair36658cf2013-12-06 17:53:48 -08003630 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003631 self.executor_server.release('.*-merge')
James E. Blair36658cf2013-12-06 17:53:48 -08003632 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04003633 self.executor_server.release('.*-merge')
James E. Blair36658cf2013-12-06 17:53:48 -08003634 self.waitUntilSettled()
3635
3636 self.assertEqual(len(self.builds), 6)
3637 self.assertEqual(self.builds[0].name, 'project-test1')
3638 self.assertEqual(self.builds[1].name, 'project-test2')
3639 self.assertEqual(self.builds[2].name, 'project-test1')
3640 self.assertEqual(self.builds[3].name, 'project-test2')
3641 self.assertEqual(self.builds[4].name, 'project-test1')
3642 self.assertEqual(self.builds[5].name, 'project-test2')
3643
Paul Belangerbaca3132016-11-04 12:49:54 -04003644 self.assertTrue(self.builds[0].hasChanges(B))
3645 self.assertFalse(self.builds[0].hasChanges(A))
3646 self.assertFalse(self.builds[0].hasChanges(C))
James E. Blair36658cf2013-12-06 17:53:48 -08003647
Paul Belangerbaca3132016-11-04 12:49:54 -04003648 self.assertTrue(self.builds[2].hasChanges(B))
3649 self.assertTrue(self.builds[2].hasChanges(C))
3650 self.assertFalse(self.builds[2].hasChanges(A))
James E. Blair36658cf2013-12-06 17:53:48 -08003651
Paul Belangerbaca3132016-11-04 12:49:54 -04003652 self.assertTrue(self.builds[4].hasChanges(B))
3653 self.assertTrue(self.builds[4].hasChanges(C))
3654 self.assertTrue(self.builds[4].hasChanges(A))
James E. Blair36658cf2013-12-06 17:53:48 -08003655
Paul Belanger174a8272017-03-14 13:20:10 -04003656 self.executor_server.release()
James E. Blair36658cf2013-12-06 17:53:48 -08003657 self.waitUntilSettled()
3658
3659 self.assertEqual(A.data['status'], 'MERGED')
3660 self.assertEqual(A.reported, 2)
3661 self.assertEqual(B.data['status'], 'MERGED')
3662 self.assertEqual(B.reported, 2)
3663 self.assertEqual(C.data['status'], 'MERGED')
3664 self.assertEqual(C.reported, 2)
3665
James E. Blair36658cf2013-12-06 17:53:48 -08003666 self.assertEqual(r, True)
3667
3668 def test_client_promote_negative(self):
3669 "Test that the RPC client returns errors for promotion"
Paul Belanger174a8272017-03-14 13:20:10 -04003670 self.executor_server.hold_jobs_in_build = True
James E. Blair36658cf2013-12-06 17:53:48 -08003671 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003672 A.addApproval('Code-Review', 2)
3673 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair36658cf2013-12-06 17:53:48 -08003674 self.waitUntilSettled()
3675
3676 client = zuul.rpcclient.RPCClient('127.0.0.1',
3677 self.gearman_server.port)
Clark Boylandf4eba22017-04-24 17:37:58 -07003678 self.addCleanup(client.shutdown)
James E. Blair36658cf2013-12-06 17:53:48 -08003679
3680 with testtools.ExpectedException(zuul.rpcclient.RPCFailure):
Paul Belangerbaca3132016-11-04 12:49:54 -04003681 r = client.promote(tenant='tenant-one',
3682 pipeline='nonexistent',
James E. Blair36658cf2013-12-06 17:53:48 -08003683 change_ids=['2,1', '3,1'])
James E. Blair36658cf2013-12-06 17:53:48 -08003684 self.assertEqual(r, False)
3685
3686 with testtools.ExpectedException(zuul.rpcclient.RPCFailure):
Paul Belangerbaca3132016-11-04 12:49:54 -04003687 r = client.promote(tenant='tenant-one',
3688 pipeline='gate',
James E. Blair36658cf2013-12-06 17:53:48 -08003689 change_ids=['4,1'])
James E. Blair36658cf2013-12-06 17:53:48 -08003690 self.assertEqual(r, False)
3691
Paul Belanger174a8272017-03-14 13:20:10 -04003692 self.executor_server.hold_jobs_in_build = False
3693 self.executor_server.release()
James E. Blair36658cf2013-12-06 17:53:48 -08003694 self.waitUntilSettled()
Clark Boylan7603a372014-01-21 11:43:20 -08003695
James E. Blaircc324192017-04-19 14:36:08 -07003696 @simple_layout('layouts/rate-limit.yaml')
Clark Boylan7603a372014-01-21 11:43:20 -08003697 def test_queue_rate_limiting(self):
3698 "Test that DependentPipelines are rate limited with dep across window"
Paul Belanger174a8272017-03-14 13:20:10 -04003699 self.executor_server.hold_jobs_in_build = True
Clark Boylan7603a372014-01-21 11:43:20 -08003700 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
3701 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
3702 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
3703
3704 C.setDependsOn(B, 1)
Paul Belanger174a8272017-03-14 13:20:10 -04003705 self.executor_server.failJob('project-test1', A)
Clark Boylan7603a372014-01-21 11:43:20 -08003706
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003707 A.addApproval('Code-Review', 2)
3708 B.addApproval('Code-Review', 2)
3709 C.addApproval('Code-Review', 2)
Clark Boylan7603a372014-01-21 11:43:20 -08003710
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003711 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
3712 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
3713 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
Clark Boylan7603a372014-01-21 11:43:20 -08003714 self.waitUntilSettled()
3715
3716 # Only A and B will have their merge jobs queued because
3717 # window is 2.
3718 self.assertEqual(len(self.builds), 2)
3719 self.assertEqual(self.builds[0].name, 'project-merge')
3720 self.assertEqual(self.builds[1].name, 'project-merge')
3721
James E. Blair732ee862017-04-21 07:49:26 -07003722 # Release the merge jobs one at a time.
3723 self.builds[0].release()
Clark Boylan7603a372014-01-21 11:43:20 -08003724 self.waitUntilSettled()
James E. Blair732ee862017-04-21 07:49:26 -07003725 self.builds[0].release()
Clark Boylan7603a372014-01-21 11:43:20 -08003726 self.waitUntilSettled()
3727
3728 # Only A and B will have their test jobs queued because
3729 # window is 2.
3730 self.assertEqual(len(self.builds), 4)
3731 self.assertEqual(self.builds[0].name, 'project-test1')
3732 self.assertEqual(self.builds[1].name, 'project-test2')
3733 self.assertEqual(self.builds[2].name, 'project-test1')
3734 self.assertEqual(self.builds[3].name, 'project-test2')
3735
Paul Belanger174a8272017-03-14 13:20:10 -04003736 self.executor_server.release('project-.*')
Clark Boylan7603a372014-01-21 11:43:20 -08003737 self.waitUntilSettled()
3738
James E. Blaircc324192017-04-19 14:36:08 -07003739 tenant = self.sched.abide.tenants.get('tenant-one')
Clint Byrumb01f1ca2017-04-02 10:19:52 -07003740 queue = tenant.layout.pipelines['gate'].queues[0]
Clark Boylan7603a372014-01-21 11:43:20 -08003741 # A failed so window is reduced by 1 to 1.
3742 self.assertEqual(queue.window, 1)
3743 self.assertEqual(queue.window_floor, 1)
3744 self.assertEqual(A.data['status'], 'NEW')
3745
3746 # Gate is reset and only B's merge job is queued because
3747 # window shrunk to 1.
3748 self.assertEqual(len(self.builds), 1)
3749 self.assertEqual(self.builds[0].name, 'project-merge')
3750
Paul Belanger174a8272017-03-14 13:20:10 -04003751 self.executor_server.release('.*-merge')
Clark Boylan7603a372014-01-21 11:43:20 -08003752 self.waitUntilSettled()
3753
3754 # Only B's test jobs are queued because window is still 1.
3755 self.assertEqual(len(self.builds), 2)
3756 self.assertEqual(self.builds[0].name, 'project-test1')
3757 self.assertEqual(self.builds[1].name, 'project-test2')
3758
Paul Belanger174a8272017-03-14 13:20:10 -04003759 self.executor_server.release('project-.*')
Clark Boylan7603a372014-01-21 11:43:20 -08003760 self.waitUntilSettled()
3761
3762 # B was successfully merged so window is increased to 2.
3763 self.assertEqual(queue.window, 2)
3764 self.assertEqual(queue.window_floor, 1)
3765 self.assertEqual(B.data['status'], 'MERGED')
3766
3767 # Only C is left and its merge job is queued.
3768 self.assertEqual(len(self.builds), 1)
3769 self.assertEqual(self.builds[0].name, 'project-merge')
3770
Paul Belanger174a8272017-03-14 13:20:10 -04003771 self.executor_server.release('.*-merge')
Clark Boylan7603a372014-01-21 11:43:20 -08003772 self.waitUntilSettled()
3773
3774 # After successful merge job the test jobs for C are queued.
3775 self.assertEqual(len(self.builds), 2)
3776 self.assertEqual(self.builds[0].name, 'project-test1')
3777 self.assertEqual(self.builds[1].name, 'project-test2')
3778
Paul Belanger174a8272017-03-14 13:20:10 -04003779 self.executor_server.release('project-.*')
Clark Boylan7603a372014-01-21 11:43:20 -08003780 self.waitUntilSettled()
3781
3782 # C successfully merged so window is bumped to 3.
3783 self.assertEqual(queue.window, 3)
3784 self.assertEqual(queue.window_floor, 1)
3785 self.assertEqual(C.data['status'], 'MERGED')
3786
James E. Blaircc324192017-04-19 14:36:08 -07003787 @simple_layout('layouts/rate-limit.yaml')
Clark Boylan7603a372014-01-21 11:43:20 -08003788 def test_queue_rate_limiting_dependent(self):
3789 "Test that DependentPipelines are rate limited with dep in window"
Paul Belanger174a8272017-03-14 13:20:10 -04003790 self.executor_server.hold_jobs_in_build = True
Clark Boylan7603a372014-01-21 11:43:20 -08003791 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
3792 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
3793 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
3794
3795 B.setDependsOn(A, 1)
3796
Paul Belanger174a8272017-03-14 13:20:10 -04003797 self.executor_server.failJob('project-test1', A)
Clark Boylan7603a372014-01-21 11:43:20 -08003798
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003799 A.addApproval('Code-Review', 2)
3800 B.addApproval('Code-Review', 2)
3801 C.addApproval('Code-Review', 2)
Clark Boylan7603a372014-01-21 11:43:20 -08003802
Tobias Henkelbf24fd12017-07-27 06:13:07 +02003803 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
3804 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
3805 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
Clark Boylan7603a372014-01-21 11:43:20 -08003806 self.waitUntilSettled()
3807
3808 # Only A and B will have their merge jobs queued because
3809 # window is 2.
3810 self.assertEqual(len(self.builds), 2)
3811 self.assertEqual(self.builds[0].name, 'project-merge')
3812 self.assertEqual(self.builds[1].name, 'project-merge')
3813
James E. Blairf4a5f022017-04-18 14:01:10 -07003814 self.orderedRelease(2)
Clark Boylan7603a372014-01-21 11:43:20 -08003815
3816 # Only A and B will have their test jobs queued because
3817 # window is 2.
3818 self.assertEqual(len(self.builds), 4)
3819 self.assertEqual(self.builds[0].name, 'project-test1')
3820 self.assertEqual(self.builds[1].name, 'project-test2')
3821 self.assertEqual(self.builds[2].name, 'project-test1')
3822 self.assertEqual(self.builds[3].name, 'project-test2')
3823
Paul Belanger174a8272017-03-14 13:20:10 -04003824 self.executor_server.release('project-.*')
Clark Boylan7603a372014-01-21 11:43:20 -08003825 self.waitUntilSettled()
3826
James E. Blaircc324192017-04-19 14:36:08 -07003827 tenant = self.sched.abide.tenants.get('tenant-one')
Clint Byrum14ab6ca2017-04-03 10:15:10 -07003828 queue = tenant.layout.pipelines['gate'].queues[0]
Clark Boylan7603a372014-01-21 11:43:20 -08003829 # A failed so window is reduced by 1 to 1.
3830 self.assertEqual(queue.window, 1)
3831 self.assertEqual(queue.window_floor, 1)
3832 self.assertEqual(A.data['status'], 'NEW')
3833 self.assertEqual(B.data['status'], 'NEW')
3834
3835 # Gate is reset and only C's merge job is queued because
3836 # window shrunk to 1 and A and B were dequeued.
3837 self.assertEqual(len(self.builds), 1)
3838 self.assertEqual(self.builds[0].name, 'project-merge')
3839
James E. Blairf4a5f022017-04-18 14:01:10 -07003840 self.orderedRelease(1)
Clark Boylan7603a372014-01-21 11:43:20 -08003841
3842 # Only C's test jobs are queued because window is still 1.
3843 self.assertEqual(len(self.builds), 2)
James E. Blairdf25ddc2017-07-08 07:57:09 -07003844 builds = self.getSortedBuilds()
3845 self.assertEqual(builds[0].name, 'project-test1')
3846 self.assertEqual(builds[1].name, 'project-test2')
Clark Boylan7603a372014-01-21 11:43:20 -08003847
Paul Belanger174a8272017-03-14 13:20:10 -04003848 self.executor_server.release('project-.*')
Clark Boylan7603a372014-01-21 11:43:20 -08003849 self.waitUntilSettled()
3850
3851 # C was successfully merged so window is increased to 2.
3852 self.assertEqual(queue.window, 2)
3853 self.assertEqual(queue.window_floor, 1)
3854 self.assertEqual(C.data['status'], 'MERGED')
Joshua Heskethba8776a2014-01-12 14:35:40 +08003855
James E. Blair8fd207b2017-11-30 13:44:38 -08003856 @simple_layout('layouts/reconfigure-window.yaml')
3857 def test_reconfigure_window_shrink(self):
3858 # Test the active window shrinking during reconfiguration
3859 self.executor_server.hold_jobs_in_build = True
3860
3861 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
3862 A.addApproval('Code-Review', 2)
3863 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
3864 self.waitUntilSettled()
3865 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
3866 B.addApproval('Code-Review', 2)
3867 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
3868 self.waitUntilSettled()
3869
3870 tenant = self.sched.abide.tenants.get('tenant-one')
3871 queue = tenant.layout.pipelines['gate'].queues[0]
3872 self.assertEqual(queue.window, 20)
3873 self.assertTrue(len(self.builds), 4)
3874
3875 self.executor_server.release('job1')
3876 self.waitUntilSettled()
3877 self.commitConfigUpdate('org/common-config',
3878 'layouts/reconfigure-window2.yaml')
3879 self.sched.reconfigure(self.config)
3880 tenant = self.sched.abide.tenants.get('tenant-one')
3881 queue = tenant.layout.pipelines['gate'].queues[0]
James E. Blair74be3812017-11-30 14:17:25 -08003882 # Even though we have configured a smaller window, the value
3883 # on the existing shared queue should be used.
3884 self.assertEqual(queue.window, 20)
3885 self.assertTrue(len(self.builds), 4)
3886
3887 self.sched.reconfigure(self.config)
3888 tenant = self.sched.abide.tenants.get('tenant-one')
3889 queue = tenant.layout.pipelines['gate'].queues[0]
3890 self.assertEqual(queue.window, 20)
3891 self.assertTrue(len(self.builds), 4)
3892
3893 self.executor_server.hold_jobs_in_build = False
3894 self.executor_server.release()
3895
3896 self.waitUntilSettled()
3897 self.assertHistory([
3898 dict(name='job1', result='SUCCESS', changes='1,1'),
3899 dict(name='job1', result='SUCCESS', changes='1,1 2,1'),
3900 dict(name='job2', result='SUCCESS', changes='1,1'),
3901 dict(name='job2', result='SUCCESS', changes='1,1 2,1'),
3902 ], ordered=False)
3903
3904 @simple_layout('layouts/reconfigure-window-fixed.yaml')
3905 def test_reconfigure_window_fixed(self):
3906 # Test the active window shrinking during reconfiguration
3907 self.executor_server.hold_jobs_in_build = True
3908
3909 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
3910 A.addApproval('Code-Review', 2)
3911 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
3912 self.waitUntilSettled()
3913 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
3914 B.addApproval('Code-Review', 2)
3915 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
3916 self.waitUntilSettled()
3917
3918 tenant = self.sched.abide.tenants.get('tenant-one')
3919 queue = tenant.layout.pipelines['gate'].queues[0]
3920 self.assertEqual(queue.window, 2)
3921 self.assertTrue(len(self.builds), 4)
3922
3923 self.executor_server.release('job1')
3924 self.waitUntilSettled()
3925 self.commitConfigUpdate('org/common-config',
3926 'layouts/reconfigure-window-fixed2.yaml')
3927 self.sched.reconfigure(self.config)
3928 tenant = self.sched.abide.tenants.get('tenant-one')
3929 queue = tenant.layout.pipelines['gate'].queues[0]
3930 # Because we have configured a static window, it should
3931 # be allowed to shrink on reconfiguration.
James E. Blair8fd207b2017-11-30 13:44:38 -08003932 self.assertEqual(queue.window, 1)
James E. Blair74be3812017-11-30 14:17:25 -08003933 # B is outside the window, but still marked active until the
3934 # next pass through the queue processor, so its builds haven't
3935 # been canceled.
James E. Blair8fd207b2017-11-30 13:44:38 -08003936 self.assertTrue(len(self.builds), 4)
3937
3938 self.sched.reconfigure(self.config)
3939 tenant = self.sched.abide.tenants.get('tenant-one')
3940 queue = tenant.layout.pipelines['gate'].queues[0]
3941 self.assertEqual(queue.window, 1)
3942 # B's builds have been canceled now
3943 self.assertTrue(len(self.builds), 2)
3944
3945 self.executor_server.hold_jobs_in_build = False
3946 self.executor_server.release()
James E. Blair74be3812017-11-30 14:17:25 -08003947
3948 # B's builds will be restarted and will show up in our history
3949 # twice.
James E. Blair8fd207b2017-11-30 13:44:38 -08003950 self.waitUntilSettled()
3951 self.assertHistory([
3952 dict(name='job1', result='SUCCESS', changes='1,1'),
3953 dict(name='job1', result='SUCCESS', changes='1,1 2,1'),
3954 dict(name='job2', result='SUCCESS', changes='1,1'),
3955 dict(name='job2', result='SUCCESS', changes='1,1 2,1'),
3956 dict(name='job1', result='SUCCESS', changes='1,1 2,1'),
3957 dict(name='job2', result='SUCCESS', changes='1,1 2,1'),
3958 ], ordered=False)
3959
3960 @simple_layout('layouts/reconfigure-remove-add.yaml')
3961 def test_reconfigure_remove_add(self):
3962 # Test removing, then adding a job while in queue
3963 self.executor_server.hold_jobs_in_build = True
3964
3965 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
3966 A.addApproval('Code-Review', 2)
3967 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
3968 self.waitUntilSettled()
3969 self.assertTrue(len(self.builds), 2)
3970 self.executor_server.release('job2')
3971 self.assertTrue(len(self.builds), 1)
3972
3973 # Remove job2
3974 self.commitConfigUpdate('org/common-config',
3975 'layouts/reconfigure-remove-add2.yaml')
3976 self.sched.reconfigure(self.config)
3977 self.assertTrue(len(self.builds), 1)
3978
3979 # Add job2 back
3980 self.commitConfigUpdate('org/common-config',
3981 'layouts/reconfigure-remove-add.yaml')
3982 self.sched.reconfigure(self.config)
3983 self.assertTrue(len(self.builds), 2)
3984
3985 self.executor_server.hold_jobs_in_build = False
3986 self.executor_server.release()
3987 # This will run new builds for B
3988 self.waitUntilSettled()
3989 self.assertHistory([
3990 dict(name='job2', result='SUCCESS', changes='1,1'),
3991 dict(name='job1', result='SUCCESS', changes='1,1'),
3992 dict(name='job2', result='SUCCESS', changes='1,1'),
3993 ], ordered=False)
3994
Joshua Heskethba8776a2014-01-12 14:35:40 +08003995 def test_worker_update_metadata(self):
3996 "Test if a worker can send back metadata about itself"
Paul Belanger174a8272017-03-14 13:20:10 -04003997 self.executor_server.hold_jobs_in_build = True
Joshua Heskethba8776a2014-01-12 14:35:40 +08003998
3999 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004000 A.addApproval('Code-Review', 2)
4001 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
Joshua Heskethba8776a2014-01-12 14:35:40 +08004002 self.waitUntilSettled()
4003
Clint Byrumd4de79a2017-05-27 11:39:13 -07004004 self.assertEqual(len(self.executor_client.builds), 1)
Joshua Heskethba8776a2014-01-12 14:35:40 +08004005
4006 self.log.debug('Current builds:')
Clint Byrumd4de79a2017-05-27 11:39:13 -07004007 self.log.debug(self.executor_client.builds)
Joshua Heskethba8776a2014-01-12 14:35:40 +08004008
4009 start = time.time()
4010 while True:
4011 if time.time() - start > 10:
4012 raise Exception("Timeout waiting for gearman server to report "
4013 + "back to the client")
Clint Byrumd4de79a2017-05-27 11:39:13 -07004014 build = list(self.executor_client.builds.values())[0]
Monty Tayloreea6bb92017-06-14 11:56:54 -05004015 if build.worker.name == self.executor_server.hostname:
Joshua Heskethba8776a2014-01-12 14:35:40 +08004016 break
4017 else:
4018 time.sleep(0)
4019
4020 self.log.debug(build)
Monty Tayloreea6bb92017-06-14 11:56:54 -05004021 self.assertEqual(self.executor_server.hostname, build.worker.name)
Joshua Heskethba8776a2014-01-12 14:35:40 +08004022
Paul Belanger174a8272017-03-14 13:20:10 -04004023 self.executor_server.hold_jobs_in_build = False
4024 self.executor_server.release()
Joshua Heskethba8776a2014-01-12 14:35:40 +08004025 self.waitUntilSettled()
Joshua Hesketh3979e3e2014-03-04 11:21:10 +11004026
James E. Blaircc324192017-04-19 14:36:08 -07004027 @simple_layout('layouts/footer-message.yaml')
Joshua Hesketh3979e3e2014-03-04 11:21:10 +11004028 def test_footer_message(self):
4029 "Test a pipeline's footer message is correctly added to the report."
Joshua Hesketh3979e3e2014-03-04 11:21:10 +11004030 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004031 A.addApproval('Code-Review', 2)
Paul Belanger174a8272017-03-14 13:20:10 -04004032 self.executor_server.failJob('project-test1', A)
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004033 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
Joshua Hesketh3979e3e2014-03-04 11:21:10 +11004034 self.waitUntilSettled()
4035
4036 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004037 B.addApproval('Code-Review', 2)
4038 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
Joshua Hesketh3979e3e2014-03-04 11:21:10 +11004039 self.waitUntilSettled()
4040
4041 self.assertEqual(2, len(self.smtp_messages))
4042
Adam Gandelman82258fd2017-02-07 16:51:58 -08004043 failure_msg = """\
Joshua Hesketh3979e3e2014-03-04 11:21:10 +11004044Build failed. For information on how to proceed, see \
Adam Gandelman82258fd2017-02-07 16:51:58 -08004045http://wiki.example.org/Test_Failures"""
Joshua Hesketh3979e3e2014-03-04 11:21:10 +11004046
Adam Gandelman82258fd2017-02-07 16:51:58 -08004047 footer_msg = """\
Joshua Hesketh3979e3e2014-03-04 11:21:10 +11004048For CI problems and help debugging, contact ci@example.org"""
4049
Adam Gandelman82258fd2017-02-07 16:51:58 -08004050 self.assertTrue(self.smtp_messages[0]['body'].startswith(failure_msg))
4051 self.assertTrue(self.smtp_messages[0]['body'].endswith(footer_msg))
4052 self.assertFalse(self.smtp_messages[1]['body'].startswith(failure_msg))
4053 self.assertTrue(self.smtp_messages[1]['body'].endswith(footer_msg))
Joshua Heskethb7179772014-01-30 23:30:46 +11004054
Tobias Henkel9842bd72017-05-16 13:40:03 +02004055 @simple_layout('layouts/unmanaged-project.yaml')
4056 def test_unmanaged_project_start_message(self):
4057 "Test start reporting is not done for unmanaged projects."
4058 self.init_repo("org/project", tag='init')
4059 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
4060 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4061 self.waitUntilSettled()
4062
4063 self.assertEqual(0, len(A.messages))
4064
Clint Byrumb9de8be2017-05-27 13:49:19 -07004065 @simple_layout('layouts/merge-failure.yaml')
Joshua Heskethb7179772014-01-30 23:30:46 +11004066 def test_merge_failure_reporters(self):
4067 """Check that the config is set up correctly"""
4068
Clint Byrumb9de8be2017-05-27 13:49:19 -07004069 tenant = self.sched.abide.tenants.get('tenant-one')
Joshua Heskethb7179772014-01-30 23:30:46 +11004070 self.assertEqual(
Jeremy Stanley1c2c3c22015-06-15 21:23:19 +00004071 "Merge Failed.\n\nThis change or one of its cross-repo "
4072 "dependencies was unable to be automatically merged with the "
4073 "current state of its repository. Please rebase the change and "
4074 "upload a new patchset.",
Clint Byrumb9de8be2017-05-27 13:49:19 -07004075 tenant.layout.pipelines['check'].merge_failure_message)
Joshua Heskethb7179772014-01-30 23:30:46 +11004076 self.assertEqual(
4077 "The merge failed! For more information...",
Clint Byrumb9de8be2017-05-27 13:49:19 -07004078 tenant.layout.pipelines['gate'].merge_failure_message)
Joshua Heskethb7179772014-01-30 23:30:46 +11004079
4080 self.assertEqual(
Clint Byrumb9de8be2017-05-27 13:49:19 -07004081 len(tenant.layout.pipelines['check'].merge_failure_actions), 1)
Joshua Heskethb7179772014-01-30 23:30:46 +11004082 self.assertEqual(
Clint Byrumb9de8be2017-05-27 13:49:19 -07004083 len(tenant.layout.pipelines['gate'].merge_failure_actions), 2)
Joshua Heskethb7179772014-01-30 23:30:46 +11004084
4085 self.assertTrue(isinstance(
Clint Byrumb9de8be2017-05-27 13:49:19 -07004086 tenant.layout.pipelines['check'].merge_failure_actions[0],
4087 gerritreporter.GerritReporter))
Joshua Heskethb7179772014-01-30 23:30:46 +11004088
4089 self.assertTrue(
4090 (
Clint Byrumb9de8be2017-05-27 13:49:19 -07004091 isinstance(tenant.layout.pipelines['gate'].
Joshua Heskethde958652015-11-10 19:19:50 +11004092 merge_failure_actions[0],
Clint Byrumb9de8be2017-05-27 13:49:19 -07004093 zuul.driver.smtp.smtpreporter.SMTPReporter) and
4094 isinstance(tenant.layout.pipelines['gate'].
Joshua Heskethde958652015-11-10 19:19:50 +11004095 merge_failure_actions[1],
Clint Byrumb9de8be2017-05-27 13:49:19 -07004096 gerritreporter.GerritReporter)
Joshua Heskethb7179772014-01-30 23:30:46 +11004097 ) or (
Clint Byrumb9de8be2017-05-27 13:49:19 -07004098 isinstance(tenant.layout.pipelines['gate'].
Joshua Heskethde958652015-11-10 19:19:50 +11004099 merge_failure_actions[0],
Clint Byrumb9de8be2017-05-27 13:49:19 -07004100 gerritreporter.GerritReporter) and
4101 isinstance(tenant.layout.pipelines['gate'].
Joshua Heskethde958652015-11-10 19:19:50 +11004102 merge_failure_actions[1],
Clint Byrumb9de8be2017-05-27 13:49:19 -07004103 zuul.driver.smtp.smtpreporter.SMTPReporter)
Joshua Heskethb7179772014-01-30 23:30:46 +11004104 )
4105 )
4106
James E. Blairec056492016-07-22 09:45:56 -07004107 @skip("Disabled for early v3 development")
Joshua Heskethb7179772014-01-30 23:30:46 +11004108 def test_merge_failure_reports(self):
4109 """Check that when a change fails to merge the correct message is sent
4110 to the correct reporter"""
James E. Blairf84026c2015-12-08 16:11:46 -08004111 self.updateConfigLayout(
4112 'tests/fixtures/layout-merge-failure.yaml')
Joshua Heskethb7179772014-01-30 23:30:46 +11004113 self.sched.reconfigure(self.config)
4114 self.registerJobs()
4115
4116 # Check a test failure isn't reported to SMTP
4117 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004118 A.addApproval('Code-Review', 2)
Paul Belanger174a8272017-03-14 13:20:10 -04004119 self.executor_server.failJob('project-test1', A)
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004120 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
Joshua Heskethb7179772014-01-30 23:30:46 +11004121 self.waitUntilSettled()
4122
4123 self.assertEqual(3, len(self.history)) # 3 jobs
4124 self.assertEqual(0, len(self.smtp_messages))
4125
4126 # Check a merge failure is reported to SMTP
4127 # B should be merged, but C will conflict with B
4128 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
4129 B.addPatchset(['conflict'])
4130 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
4131 C.addPatchset(['conflict'])
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004132 B.addApproval('Code-Review', 2)
4133 C.addApproval('Code-Review', 2)
4134 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
4135 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
Joshua Heskethb7179772014-01-30 23:30:46 +11004136 self.waitUntilSettled()
4137
4138 self.assertEqual(6, len(self.history)) # A and B jobs
4139 self.assertEqual(1, len(self.smtp_messages))
4140 self.assertEqual('The merge failed! For more information...',
4141 self.smtp_messages[0]['body'])
Joshua Hesketh36c3fa52014-01-22 11:40:52 +11004142
James E. Blairec056492016-07-22 09:45:56 -07004143 @skip("Disabled for early v3 development")
James E. Blairf760f0e2016-02-09 08:44:52 -08004144 def test_default_merge_failure_reports(self):
4145 """Check that the default merge failure reports are correct."""
4146
4147 # A should report success, B should report merge failure.
4148 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
4149 A.addPatchset(['conflict'])
4150 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
4151 B.addPatchset(['conflict'])
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004152 A.addApproval('Code-Review', 2)
4153 B.addApproval('Code-Review', 2)
4154 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
4155 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blairf760f0e2016-02-09 08:44:52 -08004156 self.waitUntilSettled()
4157
4158 self.assertEqual(3, len(self.history)) # A jobs
4159 self.assertEqual(A.reported, 2)
4160 self.assertEqual(B.reported, 2)
4161 self.assertEqual(A.data['status'], 'MERGED')
4162 self.assertEqual(B.data['status'], 'NEW')
4163 self.assertIn('Build succeeded', A.messages[1])
4164 self.assertIn('Merge Failed', B.messages[1])
4165 self.assertIn('automatically merged', B.messages[1])
4166 self.assertNotIn('logs.example.com', B.messages[1])
4167 self.assertNotIn('SKIPPED', B.messages[1])
4168
Joshua Hesketh85af4e92014-02-21 08:28:58 -08004169 def test_client_get_running_jobs(self):
4170 "Test that the RPC client can get a list of running jobs"
Paul Belanger174a8272017-03-14 13:20:10 -04004171 self.executor_server.hold_jobs_in_build = True
Joshua Hesketh85af4e92014-02-21 08:28:58 -08004172 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004173 A.addApproval('Code-Review', 2)
4174 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
Joshua Hesketh85af4e92014-02-21 08:28:58 -08004175 self.waitUntilSettled()
4176
4177 client = zuul.rpcclient.RPCClient('127.0.0.1',
4178 self.gearman_server.port)
Clark Boylandf4eba22017-04-24 17:37:58 -07004179 self.addCleanup(client.shutdown)
Joshua Hesketh85af4e92014-02-21 08:28:58 -08004180
4181 # Wait for gearman server to send the initial workData back to zuul
4182 start = time.time()
4183 while True:
4184 if time.time() - start > 10:
4185 raise Exception("Timeout waiting for gearman server to report "
4186 + "back to the client")
Clint Byrum1d0c7d12017-05-10 19:40:53 -07004187 build = list(self.executor_client.builds.values())[0]
Monty Tayloreea6bb92017-06-14 11:56:54 -05004188 if build.worker.name == self.executor_server.hostname:
Joshua Hesketh85af4e92014-02-21 08:28:58 -08004189 break
4190 else:
4191 time.sleep(0)
4192
4193 running_items = client.get_running_jobs()
4194
4195 self.assertEqual(1, len(running_items))
4196 running_item = running_items[0]
4197 self.assertEqual([], running_item['failing_reasons'])
4198 self.assertEqual([], running_item['items_behind'])
4199 self.assertEqual('https://hostname/1', running_item['url'])
Monty Taylor38b553a2017-06-05 13:06:10 -05004200 self.assertIsNone(running_item['item_ahead'])
Joshua Hesketh85af4e92014-02-21 08:28:58 -08004201 self.assertEqual('org/project', running_item['project'])
Monty Taylor38b553a2017-06-05 13:06:10 -05004202 self.assertIsNone(running_item['remaining_time'])
Joshua Hesketh85af4e92014-02-21 08:28:58 -08004203 self.assertEqual(True, running_item['active'])
4204 self.assertEqual('1,1', running_item['id'])
4205
4206 self.assertEqual(3, len(running_item['jobs']))
4207 for job in running_item['jobs']:
4208 if job['name'] == 'project-merge':
4209 self.assertEqual('project-merge', job['name'])
4210 self.assertEqual('gate', job['pipeline'])
4211 self.assertEqual(False, job['retry'])
Monty Taylor51139a02016-05-24 11:28:10 -05004212 self.assertEqual(
Tobias Henkelb4407fc2017-07-07 13:52:56 +02004213 'static/stream.html?uuid={uuid}&logfile=console.log'
4214 .format(uuid=job['uuid']), job['url'])
4215 self.assertEqual(
Monty Taylor51139a02016-05-24 11:28:10 -05004216 'finger://{hostname}/{uuid}'.format(
4217 hostname=self.executor_server.hostname,
4218 uuid=job['uuid']),
Tobias Henkelb4407fc2017-07-07 13:52:56 +02004219 job['finger_url'])
Clint Byrumd4de79a2017-05-27 11:39:13 -07004220 self.assertEqual(2, len(job['worker']))
Joshua Hesketh85af4e92014-02-21 08:28:58 -08004221 self.assertEqual(False, job['canceled'])
4222 self.assertEqual(True, job['voting'])
Monty Taylor38b553a2017-06-05 13:06:10 -05004223 self.assertIsNone(job['result'])
Joshua Hesketh85af4e92014-02-21 08:28:58 -08004224 self.assertEqual('gate', job['pipeline'])
4225 break
4226
Paul Belanger174a8272017-03-14 13:20:10 -04004227 self.executor_server.hold_jobs_in_build = False
4228 self.executor_server.release()
Joshua Hesketh85af4e92014-02-21 08:28:58 -08004229 self.waitUntilSettled()
4230
4231 running_items = client.get_running_jobs()
4232 self.assertEqual(0, len(running_items))
James E. Blairbadc1ad2014-04-28 13:55:14 -07004233
James E. Blairb70e55a2017-04-19 12:57:02 -07004234 @simple_layout('layouts/nonvoting-pipeline.yaml')
James E. Blairbadc1ad2014-04-28 13:55:14 -07004235 def test_nonvoting_pipeline(self):
4236 "Test that a nonvoting pipeline (experimental) can still report"
4237
Joshua Heskethcc017ea2014-04-30 19:55:25 +10004238 A = self.fake_gerrit.addFakeChange('org/experimental-project',
4239 'master', 'A')
James E. Blairbadc1ad2014-04-28 13:55:14 -07004240 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4241 self.waitUntilSettled()
Paul Belanger15e3e202016-10-14 16:27:34 -04004242
4243 self.assertEqual(self.getJobFromHistory('project-merge').result,
4244 'SUCCESS')
Joshua Heskethcc017ea2014-04-30 19:55:25 +10004245 self.assertEqual(
4246 self.getJobFromHistory('experimental-project-test').result,
4247 'SUCCESS')
James E. Blairbadc1ad2014-04-28 13:55:14 -07004248 self.assertEqual(A.reported, 1)
James E. Blair5ee24252014-12-30 10:12:29 -08004249
4250 def test_crd_gate(self):
4251 "Test cross-repo dependencies"
4252 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4253 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004254 A.addApproval('Code-Review', 2)
4255 B.addApproval('Code-Review', 2)
James E. Blair5ee24252014-12-30 10:12:29 -08004256
4257 AM2 = self.fake_gerrit.addFakeChange('org/project1', 'master', 'AM2')
4258 AM1 = self.fake_gerrit.addFakeChange('org/project1', 'master', 'AM1')
4259 AM2.setMerged()
4260 AM1.setMerged()
4261
4262 BM2 = self.fake_gerrit.addFakeChange('org/project2', 'master', 'BM2')
4263 BM1 = self.fake_gerrit.addFakeChange('org/project2', 'master', 'BM1')
4264 BM2.setMerged()
4265 BM1.setMerged()
4266
4267 # A -> AM1 -> AM2
4268 # B -> BM1 -> BM2
4269 # A Depends-On: B
4270 # M2 is here to make sure it is never queried. If it is, it
4271 # means zuul is walking down the entire history of merged
4272 # changes.
4273
4274 B.setDependsOn(BM1, 1)
4275 BM1.setDependsOn(BM2, 1)
4276
4277 A.setDependsOn(AM1, 1)
4278 AM1.setDependsOn(AM2, 1)
4279
4280 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4281 A.subject, B.data['id'])
4282
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004283 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair5ee24252014-12-30 10:12:29 -08004284 self.waitUntilSettled()
4285
4286 self.assertEqual(A.data['status'], 'NEW')
4287 self.assertEqual(B.data['status'], 'NEW')
4288
Paul Belanger5dccbe72016-11-14 11:17:37 -05004289 for connection in self.connections.connections.values():
James E. Blairb0a95ab2017-10-18 09:39:18 -07004290 connection.maintainCache([])
James E. Blair5ee24252014-12-30 10:12:29 -08004291
Paul Belanger174a8272017-03-14 13:20:10 -04004292 self.executor_server.hold_jobs_in_build = True
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004293 B.addApproval('Approved', 1)
4294 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair5ee24252014-12-30 10:12:29 -08004295 self.waitUntilSettled()
4296
Paul Belanger174a8272017-03-14 13:20:10 -04004297 self.executor_server.release('.*-merge')
James E. Blair5ee24252014-12-30 10:12:29 -08004298 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04004299 self.executor_server.release('.*-merge')
James E. Blair5ee24252014-12-30 10:12:29 -08004300 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04004301 self.executor_server.hold_jobs_in_build = False
4302 self.executor_server.release()
James E. Blair5ee24252014-12-30 10:12:29 -08004303 self.waitUntilSettled()
4304
4305 self.assertEqual(AM2.queried, 0)
4306 self.assertEqual(BM2.queried, 0)
4307 self.assertEqual(A.data['status'], 'MERGED')
4308 self.assertEqual(B.data['status'], 'MERGED')
4309 self.assertEqual(A.reported, 2)
4310 self.assertEqual(B.reported, 2)
4311
Paul Belanger5dccbe72016-11-14 11:17:37 -05004312 changes = self.getJobFromHistory(
4313 'project-merge', 'org/project1').changes
4314 self.assertEqual(changes, '2,1 1,1')
James E. Blair8f78d882015-02-05 08:51:37 -08004315
4316 def test_crd_branch(self):
4317 "Test cross-repo dependencies in multiple branches"
Paul Belanger6379db12016-11-14 13:57:54 -05004318
4319 self.create_branch('org/project2', 'mp')
James E. Blair8f78d882015-02-05 08:51:37 -08004320 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4321 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
K Jonathan Harkerd1d51d22017-08-23 11:57:40 -04004322 C1 = self.fake_gerrit.addFakeChange('org/project2', 'mp', 'C1')
4323 C2 = self.fake_gerrit.addFakeChange('org/project2', 'mp', 'C2',
4324 status='ABANDONED')
4325 C1.data['id'] = B.data['id']
4326 C2.data['id'] = B.data['id']
4327
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004328 A.addApproval('Code-Review', 2)
4329 B.addApproval('Code-Review', 2)
K Jonathan Harkerd1d51d22017-08-23 11:57:40 -04004330 C1.addApproval('Code-Review', 2)
James E. Blair8f78d882015-02-05 08:51:37 -08004331
K Jonathan Harkerd1d51d22017-08-23 11:57:40 -04004332 # A Depends-On: B+C1
James E. Blair8f78d882015-02-05 08:51:37 -08004333 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4334 A.subject, B.data['id'])
4335
Paul Belanger174a8272017-03-14 13:20:10 -04004336 self.executor_server.hold_jobs_in_build = True
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004337 B.addApproval('Approved', 1)
K Jonathan Harkerd1d51d22017-08-23 11:57:40 -04004338 C1.addApproval('Approved', 1)
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004339 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair8f78d882015-02-05 08:51:37 -08004340 self.waitUntilSettled()
4341
Paul Belanger174a8272017-03-14 13:20:10 -04004342 self.executor_server.release('.*-merge')
James E. Blair8f78d882015-02-05 08:51:37 -08004343 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04004344 self.executor_server.release('.*-merge')
James E. Blair8f78d882015-02-05 08:51:37 -08004345 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04004346 self.executor_server.release('.*-merge')
James E. Blair8f78d882015-02-05 08:51:37 -08004347 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04004348 self.executor_server.hold_jobs_in_build = False
4349 self.executor_server.release()
James E. Blair8f78d882015-02-05 08:51:37 -08004350 self.waitUntilSettled()
4351
4352 self.assertEqual(A.data['status'], 'MERGED')
4353 self.assertEqual(B.data['status'], 'MERGED')
K Jonathan Harkerd1d51d22017-08-23 11:57:40 -04004354 self.assertEqual(C1.data['status'], 'MERGED')
James E. Blair8f78d882015-02-05 08:51:37 -08004355 self.assertEqual(A.reported, 2)
4356 self.assertEqual(B.reported, 2)
K Jonathan Harkerd1d51d22017-08-23 11:57:40 -04004357 self.assertEqual(C1.reported, 2)
James E. Blair8f78d882015-02-05 08:51:37 -08004358
Paul Belanger6379db12016-11-14 13:57:54 -05004359 changes = self.getJobFromHistory(
4360 'project-merge', 'org/project1').changes
4361 self.assertEqual(changes, '2,1 3,1 1,1')
James E. Blair8f78d882015-02-05 08:51:37 -08004362
4363 def test_crd_multiline(self):
4364 "Test multiple depends-on lines in commit"
4365 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4366 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4367 C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004368 A.addApproval('Code-Review', 2)
4369 B.addApproval('Code-Review', 2)
4370 C.addApproval('Code-Review', 2)
James E. Blair8f78d882015-02-05 08:51:37 -08004371
4372 # A Depends-On: B+C
4373 A.data['commitMessage'] = '%s\n\nDepends-On: %s\nDepends-On: %s\n' % (
4374 A.subject, B.data['id'], C.data['id'])
4375
Paul Belanger174a8272017-03-14 13:20:10 -04004376 self.executor_server.hold_jobs_in_build = True
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004377 B.addApproval('Approved', 1)
4378 C.addApproval('Approved', 1)
4379 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair8f78d882015-02-05 08:51:37 -08004380 self.waitUntilSettled()
4381
Paul Belanger174a8272017-03-14 13:20:10 -04004382 self.executor_server.release('.*-merge')
James E. Blair8f78d882015-02-05 08:51:37 -08004383 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04004384 self.executor_server.release('.*-merge')
James E. Blair8f78d882015-02-05 08:51:37 -08004385 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04004386 self.executor_server.release('.*-merge')
James E. Blair8f78d882015-02-05 08:51:37 -08004387 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04004388 self.executor_server.hold_jobs_in_build = False
4389 self.executor_server.release()
James E. Blair8f78d882015-02-05 08:51:37 -08004390 self.waitUntilSettled()
4391
4392 self.assertEqual(A.data['status'], 'MERGED')
4393 self.assertEqual(B.data['status'], 'MERGED')
4394 self.assertEqual(C.data['status'], 'MERGED')
4395 self.assertEqual(A.reported, 2)
4396 self.assertEqual(B.reported, 2)
4397 self.assertEqual(C.reported, 2)
4398
Paul Belanger5dccbe72016-11-14 11:17:37 -05004399 changes = self.getJobFromHistory(
4400 'project-merge', 'org/project1').changes
4401 self.assertEqual(changes, '2,1 3,1 1,1')
James E. Blair5ee24252014-12-30 10:12:29 -08004402
4403 def test_crd_unshared_gate(self):
4404 "Test cross-repo dependencies in unshared gate queues"
4405 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4406 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004407 A.addApproval('Code-Review', 2)
4408 B.addApproval('Code-Review', 2)
James E. Blair5ee24252014-12-30 10:12:29 -08004409
4410 # A Depends-On: B
4411 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4412 A.subject, B.data['id'])
4413
4414 # A and B do not share a queue, make sure that A is unable to
4415 # enqueue B (and therefore, A is unable to be enqueued).
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004416 B.addApproval('Approved', 1)
4417 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair5ee24252014-12-30 10:12:29 -08004418 self.waitUntilSettled()
4419
4420 self.assertEqual(A.data['status'], 'NEW')
4421 self.assertEqual(B.data['status'], 'NEW')
4422 self.assertEqual(A.reported, 0)
4423 self.assertEqual(B.reported, 0)
4424 self.assertEqual(len(self.history), 0)
4425
4426 # Enqueue and merge B alone.
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004427 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blair5ee24252014-12-30 10:12:29 -08004428 self.waitUntilSettled()
4429
4430 self.assertEqual(B.data['status'], 'MERGED')
4431 self.assertEqual(B.reported, 2)
4432
4433 # Now that B is merged, A should be able to be enqueued and
4434 # merged.
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004435 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair5ee24252014-12-30 10:12:29 -08004436 self.waitUntilSettled()
4437
4438 self.assertEqual(A.data['status'], 'MERGED')
4439 self.assertEqual(A.reported, 2)
4440
James E. Blair96698e22015-04-02 07:48:21 -07004441 def test_crd_gate_reverse(self):
4442 "Test reverse cross-repo dependencies"
4443 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4444 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004445 A.addApproval('Code-Review', 2)
4446 B.addApproval('Code-Review', 2)
James E. Blair96698e22015-04-02 07:48:21 -07004447
4448 # A Depends-On: B
4449
4450 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4451 A.subject, B.data['id'])
4452
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004453 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair96698e22015-04-02 07:48:21 -07004454 self.waitUntilSettled()
4455
4456 self.assertEqual(A.data['status'], 'NEW')
4457 self.assertEqual(B.data['status'], 'NEW')
4458
Paul Belanger174a8272017-03-14 13:20:10 -04004459 self.executor_server.hold_jobs_in_build = True
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004460 A.addApproval('Approved', 1)
4461 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blair96698e22015-04-02 07:48:21 -07004462 self.waitUntilSettled()
4463
Paul Belanger174a8272017-03-14 13:20:10 -04004464 self.executor_server.release('.*-merge')
James E. Blair96698e22015-04-02 07:48:21 -07004465 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04004466 self.executor_server.release('.*-merge')
James E. Blair96698e22015-04-02 07:48:21 -07004467 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04004468 self.executor_server.hold_jobs_in_build = False
4469 self.executor_server.release()
James E. Blair96698e22015-04-02 07:48:21 -07004470 self.waitUntilSettled()
4471
4472 self.assertEqual(A.data['status'], 'MERGED')
4473 self.assertEqual(B.data['status'], 'MERGED')
4474 self.assertEqual(A.reported, 2)
4475 self.assertEqual(B.reported, 2)
4476
Paul Belanger5dccbe72016-11-14 11:17:37 -05004477 changes = self.getJobFromHistory(
4478 'project-merge', 'org/project1').changes
4479 self.assertEqual(changes, '2,1 1,1')
James E. Blair96698e22015-04-02 07:48:21 -07004480
James E. Blair5ee24252014-12-30 10:12:29 -08004481 def test_crd_cycle(self):
4482 "Test cross-repo dependency cycles"
4483 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4484 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004485 A.addApproval('Code-Review', 2)
4486 B.addApproval('Code-Review', 2)
James E. Blair5ee24252014-12-30 10:12:29 -08004487
4488 # A -> B -> A (via commit-depends)
4489
4490 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4491 A.subject, B.data['id'])
4492 B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4493 B.subject, A.data['id'])
4494
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004495 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair5ee24252014-12-30 10:12:29 -08004496 self.waitUntilSettled()
4497
4498 self.assertEqual(A.reported, 0)
4499 self.assertEqual(B.reported, 0)
4500 self.assertEqual(A.data['status'], 'NEW')
4501 self.assertEqual(B.data['status'], 'NEW')
James E. Blairbfb8e042014-12-30 17:01:44 -08004502
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004503 def test_crd_gate_unknown(self):
4504 "Test unknown projects in dependent pipeline"
James E. Blairc4ba97a2017-04-19 16:26:24 -07004505 self.init_repo("org/unknown", tag='init')
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004506 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
4507 B = self.fake_gerrit.addFakeChange('org/unknown', 'master', 'B')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004508 A.addApproval('Code-Review', 2)
4509 B.addApproval('Code-Review', 2)
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004510
4511 # A Depends-On: B
4512 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4513 A.subject, B.data['id'])
4514
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004515 B.addApproval('Approved', 1)
4516 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004517 self.waitUntilSettled()
4518
4519 # Unknown projects cannot share a queue with any other
4520 # since they don't have common jobs with any other (they have no jobs).
4521 # Changes which depend on unknown project changes
4522 # should not be processed in dependent pipeline
4523 self.assertEqual(A.data['status'], 'NEW')
4524 self.assertEqual(B.data['status'], 'NEW')
4525 self.assertEqual(A.reported, 0)
4526 self.assertEqual(B.reported, 0)
4527 self.assertEqual(len(self.history), 0)
4528
James E. Blair7720b402017-06-07 08:51:49 -07004529 # Simulate change B being gated outside this layout Set the
4530 # change merged before submitting the event so that when the
4531 # event triggers a gerrit query to update the change, we get
4532 # the information that it was merged.
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004533 B.setMerged()
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004534 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004535 self.waitUntilSettled()
4536 self.assertEqual(len(self.history), 0)
4537
4538 # Now that B is merged, A should be able to be enqueued and
4539 # merged.
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004540 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004541 self.waitUntilSettled()
4542
4543 self.assertEqual(A.data['status'], 'MERGED')
4544 self.assertEqual(A.reported, 2)
4545 self.assertEqual(B.data['status'], 'MERGED')
4546 self.assertEqual(B.reported, 0)
4547
James E. Blairbfb8e042014-12-30 17:01:44 -08004548 def test_crd_check(self):
4549 "Test cross-repo dependencies in independent pipelines"
4550
Paul Belanger174a8272017-03-14 13:20:10 -04004551 self.executor_server.hold_jobs_in_build = True
James E. Blairbfb8e042014-12-30 17:01:44 -08004552 self.gearman_server.hold_jobs_in_queue = True
4553 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4554 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4555
4556 # A Depends-On: B
4557 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4558 A.subject, B.data['id'])
4559
4560 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4561 self.waitUntilSettled()
4562
James E. Blairbfb8e042014-12-30 17:01:44 -08004563 self.gearman_server.hold_jobs_in_queue = False
4564 self.gearman_server.release()
4565 self.waitUntilSettled()
4566
Paul Belanger174a8272017-03-14 13:20:10 -04004567 self.executor_server.release('.*-merge')
Paul Belangerb30342b2016-11-14 12:30:43 -05004568 self.waitUntilSettled()
4569
James E. Blair2a535672017-04-27 12:03:15 -07004570 self.assertTrue(self.builds[0].hasChanges(A, B))
James E. Blairbfb8e042014-12-30 17:01:44 -08004571
Paul Belanger174a8272017-03-14 13:20:10 -04004572 self.executor_server.hold_jobs_in_build = False
4573 self.executor_server.release()
Paul Belangerb30342b2016-11-14 12:30:43 -05004574 self.waitUntilSettled()
4575
James E. Blairbfb8e042014-12-30 17:01:44 -08004576 self.assertEqual(A.data['status'], 'NEW')
4577 self.assertEqual(B.data['status'], 'NEW')
4578 self.assertEqual(A.reported, 1)
4579 self.assertEqual(B.reported, 0)
4580
4581 self.assertEqual(self.history[0].changes, '2,1 1,1')
Paul Belangerb30342b2016-11-14 12:30:43 -05004582 tenant = self.sched.abide.tenants.get('tenant-one')
4583 self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
James E. Blair8f78d882015-02-05 08:51:37 -08004584
James E. Blairdbfe1cd2015-02-07 11:41:19 -08004585 def test_crd_check_git_depends(self):
4586 "Test single-repo dependencies in independent pipelines"
James E. Blairb8c16472015-05-05 14:55:26 -07004587 self.gearman_server.hold_jobs_in_build = True
James E. Blairdbfe1cd2015-02-07 11:41:19 -08004588 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4589 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
4590
4591 # Add two git-dependent changes and make sure they both report
4592 # success.
4593 B.setDependsOn(A, 1)
4594 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4595 self.waitUntilSettled()
4596 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4597 self.waitUntilSettled()
4598
James E. Blairb8c16472015-05-05 14:55:26 -07004599 self.orderedRelease()
4600 self.gearman_server.hold_jobs_in_build = False
James E. Blairdbfe1cd2015-02-07 11:41:19 -08004601 self.waitUntilSettled()
4602
4603 self.assertEqual(A.data['status'], 'NEW')
4604 self.assertEqual(B.data['status'], 'NEW')
4605 self.assertEqual(A.reported, 1)
4606 self.assertEqual(B.reported, 1)
4607
4608 self.assertEqual(self.history[0].changes, '1,1')
4609 self.assertEqual(self.history[-1].changes, '1,1 2,1')
Paul Belanger08136962016-11-03 17:57:38 -04004610 tenant = self.sched.abide.tenants.get('tenant-one')
4611 self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
James E. Blairdbfe1cd2015-02-07 11:41:19 -08004612
4613 self.assertIn('Build succeeded', A.messages[0])
4614 self.assertIn('Build succeeded', B.messages[0])
4615
4616 def test_crd_check_duplicate(self):
4617 "Test duplicate check in independent pipelines"
Paul Belanger174a8272017-03-14 13:20:10 -04004618 self.executor_server.hold_jobs_in_build = True
James E. Blairdbfe1cd2015-02-07 11:41:19 -08004619 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4620 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
Paul Belanger08136962016-11-03 17:57:38 -04004621 tenant = self.sched.abide.tenants.get('tenant-one')
4622 check_pipeline = tenant.layout.pipelines['check']
James E. Blairdbfe1cd2015-02-07 11:41:19 -08004623
4624 # Add two git-dependent changes...
4625 B.setDependsOn(A, 1)
4626 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4627 self.waitUntilSettled()
4628 self.assertEqual(len(check_pipeline.getAllItems()), 2)
4629
4630 # ...make sure the live one is not duplicated...
4631 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4632 self.waitUntilSettled()
4633 self.assertEqual(len(check_pipeline.getAllItems()), 2)
4634
4635 # ...but the non-live one is able to be.
4636 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4637 self.waitUntilSettled()
4638 self.assertEqual(len(check_pipeline.getAllItems()), 3)
4639
Clark Boylandd849822015-03-02 12:38:14 -08004640 # Release jobs in order to avoid races with change A jobs
4641 # finishing before change B jobs.
James E. Blaird7650852015-05-07 15:47:37 -07004642 self.orderedRelease()
Paul Belanger174a8272017-03-14 13:20:10 -04004643 self.executor_server.hold_jobs_in_build = False
4644 self.executor_server.release()
James E. Blairdbfe1cd2015-02-07 11:41:19 -08004645 self.waitUntilSettled()
4646
4647 self.assertEqual(A.data['status'], 'NEW')
4648 self.assertEqual(B.data['status'], 'NEW')
4649 self.assertEqual(A.reported, 1)
4650 self.assertEqual(B.reported, 1)
4651
4652 self.assertEqual(self.history[0].changes, '1,1 2,1')
4653 self.assertEqual(self.history[1].changes, '1,1')
Paul Belanger08136962016-11-03 17:57:38 -04004654 self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
James E. Blairdbfe1cd2015-02-07 11:41:19 -08004655
4656 self.assertIn('Build succeeded', A.messages[0])
4657 self.assertIn('Build succeeded', B.messages[0])
4658
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004659 def _test_crd_check_reconfiguration(self, project1, project2):
James E. Blair8f78d882015-02-05 08:51:37 -08004660 "Test cross-repo dependencies re-enqueued in independent pipelines"
4661
4662 self.gearman_server.hold_jobs_in_queue = True
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004663 A = self.fake_gerrit.addFakeChange(project1, 'master', 'A')
4664 B = self.fake_gerrit.addFakeChange(project2, 'master', 'B')
James E. Blair8f78d882015-02-05 08:51:37 -08004665
4666 # A Depends-On: B
4667 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4668 A.subject, B.data['id'])
4669
4670 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4671 self.waitUntilSettled()
4672
4673 self.sched.reconfigure(self.config)
4674
4675 # Make sure the items still share a change queue, and the
4676 # first one is not live.
Paul Belangerce4485f2016-11-11 17:01:56 -05004677 tenant = self.sched.abide.tenants.get('tenant-one')
4678 self.assertEqual(len(tenant.layout.pipelines['check'].queues), 1)
4679 queue = tenant.layout.pipelines['check'].queues[0]
James E. Blair8f78d882015-02-05 08:51:37 -08004680 first_item = queue.queue[0]
4681 for item in queue.queue:
4682 self.assertEqual(item.queue, first_item.queue)
4683 self.assertFalse(first_item.live)
4684 self.assertTrue(queue.queue[1].live)
4685
4686 self.gearman_server.hold_jobs_in_queue = False
4687 self.gearman_server.release()
4688 self.waitUntilSettled()
4689
4690 self.assertEqual(A.data['status'], 'NEW')
4691 self.assertEqual(B.data['status'], 'NEW')
4692 self.assertEqual(A.reported, 1)
4693 self.assertEqual(B.reported, 0)
4694
4695 self.assertEqual(self.history[0].changes, '2,1 1,1')
Paul Belangerce4485f2016-11-11 17:01:56 -05004696 self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
James E. Blair17dd6772015-02-09 14:45:18 -08004697
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004698 def test_crd_check_reconfiguration(self):
4699 self._test_crd_check_reconfiguration('org/project1', 'org/project2')
4700
4701 def test_crd_undefined_project(self):
4702 """Test that undefined projects in dependencies are handled for
4703 independent pipelines"""
4704 # It's a hack for fake gerrit,
4705 # as it implies repo creation upon the creation of any change
James E. Blairc4ba97a2017-04-19 16:26:24 -07004706 self.init_repo("org/unknown", tag='init')
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +00004707 self._test_crd_check_reconfiguration('org/project1', 'org/unknown')
4708
Jesse Keatingea68ab42017-04-19 15:03:30 -07004709 @simple_layout('layouts/ignore-dependencies.yaml')
James E. Blair17dd6772015-02-09 14:45:18 -08004710 def test_crd_check_ignore_dependencies(self):
4711 "Test cross-repo dependencies can be ignored"
James E. Blair17dd6772015-02-09 14:45:18 -08004712
4713 self.gearman_server.hold_jobs_in_queue = True
4714 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4715 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4716 C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
4717
4718 # A Depends-On: B
4719 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4720 A.subject, B.data['id'])
4721 # C git-depends on B
4722 C.setDependsOn(B, 1)
4723 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4724 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4725 self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
4726 self.waitUntilSettled()
4727
4728 # Make sure none of the items share a change queue, and all
4729 # are live.
Jesse Keatingea68ab42017-04-19 15:03:30 -07004730 tenant = self.sched.abide.tenants.get('tenant-one')
Clint Byrum12282ca2017-04-03 11:04:02 -07004731 check_pipeline = tenant.layout.pipelines['check']
James E. Blair17dd6772015-02-09 14:45:18 -08004732 self.assertEqual(len(check_pipeline.queues), 3)
4733 self.assertEqual(len(check_pipeline.getAllItems()), 3)
4734 for item in check_pipeline.getAllItems():
4735 self.assertTrue(item.live)
4736
4737 self.gearman_server.hold_jobs_in_queue = False
4738 self.gearman_server.release()
4739 self.waitUntilSettled()
4740
4741 self.assertEqual(A.data['status'], 'NEW')
4742 self.assertEqual(B.data['status'], 'NEW')
4743 self.assertEqual(C.data['status'], 'NEW')
4744 self.assertEqual(A.reported, 1)
4745 self.assertEqual(B.reported, 1)
4746 self.assertEqual(C.reported, 1)
4747
4748 # Each job should have tested exactly one change
4749 for job in self.history:
4750 self.assertEqual(len(job.changes.split()), 1)
James E. Blair96698e22015-04-02 07:48:21 -07004751
James E. Blair7cb84542017-04-19 13:35:05 -07004752 @simple_layout('layouts/three-projects.yaml')
James E. Blair96698e22015-04-02 07:48:21 -07004753 def test_crd_check_transitive(self):
4754 "Test transitive cross-repo dependencies"
4755 # Specifically, if A -> B -> C, and C gets a new patchset and
4756 # A gets a new patchset, ensure the test of A,2 includes B,1
4757 # and C,2 (not C,1 which would indicate stale data in the
4758 # cache for B).
4759 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4760 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4761 C = self.fake_gerrit.addFakeChange('org/project3', 'master', 'C')
4762
4763 # A Depends-On: B
4764 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4765 A.subject, B.data['id'])
4766
4767 # B Depends-On: C
4768 B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4769 B.subject, C.data['id'])
4770
4771 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4772 self.waitUntilSettled()
4773 self.assertEqual(self.history[-1].changes, '3,1 2,1 1,1')
4774
4775 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4776 self.waitUntilSettled()
4777 self.assertEqual(self.history[-1].changes, '3,1 2,1')
4778
4779 self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
4780 self.waitUntilSettled()
4781 self.assertEqual(self.history[-1].changes, '3,1')
4782
4783 C.addPatchset()
4784 self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(2))
4785 self.waitUntilSettled()
4786 self.assertEqual(self.history[-1].changes, '3,2')
4787
4788 A.addPatchset()
4789 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
4790 self.waitUntilSettled()
4791 self.assertEqual(self.history[-1].changes, '3,2 2,1 1,2')
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004792
James E. Blaira8b90b32016-08-24 15:18:50 -07004793 def test_crd_check_unknown(self):
4794 "Test unknown projects in independent pipeline"
James E. Blairc4ba97a2017-04-19 16:26:24 -07004795 self.init_repo("org/unknown", tag='init')
James E. Blaira8b90b32016-08-24 15:18:50 -07004796 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4797 B = self.fake_gerrit.addFakeChange('org/unknown', 'master', 'D')
4798 # A Depends-On: B
4799 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4800 A.subject, B.data['id'])
4801
4802 # Make sure zuul has seen an event on B.
4803 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4804 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4805 self.waitUntilSettled()
4806
4807 self.assertEqual(A.data['status'], 'NEW')
4808 self.assertEqual(A.reported, 1)
4809 self.assertEqual(B.data['status'], 'NEW')
4810 self.assertEqual(B.reported, 0)
4811
James E. Blair92464a22016-04-05 10:21:26 -07004812 def test_crd_cycle_join(self):
4813 "Test an updated change creates a cycle"
4814 A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
4815
4816 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4817 self.waitUntilSettled()
Tobias Henkel6b9390f2017-03-28 11:23:21 +02004818 self.assertEqual(A.reported, 1)
James E. Blair92464a22016-04-05 10:21:26 -07004819
4820 # Create B->A
4821 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
4822 B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4823 B.subject, A.data['id'])
4824 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4825 self.waitUntilSettled()
4826
Tobias Henkel6b9390f2017-03-28 11:23:21 +02004827 # Dep is there so zuul should have reported on B
4828 self.assertEqual(B.reported, 1)
4829
James E. Blair92464a22016-04-05 10:21:26 -07004830 # Update A to add A->B (a cycle).
4831 A.addPatchset()
4832 A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4833 A.subject, B.data['id'])
Tobias Henkel6b9390f2017-03-28 11:23:21 +02004834 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
4835 self.waitUntilSettled()
James E. Blair92464a22016-04-05 10:21:26 -07004836
Tobias Henkel6b9390f2017-03-28 11:23:21 +02004837 # Dependency cycle injected so zuul should not have reported again on A
4838 self.assertEqual(A.reported, 1)
James E. Blair92464a22016-04-05 10:21:26 -07004839
4840 # Now if we update B to remove the depends-on, everything
4841 # should be okay. B; A->B
4842
4843 B.addPatchset()
4844 B.data['commitMessage'] = '%s\n' % (B.subject,)
Tobias Henkel6b9390f2017-03-28 11:23:21 +02004845 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
4846 self.waitUntilSettled()
Paul Belanger0e155e22016-11-14 14:12:23 -05004847
Tobias Henkel6b9390f2017-03-28 11:23:21 +02004848 # Cycle was removed so now zuul should have reported again on A
4849 self.assertEqual(A.reported, 2)
4850
4851 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
4852 self.waitUntilSettled()
4853 self.assertEqual(B.reported, 2)
James E. Blair92464a22016-04-05 10:21:26 -07004854
Jesse Keatingcc67b122017-04-19 14:54:57 -07004855 @simple_layout('layouts/disable_at.yaml')
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004856 def test_disable_at(self):
4857 "Test a pipeline will only report to the disabled trigger when failing"
4858
Jesse Keatingcc67b122017-04-19 14:54:57 -07004859 tenant = self.sched.abide.tenants.get('tenant-one')
Paul Belanger7dc76e82016-11-11 16:51:08 -05004860 self.assertEqual(3, tenant.layout.pipelines['check'].disable_at)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004861 self.assertEqual(
Paul Belanger7dc76e82016-11-11 16:51:08 -05004862 0, tenant.layout.pipelines['check']._consecutive_failures)
4863 self.assertFalse(tenant.layout.pipelines['check']._disabled)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004864
4865 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
4866 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
4867 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
4868 D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
4869 E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')
4870 F = self.fake_gerrit.addFakeChange('org/project', 'master', 'F')
4871 G = self.fake_gerrit.addFakeChange('org/project', 'master', 'G')
4872 H = self.fake_gerrit.addFakeChange('org/project', 'master', 'H')
4873 I = self.fake_gerrit.addFakeChange('org/project', 'master', 'I')
4874 J = self.fake_gerrit.addFakeChange('org/project', 'master', 'J')
4875 K = self.fake_gerrit.addFakeChange('org/project', 'master', 'K')
4876
Paul Belanger174a8272017-03-14 13:20:10 -04004877 self.executor_server.failJob('project-test1', A)
4878 self.executor_server.failJob('project-test1', B)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004879 # Let C pass, resetting the counter
Paul Belanger174a8272017-03-14 13:20:10 -04004880 self.executor_server.failJob('project-test1', D)
4881 self.executor_server.failJob('project-test1', E)
4882 self.executor_server.failJob('project-test1', F)
4883 self.executor_server.failJob('project-test1', G)
4884 self.executor_server.failJob('project-test1', H)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004885 # I also passes but should only report to the disabled reporters
Paul Belanger174a8272017-03-14 13:20:10 -04004886 self.executor_server.failJob('project-test1', J)
4887 self.executor_server.failJob('project-test1', K)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004888
4889 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4890 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4891 self.waitUntilSettled()
4892
4893 self.assertEqual(
Paul Belanger7dc76e82016-11-11 16:51:08 -05004894 2, tenant.layout.pipelines['check']._consecutive_failures)
4895 self.assertFalse(tenant.layout.pipelines['check']._disabled)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004896
4897 self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
4898 self.waitUntilSettled()
4899
4900 self.assertEqual(
Paul Belanger7dc76e82016-11-11 16:51:08 -05004901 0, tenant.layout.pipelines['check']._consecutive_failures)
4902 self.assertFalse(tenant.layout.pipelines['check']._disabled)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004903
4904 self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
4905 self.fake_gerrit.addEvent(E.getPatchsetCreatedEvent(1))
4906 self.fake_gerrit.addEvent(F.getPatchsetCreatedEvent(1))
4907 self.waitUntilSettled()
4908
4909 # We should be disabled now
4910 self.assertEqual(
Paul Belanger7dc76e82016-11-11 16:51:08 -05004911 3, tenant.layout.pipelines['check']._consecutive_failures)
4912 self.assertTrue(tenant.layout.pipelines['check']._disabled)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004913
4914 # We need to wait between each of these patches to make sure the
4915 # smtp messages come back in an expected order
4916 self.fake_gerrit.addEvent(G.getPatchsetCreatedEvent(1))
4917 self.waitUntilSettled()
4918 self.fake_gerrit.addEvent(H.getPatchsetCreatedEvent(1))
4919 self.waitUntilSettled()
4920 self.fake_gerrit.addEvent(I.getPatchsetCreatedEvent(1))
4921 self.waitUntilSettled()
4922
4923 # The first 6 (ABCDEF) jobs should have reported back to gerrt thus
4924 # leaving a message on each change
4925 self.assertEqual(1, len(A.messages))
4926 self.assertIn('Build failed.', A.messages[0])
4927 self.assertEqual(1, len(B.messages))
4928 self.assertIn('Build failed.', B.messages[0])
4929 self.assertEqual(1, len(C.messages))
4930 self.assertIn('Build succeeded.', C.messages[0])
4931 self.assertEqual(1, len(D.messages))
4932 self.assertIn('Build failed.', D.messages[0])
4933 self.assertEqual(1, len(E.messages))
4934 self.assertIn('Build failed.', E.messages[0])
4935 self.assertEqual(1, len(F.messages))
4936 self.assertIn('Build failed.', F.messages[0])
4937
4938 # The last 3 (GHI) would have only reported via smtp.
4939 self.assertEqual(3, len(self.smtp_messages))
4940 self.assertEqual(0, len(G.messages))
4941 self.assertIn('Build failed.', self.smtp_messages[0]['body'])
Paul Belanger7dc76e82016-11-11 16:51:08 -05004942 self.assertIn(
Monty Taylorde8242c2017-02-23 20:29:53 -06004943 'project-test1 finger://', self.smtp_messages[0]['body'])
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004944 self.assertEqual(0, len(H.messages))
4945 self.assertIn('Build failed.', self.smtp_messages[1]['body'])
Paul Belanger7dc76e82016-11-11 16:51:08 -05004946 self.assertIn(
Monty Taylorde8242c2017-02-23 20:29:53 -06004947 'project-test1 finger://', self.smtp_messages[1]['body'])
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004948 self.assertEqual(0, len(I.messages))
4949 self.assertIn('Build succeeded.', self.smtp_messages[2]['body'])
Paul Belanger7dc76e82016-11-11 16:51:08 -05004950 self.assertIn(
Monty Taylorde8242c2017-02-23 20:29:53 -06004951 'project-test1 finger://', self.smtp_messages[2]['body'])
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004952
4953 # Now reload the configuration (simulate a HUP) to check the pipeline
4954 # comes out of disabled
4955 self.sched.reconfigure(self.config)
4956
Jesse Keatingcc67b122017-04-19 14:54:57 -07004957 tenant = self.sched.abide.tenants.get('tenant-one')
Paul Belanger7dc76e82016-11-11 16:51:08 -05004958
4959 self.assertEqual(3, tenant.layout.pipelines['check'].disable_at)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004960 self.assertEqual(
Paul Belanger7dc76e82016-11-11 16:51:08 -05004961 0, tenant.layout.pipelines['check']._consecutive_failures)
4962 self.assertFalse(tenant.layout.pipelines['check']._disabled)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004963
4964 self.fake_gerrit.addEvent(J.getPatchsetCreatedEvent(1))
4965 self.fake_gerrit.addEvent(K.getPatchsetCreatedEvent(1))
4966 self.waitUntilSettled()
4967
4968 self.assertEqual(
Paul Belanger7dc76e82016-11-11 16:51:08 -05004969 2, tenant.layout.pipelines['check']._consecutive_failures)
4970 self.assertFalse(tenant.layout.pipelines['check']._disabled)
Joshua Hesketh89e829d2015-02-10 16:29:45 +11004971
4972 # J and K went back to gerrit
4973 self.assertEqual(1, len(J.messages))
4974 self.assertIn('Build failed.', J.messages[0])
4975 self.assertEqual(1, len(K.messages))
4976 self.assertIn('Build failed.', K.messages[0])
4977 # No more messages reported via smtp
4978 self.assertEqual(3, len(self.smtp_messages))
Joshua Heskethd6dbd682015-12-22 10:06:54 +11004979
James E. Blaircc020532017-04-19 13:04:44 -07004980 @simple_layout('layouts/one-job-project.yaml')
4981 def test_one_job_project(self):
4982 "Test that queueing works with one job"
4983 A = self.fake_gerrit.addFakeChange('org/one-job-project',
4984 'master', 'A')
4985 B = self.fake_gerrit.addFakeChange('org/one-job-project',
4986 'master', 'B')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02004987 A.addApproval('Code-Review', 2)
4988 B.addApproval('Code-Review', 2)
4989 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
4990 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
James E. Blaircc020532017-04-19 13:04:44 -07004991 self.waitUntilSettled()
4992
4993 self.assertEqual(A.data['status'], 'MERGED')
4994 self.assertEqual(A.reported, 2)
4995 self.assertEqual(B.data['status'], 'MERGED')
4996 self.assertEqual(B.reported, 2)
4997
Paul Belanger32023542017-10-06 14:48:50 -04004998 def test_job_aborted(self):
4999 "Test that if a execute server aborts a job, it is run again"
5000 self.executor_server.hold_jobs_in_build = True
5001 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
5002 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5003 self.waitUntilSettled()
5004
5005 self.executor_server.release('.*-merge')
5006 self.waitUntilSettled()
5007
5008 self.assertEqual(len(self.builds), 2)
Tobias Henkel26fa8a32017-10-07 07:08:36 +00005009
5010 # first abort
5011 self.builds[0].aborted = True
5012 self.executor_server.release('.*-test*')
5013 self.waitUntilSettled()
5014 self.assertEqual(len(self.builds), 1)
5015
5016 # second abort
5017 self.builds[0].aborted = True
5018 self.executor_server.release('.*-test*')
5019 self.waitUntilSettled()
5020 self.assertEqual(len(self.builds), 1)
5021
5022 # third abort
5023 self.builds[0].aborted = True
5024 self.executor_server.release('.*-test*')
5025 self.waitUntilSettled()
5026 self.assertEqual(len(self.builds), 1)
5027
5028 # fourth abort
Paul Belanger32023542017-10-06 14:48:50 -04005029 self.builds[0].aborted = True
5030 self.executor_server.release('.*-test*')
5031 self.waitUntilSettled()
5032 self.assertEqual(len(self.builds), 1)
5033
5034 self.executor_server.hold_jobs_in_build = False
5035 self.executor_server.release()
5036 self.waitUntilSettled()
5037
Tobias Henkel26fa8a32017-10-07 07:08:36 +00005038 self.assertEqual(len(self.history), 7)
5039 self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 4)
Paul Belanger32023542017-10-06 14:48:50 -04005040 self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 3)
5041
Paul Belanger71d98172016-11-08 10:56:31 -05005042 def test_rerun_on_abort(self):
Paul Belanger174a8272017-03-14 13:20:10 -04005043 "Test that if a execute server fails to run a job, it is run again"
Paul Belanger71d98172016-11-08 10:56:31 -05005044
Paul Belanger174a8272017-03-14 13:20:10 -04005045 self.executor_server.hold_jobs_in_build = True
Paul Belanger71d98172016-11-08 10:56:31 -05005046 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
5047 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5048 self.waitUntilSettled()
5049
Paul Belanger174a8272017-03-14 13:20:10 -04005050 self.executor_server.release('.*-merge')
Paul Belanger71d98172016-11-08 10:56:31 -05005051 self.waitUntilSettled()
5052
5053 self.assertEqual(len(self.builds), 2)
5054 self.builds[0].requeue = True
Paul Belanger174a8272017-03-14 13:20:10 -04005055 self.executor_server.release('.*-test*')
Paul Belanger71d98172016-11-08 10:56:31 -05005056 self.waitUntilSettled()
5057
Clint Byrum5bb5feb2016-12-08 12:39:05 -08005058 for x in range(3):
Clint Byrumdf0a55b2016-12-05 06:39:11 -08005059 self.assertEqual(len(self.builds), 1,
5060 'len of builds at x=%d is wrong' % x)
Paul Belanger71d98172016-11-08 10:56:31 -05005061 self.builds[0].requeue = True
Paul Belanger174a8272017-03-14 13:20:10 -04005062 self.executor_server.release('.*-test1')
Paul Belanger71d98172016-11-08 10:56:31 -05005063 self.waitUntilSettled()
5064
Paul Belanger174a8272017-03-14 13:20:10 -04005065 self.executor_server.hold_jobs_in_build = False
5066 self.executor_server.release()
Paul Belanger71d98172016-11-08 10:56:31 -05005067 self.waitUntilSettled()
Clint Byrum5bb5feb2016-12-08 12:39:05 -08005068 self.assertEqual(len(self.history), 6)
Paul Belanger71d98172016-11-08 10:56:31 -05005069 self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 2)
5070 self.assertEqual(A.reported, 1)
5071 self.assertIn('RETRY_LIMIT', A.messages[0])
Joshua Hesketh3f7def32016-11-21 17:36:44 +11005072
James E. Blair15be0e12017-01-03 13:45:20 -08005073 def test_zookeeper_disconnect(self):
Paul Belanger174a8272017-03-14 13:20:10 -04005074 "Test that jobs are executed after a zookeeper disconnect"
James E. Blair15be0e12017-01-03 13:45:20 -08005075
5076 self.fake_nodepool.paused = True
5077 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02005078 A.addApproval('Code-Review', 2)
5079 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair15be0e12017-01-03 13:45:20 -08005080 self.waitUntilSettled()
5081
5082 self.zk.client.stop()
5083 self.zk.client.start()
5084 self.fake_nodepool.paused = False
5085 self.waitUntilSettled()
5086
5087 self.assertEqual(A.data['status'], 'MERGED')
5088 self.assertEqual(A.reported, 2)
5089
James E. Blair6ab79e02017-01-06 10:10:17 -08005090 def test_nodepool_failure(self):
5091 "Test that jobs are reported after a nodepool failure"
5092
5093 self.fake_nodepool.paused = True
5094 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02005095 A.addApproval('Code-Review', 2)
5096 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blair6ab79e02017-01-06 10:10:17 -08005097 self.waitUntilSettled()
5098
5099 req = self.fake_nodepool.getNodeRequests()[0]
5100 self.fake_nodepool.addFailRequest(req)
5101
5102 self.fake_nodepool.paused = False
5103 self.waitUntilSettled()
5104
5105 self.assertEqual(A.data['status'], 'NEW')
5106 self.assertEqual(A.reported, 2)
5107 self.assertIn('project-merge : NODE_FAILURE', A.messages[1])
5108 self.assertIn('project-test1 : SKIPPED', A.messages[1])
5109 self.assertIn('project-test2 : SKIPPED', A.messages[1])
5110
Monty Taylor6dc5bc12017-09-29 15:47:31 -05005111 def test_nodepool_priority(self):
5112 "Test that nodes are requested at the correct priority"
5113
5114 self.fake_nodepool.paused = True
5115
5116 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
5117 self.fake_gerrit.addEvent(A.getRefUpdatedEvent())
5118
5119 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
5120 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
5121
5122 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
5123 C.addApproval('Code-Review', 2)
5124 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
5125
5126 self.waitUntilSettled()
5127
5128 reqs = self.fake_nodepool.getNodeRequests()
5129
5130 # The requests come back sorted by oid. Since we have three requests
5131 # for the three changes each with a different priority.
5132 # Also they get a serial number based on order they were received
5133 # so the number on the endof the oid should map to order submitted.
5134
5135 # * gate first - high priority - change C
5136 self.assertEqual(reqs[0]['_oid'], '100-0000000002')
5137 self.assertEqual(reqs[0]['node_types'], ['label1'])
5138 # * check second - normal priority - change B
5139 self.assertEqual(reqs[1]['_oid'], '200-0000000001')
5140 self.assertEqual(reqs[1]['node_types'], ['label1'])
5141 # * post third - low priority - change A
5142 # additionally, the post job defined uses an ubuntu-xenial node,
5143 # so we include that check just as an extra verification
5144 self.assertEqual(reqs[2]['_oid'], '300-0000000000')
5145 self.assertEqual(reqs[2]['node_types'], ['ubuntu-xenial'])
5146
5147 self.fake_nodepool.paused = False
5148 self.waitUntilSettled()
5149
James E. Blairfceaf412017-09-29 13:44:57 -07005150 @simple_layout('layouts/multiple-templates.yaml')
5151 def test_multiple_project_templates(self):
5152 # Test that applying multiple project templates to a project
5153 # doesn't alter them when used for a second project.
5154 A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
5155 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5156 self.waitUntilSettled()
5157
5158 build = self.getJobFromHistory('py27')
5159 self.assertEqual(build.parameters['zuul']['jobtags'], [])
5160
James E. Blair29c77002017-10-05 14:56:35 -07005161 def test_pending_merge_in_reconfig(self):
5162 # Test that if we are waiting for an outstanding merge on
5163 # reconfiguration that we continue to do so.
5164 self.gearman_server.hold_merge_jobs_in_queue = True
5165 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
5166 A.setMerged()
5167 self.fake_gerrit.addEvent(A.getRefUpdatedEvent())
5168 self.waitUntilSettled()
5169 # Reconfigure while we still have an outstanding merge job
5170 self.sched.reconfigureTenant(self.sched.abide.tenants['tenant-one'],
5171 None)
5172 self.waitUntilSettled()
5173 # Verify the merge job is still running and that the item is
5174 # in the pipeline
5175 self.assertEqual(len(self.sched.merger.jobs), 1)
5176 tenant = self.sched.abide.tenants.get('tenant-one')
5177 pipeline = tenant.layout.pipelines['post']
5178 self.assertEqual(len(pipeline.getAllItems()), 1)
5179 self.gearman_server.hold_merge_jobs_in_queue = False
5180 self.gearman_server.release()
5181 self.waitUntilSettled()
5182
James E. Blairc32a8352017-10-11 16:27:50 -07005183 @simple_layout('layouts/parent-matchers.yaml')
5184 def test_parent_matchers(self):
5185 "Test that if a job's parent does not match, the job does not run"
5186 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
5187 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5188 self.waitUntilSettled()
5189
5190 self.assertHistory([])
5191
5192 files = {'foo.txt': ''}
5193 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
5194 files=files)
5195 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
5196 self.waitUntilSettled()
5197
5198 files = {'bar.txt': ''}
5199 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C',
5200 files=files)
5201 self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
5202 self.waitUntilSettled()
5203
5204 files = {'foo.txt': '', 'bar.txt': ''}
5205 D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D',
5206 files=files)
5207 self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
5208 self.waitUntilSettled()
5209
5210 self.assertHistory([
5211 dict(name='child-job', result='SUCCESS', changes='2,1'),
5212 dict(name='child-job', result='SUCCESS', changes='3,1'),
5213 dict(name='child-job', result='SUCCESS', changes='4,1'),
5214 ])
5215
Arieb6f068c2016-10-09 13:11:06 +03005216
James E. Blaira002b032017-04-18 10:35:48 -07005217class TestExecutor(ZuulTestCase):
5218 tenant_config_file = 'config/single-tenant/main.yaml'
5219
5220 def assertFinalState(self):
5221 # In this test, we expect to shut down in a non-final state,
5222 # so skip these checks.
5223 pass
5224
5225 def assertCleanShutdown(self):
5226 self.log.debug("Assert clean shutdown")
5227
5228 # After shutdown, make sure no jobs are running
5229 self.assertEqual({}, self.executor_server.job_workers)
5230
5231 # Make sure that git.Repo objects have been garbage collected.
5232 repos = []
5233 gc.collect()
5234 for obj in gc.get_objects():
5235 if isinstance(obj, git.Repo):
5236 self.log.debug("Leaked git repo object: %s" % repr(obj))
5237 repos.append(obj)
5238 self.assertEqual(len(repos), 0)
5239
5240 def test_executor_shutdown(self):
5241 "Test that the executor can shut down with jobs running"
5242
5243 self.executor_server.hold_jobs_in_build = True
5244 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02005245 A.addApproval('Code-Review', 2)
5246 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
James E. Blaira002b032017-04-18 10:35:48 -07005247 self.waitUntilSettled()
5248
5249
Fredrik Medleyf8aec832015-09-28 13:40:20 +02005250class TestDependencyGraph(ZuulTestCase):
5251 tenant_config_file = 'config/dependency-graph/main.yaml'
5252
5253 def test_dependeny_graph_dispatch_jobs_once(self):
5254 "Test a job in a dependency graph is queued only once"
5255 # Job dependencies, starting with A
5256 # A
5257 # / \
5258 # B C
5259 # / \ / \
5260 # D F E
5261 # |
5262 # G
5263
5264 self.executor_server.hold_jobs_in_build = True
5265 change = self.fake_gerrit.addFakeChange(
5266 'org/project', 'master', 'change')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02005267 change.addApproval('Code-Review', 2)
5268 self.fake_gerrit.addEvent(change.addApproval('Approved', 1))
Fredrik Medleyf8aec832015-09-28 13:40:20 +02005269
5270 self.waitUntilSettled()
5271 self.assertEqual([b.name for b in self.builds], ['A'])
5272
5273 self.executor_server.release('A')
5274 self.waitUntilSettled()
5275 self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C'])
5276
5277 self.executor_server.release('B')
5278 self.waitUntilSettled()
5279 self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D'])
5280
5281 self.executor_server.release('D')
5282 self.waitUntilSettled()
5283 self.assertEqual([b.name for b in self.builds], ['C'])
5284
5285 self.executor_server.release('C')
5286 self.waitUntilSettled()
5287 self.assertEqual(sorted(b.name for b in self.builds), ['E', 'F'])
5288
5289 self.executor_server.release('F')
5290 self.waitUntilSettled()
5291 self.assertEqual(sorted(b.name for b in self.builds), ['E', 'G'])
5292
5293 self.executor_server.release('G')
5294 self.waitUntilSettled()
5295 self.assertEqual([b.name for b in self.builds], ['E'])
5296
5297 self.executor_server.release('E')
5298 self.waitUntilSettled()
5299 self.assertEqual(len(self.builds), 0)
5300
5301 self.executor_server.hold_jobs_in_build = False
5302 self.executor_server.release()
5303 self.waitUntilSettled()
5304
5305 self.assertEqual(len(self.builds), 0)
5306 self.assertEqual(len(self.history), 7)
5307
5308 self.assertEqual(change.data['status'], 'MERGED')
5309 self.assertEqual(change.reported, 2)
5310
5311 def test_jobs_launched_only_if_all_dependencies_are_successful(self):
5312 "Test that a job waits till all dependencies are successful"
5313 # Job dependencies, starting with A
5314 # A
5315 # / \
5316 # B C*
5317 # / \ / \
5318 # D F E
5319 # |
5320 # G
5321
5322 self.executor_server.hold_jobs_in_build = True
5323 change = self.fake_gerrit.addFakeChange(
5324 'org/project', 'master', 'change')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02005325 change.addApproval('Code-Review', 2)
Fredrik Medleyf8aec832015-09-28 13:40:20 +02005326
5327 self.executor_server.failJob('C', change)
5328
Tobias Henkelbf24fd12017-07-27 06:13:07 +02005329 self.fake_gerrit.addEvent(change.addApproval('Approved', 1))
Fredrik Medleyf8aec832015-09-28 13:40:20 +02005330
5331 self.waitUntilSettled()
5332 self.assertEqual([b.name for b in self.builds], ['A'])
5333
5334 self.executor_server.release('A')
5335 self.waitUntilSettled()
5336 self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C'])
5337
5338 self.executor_server.release('B')
5339 self.waitUntilSettled()
5340 self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D'])
5341
5342 self.executor_server.release('D')
5343 self.waitUntilSettled()
5344 self.assertEqual([b.name for b in self.builds], ['C'])
5345
5346 self.executor_server.release('C')
5347 self.waitUntilSettled()
5348 self.assertEqual(len(self.builds), 0)
5349
5350 self.executor_server.hold_jobs_in_build = False
5351 self.executor_server.release()
5352 self.waitUntilSettled()
5353
5354 self.assertEqual(len(self.builds), 0)
5355 self.assertEqual(len(self.history), 4)
5356
5357 self.assertEqual(change.data['status'], 'NEW')
5358 self.assertEqual(change.reported, 2)
5359
5360
Arieb6f068c2016-10-09 13:11:06 +03005361class TestDuplicatePipeline(ZuulTestCase):
5362 tenant_config_file = 'config/duplicate-pipeline/main.yaml'
5363
5364 def test_duplicate_pipelines(self):
5365 "Test that a change matching multiple pipelines works"
5366
5367 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
5368 self.fake_gerrit.addEvent(A.getChangeRestoredEvent())
5369 self.waitUntilSettled()
5370
5371 self.assertHistory([
5372 dict(name='project-test1', result='SUCCESS', changes='1,1',
5373 pipeline='dup1'),
5374 dict(name='project-test1', result='SUCCESS', changes='1,1',
5375 pipeline='dup2'),
Paul Belanger73a7d8e2016-11-09 11:12:42 -05005376 ], ordered=False)
Arieb6f068c2016-10-09 13:11:06 +03005377
5378 self.assertEqual(len(A.messages), 2)
5379
Paul Belangera46a3742016-11-09 14:23:03 -05005380 if 'dup1' in A.messages[0]:
5381 self.assertIn('dup1', A.messages[0])
5382 self.assertNotIn('dup2', A.messages[0])
5383 self.assertIn('project-test1', A.messages[0])
5384 self.assertIn('dup2', A.messages[1])
5385 self.assertNotIn('dup1', A.messages[1])
5386 self.assertIn('project-test1', A.messages[1])
5387 else:
5388 self.assertIn('dup1', A.messages[1])
5389 self.assertNotIn('dup2', A.messages[1])
5390 self.assertIn('project-test1', A.messages[1])
5391 self.assertIn('dup2', A.messages[0])
5392 self.assertNotIn('dup1', A.messages[0])
5393 self.assertIn('project-test1', A.messages[0])
Paul Belangerfac69ba2016-11-03 09:03:13 -04005394
5395
Paul Belanger86085b32016-11-03 12:48:57 -04005396class TestSchedulerTemplatedProject(ZuulTestCase):
5397 tenant_config_file = 'config/templated-project/main.yaml'
5398
Paul Belanger174a8272017-03-14 13:20:10 -04005399 def test_job_from_templates_executed(self):
5400 "Test whether a job generated via a template can be executed"
Paul Belanger86085b32016-11-03 12:48:57 -04005401
5402 A = self.fake_gerrit.addFakeChange(
5403 'org/templated-project', 'master', 'A')
5404 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5405 self.waitUntilSettled()
5406
5407 self.assertEqual(self.getJobFromHistory('project-test1').result,
5408 'SUCCESS')
5409 self.assertEqual(self.getJobFromHistory('project-test2').result,
5410 'SUCCESS')
Paul Belanger3adf72f2016-11-03 14:57:31 -04005411
5412 def test_layered_templates(self):
Paul Belanger174a8272017-03-14 13:20:10 -04005413 "Test whether a job generated via a template can be executed"
Paul Belanger3adf72f2016-11-03 14:57:31 -04005414
5415 A = self.fake_gerrit.addFakeChange(
5416 'org/layered-project', 'master', 'A')
5417 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5418 self.waitUntilSettled()
5419
5420 self.assertEqual(self.getJobFromHistory('project-test1').result,
5421 'SUCCESS')
5422 self.assertEqual(self.getJobFromHistory('project-test2').result,
5423 'SUCCESS')
5424 self.assertEqual(self.getJobFromHistory('layered-project-test3'
5425 ).result, 'SUCCESS')
5426 self.assertEqual(self.getJobFromHistory('layered-project-test4'
5427 ).result, 'SUCCESS')
5428 self.assertEqual(self.getJobFromHistory('layered-project-foo-test5'
5429 ).result, 'SUCCESS')
5430 self.assertEqual(self.getJobFromHistory('project-test6').result,
5431 'SUCCESS')
Adam Gandelman94a60062016-11-21 16:43:14 -08005432
James E. Blaire74f5712017-09-29 15:14:31 -07005433 def test_unimplied_branch_matchers(self):
5434 # This tests that there are no implied branch matchers added
5435 # by project templates.
5436 self.create_branch('org/layered-project', 'stable')
5437
5438 A = self.fake_gerrit.addFakeChange(
5439 'org/layered-project', 'stable', 'A')
5440
5441 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5442 self.waitUntilSettled()
5443
5444 self.assertEqual(self.getJobFromHistory('project-test1').result,
5445 'SUCCESS')
5446 print(self.getJobFromHistory('project-test1').
5447 parameters['zuul']['_inheritance_path'])
5448
5449 def test_implied_branch_matchers(self):
5450 # This tests that there is an implied branch matcher when a
5451 # template is used on an in-repo project pipeline definition.
5452 self.create_branch('untrusted-config', 'stable')
5453 self.fake_gerrit.addEvent(
5454 self.fake_gerrit.getFakeBranchCreatedEvent(
5455 'untrusted-config', 'stable'))
5456 self.waitUntilSettled()
5457
5458 A = self.fake_gerrit.addFakeChange(
5459 'untrusted-config', 'stable', 'A')
5460
5461 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5462 self.waitUntilSettled()
5463
5464 self.assertEqual(self.getJobFromHistory('project-test1').result,
5465 'SUCCESS')
5466 print(self.getJobFromHistory('project-test1').
5467 parameters['zuul']['_inheritance_path'])
5468
Adam Gandelman94a60062016-11-21 16:43:14 -08005469
5470class TestSchedulerSuccessURL(ZuulTestCase):
5471 tenant_config_file = 'config/success-url/main.yaml'
5472
5473 def test_success_url(self):
5474 "Ensure bad build params are ignored"
5475 self.sched.reconfigure(self.config)
5476 self.init_repo('org/docs')
5477
5478 A = self.fake_gerrit.addFakeChange('org/docs', 'master', 'A')
5479 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5480 self.waitUntilSettled()
5481
5482 # Both builds ran: docs-draft-test + docs-draft-test2
5483 self.assertEqual(len(self.history), 2)
5484
5485 # Grab build id
James E. Blaire7576802016-12-21 16:15:00 -08005486 for build in self.history:
5487 if build.name == 'docs-draft-test':
5488 uuid = build.uuid[:7]
Monty Taylorde8242c2017-02-23 20:29:53 -06005489 elif build.name == 'docs-draft-test2':
5490 uuid_test2 = build.uuid
Adam Gandelman94a60062016-11-21 16:43:14 -08005491
5492 # Two msgs: 'Starting...' + results
5493 self.assertEqual(len(self.smtp_messages), 2)
5494 body = self.smtp_messages[1]['body'].splitlines()
5495 self.assertEqual('Build succeeded.', body[0])
5496
5497 self.assertIn(
5498 '- docs-draft-test http://docs-draft.example.org/1/1/1/check/'
5499 'docs-draft-test/%s/publish-docs/' % uuid,
5500 body[2])
5501
Paul Belanger174a8272017-03-14 13:20:10 -04005502 # NOTE: This default URL is currently hard-coded in executor/server.py
Adam Gandelman94a60062016-11-21 16:43:14 -08005503 self.assertIn(
Monty Taylor51139a02016-05-24 11:28:10 -05005504 '- docs-draft-test2 finger://{hostname}/{uuid}'.format(
5505 hostname=self.executor_server.hostname,
Monty Taylorde8242c2017-02-23 20:29:53 -06005506 uuid=uuid_test2),
Adam Gandelman94a60062016-11-21 16:43:14 -08005507 body[3])
Adam Gandelman8bd57102016-12-02 12:58:42 -08005508
5509
Adam Gandelman4da00f62016-12-09 15:47:33 -08005510class TestSchedulerMerges(ZuulTestCase):
5511 tenant_config_file = 'config/merges/main.yaml'
Adam Gandelman8bd57102016-12-02 12:58:42 -08005512
5513 def _test_project_merge_mode(self, mode):
Paul Belanger174a8272017-03-14 13:20:10 -04005514 self.executor_server.keep_jobdir = False
Adam Gandelman8bd57102016-12-02 12:58:42 -08005515 project = 'org/project-%s' % mode
Paul Belanger174a8272017-03-14 13:20:10 -04005516 self.executor_server.hold_jobs_in_build = True
Adam Gandelman8bd57102016-12-02 12:58:42 -08005517 A = self.fake_gerrit.addFakeChange(project, 'master', 'A')
5518 B = self.fake_gerrit.addFakeChange(project, 'master', 'B')
5519 C = self.fake_gerrit.addFakeChange(project, 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02005520 A.addApproval('Code-Review', 2)
5521 B.addApproval('Code-Review', 2)
5522 C.addApproval('Code-Review', 2)
5523 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
5524 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
5525 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
Adam Gandelman8bd57102016-12-02 12:58:42 -08005526 self.waitUntilSettled()
5527
5528 build = self.builds[-1]
James E. Blair2a535672017-04-27 12:03:15 -07005529 path = os.path.join(build.jobdir.src_root, 'review.example.com',
5530 project)
Adam Gandelman8bd57102016-12-02 12:58:42 -08005531 repo = git.Repo(path)
James E. Blair247cab72017-07-20 16:52:36 -07005532 repo_messages = [c.message.strip() for c in repo.iter_commits()]
Adam Gandelman8bd57102016-12-02 12:58:42 -08005533 repo_messages.reverse()
5534
Paul Belanger174a8272017-03-14 13:20:10 -04005535 self.executor_server.hold_jobs_in_build = False
5536 self.executor_server.release()
Adam Gandelman8bd57102016-12-02 12:58:42 -08005537 self.waitUntilSettled()
5538
5539 return repo_messages
5540
5541 def _test_merge(self, mode):
5542 us_path = os.path.join(
5543 self.upstream_root, 'org/project-%s' % mode)
5544 expected_messages = [
5545 'initial commit',
5546 'add content from fixture',
5547 # the intermediate commits order is nondeterministic
5548 "Merge commit 'refs/changes/1/2/1' of %s into HEAD" % us_path,
5549 "Merge commit 'refs/changes/1/3/1' of %s into HEAD" % us_path,
5550 ]
5551 result = self._test_project_merge_mode(mode)
5552 self.assertEqual(result[:2], expected_messages[:2])
5553 self.assertEqual(result[-2:], expected_messages[-2:])
5554
5555 def test_project_merge_mode_merge(self):
5556 self._test_merge('merge')
5557
5558 def test_project_merge_mode_merge_resolve(self):
5559 self._test_merge('merge-resolve')
5560
5561 def test_project_merge_mode_cherrypick(self):
5562 expected_messages = [
5563 'initial commit',
5564 'add content from fixture',
5565 'A-1',
5566 'B-1',
5567 'C-1']
5568 result = self._test_project_merge_mode('cherry-pick')
5569 self.assertEqual(result, expected_messages)
Adam Gandelman4da00f62016-12-09 15:47:33 -08005570
5571 def test_merge_branch(self):
5572 "Test that the right commits are on alternate branches"
5573 self.create_branch('org/project-merge-branches', 'mp')
5574
Paul Belanger174a8272017-03-14 13:20:10 -04005575 self.executor_server.hold_jobs_in_build = True
Adam Gandelman4da00f62016-12-09 15:47:33 -08005576 A = self.fake_gerrit.addFakeChange(
5577 'org/project-merge-branches', 'mp', 'A')
5578 B = self.fake_gerrit.addFakeChange(
5579 'org/project-merge-branches', 'mp', 'B')
5580 C = self.fake_gerrit.addFakeChange(
5581 'org/project-merge-branches', 'mp', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02005582 A.addApproval('Code-Review', 2)
5583 B.addApproval('Code-Review', 2)
5584 C.addApproval('Code-Review', 2)
5585 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
5586 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
5587 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
Adam Gandelman4da00f62016-12-09 15:47:33 -08005588 self.waitUntilSettled()
5589
Paul Belanger174a8272017-03-14 13:20:10 -04005590 self.executor_server.release('.*-merge')
Adam Gandelman4da00f62016-12-09 15:47:33 -08005591 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04005592 self.executor_server.release('.*-merge')
Adam Gandelman4da00f62016-12-09 15:47:33 -08005593 self.waitUntilSettled()
Paul Belanger174a8272017-03-14 13:20:10 -04005594 self.executor_server.release('.*-merge')
Adam Gandelman4da00f62016-12-09 15:47:33 -08005595 self.waitUntilSettled()
5596
5597 build = self.builds[-1]
James E. Blair3b222492017-07-21 15:17:37 -07005598 self.assertEqual(build.parameters['zuul']['branch'], 'mp')
James E. Blair2a535672017-04-27 12:03:15 -07005599 path = os.path.join(build.jobdir.src_root, 'review.example.com',
5600 'org/project-merge-branches')
Adam Gandelman4da00f62016-12-09 15:47:33 -08005601 repo = git.Repo(path)
5602
James E. Blair247cab72017-07-20 16:52:36 -07005603 repo_messages = [c.message.strip() for c in repo.iter_commits()]
Adam Gandelman4da00f62016-12-09 15:47:33 -08005604 repo_messages.reverse()
5605 correct_messages = [
5606 'initial commit',
5607 'add content from fixture',
5608 'mp commit',
5609 'A-1', 'B-1', 'C-1']
5610 self.assertEqual(repo_messages, correct_messages)
5611
Paul Belanger174a8272017-03-14 13:20:10 -04005612 self.executor_server.hold_jobs_in_build = False
5613 self.executor_server.release()
Adam Gandelman4da00f62016-12-09 15:47:33 -08005614 self.waitUntilSettled()
5615
5616 def test_merge_multi_branch(self):
5617 "Test that dependent changes on multiple branches are merged"
5618 self.create_branch('org/project-merge-branches', 'mp')
5619
Paul Belanger174a8272017-03-14 13:20:10 -04005620 self.executor_server.hold_jobs_in_build = True
Adam Gandelman4da00f62016-12-09 15:47:33 -08005621 A = self.fake_gerrit.addFakeChange(
5622 'org/project-merge-branches', 'master', 'A')
5623 B = self.fake_gerrit.addFakeChange(
5624 'org/project-merge-branches', 'mp', 'B')
5625 C = self.fake_gerrit.addFakeChange(
5626 'org/project-merge-branches', 'master', 'C')
Tobias Henkelbf24fd12017-07-27 06:13:07 +02005627 A.addApproval('Code-Review', 2)
5628 B.addApproval('Code-Review', 2)
5629 C.addApproval('Code-Review', 2)
5630 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
5631 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
5632 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
Adam Gandelman4da00f62016-12-09 15:47:33 -08005633 self.waitUntilSettled()
5634
5635 job_A = None
5636 for job in self.builds:
5637 if 'project-merge' in job.name:
5638 job_A = job
Adam Gandelman4da00f62016-12-09 15:47:33 -08005639
James E. Blair2a535672017-04-27 12:03:15 -07005640 path = os.path.join(job_A.jobdir.src_root, 'review.example.com',
5641 'org/project-merge-branches')
Adam Gandelman4da00f62016-12-09 15:47:33 -08005642 repo = git.Repo(path)
5643 repo_messages = [c.message.strip()
James E. Blair247cab72017-07-20 16:52:36 -07005644 for c in repo.iter_commits()]
Adam Gandelman4da00f62016-12-09 15:47:33 -08005645 repo_messages.reverse()
5646 correct_messages = [
5647 'initial commit', 'add content from fixture', 'A-1']
5648 self.assertEqual(repo_messages, correct_messages)
5649
Paul Belanger174a8272017-03-14 13:20:10 -04005650 self.executor_server.release('.*-merge')
Adam Gandelman4da00f62016-12-09 15:47:33 -08005651 self.waitUntilSettled()
5652
5653 job_B = None
5654 for job in self.builds:
5655 if 'project-merge' in job.name:
5656 job_B = job
Adam Gandelman4da00f62016-12-09 15:47:33 -08005657
James E. Blair2a535672017-04-27 12:03:15 -07005658 path = os.path.join(job_B.jobdir.src_root, 'review.example.com',
5659 'org/project-merge-branches')
Adam Gandelman4da00f62016-12-09 15:47:33 -08005660 repo = git.Repo(path)
James E. Blair247cab72017-07-20 16:52:36 -07005661 repo_messages = [c.message.strip() for c in repo.iter_commits()]
Adam Gandelman4da00f62016-12-09 15:47:33 -08005662 repo_messages.reverse()
5663 correct_messages = [
5664 'initial commit', 'add content from fixture', 'mp commit', 'B-1']
5665 self.assertEqual(repo_messages, correct_messages)
5666
Paul Belanger174a8272017-03-14 13:20:10 -04005667 self.executor_server.release('.*-merge')
Adam Gandelman4da00f62016-12-09 15:47:33 -08005668 self.waitUntilSettled()
5669
5670 job_C = None
5671 for job in self.builds:
5672 if 'project-merge' in job.name:
5673 job_C = job
James E. Blairb1345832017-07-21 14:51:35 -07005674
James E. Blair2a535672017-04-27 12:03:15 -07005675 path = os.path.join(job_C.jobdir.src_root, 'review.example.com',
5676 'org/project-merge-branches')
Adam Gandelman4da00f62016-12-09 15:47:33 -08005677 repo = git.Repo(path)
James E. Blair247cab72017-07-20 16:52:36 -07005678 repo_messages = [c.message.strip() for c in repo.iter_commits()]
Adam Gandelman4da00f62016-12-09 15:47:33 -08005679
5680 repo_messages.reverse()
5681 correct_messages = [
5682 'initial commit', 'add content from fixture',
5683 'A-1', 'C-1']
5684 # Ensure the right commits are in the history for this ref
5685 self.assertEqual(repo_messages, correct_messages)
5686
Paul Belanger174a8272017-03-14 13:20:10 -04005687 self.executor_server.hold_jobs_in_build = False
5688 self.executor_server.release()
Adam Gandelman4da00f62016-12-09 15:47:33 -08005689 self.waitUntilSettled()
Tobias Henkel9a0e1942017-03-20 16:16:02 +01005690
5691
James E. Blair9ea0d0b2017-04-20 09:27:15 -07005692class TestSemaphore(ZuulTestCase):
5693 tenant_config_file = 'config/semaphore/main.yaml'
5694
5695 def test_semaphore_one(self):
5696 "Test semaphores with max=1 (mutex)"
5697 tenant = self.sched.abide.tenants.get('tenant-one')
5698
5699 self.executor_server.hold_jobs_in_build = True
5700
5701 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
5702 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
5703 self.assertFalse('test-semaphore' in
5704 tenant.semaphore_handler.semaphores)
5705
5706 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5707 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
5708 self.waitUntilSettled()
5709
5710 self.assertEqual(len(self.builds), 3)
5711 self.assertEqual(self.builds[0].name, 'project-test1')
5712 self.assertEqual(self.builds[1].name, 'semaphore-one-test1')
5713 self.assertEqual(self.builds[2].name, 'project-test1')
5714
5715 self.executor_server.release('semaphore-one-test1')
5716 self.waitUntilSettled()
5717
5718 self.assertEqual(len(self.builds), 3)
5719 self.assertEqual(self.builds[0].name, 'project-test1')
5720 self.assertEqual(self.builds[1].name, 'project-test1')
5721 self.assertEqual(self.builds[2].name, 'semaphore-one-test2')
5722 self.assertTrue('test-semaphore' in
5723 tenant.semaphore_handler.semaphores)
5724
5725 self.executor_server.release('semaphore-one-test2')
5726 self.waitUntilSettled()
5727
5728 self.assertEqual(len(self.builds), 3)
5729 self.assertEqual(self.builds[0].name, 'project-test1')
5730 self.assertEqual(self.builds[1].name, 'project-test1')
5731 self.assertEqual(self.builds[2].name, 'semaphore-one-test1')
5732 self.assertTrue('test-semaphore' in
5733 tenant.semaphore_handler.semaphores)
5734
5735 self.executor_server.release('semaphore-one-test1')
5736 self.waitUntilSettled()
5737
5738 self.assertEqual(len(self.builds), 3)
5739 self.assertEqual(self.builds[0].name, 'project-test1')
5740 self.assertEqual(self.builds[1].name, 'project-test1')
5741 self.assertEqual(self.builds[2].name, 'semaphore-one-test2')
5742 self.assertTrue('test-semaphore' in
5743 tenant.semaphore_handler.semaphores)
5744
5745 self.executor_server.release('semaphore-one-test2')
5746 self.waitUntilSettled()
5747
5748 self.assertEqual(len(self.builds), 2)
5749 self.assertEqual(self.builds[0].name, 'project-test1')
5750 self.assertEqual(self.builds[1].name, 'project-test1')
5751 self.assertFalse('test-semaphore' in
5752 tenant.semaphore_handler.semaphores)
5753
5754 self.executor_server.hold_jobs_in_build = False
5755 self.executor_server.release()
5756
5757 self.waitUntilSettled()
5758 self.assertEqual(len(self.builds), 0)
5759
5760 self.assertEqual(A.reported, 1)
5761 self.assertEqual(B.reported, 1)
5762 self.assertFalse('test-semaphore' in
5763 tenant.semaphore_handler.semaphores)
5764
5765 def test_semaphore_two(self):
5766 "Test semaphores with max>1"
5767 tenant = self.sched.abide.tenants.get('tenant-one')
5768
5769 self.executor_server.hold_jobs_in_build = True
5770 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
5771 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
5772 self.assertFalse('test-semaphore-two' in
5773 tenant.semaphore_handler.semaphores)
5774
5775 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5776 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
5777 self.waitUntilSettled()
5778
5779 self.assertEqual(len(self.builds), 4)
5780 self.assertEqual(self.builds[0].name, 'project-test1')
5781 self.assertEqual(self.builds[1].name, 'semaphore-two-test1')
5782 self.assertEqual(self.builds[2].name, 'semaphore-two-test2')
5783 self.assertEqual(self.builds[3].name, 'project-test1')
5784 self.assertTrue('test-semaphore-two' in
5785 tenant.semaphore_handler.semaphores)
5786 self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
5787 'test-semaphore-two', [])), 2)
5788
5789 self.executor_server.release('semaphore-two-test1')
5790 self.waitUntilSettled()
5791
5792 self.assertEqual(len(self.builds), 4)
5793 self.assertEqual(self.builds[0].name, 'project-test1')
5794 self.assertEqual(self.builds[1].name, 'semaphore-two-test2')
5795 self.assertEqual(self.builds[2].name, 'project-test1')
5796 self.assertEqual(self.builds[3].name, 'semaphore-two-test1')
5797 self.assertTrue('test-semaphore-two' in
5798 tenant.semaphore_handler.semaphores)
5799 self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
5800 'test-semaphore-two', [])), 2)
5801
5802 self.executor_server.release('semaphore-two-test2')
5803 self.waitUntilSettled()
5804
5805 self.assertEqual(len(self.builds), 4)
5806 self.assertEqual(self.builds[0].name, 'project-test1')
5807 self.assertEqual(self.builds[1].name, 'project-test1')
5808 self.assertEqual(self.builds[2].name, 'semaphore-two-test1')
5809 self.assertEqual(self.builds[3].name, 'semaphore-two-test2')
5810 self.assertTrue('test-semaphore-two' in
5811 tenant.semaphore_handler.semaphores)
5812 self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
5813 'test-semaphore-two', [])), 2)
5814
5815 self.executor_server.release('semaphore-two-test1')
5816 self.waitUntilSettled()
5817
5818 self.assertEqual(len(self.builds), 3)
5819 self.assertEqual(self.builds[0].name, 'project-test1')
5820 self.assertEqual(self.builds[1].name, 'project-test1')
5821 self.assertEqual(self.builds[2].name, 'semaphore-two-test2')
5822 self.assertTrue('test-semaphore-two' in
5823 tenant.semaphore_handler.semaphores)
5824 self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
5825 'test-semaphore-two', [])), 1)
5826
5827 self.executor_server.release('semaphore-two-test2')
5828 self.waitUntilSettled()
5829
5830 self.assertEqual(len(self.builds), 2)
5831 self.assertEqual(self.builds[0].name, 'project-test1')
5832 self.assertEqual(self.builds[1].name, 'project-test1')
5833 self.assertFalse('test-semaphore-two' in
5834 tenant.semaphore_handler.semaphores)
5835
5836 self.executor_server.hold_jobs_in_build = False
5837 self.executor_server.release()
5838
5839 self.waitUntilSettled()
5840 self.assertEqual(len(self.builds), 0)
5841
5842 self.assertEqual(A.reported, 1)
5843 self.assertEqual(B.reported, 1)
5844
5845 def test_semaphore_abandon(self):
5846 "Test abandon with job semaphores"
5847 self.executor_server.hold_jobs_in_build = True
5848 tenant = self.sched.abide.tenants.get('tenant-one')
5849 check_pipeline = tenant.layout.pipelines['check']
5850
5851 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
5852 self.assertFalse('test-semaphore' in
5853 tenant.semaphore_handler.semaphores)
5854
5855 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5856 self.waitUntilSettled()
5857
5858 self.assertTrue('test-semaphore' in
5859 tenant.semaphore_handler.semaphores)
5860
5861 self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
5862 self.waitUntilSettled()
5863
5864 # The check pipeline should be empty
5865 items = check_pipeline.getAllItems()
5866 self.assertEqual(len(items), 0)
5867
5868 # The semaphore should be released
5869 self.assertFalse('test-semaphore' in
5870 tenant.semaphore_handler.semaphores)
5871
5872 self.executor_server.hold_jobs_in_build = False
5873 self.executor_server.release()
5874 self.waitUntilSettled()
5875
Tobias Henkel0f714002017-06-30 23:30:52 +02005876 def test_semaphore_new_patchset(self):
5877 "Test new patchset with job semaphores"
5878 self.executor_server.hold_jobs_in_build = True
5879 tenant = self.sched.abide.tenants.get('tenant-one')
5880 check_pipeline = tenant.layout.pipelines['check']
5881
5882 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
5883 self.assertFalse('test-semaphore' in
5884 tenant.semaphore_handler.semaphores)
5885
5886 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5887 self.waitUntilSettled()
5888
5889 self.assertTrue('test-semaphore' in
5890 tenant.semaphore_handler.semaphores)
5891 semaphore = tenant.semaphore_handler.semaphores['test-semaphore']
5892 self.assertEqual(len(semaphore), 1)
5893
5894 A.addPatchset()
5895 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
5896 self.waitUntilSettled()
5897
5898 self.assertTrue('test-semaphore' in
5899 tenant.semaphore_handler.semaphores)
5900 semaphore = tenant.semaphore_handler.semaphores['test-semaphore']
5901 self.assertEqual(len(semaphore), 1)
5902
5903 items = check_pipeline.getAllItems()
5904 self.assertEqual(items[0].change.number, '1')
5905 self.assertEqual(items[0].change.patchset, '2')
5906 self.assertTrue(items[0].live)
5907
5908 self.executor_server.hold_jobs_in_build = False
5909 self.executor_server.release()
5910 self.waitUntilSettled()
5911
5912 # The semaphore should be released
5913 self.assertFalse('test-semaphore' in
5914 tenant.semaphore_handler.semaphores)
5915
James E. Blair9ea0d0b2017-04-20 09:27:15 -07005916 def test_semaphore_reconfigure(self):
5917 "Test reconfigure with job semaphores"
5918 self.executor_server.hold_jobs_in_build = True
5919 tenant = self.sched.abide.tenants.get('tenant-one')
5920 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
5921 self.assertFalse('test-semaphore' in
5922 tenant.semaphore_handler.semaphores)
5923
5924 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5925 self.waitUntilSettled()
5926
5927 self.assertTrue('test-semaphore' in
5928 tenant.semaphore_handler.semaphores)
5929
5930 # reconfigure without layout change
5931 self.sched.reconfigure(self.config)
5932 self.waitUntilSettled()
5933 tenant = self.sched.abide.tenants.get('tenant-one')
5934
5935 # semaphore still must be held
5936 self.assertTrue('test-semaphore' in
5937 tenant.semaphore_handler.semaphores)
5938
5939 self.commitConfigUpdate(
5940 'common-config',
5941 'config/semaphore/zuul-reconfiguration.yaml')
5942 self.sched.reconfigure(self.config)
5943 self.waitUntilSettled()
5944 tenant = self.sched.abide.tenants.get('tenant-one')
5945
5946 self.executor_server.release('project-test1')
5947 self.waitUntilSettled()
5948
5949 # There should be no builds anymore
5950 self.assertEqual(len(self.builds), 0)
5951
5952 # The semaphore should be released
5953 self.assertFalse('test-semaphore' in
5954 tenant.semaphore_handler.semaphores)
5955
5956
Tobias Henkel9a0e1942017-03-20 16:16:02 +01005957class TestSemaphoreMultiTenant(ZuulTestCase):
5958 tenant_config_file = 'config/multi-tenant-semaphore/main.yaml'
5959
5960 def test_semaphore_tenant_isolation(self):
5961 "Test semaphores in multiple tenants"
5962
5963 self.waitUntilSettled()
5964 tenant_one = self.sched.abide.tenants.get('tenant-one')
5965 tenant_two = self.sched.abide.tenants.get('tenant-two')
5966
5967 self.executor_server.hold_jobs_in_build = True
5968 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
5969 B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
5970 C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
5971 D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')
5972 E = self.fake_gerrit.addFakeChange('org/project2', 'master', 'E')
5973 self.assertFalse('test-semaphore' in
5974 tenant_one.semaphore_handler.semaphores)
5975 self.assertFalse('test-semaphore' in
5976 tenant_two.semaphore_handler.semaphores)
5977
5978 # add patches to project1 of tenant-one
5979 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5980 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
5981 self.waitUntilSettled()
5982
5983 # one build of project1-test1 must run
5984 # semaphore of tenant-one must be acquired once
5985 # semaphore of tenant-two must not be acquired
5986 self.assertEqual(len(self.builds), 1)
5987 self.assertEqual(self.builds[0].name, 'project1-test1')
5988 self.assertTrue('test-semaphore' in
5989 tenant_one.semaphore_handler.semaphores)
5990 self.assertEqual(len(tenant_one.semaphore_handler.semaphores.get(
5991 'test-semaphore', [])), 1)
5992 self.assertFalse('test-semaphore' in
5993 tenant_two.semaphore_handler.semaphores)
5994
5995 # add patches to project2 of tenant-two
5996 self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
5997 self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
5998 self.fake_gerrit.addEvent(E.getPatchsetCreatedEvent(1))
5999 self.waitUntilSettled()
6000
6001 # one build of project1-test1 must run
6002 # two builds of project2-test1 must run
6003 # semaphore of tenant-one must be acquired once
6004 # semaphore of tenant-two must be acquired twice
6005 self.assertEqual(len(self.builds), 3)
6006 self.assertEqual(self.builds[0].name, 'project1-test1')
6007 self.assertEqual(self.builds[1].name, 'project2-test1')
6008 self.assertEqual(self.builds[2].name, 'project2-test1')
6009 self.assertTrue('test-semaphore' in
6010 tenant_one.semaphore_handler.semaphores)
6011 self.assertEqual(len(tenant_one.semaphore_handler.semaphores.get(
6012 'test-semaphore', [])), 1)
6013 self.assertTrue('test-semaphore' in
6014 tenant_two.semaphore_handler.semaphores)
6015 self.assertEqual(len(tenant_two.semaphore_handler.semaphores.get(
6016 'test-semaphore', [])), 2)
6017
6018 self.executor_server.release('project1-test1')
6019 self.waitUntilSettled()
6020
6021 # one build of project1-test1 must run
6022 # two builds of project2-test1 must run
6023 # semaphore of tenant-one must be acquired once
6024 # semaphore of tenant-two must be acquired twice
6025 self.assertEqual(len(self.builds), 3)
6026 self.assertEqual(self.builds[0].name, 'project2-test1')
6027 self.assertEqual(self.builds[1].name, 'project2-test1')
6028 self.assertEqual(self.builds[2].name, 'project1-test1')
6029 self.assertTrue('test-semaphore' in
6030 tenant_one.semaphore_handler.semaphores)
6031 self.assertEqual(len(tenant_one.semaphore_handler.semaphores.get(
6032 'test-semaphore', [])), 1)
6033 self.assertTrue('test-semaphore' in
6034 tenant_two.semaphore_handler.semaphores)
6035 self.assertEqual(len(tenant_two.semaphore_handler.semaphores.get(
6036 'test-semaphore', [])), 2)
6037
6038 self.executor_server.release('project2-test1')
6039 self.waitUntilSettled()
6040
6041 # one build of project1-test1 must run
6042 # one build of project2-test1 must run
6043 # semaphore of tenant-one must be acquired once
6044 # semaphore of tenant-two must be acquired once
6045 self.assertEqual(len(self.builds), 2)
6046 self.assertTrue('test-semaphore' in
6047 tenant_one.semaphore_handler.semaphores)
6048 self.assertEqual(len(tenant_one.semaphore_handler.semaphores.get(
6049 'test-semaphore', [])), 1)
6050 self.assertTrue('test-semaphore' in
6051 tenant_two.semaphore_handler.semaphores)
6052 self.assertEqual(len(tenant_two.semaphore_handler.semaphores.get(
6053 'test-semaphore', [])), 1)
6054
6055 self.executor_server.hold_jobs_in_build = False
6056 self.executor_server.release()
6057
6058 self.waitUntilSettled()
6059
6060 # no build must run
6061 # semaphore of tenant-one must not be acquired
6062 # semaphore of tenant-two must not be acquired
6063 self.assertEqual(len(self.builds), 0)
6064 self.assertFalse('test-semaphore' in
6065 tenant_one.semaphore_handler.semaphores)
6066 self.assertFalse('test-semaphore' in
6067 tenant_two.semaphore_handler.semaphores)
6068
6069 self.assertEqual(A.reported, 1)
6070 self.assertEqual(B.reported, 1)
6071
6072
6073class TestSemaphoreInRepo(ZuulTestCase):
Tobias Henkelabf973e2017-07-28 10:07:34 +02006074 config_file = 'zuul-connections-gerrit-and-github.conf'
Tobias Henkel9a0e1942017-03-20 16:16:02 +01006075 tenant_config_file = 'config/in-repo/main.yaml'
6076
6077 def test_semaphore_in_repo(self):
6078 "Test semaphores in repo config"
6079
6080 # This tests dynamic semaphore handling in project repos. The semaphore
6081 # max value should not be evaluated dynamically but must be updated
6082 # after the change lands.
6083
6084 self.waitUntilSettled()
6085 tenant = self.sched.abide.tenants.get('tenant-one')
6086
6087 in_repo_conf = textwrap.dedent(
6088 """
6089 - job:
Tobias Henkelf02cf512017-07-21 22:55:34 +02006090 name: project-test1
6091
6092 - job:
Tobias Henkel9a0e1942017-03-20 16:16:02 +01006093 name: project-test2
James E. Blair2f589fe2017-10-26 12:57:41 -07006094 run: playbooks/project-test2.yaml
Tobias Henkel9a0e1942017-03-20 16:16:02 +01006095 semaphore: test-semaphore
6096
6097 - project:
6098 name: org/project
6099 tenant-one-gate:
6100 jobs:
6101 - project-test2
6102
6103 # the max value in dynamic layout must be ignored
6104 - semaphore:
6105 name: test-semaphore
6106 max: 2
6107 """)
6108
6109 in_repo_playbook = textwrap.dedent(
6110 """
6111 - hosts: all
6112 tasks: []
6113 """)
6114
6115 file_dict = {'.zuul.yaml': in_repo_conf,
6116 'playbooks/project-test2.yaml': in_repo_playbook}
6117 A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
6118 files=file_dict)
6119 B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
6120 C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
6121 B.setDependsOn(A, 1)
6122 C.setDependsOn(A, 1)
6123
6124 self.executor_server.hold_jobs_in_build = True
6125
Tobias Henkelbf24fd12017-07-27 06:13:07 +02006126 A.addApproval('Code-Review', 2)
6127 B.addApproval('Code-Review', 2)
6128 C.addApproval('Code-Review', 2)
6129 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
6130 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
6131 self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
Tobias Henkel9a0e1942017-03-20 16:16:02 +01006132 self.waitUntilSettled()
6133
6134 # check that the layout in a queue item still has max value of 1
6135 # for test-semaphore
6136 pipeline = tenant.layout.pipelines.get('tenant-one-gate')
6137 queue = None
6138 for queue_candidate in pipeline.queues:
6139 if queue_candidate.name == 'org/project':
6140 queue = queue_candidate
6141 break
6142 queue_item = queue.queue[0]
James E. Blair29a24fd2017-10-02 15:04:56 -07006143 item_dynamic_layout = queue_item.layout
Tobias Henkel9a0e1942017-03-20 16:16:02 +01006144 dynamic_test_semaphore = \
6145 item_dynamic_layout.semaphores.get('test-semaphore')
6146 self.assertEqual(dynamic_test_semaphore.max, 1)
6147
6148 # one build must be in queue, one semaphores acquired
6149 self.assertEqual(len(self.builds), 1)
6150 self.assertEqual(self.builds[0].name, 'project-test2')
6151 self.assertTrue('test-semaphore' in
6152 tenant.semaphore_handler.semaphores)
6153 self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
6154 'test-semaphore', [])), 1)
6155
6156 self.executor_server.release('project-test2')
6157 self.waitUntilSettled()
6158
6159 # change A must be merged
6160 self.assertEqual(A.data['status'], 'MERGED')
6161 self.assertEqual(A.reported, 2)
6162
6163 # send change-merged event as the gerrit mock doesn't send it
6164 self.fake_gerrit.addEvent(A.getChangeMergedEvent())
6165 self.waitUntilSettled()
6166
6167 # now that change A was merged, the new semaphore max must be effective
6168 tenant = self.sched.abide.tenants.get('tenant-one')
6169 self.assertEqual(tenant.layout.semaphores.get('test-semaphore').max, 2)
6170
6171 # two builds must be in queue, two semaphores acquired
6172 self.assertEqual(len(self.builds), 2)
6173 self.assertEqual(self.builds[0].name, 'project-test2')
6174 self.assertEqual(self.builds[1].name, 'project-test2')
6175 self.assertTrue('test-semaphore' in
6176 tenant.semaphore_handler.semaphores)
6177 self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
6178 'test-semaphore', [])), 2)
6179
6180 self.executor_server.release('project-test2')
6181 self.waitUntilSettled()
6182
6183 self.assertEqual(len(self.builds), 0)
6184 self.assertFalse('test-semaphore' in
6185 tenant.semaphore_handler.semaphores)
6186
6187 self.executor_server.hold_jobs_in_build = False
6188 self.executor_server.release()
6189
6190 self.waitUntilSettled()
6191 self.assertEqual(len(self.builds), 0)
6192
6193 self.assertEqual(A.reported, 2)
6194 self.assertEqual(B.reported, 2)
6195 self.assertEqual(C.reported, 2)
David Shrewsburyf6dc1762017-10-02 13:34:37 -04006196
6197
6198class TestSchedulerBranchMatcher(ZuulTestCase):
6199
6200 @simple_layout('layouts/matcher-test.yaml')
6201 def test_job_branch_ignored(self):
6202 '''
6203 Test that branch matching logic works.
6204
6205 The 'ignore-branch' job has a branch matcher that is supposed to
6206 match every branch except for the 'featureA' branch, so it should
6207 not be run on a change to that branch.
6208 '''
6209 self.create_branch('org/project', 'featureA')
6210 A = self.fake_gerrit.addFakeChange('org/project', 'featureA', 'A')
6211 A.addApproval('Code-Review', 2)
6212 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
6213 self.waitUntilSettled()
6214 self.printHistory()
6215 self.assertEqual(self.getJobFromHistory('project-test1').result,
6216 'SUCCESS')
6217 self.assertJobNotInHistory('ignore-branch')
6218 self.assertEqual(A.data['status'], 'MERGED')
6219 self.assertEqual(A.reported, 2,
6220 "A should report start and success")
6221 self.assertIn('gate', A.messages[1],
6222 "A should transit gate")