blob: 22474f18c375be579a06d34d9b8d02ed1d20997a [file] [log] [blame]
James E. Blairee743612012-05-29 14:49:32 -07001# Copyright 2012 Hewlett-Packard Development Company, L.P.
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
James E. Blair1b265312014-06-24 09:35:21 -070015import copy
James E. Blairce8a2132016-05-19 15:21:52 -070016import os
James E. Blairee743612012-05-29 14:49:32 -070017import re
James E. Blairce8a2132016-05-19 15:21:52 -070018import struct
James E. Blairff986a12012-05-30 14:56:51 -070019import time
James E. Blair4886cc12012-07-18 15:39:41 -070020from uuid import uuid4
James E. Blair5a9918a2013-08-27 10:06:27 -070021import extras
22
23OrderedDict = extras.try_imports(['collections.OrderedDict',
24 'ordereddict.OrderedDict'])
James E. Blair4886cc12012-07-18 15:39:41 -070025
26
K Jonathan Harkerf95e7232015-04-29 13:33:16 -070027EMPTY_GIT_REF = '0' * 40 # git sha of all zeros, used during creates/deletes
28
James E. Blair19deff22013-08-25 13:17:35 -070029MERGER_MERGE = 1 # "git merge"
30MERGER_MERGE_RESOLVE = 2 # "git merge -s resolve"
31MERGER_CHERRY_PICK = 3 # "git cherry-pick"
32
33MERGER_MAP = {
34 'merge': MERGER_MERGE,
35 'merge-resolve': MERGER_MERGE_RESOLVE,
36 'cherry-pick': MERGER_CHERRY_PICK,
37}
James E. Blairee743612012-05-29 14:49:32 -070038
James E. Blair64ed6f22013-07-10 14:07:23 -070039PRECEDENCE_NORMAL = 0
40PRECEDENCE_LOW = 1
41PRECEDENCE_HIGH = 2
42
43PRECEDENCE_MAP = {
44 None: PRECEDENCE_NORMAL,
45 'low': PRECEDENCE_LOW,
46 'normal': PRECEDENCE_NORMAL,
47 'high': PRECEDENCE_HIGH,
48}
49
James E. Blair803e94f2017-01-06 09:18:59 -080050# Request states
51STATE_REQUESTED = 'requested'
52STATE_PENDING = 'pending'
53STATE_FULFILLED = 'fulfilled'
54STATE_FAILED = 'failed'
55REQUEST_STATES = set([STATE_REQUESTED,
56 STATE_PENDING,
57 STATE_FULFILLED,
58 STATE_FAILED])
59
60# Node states
61STATE_BUILDING = 'building'
62STATE_TESTING = 'testing'
63STATE_READY = 'ready'
64STATE_IN_USE = 'in-use'
65STATE_USED = 'used'
66STATE_HOLD = 'hold'
67STATE_DELETING = 'deleting'
68NODE_STATES = set([STATE_BUILDING,
69 STATE_TESTING,
70 STATE_READY,
71 STATE_IN_USE,
72 STATE_USED,
73 STATE_HOLD,
74 STATE_DELETING])
75
James E. Blair1e8dd892012-05-30 09:15:05 -070076
James E. Blairc053d022014-01-22 14:57:33 -080077def time_to_seconds(s):
78 if s.endswith('s'):
79 return int(s[:-1])
80 if s.endswith('m'):
81 return int(s[:-1]) * 60
82 if s.endswith('h'):
83 return int(s[:-1]) * 60 * 60
84 if s.endswith('d'):
85 return int(s[:-1]) * 24 * 60 * 60
86 if s.endswith('w'):
87 return int(s[:-1]) * 7 * 24 * 60 * 60
88 raise Exception("Unable to parse time value: %s" % s)
89
90
James E. Blair11041d22014-05-02 14:49:53 -070091def normalizeCategory(name):
92 name = name.lower()
93 return re.sub(' ', '-', name)
94
95
James E. Blair4aea70c2012-07-26 14:23:24 -070096class Pipeline(object):
Monty Taylora42a55b2016-07-29 07:53:33 -070097 """A configuration that ties triggers, reporters, managers and sources.
98
Monty Taylor82dfd412016-07-29 12:01:28 -070099 Source
100 Where changes should come from. It is a named connection to
Monty Taylora42a55b2016-07-29 07:53:33 -0700101 an external service defined in zuul.conf
Monty Taylor82dfd412016-07-29 12:01:28 -0700102
103 Trigger
104 A description of which events should be processed
105
106 Manager
107 Responsible for enqueing and dequeing Changes
108
109 Reporter
110 Communicates success and failure results somewhere
Monty Taylora42a55b2016-07-29 07:53:33 -0700111 """
James E. Blair83005782015-12-11 14:46:03 -0800112 def __init__(self, name, layout):
James E. Blair4aea70c2012-07-26 14:23:24 -0700113 self.name = name
James E. Blair83005782015-12-11 14:46:03 -0800114 self.layout = layout
James E. Blair8dbd56a2012-12-22 10:55:10 -0800115 self.description = None
James E. Blair56370192013-01-14 15:47:28 -0800116 self.failure_message = None
Joshua Heskethb7179772014-01-30 23:30:46 +1100117 self.merge_failure_message = None
James E. Blair56370192013-01-14 15:47:28 -0800118 self.success_message = None
Joshua Hesketh3979e3e2014-03-04 11:21:10 +1100119 self.footer_message = None
James E. Blair60af7f42016-03-11 16:11:06 -0800120 self.start_message = None
James E. Blair2fa50962013-01-30 21:50:41 -0800121 self.dequeue_on_new_patchset = True
James E. Blair17dd6772015-02-09 14:45:18 -0800122 self.ignore_dependencies = False
James E. Blair4aea70c2012-07-26 14:23:24 -0700123 self.job_trees = {} # project -> JobTree
124 self.manager = None
James E. Blaire0487072012-08-29 17:38:31 -0700125 self.queues = []
James E. Blair64ed6f22013-07-10 14:07:23 -0700126 self.precedence = PRECEDENCE_NORMAL
James E. Blairc0dedf82014-08-06 09:37:52 -0700127 self.source = None
James E. Blair83005782015-12-11 14:46:03 -0800128 self.triggers = []
Joshua Hesketh352264b2015-08-11 23:42:08 +1000129 self.start_actions = []
130 self.success_actions = []
131 self.failure_actions = []
132 self.merge_failure_actions = []
133 self.disabled_actions = []
Joshua Hesketh89e829d2015-02-10 16:29:45 +1100134 self.disable_at = None
135 self._consecutive_failures = 0
136 self._disabled = False
Clark Boylan7603a372014-01-21 11:43:20 -0800137 self.window = None
138 self.window_floor = None
139 self.window_increase_type = None
140 self.window_increase_factor = None
141 self.window_decrease_type = None
142 self.window_decrease_factor = None
James E. Blair4aea70c2012-07-26 14:23:24 -0700143
James E. Blair83005782015-12-11 14:46:03 -0800144 @property
145 def actions(self):
146 return (
147 self.start_actions +
148 self.success_actions +
149 self.failure_actions +
150 self.merge_failure_actions +
151 self.disabled_actions
152 )
153
James E. Blaird09c17a2012-08-07 09:23:14 -0700154 def __repr__(self):
155 return '<Pipeline %s>' % self.name
156
James E. Blair4aea70c2012-07-26 14:23:24 -0700157 def setManager(self, manager):
158 self.manager = manager
159
James E. Blair4aea70c2012-07-26 14:23:24 -0700160 def getProjects(self):
Monty Taylor74fa3862016-06-02 07:39:49 +0300161 # cmp is not in python3, applied idiom from
162 # http://python-future.org/compatible_idioms.html#cmp
163 return sorted(
164 self.job_trees.keys(),
165 key=lambda p: p.name)
James E. Blair4aea70c2012-07-26 14:23:24 -0700166
James E. Blaire0487072012-08-29 17:38:31 -0700167 def addQueue(self, queue):
168 self.queues.append(queue)
169
170 def getQueue(self, project):
171 for queue in self.queues:
172 if project in queue.projects:
173 return queue
174 return None
175
James E. Blairbfb8e042014-12-30 17:01:44 -0800176 def removeQueue(self, queue):
177 self.queues.remove(queue)
178
James E. Blair4aea70c2012-07-26 14:23:24 -0700179 def getJobTree(self, project):
180 tree = self.job_trees.get(project)
181 return tree
182
James E. Blaire0487072012-08-29 17:38:31 -0700183 def getChangesInQueue(self):
184 changes = []
185 for shared_queue in self.queues:
James E. Blairfee8d652013-06-07 08:57:52 -0700186 changes.extend([x.change for x in shared_queue.queue])
James E. Blaire0487072012-08-29 17:38:31 -0700187 return changes
188
James E. Blairfee8d652013-06-07 08:57:52 -0700189 def getAllItems(self):
190 items = []
James E. Blaire0487072012-08-29 17:38:31 -0700191 for shared_queue in self.queues:
James E. Blairfee8d652013-06-07 08:57:52 -0700192 items.extend(shared_queue.queue)
James E. Blairfee8d652013-06-07 08:57:52 -0700193 return items
James E. Blaire0487072012-08-29 17:38:31 -0700194
James E. Blairb7273ef2016-04-19 08:58:51 -0700195 def formatStatusJSON(self, url_pattern=None):
James E. Blair8dbd56a2012-12-22 10:55:10 -0800196 j_pipeline = dict(name=self.name,
197 description=self.description)
198 j_queues = []
199 j_pipeline['change_queues'] = j_queues
200 for queue in self.queues:
201 j_queue = dict(name=queue.name)
202 j_queues.append(j_queue)
203 j_queue['heads'] = []
Clark Boylanaf2476f2014-01-23 14:47:36 -0800204 j_queue['window'] = queue.window
James E. Blair972e3c72013-08-29 12:04:55 -0700205
206 j_changes = []
207 for e in queue.queue:
208 if not e.item_ahead:
209 if j_changes:
210 j_queue['heads'].append(j_changes)
211 j_changes = []
James E. Blairb7273ef2016-04-19 08:58:51 -0700212 j_changes.append(e.formatJSON(url_pattern))
James E. Blair972e3c72013-08-29 12:04:55 -0700213 if (len(j_changes) > 1 and
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000214 (j_changes[-2]['remaining_time'] is not None) and
215 (j_changes[-1]['remaining_time'] is not None)):
James E. Blair972e3c72013-08-29 12:04:55 -0700216 j_changes[-1]['remaining_time'] = max(
217 j_changes[-2]['remaining_time'],
218 j_changes[-1]['remaining_time'])
219 if j_changes:
James E. Blair8dbd56a2012-12-22 10:55:10 -0800220 j_queue['heads'].append(j_changes)
221 return j_pipeline
222
James E. Blair4aea70c2012-07-26 14:23:24 -0700223
James E. Blairee743612012-05-29 14:49:32 -0700224class ChangeQueue(object):
Monty Taylora42a55b2016-07-29 07:53:33 -0700225 """A ChangeQueue contains Changes to be processed related projects.
226
Monty Taylor82dfd412016-07-29 12:01:28 -0700227 A Pipeline with a DependentPipelineManager has multiple parallel
228 ChangeQueues shared by different projects. For instance, there may a
229 ChangeQueue shared by interrelated projects foo and bar, and a second queue
230 for independent project baz.
Monty Taylora42a55b2016-07-29 07:53:33 -0700231
Monty Taylor82dfd412016-07-29 12:01:28 -0700232 A Pipeline with an IndependentPipelineManager puts every Change into its
233 own ChangeQueue
Monty Taylora42a55b2016-07-29 07:53:33 -0700234
235 The ChangeQueue Window is inspired by TCP windows and controlls how many
236 Changes in a given ChangeQueue will be considered active and ready to
237 be processed. If a Change succeeds, the Window is increased by
238 `window_increase_factor`. If a Change fails, the Window is decreased by
239 `window_decrease_factor`.
240 """
James E. Blairbfb8e042014-12-30 17:01:44 -0800241 def __init__(self, pipeline, window=0, window_floor=1,
Clark Boylan7603a372014-01-21 11:43:20 -0800242 window_increase_type='linear', window_increase_factor=1,
James E. Blair0dcef7a2016-08-19 09:35:17 -0700243 window_decrease_type='exponential', window_decrease_factor=2,
244 name=None):
James E. Blair4aea70c2012-07-26 14:23:24 -0700245 self.pipeline = pipeline
James E. Blair0dcef7a2016-08-19 09:35:17 -0700246 if name:
247 self.name = name
248 else:
249 self.name = ''
James E. Blairee743612012-05-29 14:49:32 -0700250 self.projects = []
251 self._jobs = set()
252 self.queue = []
Clark Boylan7603a372014-01-21 11:43:20 -0800253 self.window = window
254 self.window_floor = window_floor
255 self.window_increase_type = window_increase_type
256 self.window_increase_factor = window_increase_factor
257 self.window_decrease_type = window_decrease_type
258 self.window_decrease_factor = window_decrease_factor
James E. Blairee743612012-05-29 14:49:32 -0700259
James E. Blair9f9667e2012-06-12 17:51:08 -0700260 def __repr__(self):
James E. Blair4aea70c2012-07-26 14:23:24 -0700261 return '<ChangeQueue %s: %s>' % (self.pipeline.name, self.name)
James E. Blairee743612012-05-29 14:49:32 -0700262
263 def getJobs(self):
264 return self._jobs
265
266 def addProject(self, project):
267 if project not in self.projects:
268 self.projects.append(project)
James E. Blairc8a1e052014-02-25 09:29:26 -0800269
James E. Blair0dcef7a2016-08-19 09:35:17 -0700270 if not self.name:
271 self.name = project.name
James E. Blairee743612012-05-29 14:49:32 -0700272
273 def enqueueChange(self, change):
James E. Blairbfb8e042014-12-30 17:01:44 -0800274 item = QueueItem(self, change)
James E. Blaircdccd972013-07-01 12:10:22 -0700275 self.enqueueItem(item)
276 item.enqueue_time = time.time()
277 return item
278
279 def enqueueItem(self, item):
James E. Blair4a035d92014-01-23 13:10:48 -0800280 item.pipeline = self.pipeline
James E. Blairbfb8e042014-12-30 17:01:44 -0800281 item.queue = self
282 if self.queue:
James E. Blairfee8d652013-06-07 08:57:52 -0700283 item.item_ahead = self.queue[-1]
James E. Blair972e3c72013-08-29 12:04:55 -0700284 item.item_ahead.items_behind.append(item)
James E. Blairfee8d652013-06-07 08:57:52 -0700285 self.queue.append(item)
James E. Blairee743612012-05-29 14:49:32 -0700286
James E. Blairfee8d652013-06-07 08:57:52 -0700287 def dequeueItem(self, item):
288 if item in self.queue:
289 self.queue.remove(item)
James E. Blairfee8d652013-06-07 08:57:52 -0700290 if item.item_ahead:
James E. Blair972e3c72013-08-29 12:04:55 -0700291 item.item_ahead.items_behind.remove(item)
292 for item_behind in item.items_behind:
293 if item.item_ahead:
294 item.item_ahead.items_behind.append(item_behind)
295 item_behind.item_ahead = item.item_ahead
James E. Blairfee8d652013-06-07 08:57:52 -0700296 item.item_ahead = None
James E. Blair972e3c72013-08-29 12:04:55 -0700297 item.items_behind = []
James E. Blairfee8d652013-06-07 08:57:52 -0700298 item.dequeue_time = time.time()
James E. Blaire0487072012-08-29 17:38:31 -0700299
James E. Blair972e3c72013-08-29 12:04:55 -0700300 def moveItem(self, item, item_ahead):
James E. Blair972e3c72013-08-29 12:04:55 -0700301 if item.item_ahead == item_ahead:
302 return False
303 # Remove from current location
304 if item.item_ahead:
305 item.item_ahead.items_behind.remove(item)
306 for item_behind in item.items_behind:
307 if item.item_ahead:
308 item.item_ahead.items_behind.append(item_behind)
309 item_behind.item_ahead = item.item_ahead
310 # Add to new location
311 item.item_ahead = item_ahead
James E. Blair00451262013-09-20 11:40:17 -0700312 item.items_behind = []
James E. Blair972e3c72013-08-29 12:04:55 -0700313 if item.item_ahead:
314 item.item_ahead.items_behind.append(item)
315 return True
James E. Blairee743612012-05-29 14:49:32 -0700316
317 def mergeChangeQueue(self, other):
318 for project in other.projects:
319 self.addProject(project)
Clark Boylan7603a372014-01-21 11:43:20 -0800320 self.window = min(self.window, other.window)
321 # TODO merge semantics
322
Clark Boylan3d2f7a72014-01-23 11:07:42 -0800323 def isActionable(self, item):
James E. Blairbfb8e042014-12-30 17:01:44 -0800324 if self.window:
Clark Boylan3d2f7a72014-01-23 11:07:42 -0800325 return item in self.queue[:self.window]
Clark Boylan7603a372014-01-21 11:43:20 -0800326 else:
Clark Boylan3d2f7a72014-01-23 11:07:42 -0800327 return True
Clark Boylan7603a372014-01-21 11:43:20 -0800328
329 def increaseWindowSize(self):
James E. Blairbfb8e042014-12-30 17:01:44 -0800330 if self.window:
Clark Boylan7603a372014-01-21 11:43:20 -0800331 if self.window_increase_type == 'linear':
332 self.window += self.window_increase_factor
333 elif self.window_increase_type == 'exponential':
334 self.window *= self.window_increase_factor
335
336 def decreaseWindowSize(self):
James E. Blairbfb8e042014-12-30 17:01:44 -0800337 if self.window:
Clark Boylan7603a372014-01-21 11:43:20 -0800338 if self.window_decrease_type == 'linear':
339 self.window = max(
340 self.window_floor,
341 self.window - self.window_decrease_factor)
342 elif self.window_decrease_type == 'exponential':
343 self.window = max(
344 self.window_floor,
Morgan Fainbergfdae2252016-06-07 20:13:20 -0700345 int(self.window / self.window_decrease_factor))
James E. Blairee743612012-05-29 14:49:32 -0700346
James E. Blair1e8dd892012-05-30 09:15:05 -0700347
James E. Blair4aea70c2012-07-26 14:23:24 -0700348class Project(object):
Monty Taylora42a55b2016-07-29 07:53:33 -0700349 """A Project represents a git repository such as openstack/nova."""
350
James E. Blaircf440a22016-07-15 09:11:58 -0700351 # NOTE: Projects should only be instantiated via a Source object
352 # so that they are associated with and cached by their Connection.
353 # This makes a Project instance a unique identifier for a given
354 # project from a given source.
355
James E. Blairc73c73a2017-01-20 15:15:15 -0800356 def __init__(self, name, connection_name, foreign=False):
James E. Blair4aea70c2012-07-26 14:23:24 -0700357 self.name = name
James E. Blairc73c73a2017-01-20 15:15:15 -0800358 self.connection_name = connection_name
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +0000359 # foreign projects are those referenced in dependencies
360 # of layout projects, this should matter
361 # when deciding whether to enqueue their changes
James E. Blaircf440a22016-07-15 09:11:58 -0700362 # TODOv3 (jeblair): re-add support for foreign projects if needed
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +0000363 self.foreign = foreign
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700364 self.unparsed_config = None
James E. Blair4aea70c2012-07-26 14:23:24 -0700365
366 def __str__(self):
367 return self.name
368
369 def __repr__(self):
370 return '<Project %s>' % (self.name)
371
372
James E. Blair34776ee2016-08-25 13:53:54 -0700373class Node(object):
James E. Blaira98340f2016-09-02 11:33:49 -0700374 """A single node for use by a job.
375
376 This may represent a request for a node, or an actual node
377 provided by Nodepool.
378 """
379
James E. Blair34776ee2016-08-25 13:53:54 -0700380 def __init__(self, name, image):
381 self.name = name
382 self.image = image
James E. Blaircbf43672017-01-04 14:33:41 -0800383 self.id = None
James E. Blaira38c28e2017-01-04 10:33:20 -0800384 self.lock = None
385 # Attributes from Nodepool
386 self._state = 'unknown'
387 self.state_time = time.time()
388 self.public_ipv4 = None
389 self.private_ipv4 = None
390 self.public_ipv6 = None
James E. Blaircacdf2b2017-01-04 13:14:37 -0800391 self._keys = []
James E. Blaira38c28e2017-01-04 10:33:20 -0800392
393 @property
394 def state(self):
395 return self._state
396
397 @state.setter
398 def state(self, value):
James E. Blair803e94f2017-01-06 09:18:59 -0800399 if value not in NODE_STATES:
400 raise TypeError("'%s' is not a valid state" % value)
James E. Blaira38c28e2017-01-04 10:33:20 -0800401 self._state = value
402 self.state_time = time.time()
James E. Blair34776ee2016-08-25 13:53:54 -0700403
404 def __repr__(self):
James E. Blaircbf43672017-01-04 14:33:41 -0800405 return '<Node %s %s:%s>' % (self.id, self.name, self.image)
James E. Blair34776ee2016-08-25 13:53:54 -0700406
James E. Blaircacdf2b2017-01-04 13:14:37 -0800407 def toDict(self):
408 d = {}
409 d['state'] = self.state
410 for k in self._keys:
411 d[k] = getattr(self, k)
412 return d
413
James E. Blaira38c28e2017-01-04 10:33:20 -0800414 def updateFromDict(self, data):
415 self._state = data['state']
James E. Blaircacdf2b2017-01-04 13:14:37 -0800416 keys = []
417 for k, v in data.items():
418 if k == 'state':
419 continue
420 keys.append(k)
421 setattr(self, k, v)
422 self._keys = keys
James E. Blaira38c28e2017-01-04 10:33:20 -0800423
James E. Blair34776ee2016-08-25 13:53:54 -0700424
James E. Blaira98340f2016-09-02 11:33:49 -0700425class NodeSet(object):
426 """A set of nodes.
427
428 In configuration, NodeSets are attributes of Jobs indicating that
429 a Job requires nodes matching this description.
430
431 They may appear as top-level configuration objects and be named,
432 or they may appears anonymously in in-line job definitions.
433 """
434
435 def __init__(self, name=None):
436 self.name = name or ''
437 self.nodes = OrderedDict()
438
James E. Blaircbf43672017-01-04 14:33:41 -0800439 def copy(self):
440 n = NodeSet(self.name)
441 for name, node in self.nodes.items():
442 n.addNode(Node(node.name, node.image))
443 return n
444
James E. Blaira98340f2016-09-02 11:33:49 -0700445 def addNode(self, node):
446 if node.name in self.nodes:
447 raise Exception("Duplicate node in %s" % (self,))
448 self.nodes[node.name] = node
449
James E. Blair0eaad552016-09-02 12:09:54 -0700450 def getNodes(self):
451 return self.nodes.values()
452
James E. Blaira98340f2016-09-02 11:33:49 -0700453 def __repr__(self):
454 if self.name:
455 name = self.name + ' '
456 else:
457 name = ''
458 return '<NodeSet %s%s>' % (name, self.nodes)
459
460
James E. Blair34776ee2016-08-25 13:53:54 -0700461class NodeRequest(object):
James E. Blaira98340f2016-09-02 11:33:49 -0700462 """A request for a set of nodes."""
463
James E. Blair0eaad552016-09-02 12:09:54 -0700464 def __init__(self, build_set, job, nodeset):
James E. Blair34776ee2016-08-25 13:53:54 -0700465 self.build_set = build_set
466 self.job = job
James E. Blair0eaad552016-09-02 12:09:54 -0700467 self.nodeset = nodeset
James E. Blair803e94f2017-01-06 09:18:59 -0800468 self._state = STATE_REQUESTED
James E. Blairdce6cea2016-12-20 16:45:32 -0800469 self.state_time = time.time()
470 self.stat = None
471 self.uid = uuid4().hex
James E. Blaircbf43672017-01-04 14:33:41 -0800472 self.id = None
James E. Blaira38c28e2017-01-04 10:33:20 -0800473 # Zuul internal failure flag (not stored in ZK so it's not
474 # overwritten).
475 self.failed = False
James E. Blairdce6cea2016-12-20 16:45:32 -0800476
477 @property
James E. Blair6ab79e02017-01-06 10:10:17 -0800478 def fulfilled(self):
479 return (self._state == STATE_FULFILLED) and not self.failed
480
481 @property
James E. Blairdce6cea2016-12-20 16:45:32 -0800482 def state(self):
483 return self._state
484
485 @state.setter
486 def state(self, value):
James E. Blair803e94f2017-01-06 09:18:59 -0800487 if value not in REQUEST_STATES:
488 raise TypeError("'%s' is not a valid state" % value)
James E. Blairdce6cea2016-12-20 16:45:32 -0800489 self._state = value
490 self.state_time = time.time()
James E. Blair34776ee2016-08-25 13:53:54 -0700491
492 def __repr__(self):
James E. Blaircbf43672017-01-04 14:33:41 -0800493 return '<NodeRequest %s %s>' % (self.id, self.nodeset)
James E. Blair34776ee2016-08-25 13:53:54 -0700494
James E. Blairdce6cea2016-12-20 16:45:32 -0800495 def toDict(self):
496 d = {}
497 nodes = [n.image for n in self.nodeset.getNodes()]
498 d['node_types'] = nodes
499 d['requestor'] = 'zuul' # TODOv3(jeblair): better descriptor
500 d['state'] = self.state
501 d['state_time'] = self.state_time
502 return d
503
504 def updateFromDict(self, data):
505 self._state = data['state']
506 self.state_time = data['state_time']
507
James E. Blair34776ee2016-08-25 13:53:54 -0700508
James E. Blairee743612012-05-29 14:49:32 -0700509class Job(object):
Monty Taylora42a55b2016-07-29 07:53:33 -0700510 """A Job represents the defintion of actions to perform."""
511
James E. Blair83005782015-12-11 14:46:03 -0800512 attributes = dict(
513 timeout=None,
514 # variables={},
James E. Blair0eaad552016-09-02 12:09:54 -0700515 nodeset=NodeSet(),
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000516 auth={},
James E. Blair83005782015-12-11 14:46:03 -0800517 workspace=None,
518 pre_run=None,
519 post_run=None,
520 voting=None,
James E. Blair791b5392016-08-03 11:25:56 -0700521 hold_following_changes=None,
James E. Blair83005782015-12-11 14:46:03 -0800522 failure_message=None,
523 success_message=None,
524 failure_url=None,
525 success_url=None,
526 # Matchers. These are separate so they can be individually
527 # overidden.
528 branch_matcher=None,
529 file_matcher=None,
530 irrelevant_file_matcher=None, # skip-if
Joshua Heskethdc7820c2016-03-11 13:14:28 +1100531 tags=set(),
Joshua Hesketh89b67f62016-02-11 21:22:14 +1100532 mutex=None,
Joshua Hesketh3f7def32016-11-21 17:36:44 +1100533 attempts=3,
James E. Blairc73c73a2017-01-20 15:15:15 -0800534 source_project=None,
535 source_branch=None,
536 source_configrepo=None,
537 playbook=None,
James E. Blair83005782015-12-11 14:46:03 -0800538 )
539
James E. Blairee743612012-05-29 14:49:32 -0700540 def __init__(self, name):
541 self.name = name
James E. Blair83005782015-12-11 14:46:03 -0800542 for k, v in self.attributes.items():
543 setattr(self, k, v)
544
Paul Belangere22baea2016-11-03 16:59:27 -0400545 def __eq__(self, other):
James E. Blair83005782015-12-11 14:46:03 -0800546 # Compare the name and all inheritable attributes to determine
547 # whether two jobs with the same name are identically
548 # configured. Useful upon reconfiguration.
549 if not isinstance(other, Job):
550 return False
551 if self.name != other.name:
552 return False
553 for k, v in self.attributes.items():
554 if getattr(self, k) != getattr(other, k):
555 return False
556 return True
James E. Blairee743612012-05-29 14:49:32 -0700557
558 def __str__(self):
559 return self.name
560
561 def __repr__(self):
James E. Blair34776ee2016-08-25 13:53:54 -0700562 return '<Job %s branches: %s>' % (self.name, self.branch_matcher)
James E. Blair83005782015-12-11 14:46:03 -0800563
564 def inheritFrom(self, other):
565 """Copy the inheritable attributes which have been set on the other
566 job to this job."""
567
568 if not isinstance(other, Job):
569 raise Exception("Job unable to inherit from %s" % (other,))
570 for k, v in self.attributes.items():
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000571 if getattr(other, k) != v and k != 'auth':
James E. Blair83005782015-12-11 14:46:03 -0800572 setattr(self, k, getattr(other, k))
Ricardo Carrillo Cruz4e94f612016-07-25 16:11:56 +0000573 # Inherit auth only if explicitly allowed
574 if other.auth and 'inherit' in other.auth and other.auth['inherit']:
575 setattr(self, 'auth', getattr(other, 'auth'))
James E. Blairee743612012-05-29 14:49:32 -0700576
James E. Blaire421a232012-07-25 16:59:21 -0700577 def changeMatches(self, change):
James E. Blair83005782015-12-11 14:46:03 -0800578 if self.branch_matcher and not self.branch_matcher.matches(change):
James E. Blair70c71582013-03-06 08:50:50 -0800579 return False
580
James E. Blair83005782015-12-11 14:46:03 -0800581 if self.file_matcher and not self.file_matcher.matches(change):
James E. Blair70c71582013-03-06 08:50:50 -0800582 return False
583
James E. Blair83005782015-12-11 14:46:03 -0800584 # NB: This is a negative match.
585 if (self.irrelevant_file_matcher and
586 self.irrelevant_file_matcher.matches(change)):
Maru Newby3fe5f852015-01-13 04:22:14 +0000587 return False
588
James E. Blair70c71582013-03-06 08:50:50 -0800589 return True
James E. Blaire5a847f2012-07-10 15:29:14 -0700590
James E. Blair1e8dd892012-05-30 09:15:05 -0700591
James E. Blairee743612012-05-29 14:49:32 -0700592class JobTree(object):
Monty Taylora42a55b2016-07-29 07:53:33 -0700593 """A JobTree holds one or more Jobs to represent Job dependencies.
594
595 If Job foo should only execute if Job bar succeeds, then there will
596 be a JobTree for foo, which will contain a JobTree for bar. A JobTree
597 can hold more than one dependent JobTrees, such that jobs bar and bang
598 both depend on job foo being successful.
599
600 A root node of a JobTree will have no associated Job."""
James E. Blairee743612012-05-29 14:49:32 -0700601
602 def __init__(self, job):
603 self.job = job
604 self.job_trees = []
605
606 def addJob(self, job):
James E. Blair12a92b12014-03-26 11:54:53 -0700607 if job not in [x.job for x in self.job_trees]:
608 t = JobTree(job)
609 self.job_trees.append(t)
610 return t
James E. Blaire4ad55a2015-06-11 08:22:43 -0700611 for tree in self.job_trees:
612 if tree.job == job:
613 return tree
James E. Blairee743612012-05-29 14:49:32 -0700614
615 def getJobs(self):
616 jobs = []
617 for x in self.job_trees:
618 jobs.append(x.job)
619 jobs.extend(x.getJobs())
620 return jobs
621
622 def getJobTreeForJob(self, job):
623 if self.job == job:
624 return self
625 for tree in self.job_trees:
626 ret = tree.getJobTreeForJob(job)
627 if ret:
628 return ret
629 return None
630
James E. Blairb97ed802015-12-21 15:55:35 -0800631 def inheritFrom(self, other):
632 if other.job:
633 self.job = Job(other.job.name)
634 self.job.inheritFrom(other.job)
635 for other_tree in other.job_trees:
636 this_tree = self.getJobTreeForJob(other_tree.job)
637 if not this_tree:
638 this_tree = JobTree(None)
639 self.job_trees.append(this_tree)
640 this_tree.inheritFrom(other_tree)
641
James E. Blair1e8dd892012-05-30 09:15:05 -0700642
James E. Blair4aea70c2012-07-26 14:23:24 -0700643class Build(object):
Monty Taylora42a55b2016-07-29 07:53:33 -0700644 """A Build is an instance of a single running Job."""
645
James E. Blair4aea70c2012-07-26 14:23:24 -0700646 def __init__(self, job, uuid):
647 self.job = job
648 self.uuid = uuid
James E. Blair4aea70c2012-07-26 14:23:24 -0700649 self.url = None
James E. Blair4aea70c2012-07-26 14:23:24 -0700650 self.result = None
651 self.build_set = None
652 self.launch_time = time.time()
James E. Blair71e94122012-12-24 17:53:08 -0800653 self.start_time = None
654 self.end_time = None
James E. Blairbea9ef12013-07-15 11:52:23 -0700655 self.estimated_time = None
James E. Blair66eeebf2013-07-27 17:44:32 -0700656 self.pipeline = None
James E. Blair0aac4872013-08-23 14:02:38 -0700657 self.canceled = False
James E. Blair4a28a882013-08-23 15:17:33 -0700658 self.retry = False
James E. Blaird78576a2013-07-09 10:39:17 -0700659 self.parameters = {}
Joshua Heskethba8776a2014-01-12 14:35:40 +0800660 self.worker = Worker()
Timothy Chavezb2332082015-08-07 20:08:04 -0500661 self.node_labels = []
662 self.node_name = None
James E. Blairee743612012-05-29 14:49:32 -0700663
664 def __repr__(self):
Joshua Heskethba8776a2014-01-12 14:35:40 +0800665 return ('<Build %s of %s on %s>' %
666 (self.uuid, self.job.name, self.worker))
667
668
669class Worker(object):
Monty Taylora42a55b2016-07-29 07:53:33 -0700670 """Information about the specific worker executing a Build."""
Joshua Heskethba8776a2014-01-12 14:35:40 +0800671 def __init__(self):
672 self.name = "Unknown"
673 self.hostname = None
674 self.ips = []
675 self.fqdn = None
676 self.program = None
677 self.version = None
678 self.extra = {}
679
680 def updateFromData(self, data):
681 """Update worker information if contained in the WORK_DATA response."""
682 self.name = data.get('worker_name', self.name)
683 self.hostname = data.get('worker_hostname', self.hostname)
684 self.ips = data.get('worker_ips', self.ips)
685 self.fqdn = data.get('worker_fqdn', self.fqdn)
686 self.program = data.get('worker_program', self.program)
687 self.version = data.get('worker_version', self.version)
688 self.extra = data.get('worker_extra', self.extra)
689
690 def __repr__(self):
691 return '<Worker %s>' % self.name
James E. Blairee743612012-05-29 14:49:32 -0700692
James E. Blair1e8dd892012-05-30 09:15:05 -0700693
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700694class RepoFiles(object):
Monty Taylora42a55b2016-07-29 07:53:33 -0700695 """RepoFiles holds config-file content for per-project job config."""
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700696 # When we ask a merger to prepare a future multiple-repo state and
697 # collect files so that we can dynamically load our configuration,
698 # this class provides easy access to that data.
699 def __init__(self):
700 self.projects = {}
701
702 def __repr__(self):
703 return '<RepoFiles %s>' % self.projects
704
705 def setFiles(self, items):
706 self.projects = {}
707 for item in items:
708 project = self.projects.setdefault(item['project'], {})
709 branch = project.setdefault(item['branch'], {})
710 branch.update(item['files'])
711
712 def getFile(self, project, branch, fn):
713 return self.projects.get(project, {}).get(branch, {}).get(fn)
714
715
James E. Blair7e530ad2012-07-03 16:12:28 -0700716class BuildSet(object):
Monty Taylora42a55b2016-07-29 07:53:33 -0700717 """Contains the Builds for a Change representing potential future state.
718
719 A BuildSet also holds the UUID used to produce the Zuul Ref that builders
720 check out.
721 """
James E. Blair4076e2b2014-01-28 12:42:20 -0800722 # Merge states:
723 NEW = 1
724 PENDING = 2
725 COMPLETE = 3
726
Antoine Musso9b229282014-08-18 23:45:43 +0200727 states_map = {
728 1: 'NEW',
729 2: 'PENDING',
730 3: 'COMPLETE',
731 }
732
James E. Blairfee8d652013-06-07 08:57:52 -0700733 def __init__(self, item):
734 self.item = item
James E. Blair11700c32012-07-05 17:50:05 -0700735 self.other_changes = []
James E. Blair7e530ad2012-07-03 16:12:28 -0700736 self.builds = {}
James E. Blair11700c32012-07-05 17:50:05 -0700737 self.result = None
738 self.next_build_set = None
739 self.previous_build_set = None
James E. Blair4886cc12012-07-18 15:39:41 -0700740 self.ref = None
James E. Blair81515ad2012-10-01 18:29:08 -0700741 self.commit = None
James E. Blair4076e2b2014-01-28 12:42:20 -0800742 self.zuul_url = None
James E. Blair973721f2012-08-15 10:19:43 -0700743 self.unable_to_merge = False
James E. Blair972e3c72013-08-29 12:04:55 -0700744 self.failing_reasons = []
James E. Blair4076e2b2014-01-28 12:42:20 -0800745 self.merge_state = self.NEW
James E. Blair0eaad552016-09-02 12:09:54 -0700746 self.nodesets = {} # job -> nodeset
James E. Blair8d692392016-04-08 17:47:58 -0700747 self.node_requests = {} # job -> reqs
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700748 self.files = RepoFiles()
749 self.layout = None
Paul Belanger71d98172016-11-08 10:56:31 -0500750 self.tries = {}
James E. Blair7e530ad2012-07-03 16:12:28 -0700751
Antoine Musso9b229282014-08-18 23:45:43 +0200752 def __repr__(self):
753 return '<BuildSet item: %s #builds: %s merge state: %s>' % (
754 self.item,
755 len(self.builds),
756 self.getStateName(self.merge_state))
757
James E. Blair4886cc12012-07-18 15:39:41 -0700758 def setConfiguration(self):
James E. Blair11700c32012-07-05 17:50:05 -0700759 # The change isn't enqueued until after it's created
760 # so we don't know what the other changes ahead will be
761 # until jobs start.
762 if not self.other_changes:
James E. Blairfee8d652013-06-07 08:57:52 -0700763 next_item = self.item.item_ahead
764 while next_item:
765 self.other_changes.append(next_item.change)
766 next_item = next_item.item_ahead
James E. Blair4886cc12012-07-18 15:39:41 -0700767 if not self.ref:
768 self.ref = 'Z' + uuid4().hex
769
Antoine Musso9b229282014-08-18 23:45:43 +0200770 def getStateName(self, state_num):
771 return self.states_map.get(
772 state_num, 'UNKNOWN (%s)' % state_num)
773
James E. Blair4886cc12012-07-18 15:39:41 -0700774 def addBuild(self, build):
775 self.builds[build.job.name] = build
Paul Belanger71d98172016-11-08 10:56:31 -0500776 if build.job.name not in self.tries:
James E. Blair5aed1112016-12-15 09:18:05 -0800777 self.tries[build.job.name] = 1
James E. Blair4886cc12012-07-18 15:39:41 -0700778 build.build_set = self
James E. Blair11700c32012-07-05 17:50:05 -0700779
James E. Blair4a28a882013-08-23 15:17:33 -0700780 def removeBuild(self, build):
Paul Belanger71d98172016-11-08 10:56:31 -0500781 self.tries[build.job.name] += 1
James E. Blair4a28a882013-08-23 15:17:33 -0700782 del self.builds[build.job.name]
783
James E. Blair7e530ad2012-07-03 16:12:28 -0700784 def getBuild(self, job_name):
785 return self.builds.get(job_name)
786
James E. Blair11700c32012-07-05 17:50:05 -0700787 def getBuilds(self):
788 keys = self.builds.keys()
789 keys.sort()
790 return [self.builds.get(x) for x in keys]
791
James E. Blair0eaad552016-09-02 12:09:54 -0700792 def getJobNodeSet(self, job_name):
793 # Return None if not provisioned; empty NodeSet if no nodes
794 # required
795 return self.nodesets.get(job_name)
James E. Blair8d692392016-04-08 17:47:58 -0700796
James E. Blaire18d4602017-01-05 11:17:28 -0800797 def removeJobNodeSet(self, job_name):
798 if job_name not in self.nodesets:
799 raise Exception("No job set for %s" % (job_name))
800 del self.nodesets[job_name]
801
James E. Blair8d692392016-04-08 17:47:58 -0700802 def setJobNodeRequest(self, job_name, req):
803 if job_name in self.node_requests:
804 raise Exception("Prior node request for %s" % (job_name))
805 self.node_requests[job_name] = req
806
807 def getJobNodeRequest(self, job_name):
808 return self.node_requests.get(job_name)
809
James E. Blair0eaad552016-09-02 12:09:54 -0700810 def jobNodeRequestComplete(self, job_name, req, nodeset):
811 if job_name in self.nodesets:
James E. Blair8d692392016-04-08 17:47:58 -0700812 raise Exception("Prior node request for %s" % (job_name))
James E. Blair0eaad552016-09-02 12:09:54 -0700813 self.nodesets[job_name] = nodeset
James E. Blair8d692392016-04-08 17:47:58 -0700814 del self.node_requests[job_name]
815
Paul Belanger71d98172016-11-08 10:56:31 -0500816 def getTries(self, job_name):
817 return self.tries.get(job_name)
818
Adam Gandelman8bd57102016-12-02 12:58:42 -0800819 def getMergeMode(self, job_name):
820 if not self.layout or job_name not in self.layout.project_configs:
821 return MERGER_MERGE_RESOLVE
822 return self.layout.project_configs[job_name].merge_mode
823
James E. Blair7e530ad2012-07-03 16:12:28 -0700824
James E. Blairfee8d652013-06-07 08:57:52 -0700825class QueueItem(object):
Monty Taylora42a55b2016-07-29 07:53:33 -0700826 """Represents the position of a Change in a ChangeQueue.
827
828 All Changes are enqueued into ChangeQueue in a QueueItem. The QueueItem
829 holds the current `BuildSet` as well as all previous `BuildSets` that were
830 produced for this `QueueItem`.
831 """
James E. Blair32663402012-06-01 10:04:18 -0700832
James E. Blairbfb8e042014-12-30 17:01:44 -0800833 def __init__(self, queue, change):
834 self.pipeline = queue.pipeline
835 self.queue = queue
James E. Blairfee8d652013-06-07 08:57:52 -0700836 self.change = change # a changeish
James E. Blair7e530ad2012-07-03 16:12:28 -0700837 self.build_sets = []
James E. Blaircaec0c52012-08-22 14:52:22 -0700838 self.dequeued_needing_change = False
James E. Blair11700c32012-07-05 17:50:05 -0700839 self.current_build_set = BuildSet(self)
840 self.build_sets.append(self.current_build_set)
James E. Blairfee8d652013-06-07 08:57:52 -0700841 self.item_ahead = None
James E. Blair972e3c72013-08-29 12:04:55 -0700842 self.items_behind = []
James E. Blair8fa16972013-01-15 16:57:20 -0800843 self.enqueue_time = None
844 self.dequeue_time = None
James E. Blairfee8d652013-06-07 08:57:52 -0700845 self.reported = False
James E. Blairbfb8e042014-12-30 17:01:44 -0800846 self.active = False # Whether an item is within an active window
847 self.live = True # Whether an item is intended to be processed at all
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700848 self.layout = None # This item's shadow layout
James E. Blair83005782015-12-11 14:46:03 -0800849 self.job_tree = None
James E. Blaire5a847f2012-07-10 15:29:14 -0700850
James E. Blair972e3c72013-08-29 12:04:55 -0700851 def __repr__(self):
852 if self.pipeline:
853 pipeline = self.pipeline.name
854 else:
855 pipeline = None
856 return '<QueueItem 0x%x for %s in %s>' % (
857 id(self), self.change, pipeline)
858
James E. Blairee743612012-05-29 14:49:32 -0700859 def resetAllBuilds(self):
James E. Blair11700c32012-07-05 17:50:05 -0700860 old = self.current_build_set
861 self.current_build_set.result = 'CANCELED'
862 self.current_build_set = BuildSet(self)
863 old.next_build_set = self.current_build_set
864 self.current_build_set.previous_build_set = old
James E. Blair7e530ad2012-07-03 16:12:28 -0700865 self.build_sets.append(self.current_build_set)
James E. Blairee743612012-05-29 14:49:32 -0700866
867 def addBuild(self, build):
James E. Blair7e530ad2012-07-03 16:12:28 -0700868 self.current_build_set.addBuild(build)
James E. Blair66eeebf2013-07-27 17:44:32 -0700869 build.pipeline = self.pipeline
James E. Blairee743612012-05-29 14:49:32 -0700870
James E. Blair4a28a882013-08-23 15:17:33 -0700871 def removeBuild(self, build):
872 self.current_build_set.removeBuild(build)
873
James E. Blairfee8d652013-06-07 08:57:52 -0700874 def setReportedResult(self, result):
875 self.current_build_set.result = result
876
James E. Blair83005782015-12-11 14:46:03 -0800877 def freezeJobTree(self):
878 """Find or create actual matching jobs for this item's change and
879 store the resulting job tree."""
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700880 layout = self.current_build_set.layout
881 self.job_tree = layout.createJobTree(self)
882
883 def hasJobTree(self):
884 """Returns True if the item has a job tree."""
885 return self.job_tree is not None
James E. Blair83005782015-12-11 14:46:03 -0800886
887 def getJobs(self):
888 if not self.live or not self.job_tree:
889 return []
890 return self.job_tree.getJobs()
891
James E. Blairdbfd3282016-07-21 10:46:19 -0700892 def haveAllJobsStarted(self):
893 if not self.hasJobTree():
894 return False
895 for job in self.getJobs():
896 build = self.current_build_set.getBuild(job.name)
897 if not build or not build.start_time:
898 return False
899 return True
900
901 def areAllJobsComplete(self):
902 if not self.hasJobTree():
903 return False
904 for job in self.getJobs():
905 build = self.current_build_set.getBuild(job.name)
906 if not build or not build.result:
907 return False
908 return True
909
910 def didAllJobsSucceed(self):
911 if not self.hasJobTree():
912 return False
913 for job in self.getJobs():
914 if not job.voting:
915 continue
916 build = self.current_build_set.getBuild(job.name)
917 if not build:
918 return False
919 if build.result != 'SUCCESS':
920 return False
921 return True
922
923 def didAnyJobFail(self):
924 if not self.hasJobTree():
925 return False
926 for job in self.getJobs():
927 if not job.voting:
928 continue
929 build = self.current_build_set.getBuild(job.name)
930 if build and build.result and (build.result != 'SUCCESS'):
931 return True
932 return False
933
934 def didMergerFail(self):
935 if self.current_build_set.unable_to_merge:
936 return True
937 return False
938
James E. Blairdbfd3282016-07-21 10:46:19 -0700939 def isHoldingFollowingChanges(self):
940 if not self.live:
941 return False
942 if not self.hasJobTree():
943 return False
944 for job in self.getJobs():
945 if not job.hold_following_changes:
946 continue
947 build = self.current_build_set.getBuild(job.name)
948 if not build:
949 return True
950 if build.result != 'SUCCESS':
951 return True
952
953 if not self.item_ahead:
954 return False
955 return self.item_ahead.isHoldingFollowingChanges()
956
957 def _findJobsToRun(self, job_trees, mutex):
958 torun = []
James E. Blair791b5392016-08-03 11:25:56 -0700959 if self.item_ahead:
960 # Only run jobs if any 'hold' jobs on the change ahead
961 # have completed successfully.
962 if self.item_ahead.isHoldingFollowingChanges():
963 return []
James E. Blairdbfd3282016-07-21 10:46:19 -0700964 for tree in job_trees:
965 job = tree.job
966 result = None
967 if job:
968 if not job.changeMatches(self.change):
969 continue
970 build = self.current_build_set.getBuild(job.name)
971 if build:
972 result = build.result
973 else:
974 # There is no build for the root of this job tree,
James E. Blair34776ee2016-08-25 13:53:54 -0700975 # so it has not run yet.
James E. Blair0eaad552016-09-02 12:09:54 -0700976 nodeset = self.current_build_set.getJobNodeSet(job.name)
977 if nodeset is None:
James E. Blair34776ee2016-08-25 13:53:54 -0700978 # The nodes for this job are not ready, skip
979 # it for now.
980 continue
James E. Blairdbfd3282016-07-21 10:46:19 -0700981 if mutex.acquire(self, job):
982 # If this job needs a mutex, either acquire it or make
983 # sure that we have it before running the job.
984 torun.append(job)
985 # If there is no job, this is a null job tree, and we should
986 # run all of its jobs.
987 if result == 'SUCCESS' or not job:
988 torun.extend(self._findJobsToRun(tree.job_trees, mutex))
989 return torun
990
991 def findJobsToRun(self, mutex):
992 if not self.live:
993 return []
994 tree = self.job_tree
995 if not tree:
996 return []
997 return self._findJobsToRun(tree.job_trees, mutex)
998
999 def _findJobsToRequest(self, job_trees):
James E. Blair6ab79e02017-01-06 10:10:17 -08001000 build_set = self.current_build_set
James E. Blairdbfd3282016-07-21 10:46:19 -07001001 toreq = []
James E. Blair6ab79e02017-01-06 10:10:17 -08001002 if self.item_ahead:
1003 if self.item_ahead.isHoldingFollowingChanges():
1004 return []
James E. Blairdbfd3282016-07-21 10:46:19 -07001005 for tree in job_trees:
1006 job = tree.job
James E. Blair6ab79e02017-01-06 10:10:17 -08001007 result = None
James E. Blairdbfd3282016-07-21 10:46:19 -07001008 if job:
1009 if not job.changeMatches(self.change):
1010 continue
James E. Blair6ab79e02017-01-06 10:10:17 -08001011 build = build_set.getBuild(job.name)
1012 if build:
1013 result = build.result
1014 else:
1015 nodeset = build_set.getJobNodeSet(job.name)
1016 if nodeset is None:
1017 req = build_set.getJobNodeRequest(job.name)
1018 if req is None:
1019 toreq.append(job)
1020 if result == 'SUCCESS' or not job:
1021 toreq.extend(self._findJobsToRequest(tree.job_trees))
James E. Blairdbfd3282016-07-21 10:46:19 -07001022 return toreq
1023
1024 def findJobsToRequest(self):
1025 if not self.live:
1026 return []
1027 tree = self.job_tree
1028 if not tree:
1029 return []
1030 return self._findJobsToRequest(tree.job_trees)
1031
1032 def setResult(self, build):
1033 if build.retry:
1034 self.removeBuild(build)
1035 elif build.result != 'SUCCESS':
1036 # Get a JobTree from a Job so we can find only its dependent jobs
1037 tree = self.job_tree.getJobTreeForJob(build.job)
1038 for job in tree.getJobs():
1039 fakebuild = Build(job, None)
1040 fakebuild.result = 'SKIPPED'
1041 self.addBuild(fakebuild)
1042
James E. Blair6ab79e02017-01-06 10:10:17 -08001043 def setNodeRequestFailure(self, job):
1044 fakebuild = Build(job, None)
1045 self.addBuild(fakebuild)
1046 fakebuild.result = 'NODE_FAILURE'
1047 self.setResult(fakebuild)
1048
James E. Blairdbfd3282016-07-21 10:46:19 -07001049 def setDequeuedNeedingChange(self):
1050 self.dequeued_needing_change = True
1051 self._setAllJobsSkipped()
1052
1053 def setUnableToMerge(self):
1054 self.current_build_set.unable_to_merge = True
1055 self._setAllJobsSkipped()
1056
1057 def _setAllJobsSkipped(self):
1058 for job in self.getJobs():
1059 fakebuild = Build(job, None)
1060 fakebuild.result = 'SKIPPED'
1061 self.addBuild(fakebuild)
1062
James E. Blairb7273ef2016-04-19 08:58:51 -07001063 def formatJobResult(self, job, url_pattern=None):
1064 build = self.current_build_set.getBuild(job.name)
1065 result = build.result
1066 pattern = url_pattern
1067 if result == 'SUCCESS':
1068 if job.success_message:
1069 result = job.success_message
James E. Blaira5dba232016-08-08 15:53:24 -07001070 if job.success_url:
1071 pattern = job.success_url
James E. Blairb7273ef2016-04-19 08:58:51 -07001072 elif result == 'FAILURE':
1073 if job.failure_message:
1074 result = job.failure_message
James E. Blaira5dba232016-08-08 15:53:24 -07001075 if job.failure_url:
1076 pattern = job.failure_url
James E. Blairb7273ef2016-04-19 08:58:51 -07001077 url = None
1078 if pattern:
1079 try:
1080 url = pattern.format(change=self.change,
1081 pipeline=self.pipeline,
1082 job=job,
1083 build=build)
1084 except Exception:
1085 pass # FIXME: log this or something?
1086 if not url:
1087 url = build.url or job.name
1088 return (result, url)
1089
1090 def formatJSON(self, url_pattern=None):
Joshua Hesketh85af4e92014-02-21 08:28:58 -08001091 changeish = self.change
1092 ret = {}
1093 ret['active'] = self.active
James E. Blair107c3852015-02-07 08:23:10 -08001094 ret['live'] = self.live
Joshua Hesketh85af4e92014-02-21 08:28:58 -08001095 if hasattr(changeish, 'url') and changeish.url is not None:
1096 ret['url'] = changeish.url
1097 else:
1098 ret['url'] = None
1099 ret['id'] = changeish._id()
1100 if self.item_ahead:
1101 ret['item_ahead'] = self.item_ahead.change._id()
1102 else:
1103 ret['item_ahead'] = None
1104 ret['items_behind'] = [i.change._id() for i in self.items_behind]
1105 ret['failing_reasons'] = self.current_build_set.failing_reasons
1106 ret['zuul_ref'] = self.current_build_set.ref
Ramy Asselin07cc33c2015-06-12 14:06:34 -07001107 if changeish.project:
1108 ret['project'] = changeish.project.name
1109 else:
1110 # For cross-project dependencies with the depends-on
1111 # project not known to zuul, the project is None
1112 # Set it to a static value
1113 ret['project'] = "Unknown Project"
Joshua Hesketh85af4e92014-02-21 08:28:58 -08001114 ret['enqueue_time'] = int(self.enqueue_time * 1000)
1115 ret['jobs'] = []
Davanum Srinivasb6bfbcc2014-11-18 13:26:52 -05001116 if hasattr(changeish, 'owner'):
1117 ret['owner'] = changeish.owner
1118 else:
1119 ret['owner'] = None
Joshua Hesketh85af4e92014-02-21 08:28:58 -08001120 max_remaining = 0
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001121 for job in self.getJobs():
Joshua Hesketh85af4e92014-02-21 08:28:58 -08001122 now = time.time()
1123 build = self.current_build_set.getBuild(job.name)
1124 elapsed = None
1125 remaining = None
1126 result = None
James E. Blairb7273ef2016-04-19 08:58:51 -07001127 build_url = None
1128 report_url = None
Joshua Hesketh85af4e92014-02-21 08:28:58 -08001129 worker = None
1130 if build:
1131 result = build.result
James E. Blairb7273ef2016-04-19 08:58:51 -07001132 build_url = build.url
1133 (unused, report_url) = self.formatJobResult(job, url_pattern)
Joshua Hesketh85af4e92014-02-21 08:28:58 -08001134 if build.start_time:
1135 if build.end_time:
1136 elapsed = int((build.end_time -
1137 build.start_time) * 1000)
1138 remaining = 0
1139 else:
1140 elapsed = int((now - build.start_time) * 1000)
1141 if build.estimated_time:
1142 remaining = max(
1143 int(build.estimated_time * 1000) - elapsed,
1144 0)
1145 worker = {
1146 'name': build.worker.name,
1147 'hostname': build.worker.hostname,
1148 'ips': build.worker.ips,
1149 'fqdn': build.worker.fqdn,
1150 'program': build.worker.program,
1151 'version': build.worker.version,
1152 'extra': build.worker.extra
1153 }
1154 if remaining and remaining > max_remaining:
1155 max_remaining = remaining
1156
1157 ret['jobs'].append({
1158 'name': job.name,
1159 'elapsed_time': elapsed,
1160 'remaining_time': remaining,
James E. Blairb7273ef2016-04-19 08:58:51 -07001161 'url': build_url,
1162 'report_url': report_url,
Joshua Hesketh85af4e92014-02-21 08:28:58 -08001163 'result': result,
1164 'voting': job.voting,
1165 'uuid': build.uuid if build else None,
1166 'launch_time': build.launch_time if build else None,
1167 'start_time': build.start_time if build else None,
1168 'end_time': build.end_time if build else None,
1169 'estimated_time': build.estimated_time if build else None,
1170 'pipeline': build.pipeline.name if build else None,
1171 'canceled': build.canceled if build else None,
1172 'retry': build.retry if build else None,
Timothy Chavezb2332082015-08-07 20:08:04 -05001173 'node_labels': build.node_labels if build else [],
1174 'node_name': build.node_name if build else None,
1175 'worker': worker,
Joshua Hesketh85af4e92014-02-21 08:28:58 -08001176 })
1177
James E. Blairdbfd3282016-07-21 10:46:19 -07001178 if self.haveAllJobsStarted():
Joshua Hesketh85af4e92014-02-21 08:28:58 -08001179 ret['remaining_time'] = max_remaining
1180 else:
1181 ret['remaining_time'] = None
1182 return ret
1183
1184 def formatStatus(self, indent=0, html=False):
1185 changeish = self.change
1186 indent_str = ' ' * indent
1187 ret = ''
1188 if html and hasattr(changeish, 'url') and changeish.url is not None:
1189 ret += '%sProject %s change <a href="%s">%s</a>\n' % (
1190 indent_str,
1191 changeish.project.name,
1192 changeish.url,
1193 changeish._id())
1194 else:
1195 ret += '%sProject %s change %s based on %s\n' % (
1196 indent_str,
1197 changeish.project.name,
1198 changeish._id(),
1199 self.item_ahead)
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001200 for job in self.getJobs():
Joshua Hesketh85af4e92014-02-21 08:28:58 -08001201 build = self.current_build_set.getBuild(job.name)
1202 if build:
1203 result = build.result
1204 else:
1205 result = None
1206 job_name = job.name
1207 if not job.voting:
1208 voting = ' (non-voting)'
1209 else:
1210 voting = ''
1211 if html:
1212 if build:
1213 url = build.url
1214 else:
1215 url = None
1216 if url is not None:
1217 job_name = '<a href="%s">%s</a>' % (url, job_name)
1218 ret += '%s %s: %s%s' % (indent_str, job_name, result, voting)
1219 ret += '\n'
1220 return ret
1221
James E. Blairfee8d652013-06-07 08:57:52 -07001222
1223class Changeish(object):
Monty Taylora42a55b2016-07-29 07:53:33 -07001224 """Base class for Change and Ref."""
James E. Blairfee8d652013-06-07 08:57:52 -07001225
1226 def __init__(self, project):
1227 self.project = project
1228
Joshua Hesketh36c3fa52014-01-22 11:40:52 +11001229 def getBasePath(self):
1230 base_path = ''
1231 if hasattr(self, 'refspec'):
1232 base_path = "%s/%s/%s" % (
1233 self.number[-2:], self.number, self.patchset)
1234 elif hasattr(self, 'ref'):
1235 base_path = "%s/%s" % (self.newrev[:2], self.newrev)
1236
1237 return base_path
1238
James E. Blairfee8d652013-06-07 08:57:52 -07001239 def equals(self, other):
1240 raise NotImplementedError()
1241
1242 def isUpdateOf(self, other):
1243 raise NotImplementedError()
1244
1245 def filterJobs(self, jobs):
1246 return filter(lambda job: job.changeMatches(self), jobs)
1247
1248 def getRelatedChanges(self):
1249 return set()
1250
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001251 def updatesConfig(self):
1252 return False
1253
James E. Blair1e8dd892012-05-30 09:15:05 -07001254
James E. Blair4aea70c2012-07-26 14:23:24 -07001255class Change(Changeish):
Monty Taylora42a55b2016-07-29 07:53:33 -07001256 """A proposed new state for a Project."""
James E. Blair4aea70c2012-07-26 14:23:24 -07001257 def __init__(self, project):
1258 super(Change, self).__init__(project)
1259 self.branch = None
1260 self.number = None
1261 self.url = None
1262 self.patchset = None
1263 self.refspec = None
1264
James E. Blair70c71582013-03-06 08:50:50 -08001265 self.files = []
James E. Blair6965a4b2014-12-16 17:19:04 -08001266 self.needs_changes = []
James E. Blair4aea70c2012-07-26 14:23:24 -07001267 self.needed_by_changes = []
1268 self.is_current_patchset = True
1269 self.can_merge = False
1270 self.is_merged = False
James E. Blairfee8d652013-06-07 08:57:52 -07001271 self.failed_to_merge = False
James E. Blairc053d022014-01-22 14:57:33 -08001272 self.approvals = []
James E. Blair11041d22014-05-02 14:49:53 -07001273 self.open = None
1274 self.status = None
Davanum Srinivasb6bfbcc2014-11-18 13:26:52 -05001275 self.owner = None
James E. Blair4aea70c2012-07-26 14:23:24 -07001276
1277 def _id(self):
James E. Blairbe765db2012-08-07 08:36:20 -07001278 return '%s,%s' % (self.number, self.patchset)
James E. Blair4aea70c2012-07-26 14:23:24 -07001279
1280 def __repr__(self):
1281 return '<Change 0x%x %s>' % (id(self), self._id())
1282
1283 def equals(self, other):
Zhongyue Luoaa85ebf2012-09-21 16:38:33 +08001284 if self.number == other.number and self.patchset == other.patchset:
James E. Blair4aea70c2012-07-26 14:23:24 -07001285 return True
1286 return False
1287
James E. Blair2fa50962013-01-30 21:50:41 -08001288 def isUpdateOf(self, other):
Clark Boylan01976242013-02-17 18:41:48 -08001289 if ((hasattr(other, 'number') and self.number == other.number) and
James E. Blair7a192e42013-07-11 14:10:36 -07001290 (hasattr(other, 'patchset') and
1291 self.patchset is not None and
1292 other.patchset is not None and
1293 int(self.patchset) > int(other.patchset))):
James E. Blair2fa50962013-01-30 21:50:41 -08001294 return True
1295 return False
1296
James E. Blairfee8d652013-06-07 08:57:52 -07001297 def getRelatedChanges(self):
1298 related = set()
James E. Blair6965a4b2014-12-16 17:19:04 -08001299 for c in self.needs_changes:
1300 related.add(c)
James E. Blairfee8d652013-06-07 08:57:52 -07001301 for c in self.needed_by_changes:
1302 related.add(c)
1303 related.update(c.getRelatedChanges())
1304 return related
James E. Blair4aea70c2012-07-26 14:23:24 -07001305
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001306 def updatesConfig(self):
1307 if 'zuul.yaml' in self.files or '.zuul.yaml' in self.files:
1308 return True
1309 return False
1310
James E. Blair4aea70c2012-07-26 14:23:24 -07001311
1312class Ref(Changeish):
Monty Taylora42a55b2016-07-29 07:53:33 -07001313 """An existing state of a Project."""
James E. Blair4aea70c2012-07-26 14:23:24 -07001314 def __init__(self, project):
James E. Blairbe765db2012-08-07 08:36:20 -07001315 super(Ref, self).__init__(project)
James E. Blair4aea70c2012-07-26 14:23:24 -07001316 self.ref = None
1317 self.oldrev = None
1318 self.newrev = None
1319
James E. Blairbe765db2012-08-07 08:36:20 -07001320 def _id(self):
1321 return self.newrev
1322
Antoine Musso68bdcd72013-01-17 12:31:28 +01001323 def __repr__(self):
1324 rep = None
1325 if self.newrev == '0000000000000000000000000000000000000000':
1326 rep = '<Ref 0x%x deletes %s from %s' % (
1327 id(self), self.ref, self.oldrev)
1328 elif self.oldrev == '0000000000000000000000000000000000000000':
1329 rep = '<Ref 0x%x creates %s on %s>' % (
1330 id(self), self.ref, self.newrev)
1331 else:
1332 # Catch all
1333 rep = '<Ref 0x%x %s updated %s..%s>' % (
1334 id(self), self.ref, self.oldrev, self.newrev)
1335
1336 return rep
1337
James E. Blair4aea70c2012-07-26 14:23:24 -07001338 def equals(self, other):
James E. Blair9358c612012-09-28 08:29:39 -07001339 if (self.project == other.project
1340 and self.ref == other.ref
1341 and self.newrev == other.newrev):
James E. Blair4aea70c2012-07-26 14:23:24 -07001342 return True
1343 return False
1344
James E. Blair2fa50962013-01-30 21:50:41 -08001345 def isUpdateOf(self, other):
1346 return False
1347
James E. Blair4aea70c2012-07-26 14:23:24 -07001348
James E. Blair63bb0ef2013-07-29 17:14:51 -07001349class NullChange(Changeish):
James E. Blair23161912016-07-28 15:42:14 -07001350 # TODOv3(jeblair): remove this in favor of enqueueing Refs (eg
1351 # current master) instead.
James E. Blaire5910202013-12-27 09:50:31 -08001352 def __repr__(self):
1353 return '<NullChange for %s>' % (self.project)
James E. Blair63bb0ef2013-07-29 17:14:51 -07001354
James E. Blair63bb0ef2013-07-29 17:14:51 -07001355 def _id(self):
Alex Gaynorddb9ef32013-09-16 21:04:58 -07001356 return None
James E. Blair63bb0ef2013-07-29 17:14:51 -07001357
1358 def equals(self, other):
Steve Varnau7b78b312015-04-03 14:49:46 -07001359 if (self.project == other.project
1360 and other._id() is None):
James E. Blair4f6033c2014-03-27 15:49:09 -07001361 return True
James E. Blair63bb0ef2013-07-29 17:14:51 -07001362 return False
1363
1364 def isUpdateOf(self, other):
1365 return False
1366
1367
James E. Blairee743612012-05-29 14:49:32 -07001368class TriggerEvent(object):
Monty Taylora42a55b2016-07-29 07:53:33 -07001369 """Incoming event from an external system."""
James E. Blairee743612012-05-29 14:49:32 -07001370 def __init__(self):
1371 self.data = None
James E. Blair32663402012-06-01 10:04:18 -07001372 # common
James E. Blairee743612012-05-29 14:49:32 -07001373 self.type = None
Paul Belangerbaca3132016-11-04 12:49:54 -04001374 # For management events (eg: enqueue / promote)
1375 self.tenant_name = None
James E. Blairee743612012-05-29 14:49:32 -07001376 self.project_name = None
James E. Blair6c358e72013-07-29 17:06:47 -07001377 self.trigger_name = None
Antoine Mussob4e809e2012-12-06 16:58:06 +01001378 # Representation of the user account that performed the event.
1379 self.account = None
James E. Blair32663402012-06-01 10:04:18 -07001380 # patchset-created, comment-added, etc.
James E. Blairee743612012-05-29 14:49:32 -07001381 self.change_number = None
Clark Boylanfc56df32012-06-28 15:25:57 -07001382 self.change_url = None
James E. Blairee743612012-05-29 14:49:32 -07001383 self.patch_number = None
James E. Blaira03262c2012-05-30 09:41:16 -07001384 self.refspec = None
James E. Blairee743612012-05-29 14:49:32 -07001385 self.approvals = []
1386 self.branch = None
Clark Boylanb9bcb402012-06-29 17:44:05 -07001387 self.comment = None
James E. Blair32663402012-06-01 10:04:18 -07001388 # ref-updated
James E. Blairee743612012-05-29 14:49:32 -07001389 self.ref = None
James E. Blair32663402012-06-01 10:04:18 -07001390 self.oldrev = None
James E. Blair89cae0f2012-07-18 11:18:32 -07001391 self.newrev = None
James E. Blair63bb0ef2013-07-29 17:14:51 -07001392 # timer
1393 self.timespec = None
James E. Blairc494d542014-08-06 09:23:52 -07001394 # zuultrigger
1395 self.pipeline_name = None
James E. Blairad28e912013-11-27 10:43:22 -08001396 # For events that arrive with a destination pipeline (eg, from
1397 # an admin command, etc):
1398 self.forced_pipeline = None
James E. Blairee743612012-05-29 14:49:32 -07001399
James E. Blair9f9667e2012-06-12 17:51:08 -07001400 def __repr__(self):
James E. Blairee743612012-05-29 14:49:32 -07001401 ret = '<TriggerEvent %s %s' % (self.type, self.project_name)
James E. Blair1e8dd892012-05-30 09:15:05 -07001402
James E. Blairee743612012-05-29 14:49:32 -07001403 if self.branch:
1404 ret += " %s" % self.branch
1405 if self.change_number:
1406 ret += " %s,%s" % (self.change_number, self.patch_number)
1407 if self.approvals:
James E. Blair1e8dd892012-05-30 09:15:05 -07001408 ret += ' ' + ', '.join(
1409 ['%s:%s' % (a['type'], a['value']) for a in self.approvals])
James E. Blairee743612012-05-29 14:49:32 -07001410 ret += '>'
1411
1412 return ret
1413
James E. Blair1e8dd892012-05-30 09:15:05 -07001414
James E. Blair9c17dbf2014-06-23 14:21:58 -07001415class BaseFilter(object):
Monty Taylora42a55b2016-07-29 07:53:33 -07001416 """Base Class for filtering which Changes and Events to process."""
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001417 def __init__(self, required_approvals=[], reject_approvals=[]):
James E. Blair5bf78a32015-07-30 18:08:24 +00001418 self._required_approvals = copy.deepcopy(required_approvals)
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001419 self.required_approvals = self._tidy_approvals(required_approvals)
1420 self._reject_approvals = copy.deepcopy(reject_approvals)
1421 self.reject_approvals = self._tidy_approvals(reject_approvals)
James E. Blair9c17dbf2014-06-23 14:21:58 -07001422
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001423 def _tidy_approvals(self, approvals):
1424 for a in approvals:
James E. Blair9c17dbf2014-06-23 14:21:58 -07001425 for k, v in a.items():
1426 if k == 'username':
James E. Blairb01ec542016-06-16 09:46:49 -07001427 a['username'] = re.compile(v)
James E. Blair1fbfceb2014-06-23 14:42:53 -07001428 elif k in ['email', 'email-filter']:
James E. Blair5bf78a32015-07-30 18:08:24 +00001429 a['email'] = re.compile(v)
James E. Blair9c17dbf2014-06-23 14:21:58 -07001430 elif k == 'newer-than':
James E. Blair5bf78a32015-07-30 18:08:24 +00001431 a[k] = time_to_seconds(v)
James E. Blair9c17dbf2014-06-23 14:21:58 -07001432 elif k == 'older-than':
James E. Blair5bf78a32015-07-30 18:08:24 +00001433 a[k] = time_to_seconds(v)
James E. Blair1fbfceb2014-06-23 14:42:53 -07001434 if 'email-filter' in a:
1435 del a['email-filter']
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001436 return approvals
1437
1438 def _match_approval_required_approval(self, rapproval, approval):
1439 # Check if the required approval and approval match
1440 if 'description' not in approval:
1441 return False
1442 now = time.time()
1443 by = approval.get('by', {})
1444 for k, v in rapproval.items():
1445 if k == 'username':
James E. Blairb01ec542016-06-16 09:46:49 -07001446 if (not v.search(by.get('username', ''))):
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001447 return False
1448 elif k == 'email':
1449 if (not v.search(by.get('email', ''))):
1450 return False
1451 elif k == 'newer-than':
1452 t = now - v
1453 if (approval['grantedOn'] < t):
1454 return False
1455 elif k == 'older-than':
1456 t = now - v
1457 if (approval['grantedOn'] >= t):
1458 return False
1459 else:
1460 if not isinstance(v, list):
1461 v = [v]
1462 if (normalizeCategory(approval['description']) != k or
1463 int(approval['value']) not in v):
1464 return False
1465 return True
1466
1467 def matchesApprovals(self, change):
1468 if (self.required_approvals and not change.approvals
1469 or self.reject_approvals and not change.approvals):
1470 # A change with no approvals can not match
1471 return False
1472
1473 # TODO(jhesketh): If we wanted to optimise this slightly we could
1474 # analyse both the REQUIRE and REJECT filters by looping over the
1475 # approvals on the change and keeping track of what we have checked
1476 # rather than needing to loop on the change approvals twice
1477 return (self.matchesRequiredApprovals(change) and
1478 self.matchesNoRejectApprovals(change))
James E. Blair9c17dbf2014-06-23 14:21:58 -07001479
1480 def matchesRequiredApprovals(self, change):
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001481 # Check if any approvals match the requirements
James E. Blair5bf78a32015-07-30 18:08:24 +00001482 for rapproval in self.required_approvals:
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001483 matches_rapproval = False
James E. Blair9c17dbf2014-06-23 14:21:58 -07001484 for approval in change.approvals:
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001485 if self._match_approval_required_approval(rapproval, approval):
1486 # We have a matching approval so this requirement is
1487 # fulfilled
1488 matches_rapproval = True
James E. Blair5bf78a32015-07-30 18:08:24 +00001489 break
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001490 if not matches_rapproval:
James E. Blair5bf78a32015-07-30 18:08:24 +00001491 return False
James E. Blair9c17dbf2014-06-23 14:21:58 -07001492 return True
1493
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001494 def matchesNoRejectApprovals(self, change):
1495 # Check to make sure no approvals match a reject criteria
1496 for rapproval in self.reject_approvals:
1497 for approval in change.approvals:
1498 if self._match_approval_required_approval(rapproval, approval):
1499 # A reject approval has been matched, so we reject
1500 # immediately
1501 return False
1502 # To get here no rejects can have been matched so we should be good to
1503 # queue
1504 return True
1505
James E. Blair9c17dbf2014-06-23 14:21:58 -07001506
1507class EventFilter(BaseFilter):
Monty Taylora42a55b2016-07-29 07:53:33 -07001508 """Allows a Pipeline to only respond to certain events."""
James E. Blairc0dedf82014-08-06 09:37:52 -07001509 def __init__(self, trigger, types=[], branches=[], refs=[],
1510 event_approvals={}, comments=[], emails=[], usernames=[],
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001511 timespecs=[], required_approvals=[], reject_approvals=[],
1512 pipelines=[], ignore_deletes=True):
James E. Blair9c17dbf2014-06-23 14:21:58 -07001513 super(EventFilter, self).__init__(
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001514 required_approvals=required_approvals,
1515 reject_approvals=reject_approvals)
James E. Blairc0dedf82014-08-06 09:37:52 -07001516 self.trigger = trigger
James E. Blairee743612012-05-29 14:49:32 -07001517 self._types = types
1518 self._branches = branches
1519 self._refs = refs
James E. Blair1fbfceb2014-06-23 14:42:53 -07001520 self._comments = comments
1521 self._emails = emails
1522 self._usernames = usernames
James E. Blairc494d542014-08-06 09:23:52 -07001523 self._pipelines = pipelines
James E. Blairee743612012-05-29 14:49:32 -07001524 self.types = [re.compile(x) for x in types]
1525 self.branches = [re.compile(x) for x in branches]
1526 self.refs = [re.compile(x) for x in refs]
James E. Blair1fbfceb2014-06-23 14:42:53 -07001527 self.comments = [re.compile(x) for x in comments]
1528 self.emails = [re.compile(x) for x in emails]
1529 self.usernames = [re.compile(x) for x in usernames]
James E. Blairc494d542014-08-06 09:23:52 -07001530 self.pipelines = [re.compile(x) for x in pipelines]
James E. Blairc053d022014-01-22 14:57:33 -08001531 self.event_approvals = event_approvals
James E. Blair63bb0ef2013-07-29 17:14:51 -07001532 self.timespecs = timespecs
K Jonathan Harkerf95e7232015-04-29 13:33:16 -07001533 self.ignore_deletes = ignore_deletes
James E. Blairee743612012-05-29 14:49:32 -07001534
James E. Blair9f9667e2012-06-12 17:51:08 -07001535 def __repr__(self):
James E. Blairee743612012-05-29 14:49:32 -07001536 ret = '<EventFilter'
James E. Blair1e8dd892012-05-30 09:15:05 -07001537
James E. Blairee743612012-05-29 14:49:32 -07001538 if self._types:
1539 ret += ' types: %s' % ', '.join(self._types)
James E. Blairc494d542014-08-06 09:23:52 -07001540 if self._pipelines:
1541 ret += ' pipelines: %s' % ', '.join(self._pipelines)
James E. Blairee743612012-05-29 14:49:32 -07001542 if self._branches:
1543 ret += ' branches: %s' % ', '.join(self._branches)
1544 if self._refs:
1545 ret += ' refs: %s' % ', '.join(self._refs)
K Jonathan Harkerf95e7232015-04-29 13:33:16 -07001546 if self.ignore_deletes:
1547 ret += ' ignore_deletes: %s' % self.ignore_deletes
James E. Blairc053d022014-01-22 14:57:33 -08001548 if self.event_approvals:
1549 ret += ' event_approvals: %s' % ', '.join(
1550 ['%s:%s' % a for a in self.event_approvals.items()])
James E. Blair5bf78a32015-07-30 18:08:24 +00001551 if self.required_approvals:
1552 ret += ' required_approvals: %s' % ', '.join(
1553 ['%s' % a for a in self._required_approvals])
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001554 if self.reject_approvals:
1555 ret += ' reject_approvals: %s' % ', '.join(
1556 ['%s' % a for a in self._reject_approvals])
James E. Blair1fbfceb2014-06-23 14:42:53 -07001557 if self._comments:
1558 ret += ' comments: %s' % ', '.join(self._comments)
1559 if self._emails:
1560 ret += ' emails: %s' % ', '.join(self._emails)
1561 if self._usernames:
1562 ret += ' username_filters: %s' % ', '.join(self._usernames)
James E. Blair63bb0ef2013-07-29 17:14:51 -07001563 if self.timespecs:
1564 ret += ' timespecs: %s' % ', '.join(self.timespecs)
James E. Blairee743612012-05-29 14:49:32 -07001565 ret += '>'
1566
1567 return ret
1568
James E. Blairc053d022014-01-22 14:57:33 -08001569 def matches(self, event, change):
James E. Blairee743612012-05-29 14:49:32 -07001570 # event types are ORed
1571 matches_type = False
1572 for etype in self.types:
1573 if etype.match(event.type):
1574 matches_type = True
1575 if self.types and not matches_type:
1576 return False
1577
James E. Blairc494d542014-08-06 09:23:52 -07001578 # pipelines are ORed
1579 matches_pipeline = False
1580 for epipe in self.pipelines:
1581 if epipe.match(event.pipeline_name):
1582 matches_pipeline = True
1583 if self.pipelines and not matches_pipeline:
1584 return False
1585
James E. Blairee743612012-05-29 14:49:32 -07001586 # branches are ORed
1587 matches_branch = False
1588 for branch in self.branches:
1589 if branch.match(event.branch):
1590 matches_branch = True
1591 if self.branches and not matches_branch:
1592 return False
1593
1594 # refs are ORed
1595 matches_ref = False
Yolanda Robla16698872014-08-25 11:59:27 +02001596 if event.ref is not None:
1597 for ref in self.refs:
1598 if ref.match(event.ref):
1599 matches_ref = True
James E. Blairee743612012-05-29 14:49:32 -07001600 if self.refs and not matches_ref:
1601 return False
K Jonathan Harkerf95e7232015-04-29 13:33:16 -07001602 if self.ignore_deletes and event.newrev == EMPTY_GIT_REF:
1603 # If the updated ref has an empty git sha (all 0s),
1604 # then the ref is being deleted
1605 return False
James E. Blairee743612012-05-29 14:49:32 -07001606
James E. Blair1fbfceb2014-06-23 14:42:53 -07001607 # comments are ORed
1608 matches_comment_re = False
1609 for comment_re in self.comments:
Clark Boylanb9bcb402012-06-29 17:44:05 -07001610 if (event.comment is not None and
James E. Blair1fbfceb2014-06-23 14:42:53 -07001611 comment_re.search(event.comment)):
1612 matches_comment_re = True
1613 if self.comments and not matches_comment_re:
Clark Boylanb9bcb402012-06-29 17:44:05 -07001614 return False
1615
Antoine Mussob4e809e2012-12-06 16:58:06 +01001616 # We better have an account provided by Gerrit to do
1617 # email filtering.
1618 if event.account is not None:
James E. Blaircf429f32012-12-20 14:28:24 -08001619 account_email = event.account.get('email')
James E. Blair1fbfceb2014-06-23 14:42:53 -07001620 # emails are ORed
1621 matches_email_re = False
1622 for email_re in self.emails:
Antoine Mussob4e809e2012-12-06 16:58:06 +01001623 if (account_email is not None and
Joshua Hesketh29d99b72014-08-19 16:27:42 +10001624 email_re.search(account_email)):
James E. Blair1fbfceb2014-06-23 14:42:53 -07001625 matches_email_re = True
1626 if self.emails and not matches_email_re:
Antoine Mussob4e809e2012-12-06 16:58:06 +01001627 return False
1628
James E. Blair1fbfceb2014-06-23 14:42:53 -07001629 # usernames are ORed
Joshua Heskethb8a817e2013-12-27 11:21:38 +11001630 account_username = event.account.get('username')
James E. Blair1fbfceb2014-06-23 14:42:53 -07001631 matches_username_re = False
1632 for username_re in self.usernames:
Joshua Heskethb8a817e2013-12-27 11:21:38 +11001633 if (account_username is not None and
James E. Blair1fbfceb2014-06-23 14:42:53 -07001634 username_re.search(account_username)):
1635 matches_username_re = True
1636 if self.usernames and not matches_username_re:
Joshua Heskethb8a817e2013-12-27 11:21:38 +11001637 return False
1638
James E. Blairee743612012-05-29 14:49:32 -07001639 # approvals are ANDed
James E. Blairc053d022014-01-22 14:57:33 -08001640 for category, value in self.event_approvals.items():
James E. Blairee743612012-05-29 14:49:32 -07001641 matches_approval = False
1642 for eapproval in event.approvals:
1643 if (normalizeCategory(eapproval['description']) == category and
1644 int(eapproval['value']) == int(value)):
1645 matches_approval = True
James E. Blair1e8dd892012-05-30 09:15:05 -07001646 if not matches_approval:
1647 return False
James E. Blair63bb0ef2013-07-29 17:14:51 -07001648
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001649 # required approvals are ANDed (reject approvals are ORed)
1650 if not self.matchesApprovals(change):
James E. Blair9c17dbf2014-06-23 14:21:58 -07001651 return False
James E. Blairc053d022014-01-22 14:57:33 -08001652
James E. Blair63bb0ef2013-07-29 17:14:51 -07001653 # timespecs are ORed
1654 matches_timespec = False
1655 for timespec in self.timespecs:
1656 if (event.timespec == timespec):
1657 matches_timespec = True
1658 if self.timespecs and not matches_timespec:
1659 return False
1660
James E. Blairee743612012-05-29 14:49:32 -07001661 return True
James E. Blaireff88162013-07-01 12:44:14 -04001662
1663
James E. Blair9c17dbf2014-06-23 14:21:58 -07001664class ChangeishFilter(BaseFilter):
Monty Taylora42a55b2016-07-29 07:53:33 -07001665 """Allows a Manager to only enqueue Changes that meet certain criteria."""
Clark Boylana9702ad2014-05-08 17:17:24 -07001666 def __init__(self, open=None, current_patchset=None,
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001667 statuses=[], required_approvals=[],
1668 reject_approvals=[]):
James E. Blair9c17dbf2014-06-23 14:21:58 -07001669 super(ChangeishFilter, self).__init__(
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001670 required_approvals=required_approvals,
1671 reject_approvals=reject_approvals)
James E. Blair11041d22014-05-02 14:49:53 -07001672 self.open = open
Clark Boylana9702ad2014-05-08 17:17:24 -07001673 self.current_patchset = current_patchset
James E. Blair11041d22014-05-02 14:49:53 -07001674 self.statuses = statuses
James E. Blair11041d22014-05-02 14:49:53 -07001675
1676 def __repr__(self):
1677 ret = '<ChangeishFilter'
1678
1679 if self.open is not None:
1680 ret += ' open: %s' % self.open
Clark Boylana9702ad2014-05-08 17:17:24 -07001681 if self.current_patchset is not None:
1682 ret += ' current-patchset: %s' % self.current_patchset
James E. Blair11041d22014-05-02 14:49:53 -07001683 if self.statuses:
1684 ret += ' statuses: %s' % ', '.join(self.statuses)
James E. Blair5bf78a32015-07-30 18:08:24 +00001685 if self.required_approvals:
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001686 ret += (' required_approvals: %s' %
1687 str(self.required_approvals))
1688 if self.reject_approvals:
1689 ret += (' reject_approvals: %s' %
1690 str(self.reject_approvals))
James E. Blair11041d22014-05-02 14:49:53 -07001691 ret += '>'
1692
1693 return ret
1694
1695 def matches(self, change):
1696 if self.open is not None:
1697 if self.open != change.open:
1698 return False
1699
Clark Boylana9702ad2014-05-08 17:17:24 -07001700 if self.current_patchset is not None:
1701 if self.current_patchset != change.is_current_patchset:
1702 return False
1703
James E. Blair11041d22014-05-02 14:49:53 -07001704 if self.statuses:
1705 if change.status not in self.statuses:
1706 return False
1707
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001708 # required approvals are ANDed (reject approvals are ORed)
1709 if not self.matchesApprovals(change):
James E. Blair9c17dbf2014-06-23 14:21:58 -07001710 return False
James E. Blair11041d22014-05-02 14:49:53 -07001711
1712 return True
1713
1714
James E. Blairb97ed802015-12-21 15:55:35 -08001715class ProjectPipelineConfig(object):
1716 # Represents a project cofiguration in the context of a pipeline
1717 def __init__(self):
1718 self.job_tree = None
1719 self.queue_name = None
Adam Gandelman8bd57102016-12-02 12:58:42 -08001720 self.merge_mode = None
James E. Blairb97ed802015-12-21 15:55:35 -08001721
1722
1723class ProjectConfig(object):
1724 # Represents a project cofiguration
1725 def __init__(self, name):
1726 self.name = name
Adam Gandelman8bd57102016-12-02 12:58:42 -08001727 self.merge_mode = None
James E. Blairb97ed802015-12-21 15:55:35 -08001728 self.pipelines = {}
1729
1730
James E. Blaird8e778f2015-12-22 14:09:20 -08001731class UnparsedAbideConfig(object):
Monty Taylora42a55b2016-07-29 07:53:33 -07001732 """A collection of yaml lists that has not yet been parsed into objects.
1733
1734 An Abide is a collection of tenants.
1735 """
1736
James E. Blaird8e778f2015-12-22 14:09:20 -08001737 def __init__(self):
1738 self.tenants = []
1739
1740 def extend(self, conf):
1741 if isinstance(conf, UnparsedAbideConfig):
1742 self.tenants.extend(conf.tenants)
1743 return
1744
1745 if not isinstance(conf, list):
1746 raise Exception("Configuration items must be in the form of "
1747 "a list of dictionaries (when parsing %s)" %
1748 (conf,))
1749 for item in conf:
1750 if not isinstance(item, dict):
1751 raise Exception("Configuration items must be in the form of "
1752 "a list of dictionaries (when parsing %s)" %
1753 (conf,))
1754 if len(item.keys()) > 1:
1755 raise Exception("Configuration item dictionaries must have "
1756 "a single key (when parsing %s)" %
1757 (conf,))
1758 key, value = item.items()[0]
1759 if key == 'tenant':
1760 self.tenants.append(value)
1761 else:
1762 raise Exception("Configuration item not recognized "
1763 "(when parsing %s)" %
1764 (conf,))
1765
1766
1767class UnparsedTenantConfig(object):
Monty Taylora42a55b2016-07-29 07:53:33 -07001768 """A collection of yaml lists that has not yet been parsed into objects."""
1769
James E. Blaird8e778f2015-12-22 14:09:20 -08001770 def __init__(self):
1771 self.pipelines = []
1772 self.jobs = []
1773 self.project_templates = []
1774 self.projects = []
James E. Blaira98340f2016-09-02 11:33:49 -07001775 self.nodesets = []
James E. Blaird8e778f2015-12-22 14:09:20 -08001776
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001777 def copy(self):
1778 r = UnparsedTenantConfig()
1779 r.pipelines = copy.deepcopy(self.pipelines)
1780 r.jobs = copy.deepcopy(self.jobs)
1781 r.project_templates = copy.deepcopy(self.project_templates)
1782 r.projects = copy.deepcopy(self.projects)
James E. Blaira98340f2016-09-02 11:33:49 -07001783 r.nodesets = copy.deepcopy(self.nodesets)
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001784 return r
1785
James E. Blairc73c73a2017-01-20 15:15:15 -08001786 def extend(self, conf, source_project=None, source_branch=None,
1787 source_configrepo=None):
James E. Blaird8e778f2015-12-22 14:09:20 -08001788 if isinstance(conf, UnparsedTenantConfig):
1789 self.pipelines.extend(conf.pipelines)
1790 self.jobs.extend(conf.jobs)
1791 self.project_templates.extend(conf.project_templates)
1792 self.projects.extend(conf.projects)
James E. Blaira98340f2016-09-02 11:33:49 -07001793 self.nodesets.extend(conf.nodesets)
James E. Blaird8e778f2015-12-22 14:09:20 -08001794 return
1795
1796 if not isinstance(conf, list):
1797 raise Exception("Configuration items must be in the form of "
1798 "a list of dictionaries (when parsing %s)" %
1799 (conf,))
1800 for item in conf:
1801 if not isinstance(item, dict):
1802 raise Exception("Configuration items must be in the form of "
1803 "a list of dictionaries (when parsing %s)" %
1804 (conf,))
1805 if len(item.keys()) > 1:
1806 raise Exception("Configuration item dictionaries must have "
1807 "a single key (when parsing %s)" %
1808 (conf,))
1809 key, value = item.items()[0]
1810 if key == 'project':
1811 self.projects.append(value)
1812 elif key == 'job':
James E. Blair4317e9f2016-07-15 10:05:47 -07001813 if source_project is not None:
1814 value['_source_project'] = source_project
James E. Blaire208c482016-10-04 14:35:30 -07001815 if source_branch is not None:
1816 value['_source_branch'] = source_branch
James E. Blairc73c73a2017-01-20 15:15:15 -08001817 if source_configrepo is not None:
1818 value['_source_configrepo'] = source_configrepo
James E. Blaird8e778f2015-12-22 14:09:20 -08001819 self.jobs.append(value)
1820 elif key == 'project-template':
1821 self.project_templates.append(value)
1822 elif key == 'pipeline':
1823 self.pipelines.append(value)
James E. Blaira98340f2016-09-02 11:33:49 -07001824 elif key == 'nodeset':
1825 self.nodesets.append(value)
James E. Blaird8e778f2015-12-22 14:09:20 -08001826 else:
Morgan Fainberg4245a422016-08-05 16:20:12 -07001827 raise Exception("Configuration item `%s` not recognized "
James E. Blaird8e778f2015-12-22 14:09:20 -08001828 "(when parsing %s)" %
Morgan Fainberg4245a422016-08-05 16:20:12 -07001829 (item, conf,))
James E. Blaird8e778f2015-12-22 14:09:20 -08001830
1831
James E. Blaireff88162013-07-01 12:44:14 -04001832class Layout(object):
Monty Taylora42a55b2016-07-29 07:53:33 -07001833 """Holds all of the Pipelines."""
1834
James E. Blaireff88162013-07-01 12:44:14 -04001835 def __init__(self):
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001836 self.tenant = None
James E. Blaireff88162013-07-01 12:44:14 -04001837 self.projects = {}
James E. Blairb97ed802015-12-21 15:55:35 -08001838 self.project_configs = {}
1839 self.project_templates = {}
James E. Blair5a9918a2013-08-27 10:06:27 -07001840 self.pipelines = OrderedDict()
James E. Blair83005782015-12-11 14:46:03 -08001841 # This is a dictionary of name -> [jobs]. The first element
1842 # of the list is the first job added with that name. It is
1843 # the reference definition for a given job. Subsequent
1844 # elements are aspects of that job with different matchers
1845 # that override some attribute of the job. These aspects all
1846 # inherit from the reference definition.
James E. Blairfef88ec2016-11-30 08:38:27 -08001847 self.jobs = {'noop': [Job('noop')]}
James E. Blaira98340f2016-09-02 11:33:49 -07001848 self.nodesets = {}
James E. Blaireff88162013-07-01 12:44:14 -04001849
1850 def getJob(self, name):
1851 if name in self.jobs:
James E. Blair83005782015-12-11 14:46:03 -08001852 return self.jobs[name][0]
James E. Blairb97ed802015-12-21 15:55:35 -08001853 raise Exception("Job %s not defined" % (name,))
James E. Blair83005782015-12-11 14:46:03 -08001854
1855 def getJobs(self, name):
1856 return self.jobs.get(name, [])
1857
1858 def addJob(self, job):
James E. Blair4317e9f2016-07-15 10:05:47 -07001859 # We can have multiple variants of a job all with the same
1860 # name, but these variants must all be defined in the same repo.
1861 prior_jobs = [j for j in self.getJobs(job.name)
1862 if j.source_project != job.source_project]
1863 if prior_jobs:
1864 raise Exception("Job %s in %s is not permitted to shadow "
1865 "job %s in %s" % (job, job.source_project,
1866 prior_jobs[0],
1867 prior_jobs[0].source_project))
1868
James E. Blair83005782015-12-11 14:46:03 -08001869 if job.name in self.jobs:
1870 self.jobs[job.name].append(job)
1871 else:
1872 self.jobs[job.name] = [job]
1873
James E. Blaira98340f2016-09-02 11:33:49 -07001874 def addNodeSet(self, nodeset):
1875 if nodeset.name in self.nodesets:
1876 raise Exception("NodeSet %s already defined" % (nodeset.name,))
1877 self.nodesets[nodeset.name] = nodeset
1878
James E. Blair83005782015-12-11 14:46:03 -08001879 def addPipeline(self, pipeline):
1880 self.pipelines[pipeline.name] = pipeline
James E. Blair59fdbac2015-12-07 17:08:06 -08001881
James E. Blairb97ed802015-12-21 15:55:35 -08001882 def addProjectTemplate(self, project_template):
1883 self.project_templates[project_template.name] = project_template
1884
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001885 def addProjectConfig(self, project_config, update_pipeline=True):
James E. Blairb97ed802015-12-21 15:55:35 -08001886 self.project_configs[project_config.name] = project_config
1887 # TODOv3(jeblair): tidy up the relationship between pipelines
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001888 # and projects and projectconfigs. Specifically, move
1889 # job_trees out of the pipeline since they are more dynamic
1890 # than pipelines. Remove the update_pipeline argument
1891 if not update_pipeline:
1892 return
James E. Blairb97ed802015-12-21 15:55:35 -08001893 for pipeline_name, pipeline_config in project_config.pipelines.items():
1894 pipeline = self.pipelines[pipeline_name]
1895 project = pipeline.source.getProject(project_config.name)
1896 pipeline.job_trees[project] = pipeline_config.job_tree
1897
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001898 def _createJobTree(self, change, job_trees, parent):
1899 for tree in job_trees:
1900 job = tree.job
1901 if not job.changeMatches(change):
1902 continue
1903 frozen_job = Job(job.name)
1904 frozen_tree = JobTree(frozen_job)
1905 inherited = set()
1906 for variant in self.getJobs(job.name):
1907 if variant.changeMatches(change):
1908 if variant not in inherited:
1909 frozen_job.inheritFrom(variant)
1910 inherited.add(variant)
James E. Blair6e85c2b2016-11-21 16:47:01 -08001911 if not inherited:
1912 # A change must match at least one defined job variant
1913 # (that is to say that it must match more than just
1914 # the job that is defined in the tree).
1915 continue
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001916 if job not in inherited:
1917 # Only update from the job in the tree if it is
1918 # unique, otherwise we might unset an attribute we
1919 # have overloaded.
1920 frozen_job.inheritFrom(job)
1921 parent.job_trees.append(frozen_tree)
1922 self._createJobTree(change, tree.job_trees, frozen_tree)
1923
1924 def createJobTree(self, item):
Paul Belanger160cb8e2016-11-11 19:04:24 -05001925 project_config = self.project_configs.get(
1926 item.change.project.name, None)
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001927 ret = JobTree(None)
Paul Belanger15e3e202016-10-14 16:27:34 -04001928 # NOTE(pabelanger): It is possible for a foreign project not to have a
1929 # configured pipeline, if so return an empty JobTree.
Paul Belanger160cb8e2016-11-11 19:04:24 -05001930 if project_config and item.pipeline.name in project_config.pipelines:
Paul Belanger15e3e202016-10-14 16:27:34 -04001931 project_tree = \
1932 project_config.pipelines[item.pipeline.name].job_tree
1933 self._createJobTree(item.change, project_tree.job_trees, ret)
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001934 return ret
1935
James E. Blair59fdbac2015-12-07 17:08:06 -08001936
1937class Tenant(object):
1938 def __init__(self, name):
1939 self.name = name
1940 self.layout = None
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001941 # The list of repos from which we will read main
1942 # configuration. (source, project)
1943 self.config_repos = []
1944 # The unparsed config from those repos.
1945 self.config_repos_config = None
1946 # The list of projects from which we will read in-repo
1947 # configuration. (source, project)
1948 self.project_repos = []
1949 # The unparsed config from those repos.
1950 self.project_repos_config = None
James E. Blair59fdbac2015-12-07 17:08:06 -08001951
1952
1953class Abide(object):
1954 def __init__(self):
1955 self.tenants = OrderedDict()
James E. Blairce8a2132016-05-19 15:21:52 -07001956
1957
1958class JobTimeData(object):
1959 format = 'B10H10H10B'
1960 version = 0
1961
1962 def __init__(self, path):
1963 self.path = path
1964 self.success_times = [0 for x in range(10)]
1965 self.failure_times = [0 for x in range(10)]
1966 self.results = [0 for x in range(10)]
1967
1968 def load(self):
1969 if not os.path.exists(self.path):
1970 return
1971 with open(self.path) as f:
1972 data = struct.unpack(self.format, f.read())
1973 version = data[0]
1974 if version != self.version:
1975 raise Exception("Unkown data version")
1976 self.success_times = list(data[1:11])
1977 self.failure_times = list(data[11:21])
1978 self.results = list(data[21:32])
1979
1980 def save(self):
1981 tmpfile = self.path + '.tmp'
1982 data = [self.version]
1983 data.extend(self.success_times)
1984 data.extend(self.failure_times)
1985 data.extend(self.results)
1986 data = struct.pack(self.format, *data)
1987 with open(tmpfile, 'w') as f:
1988 f.write(data)
1989 os.rename(tmpfile, self.path)
1990
1991 def add(self, elapsed, result):
1992 elapsed = int(elapsed)
1993 if result == 'SUCCESS':
1994 self.success_times.append(elapsed)
1995 self.success_times.pop(0)
1996 result = 0
1997 else:
1998 self.failure_times.append(elapsed)
1999 self.failure_times.pop(0)
2000 result = 1
2001 self.results.append(result)
2002 self.results.pop(0)
2003
2004 def getEstimatedTime(self):
2005 times = [x for x in self.success_times if x]
2006 if times:
2007 return float(sum(times)) / len(times)
2008 return 0.0
2009
2010
2011class TimeDataBase(object):
2012 def __init__(self, root):
2013 self.root = root
2014 self.jobs = {}
2015
2016 def _getTD(self, name):
2017 td = self.jobs.get(name)
2018 if not td:
2019 td = JobTimeData(os.path.join(self.root, name))
2020 self.jobs[name] = td
2021 td.load()
2022 return td
2023
2024 def getEstimatedTime(self, name):
2025 return self._getTD(name).getEstimatedTime()
2026
2027 def update(self, name, elapsed, result):
2028 td = self._getTD(name)
2029 td.add(elapsed, result)
2030 td.save()