blob: 2571dc0ab2425f5e4af806cc762b649c0b0e7d12 [file] [log] [blame]
James E. Blairee743612012-05-29 14:49:32 -07001# Copyright 2012 Hewlett-Packard Development Company, L.P.
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
James E. Blair1b265312014-06-24 09:35:21 -070015import copy
James E. Blairee743612012-05-29 14:49:32 -070016import re
James E. Blairff986a12012-05-30 14:56:51 -070017import time
James E. Blair4886cc12012-07-18 15:39:41 -070018from uuid import uuid4
James E. Blair5a9918a2013-08-27 10:06:27 -070019import extras
20
21OrderedDict = extras.try_imports(['collections.OrderedDict',
22 'ordereddict.OrderedDict'])
James E. Blair4886cc12012-07-18 15:39:41 -070023
24
K Jonathan Harkerf95e7232015-04-29 13:33:16 -070025EMPTY_GIT_REF = '0' * 40 # git sha of all zeros, used during creates/deletes
26
James E. Blair19deff22013-08-25 13:17:35 -070027MERGER_MERGE = 1 # "git merge"
28MERGER_MERGE_RESOLVE = 2 # "git merge -s resolve"
29MERGER_CHERRY_PICK = 3 # "git cherry-pick"
30
31MERGER_MAP = {
32 'merge': MERGER_MERGE,
33 'merge-resolve': MERGER_MERGE_RESOLVE,
34 'cherry-pick': MERGER_CHERRY_PICK,
35}
James E. Blairee743612012-05-29 14:49:32 -070036
James E. Blair64ed6f22013-07-10 14:07:23 -070037PRECEDENCE_NORMAL = 0
38PRECEDENCE_LOW = 1
39PRECEDENCE_HIGH = 2
40
41PRECEDENCE_MAP = {
42 None: PRECEDENCE_NORMAL,
43 'low': PRECEDENCE_LOW,
44 'normal': PRECEDENCE_NORMAL,
45 'high': PRECEDENCE_HIGH,
46}
47
James E. Blair1e8dd892012-05-30 09:15:05 -070048
James E. Blairc053d022014-01-22 14:57:33 -080049def time_to_seconds(s):
50 if s.endswith('s'):
51 return int(s[:-1])
52 if s.endswith('m'):
53 return int(s[:-1]) * 60
54 if s.endswith('h'):
55 return int(s[:-1]) * 60 * 60
56 if s.endswith('d'):
57 return int(s[:-1]) * 24 * 60 * 60
58 if s.endswith('w'):
59 return int(s[:-1]) * 7 * 24 * 60 * 60
60 raise Exception("Unable to parse time value: %s" % s)
61
62
James E. Blair11041d22014-05-02 14:49:53 -070063def normalizeCategory(name):
64 name = name.lower()
65 return re.sub(' ', '-', name)
66
67
James E. Blair4aea70c2012-07-26 14:23:24 -070068class Pipeline(object):
69 """A top-level pipeline such as check, gate, post, etc."""
James E. Blair83005782015-12-11 14:46:03 -080070 def __init__(self, name, layout):
James E. Blair4aea70c2012-07-26 14:23:24 -070071 self.name = name
James E. Blair83005782015-12-11 14:46:03 -080072 self.layout = layout
James E. Blair8dbd56a2012-12-22 10:55:10 -080073 self.description = None
James E. Blair56370192013-01-14 15:47:28 -080074 self.failure_message = None
Joshua Heskethb7179772014-01-30 23:30:46 +110075 self.merge_failure_message = None
James E. Blair56370192013-01-14 15:47:28 -080076 self.success_message = None
Joshua Hesketh3979e3e2014-03-04 11:21:10 +110077 self.footer_message = None
James E. Blair2fa50962013-01-30 21:50:41 -080078 self.dequeue_on_new_patchset = True
James E. Blair17dd6772015-02-09 14:45:18 -080079 self.ignore_dependencies = False
James E. Blair4aea70c2012-07-26 14:23:24 -070080 self.job_trees = {} # project -> JobTree
81 self.manager = None
James E. Blaire0487072012-08-29 17:38:31 -070082 self.queues = []
James E. Blair64ed6f22013-07-10 14:07:23 -070083 self.precedence = PRECEDENCE_NORMAL
James E. Blairc0dedf82014-08-06 09:37:52 -070084 self.source = None
James E. Blair83005782015-12-11 14:46:03 -080085 self.triggers = []
Joshua Hesketh352264b2015-08-11 23:42:08 +100086 self.start_actions = []
87 self.success_actions = []
88 self.failure_actions = []
89 self.merge_failure_actions = []
90 self.disabled_actions = []
Joshua Hesketh89e829d2015-02-10 16:29:45 +110091 self.disable_at = None
92 self._consecutive_failures = 0
93 self._disabled = False
Clark Boylan7603a372014-01-21 11:43:20 -080094 self.window = None
95 self.window_floor = None
96 self.window_increase_type = None
97 self.window_increase_factor = None
98 self.window_decrease_type = None
99 self.window_decrease_factor = None
James E. Blair4aea70c2012-07-26 14:23:24 -0700100
James E. Blair83005782015-12-11 14:46:03 -0800101 @property
102 def actions(self):
103 return (
104 self.start_actions +
105 self.success_actions +
106 self.failure_actions +
107 self.merge_failure_actions +
108 self.disabled_actions
109 )
110
James E. Blaird09c17a2012-08-07 09:23:14 -0700111 def __repr__(self):
112 return '<Pipeline %s>' % self.name
113
James E. Blair4aea70c2012-07-26 14:23:24 -0700114 def setManager(self, manager):
115 self.manager = manager
116
117 def addProject(self, project):
118 job_tree = JobTree(None) # Null job == job tree root
119 self.job_trees[project] = job_tree
120 return job_tree
121
122 def getProjects(self):
James E. Blairc3d428e2013-12-03 15:06:48 -0800123 return sorted(self.job_trees.keys(), lambda a, b: cmp(a.name, b.name))
James E. Blair4aea70c2012-07-26 14:23:24 -0700124
James E. Blaire0487072012-08-29 17:38:31 -0700125 def addQueue(self, queue):
126 self.queues.append(queue)
127
128 def getQueue(self, project):
129 for queue in self.queues:
130 if project in queue.projects:
131 return queue
132 return None
133
James E. Blairbfb8e042014-12-30 17:01:44 -0800134 def removeQueue(self, queue):
135 self.queues.remove(queue)
136
James E. Blair4aea70c2012-07-26 14:23:24 -0700137 def getJobTree(self, project):
138 tree = self.job_trees.get(project)
139 return tree
140
James E. Blair107c3852015-02-07 08:23:10 -0800141 def getJobs(self, item):
142 if not item.live:
143 return []
144 tree = self.getJobTree(item.change.project)
James E. Blair4aea70c2012-07-26 14:23:24 -0700145 if not tree:
146 return []
James E. Blair107c3852015-02-07 08:23:10 -0800147 return item.change.filterJobs(tree.getJobs())
James E. Blair4aea70c2012-07-26 14:23:24 -0700148
James E. Blairaf17a972016-02-03 15:07:18 -0800149 def _findJobsToRun(self, job_trees, item, mutex):
James E. Blair4aea70c2012-07-26 14:23:24 -0700150 torun = []
James E. Blair4aea70c2012-07-26 14:23:24 -0700151 for tree in job_trees:
152 job = tree.job
153 result = None
154 if job:
James E. Blairfee8d652013-06-07 08:57:52 -0700155 if not job.changeMatches(item.change):
James E. Blair4aea70c2012-07-26 14:23:24 -0700156 continue
James E. Blairfee8d652013-06-07 08:57:52 -0700157 build = item.current_build_set.getBuild(job.name)
James E. Blair4aea70c2012-07-26 14:23:24 -0700158 if build:
159 result = build.result
160 else:
161 # There is no build for the root of this job tree,
162 # so we should run it.
James E. Blairaf17a972016-02-03 15:07:18 -0800163 if mutex.acquire(item, job):
164 # If this job needs a mutex, either acquire it or make
165 # sure that we have it before running the job.
166 torun.append(job)
James E. Blair4aea70c2012-07-26 14:23:24 -0700167 # If there is no job, this is a null job tree, and we should
168 # run all of its jobs.
169 if result == 'SUCCESS' or not job:
James E. Blairaf17a972016-02-03 15:07:18 -0800170 torun.extend(self._findJobsToRun(tree.job_trees, item, mutex))
James E. Blair4aea70c2012-07-26 14:23:24 -0700171 return torun
172
James E. Blairaf17a972016-02-03 15:07:18 -0800173 def findJobsToRun(self, item, mutex):
James E. Blairbfb8e042014-12-30 17:01:44 -0800174 if not item.live:
175 return []
James E. Blair83005782015-12-11 14:46:03 -0800176 tree = item.job_tree
James E. Blair4aea70c2012-07-26 14:23:24 -0700177 if not tree:
178 return []
James E. Blairaf17a972016-02-03 15:07:18 -0800179 return self._findJobsToRun(tree.job_trees, item, mutex)
James E. Blair4aea70c2012-07-26 14:23:24 -0700180
James E. Blairbea9ef12013-07-15 11:52:23 -0700181 def haveAllJobsStarted(self, item):
James E. Blair107c3852015-02-07 08:23:10 -0800182 for job in self.getJobs(item):
James E. Blairbea9ef12013-07-15 11:52:23 -0700183 build = item.current_build_set.getBuild(job.name)
184 if not build or not build.start_time:
185 return False
186 return True
187
James E. Blairfee8d652013-06-07 08:57:52 -0700188 def areAllJobsComplete(self, item):
James E. Blair107c3852015-02-07 08:23:10 -0800189 for job in self.getJobs(item):
James E. Blairfee8d652013-06-07 08:57:52 -0700190 build = item.current_build_set.getBuild(job.name)
James E. Blair4aea70c2012-07-26 14:23:24 -0700191 if not build or not build.result:
192 return False
193 return True
194
James E. Blairfee8d652013-06-07 08:57:52 -0700195 def didAllJobsSucceed(self, item):
James E. Blair107c3852015-02-07 08:23:10 -0800196 for job in self.getJobs(item):
James E. Blair4ec821f2012-08-23 15:28:28 -0700197 if not job.voting:
198 continue
James E. Blairfee8d652013-06-07 08:57:52 -0700199 build = item.current_build_set.getBuild(job.name)
James E. Blair4aea70c2012-07-26 14:23:24 -0700200 if not build:
201 return False
202 if build.result != 'SUCCESS':
203 return False
204 return True
205
Joshua Heskethb7179772014-01-30 23:30:46 +1100206 def didMergerSucceed(self, item):
207 if item.current_build_set.unable_to_merge:
208 return False
209 return True
210
James E. Blairfee8d652013-06-07 08:57:52 -0700211 def didAnyJobFail(self, item):
James E. Blair107c3852015-02-07 08:23:10 -0800212 for job in self.getJobs(item):
James E. Blair4ec821f2012-08-23 15:28:28 -0700213 if not job.voting:
214 continue
James E. Blairfee8d652013-06-07 08:57:52 -0700215 build = item.current_build_set.getBuild(job.name)
James E. Blair0018a6c2013-02-27 14:11:45 -0800216 if build and build.result and (build.result != 'SUCCESS'):
James E. Blair4aea70c2012-07-26 14:23:24 -0700217 return True
218 return False
219
James E. Blairfee8d652013-06-07 08:57:52 -0700220 def isHoldingFollowingChanges(self, item):
James E. Blairbfb8e042014-12-30 17:01:44 -0800221 if not item.live:
222 return False
James E. Blair107c3852015-02-07 08:23:10 -0800223 for job in self.getJobs(item):
James E. Blair4aea70c2012-07-26 14:23:24 -0700224 if not job.hold_following_changes:
225 continue
James E. Blairfee8d652013-06-07 08:57:52 -0700226 build = item.current_build_set.getBuild(job.name)
James E. Blair4aea70c2012-07-26 14:23:24 -0700227 if not build:
228 return True
229 if build.result != 'SUCCESS':
230 return True
James E. Blair972e3c72013-08-29 12:04:55 -0700231
James E. Blairfee8d652013-06-07 08:57:52 -0700232 if not item.item_ahead:
James E. Blair4aea70c2012-07-26 14:23:24 -0700233 return False
James E. Blairfee8d652013-06-07 08:57:52 -0700234 return self.isHoldingFollowingChanges(item.item_ahead)
James E. Blair4aea70c2012-07-26 14:23:24 -0700235
James E. Blairfee8d652013-06-07 08:57:52 -0700236 def setResult(self, item, build):
James E. Blair4a28a882013-08-23 15:17:33 -0700237 if build.retry:
238 item.removeBuild(build)
239 elif build.result != 'SUCCESS':
James E. Blair4aea70c2012-07-26 14:23:24 -0700240 # Get a JobTree from a Job so we can find only its dependent jobs
James E. Blairf5dbd002015-12-23 15:26:17 -0800241 tree = item.job_tree.getJobTreeForJob(build.job)
James E. Blair4aea70c2012-07-26 14:23:24 -0700242 for job in tree.getJobs():
243 fakebuild = Build(job, None)
244 fakebuild.result = 'SKIPPED'
James E. Blairfee8d652013-06-07 08:57:52 -0700245 item.addBuild(fakebuild)
James E. Blair4aea70c2012-07-26 14:23:24 -0700246
Joshua Heskethb7179772014-01-30 23:30:46 +1100247 def setUnableToMerge(self, item):
James E. Blairfee8d652013-06-07 08:57:52 -0700248 item.current_build_set.unable_to_merge = True
249 root = self.getJobTree(item.change.project)
James E. Blair973721f2012-08-15 10:19:43 -0700250 for job in root.getJobs():
251 fakebuild = Build(job, None)
252 fakebuild.result = 'SKIPPED'
James E. Blairfee8d652013-06-07 08:57:52 -0700253 item.addBuild(fakebuild)
James E. Blair973721f2012-08-15 10:19:43 -0700254
James E. Blairfee8d652013-06-07 08:57:52 -0700255 def setDequeuedNeedingChange(self, item):
256 item.dequeued_needing_change = True
257 root = self.getJobTree(item.change.project)
James E. Blaircaec0c52012-08-22 14:52:22 -0700258 for job in root.getJobs():
259 fakebuild = Build(job, None)
260 fakebuild.result = 'SKIPPED'
James E. Blairfee8d652013-06-07 08:57:52 -0700261 item.addBuild(fakebuild)
James E. Blaircaec0c52012-08-22 14:52:22 -0700262
James E. Blaire0487072012-08-29 17:38:31 -0700263 def getChangesInQueue(self):
264 changes = []
265 for shared_queue in self.queues:
James E. Blairfee8d652013-06-07 08:57:52 -0700266 changes.extend([x.change for x in shared_queue.queue])
James E. Blaire0487072012-08-29 17:38:31 -0700267 return changes
268
James E. Blairfee8d652013-06-07 08:57:52 -0700269 def getAllItems(self):
270 items = []
James E. Blaire0487072012-08-29 17:38:31 -0700271 for shared_queue in self.queues:
James E. Blairfee8d652013-06-07 08:57:52 -0700272 items.extend(shared_queue.queue)
James E. Blairfee8d652013-06-07 08:57:52 -0700273 return items
James E. Blaire0487072012-08-29 17:38:31 -0700274
James E. Blair8dbd56a2012-12-22 10:55:10 -0800275 def formatStatusJSON(self):
276 j_pipeline = dict(name=self.name,
277 description=self.description)
278 j_queues = []
279 j_pipeline['change_queues'] = j_queues
280 for queue in self.queues:
281 j_queue = dict(name=queue.name)
282 j_queues.append(j_queue)
283 j_queue['heads'] = []
Clark Boylanaf2476f2014-01-23 14:47:36 -0800284 j_queue['window'] = queue.window
James E. Blair972e3c72013-08-29 12:04:55 -0700285
286 j_changes = []
287 for e in queue.queue:
288 if not e.item_ahead:
289 if j_changes:
290 j_queue['heads'].append(j_changes)
291 j_changes = []
Joshua Hesketh85af4e92014-02-21 08:28:58 -0800292 j_changes.append(e.formatJSON())
James E. Blair972e3c72013-08-29 12:04:55 -0700293 if (len(j_changes) > 1 and
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000294 (j_changes[-2]['remaining_time'] is not None) and
295 (j_changes[-1]['remaining_time'] is not None)):
James E. Blair972e3c72013-08-29 12:04:55 -0700296 j_changes[-1]['remaining_time'] = max(
297 j_changes[-2]['remaining_time'],
298 j_changes[-1]['remaining_time'])
299 if j_changes:
James E. Blair8dbd56a2012-12-22 10:55:10 -0800300 j_queue['heads'].append(j_changes)
301 return j_pipeline
302
James E. Blair4aea70c2012-07-26 14:23:24 -0700303
James E. Blairee743612012-05-29 14:49:32 -0700304class ChangeQueue(object):
James E. Blair4aea70c2012-07-26 14:23:24 -0700305 """DependentPipelines have multiple parallel queues shared by
306 different projects; this is one of them. For instance, there may
307 a queue shared by interrelated projects foo and bar, and a second
308 queue for independent project baz. Pipelines have one or more
James E. Blairbfb8e042014-12-30 17:01:44 -0800309 ChangeQueues."""
310 def __init__(self, pipeline, window=0, window_floor=1,
Clark Boylan7603a372014-01-21 11:43:20 -0800311 window_increase_type='linear', window_increase_factor=1,
312 window_decrease_type='exponential', window_decrease_factor=2):
James E. Blair4aea70c2012-07-26 14:23:24 -0700313 self.pipeline = pipeline
James E. Blairee743612012-05-29 14:49:32 -0700314 self.name = ''
James E. Blairc8a1e052014-02-25 09:29:26 -0800315 self.assigned_name = None
316 self.generated_name = None
James E. Blairee743612012-05-29 14:49:32 -0700317 self.projects = []
318 self._jobs = set()
319 self.queue = []
Clark Boylan7603a372014-01-21 11:43:20 -0800320 self.window = window
321 self.window_floor = window_floor
322 self.window_increase_type = window_increase_type
323 self.window_increase_factor = window_increase_factor
324 self.window_decrease_type = window_decrease_type
325 self.window_decrease_factor = window_decrease_factor
James E. Blairee743612012-05-29 14:49:32 -0700326
James E. Blair9f9667e2012-06-12 17:51:08 -0700327 def __repr__(self):
James E. Blair4aea70c2012-07-26 14:23:24 -0700328 return '<ChangeQueue %s: %s>' % (self.pipeline.name, self.name)
James E. Blairee743612012-05-29 14:49:32 -0700329
330 def getJobs(self):
331 return self._jobs
332
333 def addProject(self, project):
334 if project not in self.projects:
335 self.projects.append(project)
James E. Blairc8a1e052014-02-25 09:29:26 -0800336
James E. Blairee743612012-05-29 14:49:32 -0700337 names = [x.name for x in self.projects]
338 names.sort()
James E. Blairc8a1e052014-02-25 09:29:26 -0800339 self.generated_name = ', '.join(names)
James E. Blairc8a1e052014-02-25 09:29:26 -0800340 self.name = self.assigned_name or self.generated_name
James E. Blairee743612012-05-29 14:49:32 -0700341
342 def enqueueChange(self, change):
James E. Blairbfb8e042014-12-30 17:01:44 -0800343 item = QueueItem(self, change)
James E. Blair83005782015-12-11 14:46:03 -0800344 item.freezeJobTree()
James E. Blaircdccd972013-07-01 12:10:22 -0700345 self.enqueueItem(item)
346 item.enqueue_time = time.time()
347 return item
348
349 def enqueueItem(self, item):
James E. Blair4a035d92014-01-23 13:10:48 -0800350 item.pipeline = self.pipeline
James E. Blairbfb8e042014-12-30 17:01:44 -0800351 item.queue = self
352 if self.queue:
James E. Blairfee8d652013-06-07 08:57:52 -0700353 item.item_ahead = self.queue[-1]
James E. Blair972e3c72013-08-29 12:04:55 -0700354 item.item_ahead.items_behind.append(item)
James E. Blairfee8d652013-06-07 08:57:52 -0700355 self.queue.append(item)
James E. Blairee743612012-05-29 14:49:32 -0700356
James E. Blairfee8d652013-06-07 08:57:52 -0700357 def dequeueItem(self, item):
358 if item in self.queue:
359 self.queue.remove(item)
James E. Blairfee8d652013-06-07 08:57:52 -0700360 if item.item_ahead:
James E. Blair972e3c72013-08-29 12:04:55 -0700361 item.item_ahead.items_behind.remove(item)
362 for item_behind in item.items_behind:
363 if item.item_ahead:
364 item.item_ahead.items_behind.append(item_behind)
365 item_behind.item_ahead = item.item_ahead
James E. Blairfee8d652013-06-07 08:57:52 -0700366 item.item_ahead = None
James E. Blair972e3c72013-08-29 12:04:55 -0700367 item.items_behind = []
James E. Blairfee8d652013-06-07 08:57:52 -0700368 item.dequeue_time = time.time()
James E. Blaire0487072012-08-29 17:38:31 -0700369
James E. Blair972e3c72013-08-29 12:04:55 -0700370 def moveItem(self, item, item_ahead):
James E. Blair972e3c72013-08-29 12:04:55 -0700371 if item.item_ahead == item_ahead:
372 return False
373 # Remove from current location
374 if item.item_ahead:
375 item.item_ahead.items_behind.remove(item)
376 for item_behind in item.items_behind:
377 if item.item_ahead:
378 item.item_ahead.items_behind.append(item_behind)
379 item_behind.item_ahead = item.item_ahead
380 # Add to new location
381 item.item_ahead = item_ahead
James E. Blair00451262013-09-20 11:40:17 -0700382 item.items_behind = []
James E. Blair972e3c72013-08-29 12:04:55 -0700383 if item.item_ahead:
384 item.item_ahead.items_behind.append(item)
385 return True
James E. Blairee743612012-05-29 14:49:32 -0700386
387 def mergeChangeQueue(self, other):
388 for project in other.projects:
389 self.addProject(project)
Clark Boylan7603a372014-01-21 11:43:20 -0800390 self.window = min(self.window, other.window)
391 # TODO merge semantics
392
Clark Boylan3d2f7a72014-01-23 11:07:42 -0800393 def isActionable(self, item):
James E. Blairbfb8e042014-12-30 17:01:44 -0800394 if self.window:
Clark Boylan3d2f7a72014-01-23 11:07:42 -0800395 return item in self.queue[:self.window]
Clark Boylan7603a372014-01-21 11:43:20 -0800396 else:
Clark Boylan3d2f7a72014-01-23 11:07:42 -0800397 return True
Clark Boylan7603a372014-01-21 11:43:20 -0800398
399 def increaseWindowSize(self):
James E. Blairbfb8e042014-12-30 17:01:44 -0800400 if self.window:
Clark Boylan7603a372014-01-21 11:43:20 -0800401 if self.window_increase_type == 'linear':
402 self.window += self.window_increase_factor
403 elif self.window_increase_type == 'exponential':
404 self.window *= self.window_increase_factor
405
406 def decreaseWindowSize(self):
James E. Blairbfb8e042014-12-30 17:01:44 -0800407 if self.window:
Clark Boylan7603a372014-01-21 11:43:20 -0800408 if self.window_decrease_type == 'linear':
409 self.window = max(
410 self.window_floor,
411 self.window - self.window_decrease_factor)
412 elif self.window_decrease_type == 'exponential':
413 self.window = max(
414 self.window_floor,
415 self.window / self.window_decrease_factor)
James E. Blairee743612012-05-29 14:49:32 -0700416
James E. Blair1e8dd892012-05-30 09:15:05 -0700417
James E. Blair4aea70c2012-07-26 14:23:24 -0700418class Project(object):
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +0000419 def __init__(self, name, foreign=False):
James E. Blair4aea70c2012-07-26 14:23:24 -0700420 self.name = name
James E. Blair19deff22013-08-25 13:17:35 -0700421 self.merge_mode = MERGER_MERGE_RESOLVE
Evgeny Antyshev0deaaad2015-08-03 20:22:56 +0000422 # foreign projects are those referenced in dependencies
423 # of layout projects, this should matter
424 # when deciding whether to enqueue their changes
425 self.foreign = foreign
James E. Blair4aea70c2012-07-26 14:23:24 -0700426
427 def __str__(self):
428 return self.name
429
430 def __repr__(self):
431 return '<Project %s>' % (self.name)
432
433
James E. Blair83005782015-12-11 14:46:03 -0800434class Inheritable(object):
435 def __init__(self, parent=None):
436 self.parent = parent
437
438 def __getattribute__(self, name):
439 parent = object.__getattribute__(self, 'parent')
440 try:
441 return object.__getattribute__(self, name)
442 except AttributeError:
443 if parent:
444 return getattr(parent, name)
445 raise
446
447
James E. Blairee743612012-05-29 14:49:32 -0700448class Job(object):
James E. Blair83005782015-12-11 14:46:03 -0800449 attributes = dict(
450 timeout=None,
451 # variables={},
452 # nodes=[],
453 # auth={},
454 workspace=None,
455 pre_run=None,
456 post_run=None,
457 voting=None,
458 failure_message=None,
459 success_message=None,
460 failure_url=None,
461 success_url=None,
462 # Matchers. These are separate so they can be individually
463 # overidden.
464 branch_matcher=None,
465 file_matcher=None,
466 irrelevant_file_matcher=None, # skip-if
467 swift=None, # TODOv3(jeblair): move to auth
468 parameter_function=None, # TODOv3(jeblair): remove
469 success_pattern=None, # TODOv3(jeblair): remove
Joshua Hesketh89b67f62016-02-11 21:22:14 +1100470 mutex=None,
James E. Blair83005782015-12-11 14:46:03 -0800471 )
472
James E. Blairee743612012-05-29 14:49:32 -0700473 def __init__(self, name):
474 self.name = name
James E. Blair83005782015-12-11 14:46:03 -0800475 for k, v in self.attributes.items():
476 setattr(self, k, v)
477
478 def __equals__(self, other):
479 # Compare the name and all inheritable attributes to determine
480 # whether two jobs with the same name are identically
481 # configured. Useful upon reconfiguration.
482 if not isinstance(other, Job):
483 return False
484 if self.name != other.name:
485 return False
486 for k, v in self.attributes.items():
487 if getattr(self, k) != getattr(other, k):
488 return False
489 return True
James E. Blairee743612012-05-29 14:49:32 -0700490
491 def __str__(self):
492 return self.name
493
494 def __repr__(self):
James E. Blair83005782015-12-11 14:46:03 -0800495 return '<Job %s>' % (self.name,)
496
497 def inheritFrom(self, other):
498 """Copy the inheritable attributes which have been set on the other
499 job to this job."""
500
501 if not isinstance(other, Job):
502 raise Exception("Job unable to inherit from %s" % (other,))
503 for k, v in self.attributes.items():
504 if getattr(other, k) != v:
505 setattr(self, k, getattr(other, k))
James E. Blairee743612012-05-29 14:49:32 -0700506
James E. Blaire421a232012-07-25 16:59:21 -0700507 def changeMatches(self, change):
James E. Blair83005782015-12-11 14:46:03 -0800508 if self.branch_matcher and not self.branch_matcher.matches(change):
James E. Blair70c71582013-03-06 08:50:50 -0800509 return False
510
James E. Blair83005782015-12-11 14:46:03 -0800511 if self.file_matcher and not self.file_matcher.matches(change):
James E. Blair70c71582013-03-06 08:50:50 -0800512 return False
513
James E. Blair83005782015-12-11 14:46:03 -0800514 # NB: This is a negative match.
515 if (self.irrelevant_file_matcher and
516 self.irrelevant_file_matcher.matches(change)):
Maru Newby3fe5f852015-01-13 04:22:14 +0000517 return False
518
James E. Blair70c71582013-03-06 08:50:50 -0800519 return True
James E. Blaire5a847f2012-07-10 15:29:14 -0700520
James E. Blair1e8dd892012-05-30 09:15:05 -0700521
James E. Blairee743612012-05-29 14:49:32 -0700522class JobTree(object):
523 """ A JobTree represents an instance of one Job, and holds JobTrees
524 whose jobs should be run if that Job succeeds. A root node of a
525 JobTree will have no associated Job. """
526
527 def __init__(self, job):
528 self.job = job
529 self.job_trees = []
530
531 def addJob(self, job):
James E. Blair12a92b12014-03-26 11:54:53 -0700532 if job not in [x.job for x in self.job_trees]:
533 t = JobTree(job)
534 self.job_trees.append(t)
535 return t
James E. Blaire4ad55a2015-06-11 08:22:43 -0700536 for tree in self.job_trees:
537 if tree.job == job:
538 return tree
James E. Blairee743612012-05-29 14:49:32 -0700539
540 def getJobs(self):
541 jobs = []
542 for x in self.job_trees:
543 jobs.append(x.job)
544 jobs.extend(x.getJobs())
545 return jobs
546
547 def getJobTreeForJob(self, job):
548 if self.job == job:
549 return self
550 for tree in self.job_trees:
551 ret = tree.getJobTreeForJob(job)
552 if ret:
553 return ret
554 return None
555
James E. Blairb97ed802015-12-21 15:55:35 -0800556 def inheritFrom(self, other):
557 if other.job:
558 self.job = Job(other.job.name)
559 self.job.inheritFrom(other.job)
560 for other_tree in other.job_trees:
561 this_tree = self.getJobTreeForJob(other_tree.job)
562 if not this_tree:
563 this_tree = JobTree(None)
564 self.job_trees.append(this_tree)
565 this_tree.inheritFrom(other_tree)
566
James E. Blair1e8dd892012-05-30 09:15:05 -0700567
James E. Blair4aea70c2012-07-26 14:23:24 -0700568class Build(object):
569 def __init__(self, job, uuid):
570 self.job = job
571 self.uuid = uuid
James E. Blair4aea70c2012-07-26 14:23:24 -0700572 self.url = None
573 self.number = None
574 self.result = None
575 self.build_set = None
576 self.launch_time = time.time()
James E. Blair71e94122012-12-24 17:53:08 -0800577 self.start_time = None
578 self.end_time = None
James E. Blairbea9ef12013-07-15 11:52:23 -0700579 self.estimated_time = None
James E. Blair66eeebf2013-07-27 17:44:32 -0700580 self.pipeline = None
James E. Blair0aac4872013-08-23 14:02:38 -0700581 self.canceled = False
James E. Blair4a28a882013-08-23 15:17:33 -0700582 self.retry = False
James E. Blaird78576a2013-07-09 10:39:17 -0700583 self.parameters = {}
Joshua Heskethba8776a2014-01-12 14:35:40 +0800584 self.worker = Worker()
Timothy Chavezb2332082015-08-07 20:08:04 -0500585 self.node_labels = []
586 self.node_name = None
James E. Blairee743612012-05-29 14:49:32 -0700587
588 def __repr__(self):
Joshua Heskethba8776a2014-01-12 14:35:40 +0800589 return ('<Build %s of %s on %s>' %
590 (self.uuid, self.job.name, self.worker))
591
592
593class Worker(object):
594 """A model of the worker running a job"""
595 def __init__(self):
596 self.name = "Unknown"
597 self.hostname = None
598 self.ips = []
599 self.fqdn = None
600 self.program = None
601 self.version = None
602 self.extra = {}
603
604 def updateFromData(self, data):
605 """Update worker information if contained in the WORK_DATA response."""
606 self.name = data.get('worker_name', self.name)
607 self.hostname = data.get('worker_hostname', self.hostname)
608 self.ips = data.get('worker_ips', self.ips)
609 self.fqdn = data.get('worker_fqdn', self.fqdn)
610 self.program = data.get('worker_program', self.program)
611 self.version = data.get('worker_version', self.version)
612 self.extra = data.get('worker_extra', self.extra)
613
614 def __repr__(self):
615 return '<Worker %s>' % self.name
James E. Blairee743612012-05-29 14:49:32 -0700616
James E. Blair1e8dd892012-05-30 09:15:05 -0700617
James E. Blair7e530ad2012-07-03 16:12:28 -0700618class BuildSet(object):
James E. Blair4076e2b2014-01-28 12:42:20 -0800619 # Merge states:
620 NEW = 1
621 PENDING = 2
622 COMPLETE = 3
623
Antoine Musso9b229282014-08-18 23:45:43 +0200624 states_map = {
625 1: 'NEW',
626 2: 'PENDING',
627 3: 'COMPLETE',
628 }
629
James E. Blairfee8d652013-06-07 08:57:52 -0700630 def __init__(self, item):
631 self.item = item
James E. Blair11700c32012-07-05 17:50:05 -0700632 self.other_changes = []
James E. Blair7e530ad2012-07-03 16:12:28 -0700633 self.builds = {}
James E. Blair11700c32012-07-05 17:50:05 -0700634 self.result = None
635 self.next_build_set = None
636 self.previous_build_set = None
James E. Blair4886cc12012-07-18 15:39:41 -0700637 self.ref = None
James E. Blair81515ad2012-10-01 18:29:08 -0700638 self.commit = None
James E. Blair4076e2b2014-01-28 12:42:20 -0800639 self.zuul_url = None
James E. Blair973721f2012-08-15 10:19:43 -0700640 self.unable_to_merge = False
James E. Blair972e3c72013-08-29 12:04:55 -0700641 self.failing_reasons = []
James E. Blair4076e2b2014-01-28 12:42:20 -0800642 self.merge_state = self.NEW
James E. Blair7e530ad2012-07-03 16:12:28 -0700643
Antoine Musso9b229282014-08-18 23:45:43 +0200644 def __repr__(self):
645 return '<BuildSet item: %s #builds: %s merge state: %s>' % (
646 self.item,
647 len(self.builds),
648 self.getStateName(self.merge_state))
649
James E. Blair4886cc12012-07-18 15:39:41 -0700650 def setConfiguration(self):
James E. Blair11700c32012-07-05 17:50:05 -0700651 # The change isn't enqueued until after it's created
652 # so we don't know what the other changes ahead will be
653 # until jobs start.
654 if not self.other_changes:
James E. Blairfee8d652013-06-07 08:57:52 -0700655 next_item = self.item.item_ahead
656 while next_item:
657 self.other_changes.append(next_item.change)
658 next_item = next_item.item_ahead
James E. Blair4886cc12012-07-18 15:39:41 -0700659 if not self.ref:
660 self.ref = 'Z' + uuid4().hex
661
Antoine Musso9b229282014-08-18 23:45:43 +0200662 def getStateName(self, state_num):
663 return self.states_map.get(
664 state_num, 'UNKNOWN (%s)' % state_num)
665
James E. Blair4886cc12012-07-18 15:39:41 -0700666 def addBuild(self, build):
667 self.builds[build.job.name] = build
668 build.build_set = self
James E. Blair11700c32012-07-05 17:50:05 -0700669
James E. Blair4a28a882013-08-23 15:17:33 -0700670 def removeBuild(self, build):
671 del self.builds[build.job.name]
672
James E. Blair7e530ad2012-07-03 16:12:28 -0700673 def getBuild(self, job_name):
674 return self.builds.get(job_name)
675
James E. Blair11700c32012-07-05 17:50:05 -0700676 def getBuilds(self):
677 keys = self.builds.keys()
678 keys.sort()
679 return [self.builds.get(x) for x in keys]
680
James E. Blair7e530ad2012-07-03 16:12:28 -0700681
James E. Blairfee8d652013-06-07 08:57:52 -0700682class QueueItem(object):
683 """A changish inside of a Pipeline queue"""
James E. Blair32663402012-06-01 10:04:18 -0700684
James E. Blairbfb8e042014-12-30 17:01:44 -0800685 def __init__(self, queue, change):
686 self.pipeline = queue.pipeline
687 self.queue = queue
James E. Blairfee8d652013-06-07 08:57:52 -0700688 self.change = change # a changeish
James E. Blair7e530ad2012-07-03 16:12:28 -0700689 self.build_sets = []
James E. Blaircaec0c52012-08-22 14:52:22 -0700690 self.dequeued_needing_change = False
James E. Blair11700c32012-07-05 17:50:05 -0700691 self.current_build_set = BuildSet(self)
692 self.build_sets.append(self.current_build_set)
James E. Blairfee8d652013-06-07 08:57:52 -0700693 self.item_ahead = None
James E. Blair972e3c72013-08-29 12:04:55 -0700694 self.items_behind = []
James E. Blair8fa16972013-01-15 16:57:20 -0800695 self.enqueue_time = None
696 self.dequeue_time = None
James E. Blairfee8d652013-06-07 08:57:52 -0700697 self.reported = False
James E. Blairbfb8e042014-12-30 17:01:44 -0800698 self.active = False # Whether an item is within an active window
699 self.live = True # Whether an item is intended to be processed at all
James E. Blair83005782015-12-11 14:46:03 -0800700 self.job_tree = None
James E. Blaire5a847f2012-07-10 15:29:14 -0700701
James E. Blair972e3c72013-08-29 12:04:55 -0700702 def __repr__(self):
703 if self.pipeline:
704 pipeline = self.pipeline.name
705 else:
706 pipeline = None
707 return '<QueueItem 0x%x for %s in %s>' % (
708 id(self), self.change, pipeline)
709
James E. Blairee743612012-05-29 14:49:32 -0700710 def resetAllBuilds(self):
James E. Blair11700c32012-07-05 17:50:05 -0700711 old = self.current_build_set
712 self.current_build_set.result = 'CANCELED'
713 self.current_build_set = BuildSet(self)
714 old.next_build_set = self.current_build_set
715 self.current_build_set.previous_build_set = old
James E. Blair7e530ad2012-07-03 16:12:28 -0700716 self.build_sets.append(self.current_build_set)
James E. Blairee743612012-05-29 14:49:32 -0700717
718 def addBuild(self, build):
James E. Blair7e530ad2012-07-03 16:12:28 -0700719 self.current_build_set.addBuild(build)
James E. Blair66eeebf2013-07-27 17:44:32 -0700720 build.pipeline = self.pipeline
James E. Blairee743612012-05-29 14:49:32 -0700721
James E. Blair4a28a882013-08-23 15:17:33 -0700722 def removeBuild(self, build):
723 self.current_build_set.removeBuild(build)
724
James E. Blairfee8d652013-06-07 08:57:52 -0700725 def setReportedResult(self, result):
726 self.current_build_set.result = result
727
James E. Blair83005782015-12-11 14:46:03 -0800728 def _createJobTree(self, job_trees, parent):
729 for tree in job_trees:
730 job = tree.job
731 if not job.changeMatches(self.change):
732 continue
733 frozen_job = Job(job.name)
734 frozen_tree = JobTree(frozen_job)
735 inherited = set()
736 for variant in self.pipeline.layout.getJobs(job.name):
737 if variant.changeMatches(self.change):
738 if variant not in inherited:
739 frozen_job.inheritFrom(variant)
740 inherited.add(variant)
741 if job not in inherited:
742 # Only update from the job in the tree if it is
743 # unique, otherwise we might unset an attribute we
744 # have overloaded.
745 frozen_job.inheritFrom(job)
746 parent.job_trees.append(frozen_tree)
747 self._createJobTree(tree.job_trees, frozen_tree)
748
749 def createJobTree(self):
750 project_tree = self.pipeline.getJobTree(self.change.project)
751 ret = JobTree(None)
752 self._createJobTree(project_tree.job_trees, ret)
753 return ret
754
755 def freezeJobTree(self):
756 """Find or create actual matching jobs for this item's change and
757 store the resulting job tree."""
758 self.job_tree = self.createJobTree()
759
760 def getJobs(self):
761 if not self.live or not self.job_tree:
762 return []
763 return self.job_tree.getJobs()
764
Joshua Hesketh85af4e92014-02-21 08:28:58 -0800765 def formatJSON(self):
766 changeish = self.change
767 ret = {}
768 ret['active'] = self.active
James E. Blair107c3852015-02-07 08:23:10 -0800769 ret['live'] = self.live
Joshua Hesketh85af4e92014-02-21 08:28:58 -0800770 if hasattr(changeish, 'url') and changeish.url is not None:
771 ret['url'] = changeish.url
772 else:
773 ret['url'] = None
774 ret['id'] = changeish._id()
775 if self.item_ahead:
776 ret['item_ahead'] = self.item_ahead.change._id()
777 else:
778 ret['item_ahead'] = None
779 ret['items_behind'] = [i.change._id() for i in self.items_behind]
780 ret['failing_reasons'] = self.current_build_set.failing_reasons
781 ret['zuul_ref'] = self.current_build_set.ref
Ramy Asselin07cc33c2015-06-12 14:06:34 -0700782 if changeish.project:
783 ret['project'] = changeish.project.name
784 else:
785 # For cross-project dependencies with the depends-on
786 # project not known to zuul, the project is None
787 # Set it to a static value
788 ret['project'] = "Unknown Project"
Joshua Hesketh85af4e92014-02-21 08:28:58 -0800789 ret['enqueue_time'] = int(self.enqueue_time * 1000)
790 ret['jobs'] = []
Davanum Srinivasb6bfbcc2014-11-18 13:26:52 -0500791 if hasattr(changeish, 'owner'):
792 ret['owner'] = changeish.owner
793 else:
794 ret['owner'] = None
Joshua Hesketh85af4e92014-02-21 08:28:58 -0800795 max_remaining = 0
James E. Blair107c3852015-02-07 08:23:10 -0800796 for job in self.pipeline.getJobs(self):
Joshua Hesketh85af4e92014-02-21 08:28:58 -0800797 now = time.time()
798 build = self.current_build_set.getBuild(job.name)
799 elapsed = None
800 remaining = None
801 result = None
802 url = None
803 worker = None
804 if build:
805 result = build.result
806 url = build.url
807 if build.start_time:
808 if build.end_time:
809 elapsed = int((build.end_time -
810 build.start_time) * 1000)
811 remaining = 0
812 else:
813 elapsed = int((now - build.start_time) * 1000)
814 if build.estimated_time:
815 remaining = max(
816 int(build.estimated_time * 1000) - elapsed,
817 0)
818 worker = {
819 'name': build.worker.name,
820 'hostname': build.worker.hostname,
821 'ips': build.worker.ips,
822 'fqdn': build.worker.fqdn,
823 'program': build.worker.program,
824 'version': build.worker.version,
825 'extra': build.worker.extra
826 }
827 if remaining and remaining > max_remaining:
828 max_remaining = remaining
829
830 ret['jobs'].append({
831 'name': job.name,
832 'elapsed_time': elapsed,
833 'remaining_time': remaining,
834 'url': url,
835 'result': result,
836 'voting': job.voting,
837 'uuid': build.uuid if build else None,
838 'launch_time': build.launch_time if build else None,
839 'start_time': build.start_time if build else None,
840 'end_time': build.end_time if build else None,
841 'estimated_time': build.estimated_time if build else None,
842 'pipeline': build.pipeline.name if build else None,
843 'canceled': build.canceled if build else None,
844 'retry': build.retry if build else None,
845 'number': build.number if build else None,
Timothy Chavezb2332082015-08-07 20:08:04 -0500846 'node_labels': build.node_labels if build else [],
847 'node_name': build.node_name if build else None,
848 'worker': worker,
Joshua Hesketh85af4e92014-02-21 08:28:58 -0800849 })
850
851 if self.pipeline.haveAllJobsStarted(self):
852 ret['remaining_time'] = max_remaining
853 else:
854 ret['remaining_time'] = None
855 return ret
856
857 def formatStatus(self, indent=0, html=False):
858 changeish = self.change
859 indent_str = ' ' * indent
860 ret = ''
861 if html and hasattr(changeish, 'url') and changeish.url is not None:
862 ret += '%sProject %s change <a href="%s">%s</a>\n' % (
863 indent_str,
864 changeish.project.name,
865 changeish.url,
866 changeish._id())
867 else:
868 ret += '%sProject %s change %s based on %s\n' % (
869 indent_str,
870 changeish.project.name,
871 changeish._id(),
872 self.item_ahead)
James E. Blair107c3852015-02-07 08:23:10 -0800873 for job in self.pipeline.getJobs(self):
Joshua Hesketh85af4e92014-02-21 08:28:58 -0800874 build = self.current_build_set.getBuild(job.name)
875 if build:
876 result = build.result
877 else:
878 result = None
879 job_name = job.name
880 if not job.voting:
881 voting = ' (non-voting)'
882 else:
883 voting = ''
884 if html:
885 if build:
886 url = build.url
887 else:
888 url = None
889 if url is not None:
890 job_name = '<a href="%s">%s</a>' % (url, job_name)
891 ret += '%s %s: %s%s' % (indent_str, job_name, result, voting)
892 ret += '\n'
893 return ret
894
James E. Blairfee8d652013-06-07 08:57:52 -0700895
896class Changeish(object):
897 """Something like a change; either a change or a ref"""
James E. Blairfee8d652013-06-07 08:57:52 -0700898
899 def __init__(self, project):
900 self.project = project
901
Joshua Hesketh36c3fa52014-01-22 11:40:52 +1100902 def getBasePath(self):
903 base_path = ''
904 if hasattr(self, 'refspec'):
905 base_path = "%s/%s/%s" % (
906 self.number[-2:], self.number, self.patchset)
907 elif hasattr(self, 'ref'):
908 base_path = "%s/%s" % (self.newrev[:2], self.newrev)
909
910 return base_path
911
James E. Blairfee8d652013-06-07 08:57:52 -0700912 def equals(self, other):
913 raise NotImplementedError()
914
915 def isUpdateOf(self, other):
916 raise NotImplementedError()
917
918 def filterJobs(self, jobs):
919 return filter(lambda job: job.changeMatches(self), jobs)
920
921 def getRelatedChanges(self):
922 return set()
923
James E. Blair1e8dd892012-05-30 09:15:05 -0700924
James E. Blair4aea70c2012-07-26 14:23:24 -0700925class Change(Changeish):
James E. Blair4aea70c2012-07-26 14:23:24 -0700926 def __init__(self, project):
927 super(Change, self).__init__(project)
928 self.branch = None
929 self.number = None
930 self.url = None
931 self.patchset = None
932 self.refspec = None
933
James E. Blair70c71582013-03-06 08:50:50 -0800934 self.files = []
James E. Blair6965a4b2014-12-16 17:19:04 -0800935 self.needs_changes = []
James E. Blair4aea70c2012-07-26 14:23:24 -0700936 self.needed_by_changes = []
937 self.is_current_patchset = True
938 self.can_merge = False
939 self.is_merged = False
James E. Blairfee8d652013-06-07 08:57:52 -0700940 self.failed_to_merge = False
James E. Blairc053d022014-01-22 14:57:33 -0800941 self.approvals = []
James E. Blair11041d22014-05-02 14:49:53 -0700942 self.open = None
943 self.status = None
Davanum Srinivasb6bfbcc2014-11-18 13:26:52 -0500944 self.owner = None
James E. Blair4aea70c2012-07-26 14:23:24 -0700945
946 def _id(self):
James E. Blairbe765db2012-08-07 08:36:20 -0700947 return '%s,%s' % (self.number, self.patchset)
James E. Blair4aea70c2012-07-26 14:23:24 -0700948
949 def __repr__(self):
950 return '<Change 0x%x %s>' % (id(self), self._id())
951
952 def equals(self, other):
Zhongyue Luoaa85ebf2012-09-21 16:38:33 +0800953 if self.number == other.number and self.patchset == other.patchset:
James E. Blair4aea70c2012-07-26 14:23:24 -0700954 return True
955 return False
956
James E. Blair2fa50962013-01-30 21:50:41 -0800957 def isUpdateOf(self, other):
Clark Boylan01976242013-02-17 18:41:48 -0800958 if ((hasattr(other, 'number') and self.number == other.number) and
James E. Blair7a192e42013-07-11 14:10:36 -0700959 (hasattr(other, 'patchset') and
960 self.patchset is not None and
961 other.patchset is not None and
962 int(self.patchset) > int(other.patchset))):
James E. Blair2fa50962013-01-30 21:50:41 -0800963 return True
964 return False
965
James E. Blairfee8d652013-06-07 08:57:52 -0700966 def getRelatedChanges(self):
967 related = set()
James E. Blair6965a4b2014-12-16 17:19:04 -0800968 for c in self.needs_changes:
969 related.add(c)
James E. Blairfee8d652013-06-07 08:57:52 -0700970 for c in self.needed_by_changes:
971 related.add(c)
972 related.update(c.getRelatedChanges())
973 return related
James E. Blair4aea70c2012-07-26 14:23:24 -0700974
975
976class Ref(Changeish):
James E. Blair4aea70c2012-07-26 14:23:24 -0700977 def __init__(self, project):
James E. Blairbe765db2012-08-07 08:36:20 -0700978 super(Ref, self).__init__(project)
James E. Blair4aea70c2012-07-26 14:23:24 -0700979 self.ref = None
980 self.oldrev = None
981 self.newrev = None
982
James E. Blairbe765db2012-08-07 08:36:20 -0700983 def _id(self):
984 return self.newrev
985
Antoine Musso68bdcd72013-01-17 12:31:28 +0100986 def __repr__(self):
987 rep = None
988 if self.newrev == '0000000000000000000000000000000000000000':
989 rep = '<Ref 0x%x deletes %s from %s' % (
990 id(self), self.ref, self.oldrev)
991 elif self.oldrev == '0000000000000000000000000000000000000000':
992 rep = '<Ref 0x%x creates %s on %s>' % (
993 id(self), self.ref, self.newrev)
994 else:
995 # Catch all
996 rep = '<Ref 0x%x %s updated %s..%s>' % (
997 id(self), self.ref, self.oldrev, self.newrev)
998
999 return rep
1000
James E. Blair4aea70c2012-07-26 14:23:24 -07001001 def equals(self, other):
James E. Blair9358c612012-09-28 08:29:39 -07001002 if (self.project == other.project
1003 and self.ref == other.ref
1004 and self.newrev == other.newrev):
James E. Blair4aea70c2012-07-26 14:23:24 -07001005 return True
1006 return False
1007
James E. Blair2fa50962013-01-30 21:50:41 -08001008 def isUpdateOf(self, other):
1009 return False
1010
James E. Blair4aea70c2012-07-26 14:23:24 -07001011
James E. Blair63bb0ef2013-07-29 17:14:51 -07001012class NullChange(Changeish):
James E. Blaire5910202013-12-27 09:50:31 -08001013 def __repr__(self):
1014 return '<NullChange for %s>' % (self.project)
James E. Blair63bb0ef2013-07-29 17:14:51 -07001015
James E. Blair63bb0ef2013-07-29 17:14:51 -07001016 def _id(self):
Alex Gaynorddb9ef32013-09-16 21:04:58 -07001017 return None
James E. Blair63bb0ef2013-07-29 17:14:51 -07001018
1019 def equals(self, other):
Steve Varnau7b78b312015-04-03 14:49:46 -07001020 if (self.project == other.project
1021 and other._id() is None):
James E. Blair4f6033c2014-03-27 15:49:09 -07001022 return True
James E. Blair63bb0ef2013-07-29 17:14:51 -07001023 return False
1024
1025 def isUpdateOf(self, other):
1026 return False
1027
1028
James E. Blairee743612012-05-29 14:49:32 -07001029class TriggerEvent(object):
1030 def __init__(self):
1031 self.data = None
James E. Blair32663402012-06-01 10:04:18 -07001032 # common
James E. Blairee743612012-05-29 14:49:32 -07001033 self.type = None
1034 self.project_name = None
James E. Blair6c358e72013-07-29 17:06:47 -07001035 self.trigger_name = None
Antoine Mussob4e809e2012-12-06 16:58:06 +01001036 # Representation of the user account that performed the event.
1037 self.account = None
James E. Blair32663402012-06-01 10:04:18 -07001038 # patchset-created, comment-added, etc.
James E. Blairee743612012-05-29 14:49:32 -07001039 self.change_number = None
Clark Boylanfc56df32012-06-28 15:25:57 -07001040 self.change_url = None
James E. Blairee743612012-05-29 14:49:32 -07001041 self.patch_number = None
James E. Blaira03262c2012-05-30 09:41:16 -07001042 self.refspec = None
James E. Blairee743612012-05-29 14:49:32 -07001043 self.approvals = []
1044 self.branch = None
Clark Boylanb9bcb402012-06-29 17:44:05 -07001045 self.comment = None
James E. Blair32663402012-06-01 10:04:18 -07001046 # ref-updated
James E. Blairee743612012-05-29 14:49:32 -07001047 self.ref = None
James E. Blair32663402012-06-01 10:04:18 -07001048 self.oldrev = None
James E. Blair89cae0f2012-07-18 11:18:32 -07001049 self.newrev = None
James E. Blair63bb0ef2013-07-29 17:14:51 -07001050 # timer
1051 self.timespec = None
James E. Blairc494d542014-08-06 09:23:52 -07001052 # zuultrigger
1053 self.pipeline_name = None
James E. Blairad28e912013-11-27 10:43:22 -08001054 # For events that arrive with a destination pipeline (eg, from
1055 # an admin command, etc):
1056 self.forced_pipeline = None
James E. Blairee743612012-05-29 14:49:32 -07001057
James E. Blair9f9667e2012-06-12 17:51:08 -07001058 def __repr__(self):
James E. Blairee743612012-05-29 14:49:32 -07001059 ret = '<TriggerEvent %s %s' % (self.type, self.project_name)
James E. Blair1e8dd892012-05-30 09:15:05 -07001060
James E. Blairee743612012-05-29 14:49:32 -07001061 if self.branch:
1062 ret += " %s" % self.branch
1063 if self.change_number:
1064 ret += " %s,%s" % (self.change_number, self.patch_number)
1065 if self.approvals:
James E. Blair1e8dd892012-05-30 09:15:05 -07001066 ret += ' ' + ', '.join(
1067 ['%s:%s' % (a['type'], a['value']) for a in self.approvals])
James E. Blairee743612012-05-29 14:49:32 -07001068 ret += '>'
1069
1070 return ret
1071
James E. Blair1e8dd892012-05-30 09:15:05 -07001072
James E. Blair9c17dbf2014-06-23 14:21:58 -07001073class BaseFilter(object):
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001074 def __init__(self, required_approvals=[], reject_approvals=[]):
James E. Blair5bf78a32015-07-30 18:08:24 +00001075 self._required_approvals = copy.deepcopy(required_approvals)
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001076 self.required_approvals = self._tidy_approvals(required_approvals)
1077 self._reject_approvals = copy.deepcopy(reject_approvals)
1078 self.reject_approvals = self._tidy_approvals(reject_approvals)
James E. Blair9c17dbf2014-06-23 14:21:58 -07001079
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001080 def _tidy_approvals(self, approvals):
1081 for a in approvals:
James E. Blair9c17dbf2014-06-23 14:21:58 -07001082 for k, v in a.items():
1083 if k == 'username':
1084 pass
James E. Blair1fbfceb2014-06-23 14:42:53 -07001085 elif k in ['email', 'email-filter']:
James E. Blair5bf78a32015-07-30 18:08:24 +00001086 a['email'] = re.compile(v)
James E. Blair9c17dbf2014-06-23 14:21:58 -07001087 elif k == 'newer-than':
James E. Blair5bf78a32015-07-30 18:08:24 +00001088 a[k] = time_to_seconds(v)
James E. Blair9c17dbf2014-06-23 14:21:58 -07001089 elif k == 'older-than':
James E. Blair5bf78a32015-07-30 18:08:24 +00001090 a[k] = time_to_seconds(v)
James E. Blair1fbfceb2014-06-23 14:42:53 -07001091 if 'email-filter' in a:
1092 del a['email-filter']
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001093 return approvals
1094
1095 def _match_approval_required_approval(self, rapproval, approval):
1096 # Check if the required approval and approval match
1097 if 'description' not in approval:
1098 return False
1099 now = time.time()
1100 by = approval.get('by', {})
1101 for k, v in rapproval.items():
1102 if k == 'username':
1103 if (by.get('username', '') != v):
1104 return False
1105 elif k == 'email':
1106 if (not v.search(by.get('email', ''))):
1107 return False
1108 elif k == 'newer-than':
1109 t = now - v
1110 if (approval['grantedOn'] < t):
1111 return False
1112 elif k == 'older-than':
1113 t = now - v
1114 if (approval['grantedOn'] >= t):
1115 return False
1116 else:
1117 if not isinstance(v, list):
1118 v = [v]
1119 if (normalizeCategory(approval['description']) != k or
1120 int(approval['value']) not in v):
1121 return False
1122 return True
1123
1124 def matchesApprovals(self, change):
1125 if (self.required_approvals and not change.approvals
1126 or self.reject_approvals and not change.approvals):
1127 # A change with no approvals can not match
1128 return False
1129
1130 # TODO(jhesketh): If we wanted to optimise this slightly we could
1131 # analyse both the REQUIRE and REJECT filters by looping over the
1132 # approvals on the change and keeping track of what we have checked
1133 # rather than needing to loop on the change approvals twice
1134 return (self.matchesRequiredApprovals(change) and
1135 self.matchesNoRejectApprovals(change))
James E. Blair9c17dbf2014-06-23 14:21:58 -07001136
1137 def matchesRequiredApprovals(self, change):
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001138 # Check if any approvals match the requirements
James E. Blair5bf78a32015-07-30 18:08:24 +00001139 for rapproval in self.required_approvals:
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001140 matches_rapproval = False
James E. Blair9c17dbf2014-06-23 14:21:58 -07001141 for approval in change.approvals:
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001142 if self._match_approval_required_approval(rapproval, approval):
1143 # We have a matching approval so this requirement is
1144 # fulfilled
1145 matches_rapproval = True
James E. Blair5bf78a32015-07-30 18:08:24 +00001146 break
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001147 if not matches_rapproval:
James E. Blair5bf78a32015-07-30 18:08:24 +00001148 return False
James E. Blair9c17dbf2014-06-23 14:21:58 -07001149 return True
1150
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001151 def matchesNoRejectApprovals(self, change):
1152 # Check to make sure no approvals match a reject criteria
1153 for rapproval in self.reject_approvals:
1154 for approval in change.approvals:
1155 if self._match_approval_required_approval(rapproval, approval):
1156 # A reject approval has been matched, so we reject
1157 # immediately
1158 return False
1159 # To get here no rejects can have been matched so we should be good to
1160 # queue
1161 return True
1162
James E. Blair9c17dbf2014-06-23 14:21:58 -07001163
1164class EventFilter(BaseFilter):
James E. Blairc0dedf82014-08-06 09:37:52 -07001165 def __init__(self, trigger, types=[], branches=[], refs=[],
1166 event_approvals={}, comments=[], emails=[], usernames=[],
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001167 timespecs=[], required_approvals=[], reject_approvals=[],
1168 pipelines=[], ignore_deletes=True):
James E. Blair9c17dbf2014-06-23 14:21:58 -07001169 super(EventFilter, self).__init__(
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001170 required_approvals=required_approvals,
1171 reject_approvals=reject_approvals)
James E. Blairc0dedf82014-08-06 09:37:52 -07001172 self.trigger = trigger
James E. Blairee743612012-05-29 14:49:32 -07001173 self._types = types
1174 self._branches = branches
1175 self._refs = refs
James E. Blair1fbfceb2014-06-23 14:42:53 -07001176 self._comments = comments
1177 self._emails = emails
1178 self._usernames = usernames
James E. Blairc494d542014-08-06 09:23:52 -07001179 self._pipelines = pipelines
James E. Blairee743612012-05-29 14:49:32 -07001180 self.types = [re.compile(x) for x in types]
1181 self.branches = [re.compile(x) for x in branches]
1182 self.refs = [re.compile(x) for x in refs]
James E. Blair1fbfceb2014-06-23 14:42:53 -07001183 self.comments = [re.compile(x) for x in comments]
1184 self.emails = [re.compile(x) for x in emails]
1185 self.usernames = [re.compile(x) for x in usernames]
James E. Blairc494d542014-08-06 09:23:52 -07001186 self.pipelines = [re.compile(x) for x in pipelines]
James E. Blairc053d022014-01-22 14:57:33 -08001187 self.event_approvals = event_approvals
James E. Blair63bb0ef2013-07-29 17:14:51 -07001188 self.timespecs = timespecs
K Jonathan Harkerf95e7232015-04-29 13:33:16 -07001189 self.ignore_deletes = ignore_deletes
James E. Blairee743612012-05-29 14:49:32 -07001190
James E. Blair9f9667e2012-06-12 17:51:08 -07001191 def __repr__(self):
James E. Blairee743612012-05-29 14:49:32 -07001192 ret = '<EventFilter'
James E. Blair1e8dd892012-05-30 09:15:05 -07001193
James E. Blairee743612012-05-29 14:49:32 -07001194 if self._types:
1195 ret += ' types: %s' % ', '.join(self._types)
James E. Blairc494d542014-08-06 09:23:52 -07001196 if self._pipelines:
1197 ret += ' pipelines: %s' % ', '.join(self._pipelines)
James E. Blairee743612012-05-29 14:49:32 -07001198 if self._branches:
1199 ret += ' branches: %s' % ', '.join(self._branches)
1200 if self._refs:
1201 ret += ' refs: %s' % ', '.join(self._refs)
K Jonathan Harkerf95e7232015-04-29 13:33:16 -07001202 if self.ignore_deletes:
1203 ret += ' ignore_deletes: %s' % self.ignore_deletes
James E. Blairc053d022014-01-22 14:57:33 -08001204 if self.event_approvals:
1205 ret += ' event_approvals: %s' % ', '.join(
1206 ['%s:%s' % a for a in self.event_approvals.items()])
James E. Blair5bf78a32015-07-30 18:08:24 +00001207 if self.required_approvals:
1208 ret += ' required_approvals: %s' % ', '.join(
1209 ['%s' % a for a in self._required_approvals])
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001210 if self.reject_approvals:
1211 ret += ' reject_approvals: %s' % ', '.join(
1212 ['%s' % a for a in self._reject_approvals])
James E. Blair1fbfceb2014-06-23 14:42:53 -07001213 if self._comments:
1214 ret += ' comments: %s' % ', '.join(self._comments)
1215 if self._emails:
1216 ret += ' emails: %s' % ', '.join(self._emails)
1217 if self._usernames:
1218 ret += ' username_filters: %s' % ', '.join(self._usernames)
James E. Blair63bb0ef2013-07-29 17:14:51 -07001219 if self.timespecs:
1220 ret += ' timespecs: %s' % ', '.join(self.timespecs)
James E. Blairee743612012-05-29 14:49:32 -07001221 ret += '>'
1222
1223 return ret
1224
James E. Blairc053d022014-01-22 14:57:33 -08001225 def matches(self, event, change):
James E. Blairee743612012-05-29 14:49:32 -07001226 # event types are ORed
1227 matches_type = False
1228 for etype in self.types:
1229 if etype.match(event.type):
1230 matches_type = True
1231 if self.types and not matches_type:
1232 return False
1233
James E. Blairc494d542014-08-06 09:23:52 -07001234 # pipelines are ORed
1235 matches_pipeline = False
1236 for epipe in self.pipelines:
1237 if epipe.match(event.pipeline_name):
1238 matches_pipeline = True
1239 if self.pipelines and not matches_pipeline:
1240 return False
1241
James E. Blairee743612012-05-29 14:49:32 -07001242 # branches are ORed
1243 matches_branch = False
1244 for branch in self.branches:
1245 if branch.match(event.branch):
1246 matches_branch = True
1247 if self.branches and not matches_branch:
1248 return False
1249
1250 # refs are ORed
1251 matches_ref = False
Yolanda Robla16698872014-08-25 11:59:27 +02001252 if event.ref is not None:
1253 for ref in self.refs:
1254 if ref.match(event.ref):
1255 matches_ref = True
James E. Blairee743612012-05-29 14:49:32 -07001256 if self.refs and not matches_ref:
1257 return False
K Jonathan Harkerf95e7232015-04-29 13:33:16 -07001258 if self.ignore_deletes and event.newrev == EMPTY_GIT_REF:
1259 # If the updated ref has an empty git sha (all 0s),
1260 # then the ref is being deleted
1261 return False
James E. Blairee743612012-05-29 14:49:32 -07001262
James E. Blair1fbfceb2014-06-23 14:42:53 -07001263 # comments are ORed
1264 matches_comment_re = False
1265 for comment_re in self.comments:
Clark Boylanb9bcb402012-06-29 17:44:05 -07001266 if (event.comment is not None and
James E. Blair1fbfceb2014-06-23 14:42:53 -07001267 comment_re.search(event.comment)):
1268 matches_comment_re = True
1269 if self.comments and not matches_comment_re:
Clark Boylanb9bcb402012-06-29 17:44:05 -07001270 return False
1271
Antoine Mussob4e809e2012-12-06 16:58:06 +01001272 # We better have an account provided by Gerrit to do
1273 # email filtering.
1274 if event.account is not None:
James E. Blaircf429f32012-12-20 14:28:24 -08001275 account_email = event.account.get('email')
James E. Blair1fbfceb2014-06-23 14:42:53 -07001276 # emails are ORed
1277 matches_email_re = False
1278 for email_re in self.emails:
Antoine Mussob4e809e2012-12-06 16:58:06 +01001279 if (account_email is not None and
Joshua Hesketh29d99b72014-08-19 16:27:42 +10001280 email_re.search(account_email)):
James E. Blair1fbfceb2014-06-23 14:42:53 -07001281 matches_email_re = True
1282 if self.emails and not matches_email_re:
Antoine Mussob4e809e2012-12-06 16:58:06 +01001283 return False
1284
James E. Blair1fbfceb2014-06-23 14:42:53 -07001285 # usernames are ORed
Joshua Heskethb8a817e2013-12-27 11:21:38 +11001286 account_username = event.account.get('username')
James E. Blair1fbfceb2014-06-23 14:42:53 -07001287 matches_username_re = False
1288 for username_re in self.usernames:
Joshua Heskethb8a817e2013-12-27 11:21:38 +11001289 if (account_username is not None and
James E. Blair1fbfceb2014-06-23 14:42:53 -07001290 username_re.search(account_username)):
1291 matches_username_re = True
1292 if self.usernames and not matches_username_re:
Joshua Heskethb8a817e2013-12-27 11:21:38 +11001293 return False
1294
James E. Blairee743612012-05-29 14:49:32 -07001295 # approvals are ANDed
James E. Blairc053d022014-01-22 14:57:33 -08001296 for category, value in self.event_approvals.items():
James E. Blairee743612012-05-29 14:49:32 -07001297 matches_approval = False
1298 for eapproval in event.approvals:
1299 if (normalizeCategory(eapproval['description']) == category and
1300 int(eapproval['value']) == int(value)):
1301 matches_approval = True
James E. Blair1e8dd892012-05-30 09:15:05 -07001302 if not matches_approval:
1303 return False
James E. Blair63bb0ef2013-07-29 17:14:51 -07001304
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001305 # required approvals are ANDed (reject approvals are ORed)
1306 if not self.matchesApprovals(change):
James E. Blair9c17dbf2014-06-23 14:21:58 -07001307 return False
James E. Blairc053d022014-01-22 14:57:33 -08001308
James E. Blair63bb0ef2013-07-29 17:14:51 -07001309 # timespecs are ORed
1310 matches_timespec = False
1311 for timespec in self.timespecs:
1312 if (event.timespec == timespec):
1313 matches_timespec = True
1314 if self.timespecs and not matches_timespec:
1315 return False
1316
James E. Blairee743612012-05-29 14:49:32 -07001317 return True
James E. Blaireff88162013-07-01 12:44:14 -04001318
1319
James E. Blair9c17dbf2014-06-23 14:21:58 -07001320class ChangeishFilter(BaseFilter):
Clark Boylana9702ad2014-05-08 17:17:24 -07001321 def __init__(self, open=None, current_patchset=None,
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001322 statuses=[], required_approvals=[],
1323 reject_approvals=[]):
James E. Blair9c17dbf2014-06-23 14:21:58 -07001324 super(ChangeishFilter, self).__init__(
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001325 required_approvals=required_approvals,
1326 reject_approvals=reject_approvals)
James E. Blair11041d22014-05-02 14:49:53 -07001327 self.open = open
Clark Boylana9702ad2014-05-08 17:17:24 -07001328 self.current_patchset = current_patchset
James E. Blair11041d22014-05-02 14:49:53 -07001329 self.statuses = statuses
James E. Blair11041d22014-05-02 14:49:53 -07001330
1331 def __repr__(self):
1332 ret = '<ChangeishFilter'
1333
1334 if self.open is not None:
1335 ret += ' open: %s' % self.open
Clark Boylana9702ad2014-05-08 17:17:24 -07001336 if self.current_patchset is not None:
1337 ret += ' current-patchset: %s' % self.current_patchset
James E. Blair11041d22014-05-02 14:49:53 -07001338 if self.statuses:
1339 ret += ' statuses: %s' % ', '.join(self.statuses)
James E. Blair5bf78a32015-07-30 18:08:24 +00001340 if self.required_approvals:
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001341 ret += (' required_approvals: %s' %
1342 str(self.required_approvals))
1343 if self.reject_approvals:
1344 ret += (' reject_approvals: %s' %
1345 str(self.reject_approvals))
James E. Blair11041d22014-05-02 14:49:53 -07001346 ret += '>'
1347
1348 return ret
1349
1350 def matches(self, change):
1351 if self.open is not None:
1352 if self.open != change.open:
1353 return False
1354
Clark Boylana9702ad2014-05-08 17:17:24 -07001355 if self.current_patchset is not None:
1356 if self.current_patchset != change.is_current_patchset:
1357 return False
1358
James E. Blair11041d22014-05-02 14:49:53 -07001359 if self.statuses:
1360 if change.status not in self.statuses:
1361 return False
1362
Joshua Hesketh66c8e522014-06-26 15:30:08 +10001363 # required approvals are ANDed (reject approvals are ORed)
1364 if not self.matchesApprovals(change):
James E. Blair9c17dbf2014-06-23 14:21:58 -07001365 return False
James E. Blair11041d22014-05-02 14:49:53 -07001366
1367 return True
1368
1369
James E. Blairb97ed802015-12-21 15:55:35 -08001370class ProjectPipelineConfig(object):
1371 # Represents a project cofiguration in the context of a pipeline
1372 def __init__(self):
1373 self.job_tree = None
1374 self.queue_name = None
1375 # TODOv3(jeblair): add merge mode
1376
1377
1378class ProjectConfig(object):
1379 # Represents a project cofiguration
1380 def __init__(self, name):
1381 self.name = name
1382 self.pipelines = {}
1383
1384
James E. Blaird8e778f2015-12-22 14:09:20 -08001385class UnparsedAbideConfig(object):
1386 # A collection of yaml lists that has not yet been parsed into
1387 # objects.
1388 def __init__(self):
1389 self.tenants = []
1390
1391 def extend(self, conf):
1392 if isinstance(conf, UnparsedAbideConfig):
1393 self.tenants.extend(conf.tenants)
1394 return
1395
1396 if not isinstance(conf, list):
1397 raise Exception("Configuration items must be in the form of "
1398 "a list of dictionaries (when parsing %s)" %
1399 (conf,))
1400 for item in conf:
1401 if not isinstance(item, dict):
1402 raise Exception("Configuration items must be in the form of "
1403 "a list of dictionaries (when parsing %s)" %
1404 (conf,))
1405 if len(item.keys()) > 1:
1406 raise Exception("Configuration item dictionaries must have "
1407 "a single key (when parsing %s)" %
1408 (conf,))
1409 key, value = item.items()[0]
1410 if key == 'tenant':
1411 self.tenants.append(value)
1412 else:
1413 raise Exception("Configuration item not recognized "
1414 "(when parsing %s)" %
1415 (conf,))
1416
1417
1418class UnparsedTenantConfig(object):
1419 # A collection of yaml lists that has not yet been parsed into
1420 # objects.
1421 def __init__(self):
1422 self.pipelines = []
1423 self.jobs = []
1424 self.project_templates = []
1425 self.projects = []
1426
1427 def extend(self, conf):
1428 if isinstance(conf, UnparsedTenantConfig):
1429 self.pipelines.extend(conf.pipelines)
1430 self.jobs.extend(conf.jobs)
1431 self.project_templates.extend(conf.project_templates)
1432 self.projects.extend(conf.projects)
1433 return
1434
1435 if not isinstance(conf, list):
1436 raise Exception("Configuration items must be in the form of "
1437 "a list of dictionaries (when parsing %s)" %
1438 (conf,))
1439 for item in conf:
1440 if not isinstance(item, dict):
1441 raise Exception("Configuration items must be in the form of "
1442 "a list of dictionaries (when parsing %s)" %
1443 (conf,))
1444 if len(item.keys()) > 1:
1445 raise Exception("Configuration item dictionaries must have "
1446 "a single key (when parsing %s)" %
1447 (conf,))
1448 key, value = item.items()[0]
1449 if key == 'project':
1450 self.projects.append(value)
1451 elif key == 'job':
1452 self.jobs.append(value)
1453 elif key == 'project-template':
1454 self.project_templates.append(value)
1455 elif key == 'pipeline':
1456 self.pipelines.append(value)
1457 else:
1458 raise Exception("Configuration item not recognized "
1459 "(when parsing %s)" %
1460 (conf,))
1461
1462
James E. Blaireff88162013-07-01 12:44:14 -04001463class Layout(object):
1464 def __init__(self):
1465 self.projects = {}
James E. Blairb97ed802015-12-21 15:55:35 -08001466 self.project_configs = {}
1467 self.project_templates = {}
James E. Blair5a9918a2013-08-27 10:06:27 -07001468 self.pipelines = OrderedDict()
James E. Blair83005782015-12-11 14:46:03 -08001469 # This is a dictionary of name -> [jobs]. The first element
1470 # of the list is the first job added with that name. It is
1471 # the reference definition for a given job. Subsequent
1472 # elements are aspects of that job with different matchers
1473 # that override some attribute of the job. These aspects all
1474 # inherit from the reference definition.
James E. Blaireff88162013-07-01 12:44:14 -04001475 self.jobs = {}
James E. Blaireff88162013-07-01 12:44:14 -04001476
1477 def getJob(self, name):
1478 if name in self.jobs:
James E. Blair83005782015-12-11 14:46:03 -08001479 return self.jobs[name][0]
James E. Blairb97ed802015-12-21 15:55:35 -08001480 raise Exception("Job %s not defined" % (name,))
James E. Blair83005782015-12-11 14:46:03 -08001481
1482 def getJobs(self, name):
1483 return self.jobs.get(name, [])
1484
1485 def addJob(self, job):
1486 if job.name in self.jobs:
1487 self.jobs[job.name].append(job)
1488 else:
1489 self.jobs[job.name] = [job]
1490
1491 def addPipeline(self, pipeline):
1492 self.pipelines[pipeline.name] = pipeline
James E. Blair59fdbac2015-12-07 17:08:06 -08001493
James E. Blairb97ed802015-12-21 15:55:35 -08001494 def addProjectTemplate(self, project_template):
1495 self.project_templates[project_template.name] = project_template
1496
1497 def addProjectConfig(self, project_config):
1498 self.project_configs[project_config.name] = project_config
1499 # TODOv3(jeblair): tidy up the relationship between pipelines
1500 # and projects and projectconfigs
1501 for pipeline_name, pipeline_config in project_config.pipelines.items():
1502 pipeline = self.pipelines[pipeline_name]
1503 project = pipeline.source.getProject(project_config.name)
1504 pipeline.job_trees[project] = pipeline_config.job_tree
1505
James E. Blair59fdbac2015-12-07 17:08:06 -08001506
1507class Tenant(object):
1508 def __init__(self, name):
1509 self.name = name
1510 self.layout = None
1511
1512
1513class Abide(object):
1514 def __init__(self):
1515 self.tenants = OrderedDict()