James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 1 | # Copyright 2012 Hewlett-Packard Development Company, L.P. |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 4 | # not use this file except in compliance with the License. You may obtain |
| 5 | # a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 12 | # License for the specific language governing permissions and limitations |
| 13 | # under the License. |
| 14 | |
| 15 | import re |
James E. Blair | ff986a1 | 2012-05-30 14:56:51 -0700 | [diff] [blame] | 16 | import time |
James E. Blair | 4886cc1 | 2012-07-18 15:39:41 -0700 | [diff] [blame] | 17 | from uuid import uuid4 |
| 18 | |
| 19 | |
| 20 | FAST_FORWARD_ONLY = 1 |
| 21 | MERGE_ALWAYS = 2 |
| 22 | MERGE_IF_NECESSARY = 3 |
| 23 | CHERRY_PICK = 4 |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 24 | |
James E. Blair | 1e8dd89 | 2012-05-30 09:15:05 -0700 | [diff] [blame] | 25 | |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 26 | class Pipeline(object): |
| 27 | """A top-level pipeline such as check, gate, post, etc.""" |
| 28 | def __init__(self, name): |
| 29 | self.name = name |
James E. Blair | 8dbd56a | 2012-12-22 10:55:10 -0800 | [diff] [blame] | 30 | self.description = None |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 31 | self.job_trees = {} # project -> JobTree |
| 32 | self.manager = None |
James E. Blair | e048707 | 2012-08-29 17:38:31 -0700 | [diff] [blame] | 33 | self.queues = [] |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 34 | |
James E. Blair | d09c17a | 2012-08-07 09:23:14 -0700 | [diff] [blame] | 35 | def __repr__(self): |
| 36 | return '<Pipeline %s>' % self.name |
| 37 | |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 38 | def setManager(self, manager): |
| 39 | self.manager = manager |
| 40 | |
| 41 | def addProject(self, project): |
| 42 | job_tree = JobTree(None) # Null job == job tree root |
| 43 | self.job_trees[project] = job_tree |
| 44 | return job_tree |
| 45 | |
| 46 | def getProjects(self): |
| 47 | return self.job_trees.keys() |
| 48 | |
James E. Blair | e048707 | 2012-08-29 17:38:31 -0700 | [diff] [blame] | 49 | def addQueue(self, queue): |
| 50 | self.queues.append(queue) |
| 51 | |
| 52 | def getQueue(self, project): |
| 53 | for queue in self.queues: |
| 54 | if project in queue.projects: |
| 55 | return queue |
| 56 | return None |
| 57 | |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 58 | def getJobTree(self, project): |
| 59 | tree = self.job_trees.get(project) |
| 60 | return tree |
| 61 | |
| 62 | def getJobs(self, changeish): |
James E. Blair | d09c17a | 2012-08-07 09:23:14 -0700 | [diff] [blame] | 63 | tree = self.getJobTree(changeish.project) |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 64 | if not tree: |
| 65 | return [] |
| 66 | return changeish.filterJobs(tree.getJobs()) |
| 67 | |
| 68 | def _findJobsToRun(self, job_trees, changeish): |
| 69 | torun = [] |
| 70 | if changeish.change_ahead: |
| 71 | # Only run jobs if any 'hold' jobs on the change ahead |
| 72 | # have completed successfully. |
| 73 | if self.isHoldingFollowingChanges(changeish.change_ahead): |
| 74 | return [] |
| 75 | for tree in job_trees: |
| 76 | job = tree.job |
| 77 | result = None |
| 78 | if job: |
| 79 | if not job.changeMatches(changeish): |
| 80 | continue |
| 81 | build = changeish.current_build_set.getBuild(job.name) |
| 82 | if build: |
| 83 | result = build.result |
| 84 | else: |
| 85 | # There is no build for the root of this job tree, |
| 86 | # so we should run it. |
| 87 | torun.append(job) |
| 88 | # If there is no job, this is a null job tree, and we should |
| 89 | # run all of its jobs. |
| 90 | if result == 'SUCCESS' or not job: |
| 91 | torun.extend(self._findJobsToRun(tree.job_trees, changeish)) |
| 92 | return torun |
| 93 | |
| 94 | def findJobsToRun(self, changeish): |
James E. Blair | d09c17a | 2012-08-07 09:23:14 -0700 | [diff] [blame] | 95 | tree = self.getJobTree(changeish.project) |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 96 | if not tree: |
| 97 | return [] |
| 98 | return self._findJobsToRun(tree.job_trees, changeish) |
| 99 | |
| 100 | def areAllJobsComplete(self, changeish): |
| 101 | for job in self.getJobs(changeish): |
| 102 | build = changeish.current_build_set.getBuild(job.name) |
| 103 | if not build or not build.result: |
| 104 | return False |
| 105 | return True |
| 106 | |
| 107 | def didAllJobsSucceed(self, changeish): |
| 108 | for job in self.getJobs(changeish): |
James E. Blair | 4ec821f | 2012-08-23 15:28:28 -0700 | [diff] [blame] | 109 | if not job.voting: |
| 110 | continue |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 111 | build = changeish.current_build_set.getBuild(job.name) |
| 112 | if not build: |
| 113 | return False |
| 114 | if build.result != 'SUCCESS': |
| 115 | return False |
| 116 | return True |
| 117 | |
| 118 | def didAnyJobFail(self, changeish): |
| 119 | for job in self.getJobs(changeish): |
James E. Blair | 4ec821f | 2012-08-23 15:28:28 -0700 | [diff] [blame] | 120 | if not job.voting: |
| 121 | continue |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 122 | build = changeish.current_build_set.getBuild(job.name) |
| 123 | if build and build.result == 'FAILURE': |
| 124 | return True |
| 125 | return False |
| 126 | |
| 127 | def isHoldingFollowingChanges(self, changeish): |
| 128 | for job in self.getJobs(changeish): |
| 129 | if not job.hold_following_changes: |
| 130 | continue |
| 131 | build = changeish.current_build_set.getBuild(job.name) |
| 132 | if not build: |
| 133 | return True |
| 134 | if build.result != 'SUCCESS': |
| 135 | return True |
| 136 | if not changeish.change_ahead: |
| 137 | return False |
| 138 | return self.isHoldingFollowingChanges(changeish.change_ahead) |
| 139 | |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 140 | def setResult(self, changeish, build): |
| 141 | if build.result != 'SUCCESS': |
| 142 | # Get a JobTree from a Job so we can find only its dependent jobs |
| 143 | root = self.getJobTree(changeish.project) |
| 144 | tree = root.getJobTreeForJob(build.job) |
| 145 | for job in tree.getJobs(): |
| 146 | fakebuild = Build(job, None) |
| 147 | fakebuild.result = 'SKIPPED' |
| 148 | changeish.addBuild(fakebuild) |
| 149 | |
James E. Blair | 973721f | 2012-08-15 10:19:43 -0700 | [diff] [blame] | 150 | def setUnableToMerge(self, changeish): |
| 151 | changeish.current_build_set.unable_to_merge = True |
| 152 | root = self.getJobTree(changeish.project) |
| 153 | for job in root.getJobs(): |
| 154 | fakebuild = Build(job, None) |
| 155 | fakebuild.result = 'SKIPPED' |
| 156 | changeish.addBuild(fakebuild) |
| 157 | |
James E. Blair | caec0c5 | 2012-08-22 14:52:22 -0700 | [diff] [blame] | 158 | def setDequeuedNeedingChange(self, changeish): |
| 159 | changeish.dequeued_needing_change = True |
| 160 | root = self.getJobTree(changeish.project) |
| 161 | for job in root.getJobs(): |
| 162 | fakebuild = Build(job, None) |
| 163 | fakebuild.result = 'SKIPPED' |
| 164 | changeish.addBuild(fakebuild) |
| 165 | |
James E. Blair | e048707 | 2012-08-29 17:38:31 -0700 | [diff] [blame] | 166 | def getChangesInQueue(self): |
| 167 | changes = [] |
| 168 | for shared_queue in self.queues: |
| 169 | changes.extend(shared_queue.queue) |
| 170 | return changes |
| 171 | |
| 172 | def getAllChanges(self): |
| 173 | changes = [] |
| 174 | for shared_queue in self.queues: |
| 175 | changes.extend(shared_queue.queue) |
| 176 | changes.extend(shared_queue.severed_heads) |
| 177 | return changes |
| 178 | |
| 179 | def formatStatusHTML(self): |
| 180 | ret = '' |
| 181 | for queue in self.queues: |
| 182 | if len(self.queues) > 1: |
| 183 | s = 'Change queue: %s' % queue.name |
| 184 | ret += s + '\n' |
| 185 | ret += '-' * len(s) + '\n' |
| 186 | for head in queue.getHeads(): |
| 187 | ret += self.formatStatus(head, html=True) |
| 188 | return ret |
| 189 | |
James E. Blair | 8dbd56a | 2012-12-22 10:55:10 -0800 | [diff] [blame] | 190 | def formatStatusJSON(self): |
| 191 | j_pipeline = dict(name=self.name, |
| 192 | description=self.description) |
| 193 | j_queues = [] |
| 194 | j_pipeline['change_queues'] = j_queues |
| 195 | for queue in self.queues: |
| 196 | j_queue = dict(name=queue.name) |
| 197 | j_queues.append(j_queue) |
| 198 | j_queue['heads'] = [] |
| 199 | for head in queue.getHeads(): |
| 200 | j_changes = [] |
| 201 | c = head |
| 202 | while c: |
| 203 | j_changes.append(self.formatChangeJSON(c)) |
| 204 | c = c.change_behind |
| 205 | j_queue['heads'].append(j_changes) |
| 206 | return j_pipeline |
| 207 | |
James E. Blair | e048707 | 2012-08-29 17:38:31 -0700 | [diff] [blame] | 208 | def formatStatus(self, changeish, indent=0, html=False): |
| 209 | indent_str = ' ' * indent |
| 210 | ret = '' |
| 211 | if html and hasattr(changeish, 'url') and changeish.url is not None: |
| 212 | ret += '%sProject %s change <a href="%s">%s</a>\n' % ( |
| 213 | indent_str, |
| 214 | changeish.project.name, |
| 215 | changeish.url, |
| 216 | changeish._id()) |
| 217 | else: |
| 218 | ret += '%sProject %s change %s\n' % (indent_str, |
| 219 | changeish.project.name, |
| 220 | changeish._id()) |
| 221 | for job in self.getJobs(changeish): |
| 222 | build = changeish.current_build_set.getBuild(job.name) |
| 223 | if build: |
| 224 | result = build.result |
| 225 | else: |
| 226 | result = None |
| 227 | job_name = job.name |
| 228 | if not job.voting: |
| 229 | voting = ' (non-voting)' |
| 230 | else: |
| 231 | voting = '' |
| 232 | if html: |
| 233 | if build: |
| 234 | url = build.url |
| 235 | else: |
| 236 | url = None |
| 237 | if url is not None: |
| 238 | job_name = '<a href="%s">%s</a>' % (url, job_name) |
| 239 | ret += '%s %s: %s%s' % (indent_str, job_name, result, voting) |
| 240 | ret += '\n' |
| 241 | if changeish.change_behind: |
| 242 | ret += '%sFollowed by:\n' % (indent_str) |
| 243 | ret += self.formatStatus(changeish.change_behind, indent + 2, html) |
| 244 | return ret |
| 245 | |
James E. Blair | 8dbd56a | 2012-12-22 10:55:10 -0800 | [diff] [blame] | 246 | def formatChangeJSON(self, changeish): |
| 247 | ret = {} |
| 248 | if hasattr(changeish, 'url') and changeish.url is not None: |
| 249 | ret['url'] = changeish.url |
James E. Blair | c44b138 | 2012-12-23 09:39:55 -0800 | [diff] [blame^] | 250 | else: |
| 251 | ret['url'] = None |
James E. Blair | 8dbd56a | 2012-12-22 10:55:10 -0800 | [diff] [blame] | 252 | ret['id'] = changeish._id() |
| 253 | ret['project'] = changeish.project.name |
| 254 | ret['jobs'] = [] |
| 255 | for job in self.getJobs(changeish): |
| 256 | build = changeish.current_build_set.getBuild(job.name) |
| 257 | if build: |
| 258 | result = build.result |
| 259 | url = build.url |
| 260 | else: |
| 261 | result = None |
| 262 | url = None |
| 263 | ret['jobs'].append( |
| 264 | dict( |
| 265 | name=job.name, |
| 266 | url=url, |
| 267 | result=result, |
| 268 | voting=job.voting)) |
| 269 | return ret |
| 270 | |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 271 | |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 272 | class ChangeQueue(object): |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 273 | """DependentPipelines have multiple parallel queues shared by |
| 274 | different projects; this is one of them. For instance, there may |
| 275 | a queue shared by interrelated projects foo and bar, and a second |
| 276 | queue for independent project baz. Pipelines have one or more |
| 277 | PipelineQueues.""" |
James E. Blair | e048707 | 2012-08-29 17:38:31 -0700 | [diff] [blame] | 278 | def __init__(self, pipeline, dependent=True): |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 279 | self.pipeline = pipeline |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 280 | self.name = '' |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 281 | self.projects = [] |
| 282 | self._jobs = set() |
| 283 | self.queue = [] |
James E. Blair | e048707 | 2012-08-29 17:38:31 -0700 | [diff] [blame] | 284 | self.severed_heads = [] |
| 285 | self.dependent = dependent |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 286 | |
James E. Blair | 9f9667e | 2012-06-12 17:51:08 -0700 | [diff] [blame] | 287 | def __repr__(self): |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 288 | return '<ChangeQueue %s: %s>' % (self.pipeline.name, self.name) |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 289 | |
| 290 | def getJobs(self): |
| 291 | return self._jobs |
| 292 | |
| 293 | def addProject(self, project): |
| 294 | if project not in self.projects: |
| 295 | self.projects.append(project) |
| 296 | names = [x.name for x in self.projects] |
| 297 | names.sort() |
| 298 | self.name = ', '.join(names) |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 299 | self._jobs |= set(self.pipeline.getJobTree(project).getJobs()) |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 300 | |
| 301 | def enqueueChange(self, change): |
James E. Blair | 7524158 | 2012-08-31 12:16:55 -0700 | [diff] [blame] | 302 | if self.dependent and self.queue: |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 303 | change.change_ahead = self.queue[-1] |
James E. Blair | e048707 | 2012-08-29 17:38:31 -0700 | [diff] [blame] | 304 | change.change_ahead.change_behind = change |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 305 | self.queue.append(change) |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 306 | |
| 307 | def dequeueChange(self, change): |
| 308 | if change in self.queue: |
| 309 | self.queue.remove(change) |
James E. Blair | e048707 | 2012-08-29 17:38:31 -0700 | [diff] [blame] | 310 | if change in self.severed_heads: |
| 311 | self.severed_heads.remove(change) |
| 312 | if change.change_ahead: |
| 313 | change.change_ahead.change_behind = change.change_behind |
| 314 | if change.change_behind: |
| 315 | change.change_behind.change_ahead = change.change_ahead |
| 316 | change.change_ahead = None |
| 317 | change.change_behind = None |
| 318 | |
| 319 | def addSeveredHead(self, change): |
| 320 | self.severed_heads.append(change) |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 321 | |
| 322 | def mergeChangeQueue(self, other): |
| 323 | for project in other.projects: |
| 324 | self.addProject(project) |
| 325 | |
James E. Blair | e048707 | 2012-08-29 17:38:31 -0700 | [diff] [blame] | 326 | def getHead(self): |
| 327 | if not self.queue: |
| 328 | return None |
| 329 | return self.queue[0] |
| 330 | |
| 331 | def getHeads(self): |
| 332 | heads = [] |
| 333 | if self.dependent: |
| 334 | h = self.getHead() |
| 335 | if h: |
| 336 | heads.append(h) |
| 337 | else: |
| 338 | heads.extend(self.queue) |
| 339 | heads.extend(self.severed_heads) |
| 340 | return heads |
| 341 | |
James E. Blair | 1e8dd89 | 2012-05-30 09:15:05 -0700 | [diff] [blame] | 342 | |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 343 | class Project(object): |
| 344 | def __init__(self, name): |
| 345 | self.name = name |
James E. Blair | 4886cc1 | 2012-07-18 15:39:41 -0700 | [diff] [blame] | 346 | self.merge_mode = MERGE_IF_NECESSARY |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 347 | |
| 348 | def __str__(self): |
| 349 | return self.name |
| 350 | |
| 351 | def __repr__(self): |
| 352 | return '<Project %s>' % (self.name) |
| 353 | |
| 354 | |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 355 | class Job(object): |
| 356 | def __init__(self, name): |
James E. Blair | 222d498 | 2012-07-16 09:31:19 -0700 | [diff] [blame] | 357 | # If you add attributes here, be sure to add them to the copy method. |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 358 | self.name = name |
| 359 | self.failure_message = None |
| 360 | self.success_message = None |
James E. Blair | 6aea36d | 2012-12-17 13:03:24 -0800 | [diff] [blame] | 361 | self.failure_pattern = None |
| 362 | self.success_pattern = None |
James E. Blair | e5a847f | 2012-07-10 15:29:14 -0700 | [diff] [blame] | 363 | self.parameter_function = None |
James E. Blair | 222d498 | 2012-07-16 09:31:19 -0700 | [diff] [blame] | 364 | self.hold_following_changes = False |
James E. Blair | 4ec821f | 2012-08-23 15:28:28 -0700 | [diff] [blame] | 365 | self.voting = True |
James E. Blair | e421a23 | 2012-07-25 16:59:21 -0700 | [diff] [blame] | 366 | self.branches = [] |
| 367 | self._branches = [] |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 368 | |
| 369 | def __str__(self): |
| 370 | return self.name |
| 371 | |
| 372 | def __repr__(self): |
| 373 | return '<Job %s>' % (self.name) |
| 374 | |
James E. Blair | b095465 | 2012-06-01 11:32:01 -0700 | [diff] [blame] | 375 | def copy(self, other): |
| 376 | self.failure_message = other.failure_message |
James E. Blair | 1543741 | 2012-07-18 08:45:55 -0700 | [diff] [blame] | 377 | self.success_message = other.success_message |
James E. Blair | 6aea36d | 2012-12-17 13:03:24 -0800 | [diff] [blame] | 378 | self.failure_pattern = other.failure_pattern |
| 379 | self.success_pattern = other.success_pattern |
James E. Blair | 222d498 | 2012-07-16 09:31:19 -0700 | [diff] [blame] | 380 | self.parameter_function = other.parameter_function |
| 381 | self.hold_following_changes = other.hold_following_changes |
James E. Blair | 4ec821f | 2012-08-23 15:28:28 -0700 | [diff] [blame] | 382 | self.voting = other.voting |
James E. Blair | e421a23 | 2012-07-25 16:59:21 -0700 | [diff] [blame] | 383 | self.branches = other.branches[:] |
| 384 | self._branches = other._branches[:] |
James E. Blair | b095465 | 2012-06-01 11:32:01 -0700 | [diff] [blame] | 385 | |
James E. Blair | e421a23 | 2012-07-25 16:59:21 -0700 | [diff] [blame] | 386 | def changeMatches(self, change): |
| 387 | if not self.branches: |
James E. Blair | e5a847f | 2012-07-10 15:29:14 -0700 | [diff] [blame] | 388 | return True |
James E. Blair | e421a23 | 2012-07-25 16:59:21 -0700 | [diff] [blame] | 389 | for branch in self.branches: |
James E. Blair | 45865f3 | 2012-10-05 09:39:46 -0700 | [diff] [blame] | 390 | if hasattr(change, 'branch') and branch.match(change.branch): |
| 391 | return True |
| 392 | if hasattr(change, 'ref') and branch.match(change.ref): |
James E. Blair | e5a847f | 2012-07-10 15:29:14 -0700 | [diff] [blame] | 393 | return True |
| 394 | return False |
| 395 | |
James E. Blair | 1e8dd89 | 2012-05-30 09:15:05 -0700 | [diff] [blame] | 396 | |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 397 | class JobTree(object): |
| 398 | """ A JobTree represents an instance of one Job, and holds JobTrees |
| 399 | whose jobs should be run if that Job succeeds. A root node of a |
| 400 | JobTree will have no associated Job. """ |
| 401 | |
| 402 | def __init__(self, job): |
| 403 | self.job = job |
| 404 | self.job_trees = [] |
| 405 | |
| 406 | def addJob(self, job): |
| 407 | if job not in [x.job for x in self.job_trees]: |
| 408 | t = JobTree(job) |
| 409 | self.job_trees.append(t) |
| 410 | return t |
| 411 | |
| 412 | def getJobs(self): |
| 413 | jobs = [] |
| 414 | for x in self.job_trees: |
| 415 | jobs.append(x.job) |
| 416 | jobs.extend(x.getJobs()) |
| 417 | return jobs |
| 418 | |
| 419 | def getJobTreeForJob(self, job): |
| 420 | if self.job == job: |
| 421 | return self |
| 422 | for tree in self.job_trees: |
| 423 | ret = tree.getJobTreeForJob(job) |
| 424 | if ret: |
| 425 | return ret |
| 426 | return None |
| 427 | |
James E. Blair | 1e8dd89 | 2012-05-30 09:15:05 -0700 | [diff] [blame] | 428 | |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 429 | class Build(object): |
| 430 | def __init__(self, job, uuid): |
| 431 | self.job = job |
| 432 | self.uuid = uuid |
| 433 | self.base_url = None |
| 434 | self.url = None |
| 435 | self.number = None |
| 436 | self.result = None |
| 437 | self.build_set = None |
| 438 | self.launch_time = time.time() |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 439 | |
| 440 | def __repr__(self): |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 441 | return '<Build %s of %s>' % (self.uuid, self.job.name) |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 442 | |
James E. Blair | 1e8dd89 | 2012-05-30 09:15:05 -0700 | [diff] [blame] | 443 | |
James E. Blair | 7e530ad | 2012-07-03 16:12:28 -0700 | [diff] [blame] | 444 | class BuildSet(object): |
James E. Blair | 11700c3 | 2012-07-05 17:50:05 -0700 | [diff] [blame] | 445 | def __init__(self, change): |
| 446 | self.change = change |
| 447 | self.other_changes = [] |
James E. Blair | 7e530ad | 2012-07-03 16:12:28 -0700 | [diff] [blame] | 448 | self.builds = {} |
James E. Blair | 11700c3 | 2012-07-05 17:50:05 -0700 | [diff] [blame] | 449 | self.result = None |
| 450 | self.next_build_set = None |
| 451 | self.previous_build_set = None |
James E. Blair | 4886cc1 | 2012-07-18 15:39:41 -0700 | [diff] [blame] | 452 | self.ref = None |
James E. Blair | 81515ad | 2012-10-01 18:29:08 -0700 | [diff] [blame] | 453 | self.commit = None |
James E. Blair | 973721f | 2012-08-15 10:19:43 -0700 | [diff] [blame] | 454 | self.unable_to_merge = False |
James E. Blair | 7e530ad | 2012-07-03 16:12:28 -0700 | [diff] [blame] | 455 | |
James E. Blair | 4886cc1 | 2012-07-18 15:39:41 -0700 | [diff] [blame] | 456 | def setConfiguration(self): |
James E. Blair | 11700c3 | 2012-07-05 17:50:05 -0700 | [diff] [blame] | 457 | # The change isn't enqueued until after it's created |
| 458 | # so we don't know what the other changes ahead will be |
| 459 | # until jobs start. |
| 460 | if not self.other_changes: |
| 461 | next_change = self.change.change_ahead |
| 462 | while next_change: |
| 463 | self.other_changes.append(next_change) |
| 464 | next_change = next_change.change_ahead |
James E. Blair | 4886cc1 | 2012-07-18 15:39:41 -0700 | [diff] [blame] | 465 | if not self.ref: |
| 466 | self.ref = 'Z' + uuid4().hex |
| 467 | |
James E. Blair | 4886cc1 | 2012-07-18 15:39:41 -0700 | [diff] [blame] | 468 | def addBuild(self, build): |
| 469 | self.builds[build.job.name] = build |
| 470 | build.build_set = self |
James E. Blair | 11700c3 | 2012-07-05 17:50:05 -0700 | [diff] [blame] | 471 | |
James E. Blair | 7e530ad | 2012-07-03 16:12:28 -0700 | [diff] [blame] | 472 | def getBuild(self, job_name): |
| 473 | return self.builds.get(job_name) |
| 474 | |
James E. Blair | 11700c3 | 2012-07-05 17:50:05 -0700 | [diff] [blame] | 475 | def getBuilds(self): |
| 476 | keys = self.builds.keys() |
| 477 | keys.sort() |
| 478 | return [self.builds.get(x) for x in keys] |
| 479 | |
James E. Blair | 7e530ad | 2012-07-03 16:12:28 -0700 | [diff] [blame] | 480 | |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 481 | class Changeish(object): |
| 482 | """Something like a change; either a change or a ref""" |
| 483 | is_reportable = False |
James E. Blair | 3266340 | 2012-06-01 10:04:18 -0700 | [diff] [blame] | 484 | |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 485 | def __init__(self, project): |
| 486 | self.project = project |
James E. Blair | 7e530ad | 2012-07-03 16:12:28 -0700 | [diff] [blame] | 487 | self.build_sets = [] |
James E. Blair | caec0c5 | 2012-08-22 14:52:22 -0700 | [diff] [blame] | 488 | self.dequeued_needing_change = False |
James E. Blair | 11700c3 | 2012-07-05 17:50:05 -0700 | [diff] [blame] | 489 | self.current_build_set = BuildSet(self) |
| 490 | self.build_sets.append(self.current_build_set) |
James E. Blair | e048707 | 2012-08-29 17:38:31 -0700 | [diff] [blame] | 491 | self.change_ahead = None |
| 492 | self.change_behind = None |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 493 | |
James E. Blair | 0dc8ba9 | 2012-07-16 14:23:52 -0700 | [diff] [blame] | 494 | def equals(self, other): |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 495 | raise NotImplementedError() |
James E. Blair | 0dc8ba9 | 2012-07-16 14:23:52 -0700 | [diff] [blame] | 496 | |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 497 | def filterJobs(self, jobs): |
James E. Blair | e421a23 | 2012-07-25 16:59:21 -0700 | [diff] [blame] | 498 | return filter(lambda job: job.changeMatches(self), jobs) |
James E. Blair | e5a847f | 2012-07-10 15:29:14 -0700 | [diff] [blame] | 499 | |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 500 | def resetAllBuilds(self): |
James E. Blair | 11700c3 | 2012-07-05 17:50:05 -0700 | [diff] [blame] | 501 | old = self.current_build_set |
| 502 | self.current_build_set.result = 'CANCELED' |
| 503 | self.current_build_set = BuildSet(self) |
| 504 | old.next_build_set = self.current_build_set |
| 505 | self.current_build_set.previous_build_set = old |
James E. Blair | 7e530ad | 2012-07-03 16:12:28 -0700 | [diff] [blame] | 506 | self.build_sets.append(self.current_build_set) |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 507 | |
| 508 | def addBuild(self, build): |
James E. Blair | 7e530ad | 2012-07-03 16:12:28 -0700 | [diff] [blame] | 509 | self.current_build_set.addBuild(build) |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 510 | |
James E. Blair | 1e8dd89 | 2012-05-30 09:15:05 -0700 | [diff] [blame] | 511 | |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 512 | class Change(Changeish): |
| 513 | is_reportable = True |
| 514 | |
| 515 | def __init__(self, project): |
| 516 | super(Change, self).__init__(project) |
| 517 | self.branch = None |
| 518 | self.number = None |
| 519 | self.url = None |
| 520 | self.patchset = None |
| 521 | self.refspec = None |
| 522 | |
| 523 | self.reported = False |
| 524 | self.needs_change = None |
| 525 | self.needed_by_changes = [] |
| 526 | self.is_current_patchset = True |
| 527 | self.can_merge = False |
| 528 | self.is_merged = False |
| 529 | |
| 530 | def _id(self): |
James E. Blair | be765db | 2012-08-07 08:36:20 -0700 | [diff] [blame] | 531 | return '%s,%s' % (self.number, self.patchset) |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 532 | |
| 533 | def __repr__(self): |
| 534 | return '<Change 0x%x %s>' % (id(self), self._id()) |
| 535 | |
| 536 | def equals(self, other): |
Zhongyue Luo | aa85ebf | 2012-09-21 16:38:33 +0800 | [diff] [blame] | 537 | if self.number == other.number and self.patchset == other.patchset: |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 538 | return True |
| 539 | return False |
| 540 | |
| 541 | def setReportedResult(self, result): |
| 542 | self.current_build_set.result = result |
| 543 | |
| 544 | |
| 545 | class Ref(Changeish): |
| 546 | is_reportable = False |
| 547 | |
| 548 | def __init__(self, project): |
James E. Blair | be765db | 2012-08-07 08:36:20 -0700 | [diff] [blame] | 549 | super(Ref, self).__init__(project) |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 550 | self.ref = None |
| 551 | self.oldrev = None |
| 552 | self.newrev = None |
| 553 | |
James E. Blair | be765db | 2012-08-07 08:36:20 -0700 | [diff] [blame] | 554 | def _id(self): |
| 555 | return self.newrev |
| 556 | |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 557 | def equals(self, other): |
James E. Blair | 9358c61 | 2012-09-28 08:29:39 -0700 | [diff] [blame] | 558 | if (self.project == other.project |
| 559 | and self.ref == other.ref |
| 560 | and self.newrev == other.newrev): |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 561 | return True |
| 562 | return False |
| 563 | |
| 564 | |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 565 | class TriggerEvent(object): |
| 566 | def __init__(self): |
| 567 | self.data = None |
James E. Blair | 3266340 | 2012-06-01 10:04:18 -0700 | [diff] [blame] | 568 | # common |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 569 | self.type = None |
| 570 | self.project_name = None |
Antoine Musso | b4e809e | 2012-12-06 16:58:06 +0100 | [diff] [blame] | 571 | # Representation of the user account that performed the event. |
| 572 | self.account = None |
James E. Blair | 3266340 | 2012-06-01 10:04:18 -0700 | [diff] [blame] | 573 | # patchset-created, comment-added, etc. |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 574 | self.change_number = None |
Clark Boylan | fc56df3 | 2012-06-28 15:25:57 -0700 | [diff] [blame] | 575 | self.change_url = None |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 576 | self.patch_number = None |
James E. Blair | a03262c | 2012-05-30 09:41:16 -0700 | [diff] [blame] | 577 | self.refspec = None |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 578 | self.approvals = [] |
| 579 | self.branch = None |
Clark Boylan | b9bcb40 | 2012-06-29 17:44:05 -0700 | [diff] [blame] | 580 | self.comment = None |
James E. Blair | 3266340 | 2012-06-01 10:04:18 -0700 | [diff] [blame] | 581 | # ref-updated |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 582 | self.ref = None |
James E. Blair | 3266340 | 2012-06-01 10:04:18 -0700 | [diff] [blame] | 583 | self.oldrev = None |
James E. Blair | 89cae0f | 2012-07-18 11:18:32 -0700 | [diff] [blame] | 584 | self.newrev = None |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 585 | |
James E. Blair | 9f9667e | 2012-06-12 17:51:08 -0700 | [diff] [blame] | 586 | def __repr__(self): |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 587 | ret = '<TriggerEvent %s %s' % (self.type, self.project_name) |
James E. Blair | 1e8dd89 | 2012-05-30 09:15:05 -0700 | [diff] [blame] | 588 | |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 589 | if self.branch: |
| 590 | ret += " %s" % self.branch |
| 591 | if self.change_number: |
| 592 | ret += " %s,%s" % (self.change_number, self.patch_number) |
| 593 | if self.approvals: |
James E. Blair | 1e8dd89 | 2012-05-30 09:15:05 -0700 | [diff] [blame] | 594 | ret += ' ' + ', '.join( |
| 595 | ['%s:%s' % (a['type'], a['value']) for a in self.approvals]) |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 596 | ret += '>' |
| 597 | |
| 598 | return ret |
| 599 | |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 600 | def getChange(self, project, trigger): |
James E. Blair | e421a23 | 2012-07-25 16:59:21 -0700 | [diff] [blame] | 601 | # TODO: make the scheduler deal with events (which may have |
| 602 | # changes) rather than changes so that we don't have to create |
| 603 | # "fake" changes for events that aren't associated with changes. |
| 604 | |
| 605 | if self.change_number: |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 606 | change = trigger.getChange(self.change_number, self.patch_number) |
James E. Blair | e421a23 | 2012-07-25 16:59:21 -0700 | [diff] [blame] | 607 | if self.ref: |
James E. Blair | 4aea70c | 2012-07-26 14:23:24 -0700 | [diff] [blame] | 608 | change = Ref(project) |
James E. Blair | e421a23 | 2012-07-25 16:59:21 -0700 | [diff] [blame] | 609 | change.ref = self.ref |
| 610 | change.oldrev = self.oldrev |
| 611 | change.newrev = self.newrev |
James E. Blair | c44b138 | 2012-12-23 09:39:55 -0800 | [diff] [blame^] | 612 | change.url = trigger.getGitwebUrl(project, sha=self.newrev) |
James E. Blair | e421a23 | 2012-07-25 16:59:21 -0700 | [diff] [blame] | 613 | |
| 614 | return change |
| 615 | |
James E. Blair | 1e8dd89 | 2012-05-30 09:15:05 -0700 | [diff] [blame] | 616 | |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 617 | class EventFilter(object): |
James E. Blair | e5a847f | 2012-07-10 15:29:14 -0700 | [diff] [blame] | 618 | def __init__(self, types=[], branches=[], refs=[], approvals={}, |
Antoine Musso | b4e809e | 2012-12-06 16:58:06 +0100 | [diff] [blame] | 619 | comment_filters=[], email_filters=[]): |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 620 | self._types = types |
| 621 | self._branches = branches |
| 622 | self._refs = refs |
| 623 | self.types = [re.compile(x) for x in types] |
| 624 | self.branches = [re.compile(x) for x in branches] |
| 625 | self.refs = [re.compile(x) for x in refs] |
Clark Boylan | b9bcb40 | 2012-06-29 17:44:05 -0700 | [diff] [blame] | 626 | self.comment_filters = [re.compile(x) for x in comment_filters] |
Antoine Musso | b4e809e | 2012-12-06 16:58:06 +0100 | [diff] [blame] | 627 | self.email_filters = [re.compile(x) for x in email_filters] |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 628 | self.approvals = approvals |
| 629 | |
James E. Blair | 9f9667e | 2012-06-12 17:51:08 -0700 | [diff] [blame] | 630 | def __repr__(self): |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 631 | ret = '<EventFilter' |
James E. Blair | 1e8dd89 | 2012-05-30 09:15:05 -0700 | [diff] [blame] | 632 | |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 633 | if self._types: |
| 634 | ret += ' types: %s' % ', '.join(self._types) |
| 635 | if self._branches: |
| 636 | ret += ' branches: %s' % ', '.join(self._branches) |
| 637 | if self._refs: |
| 638 | ret += ' refs: %s' % ', '.join(self._refs) |
| 639 | if self.approvals: |
James E. Blair | 1e8dd89 | 2012-05-30 09:15:05 -0700 | [diff] [blame] | 640 | ret += ' approvals: %s' % ', '.join( |
| 641 | ['%s:%s' % a for a in self.approvals.items()]) |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 642 | ret += '>' |
| 643 | |
| 644 | return ret |
| 645 | |
| 646 | def matches(self, event): |
| 647 | def normalizeCategory(name): |
| 648 | name = name.lower() |
| 649 | return re.sub(' ', '-', name) |
| 650 | |
| 651 | # event types are ORed |
| 652 | matches_type = False |
| 653 | for etype in self.types: |
| 654 | if etype.match(event.type): |
| 655 | matches_type = True |
| 656 | if self.types and not matches_type: |
| 657 | return False |
| 658 | |
| 659 | # branches are ORed |
| 660 | matches_branch = False |
| 661 | for branch in self.branches: |
| 662 | if branch.match(event.branch): |
| 663 | matches_branch = True |
| 664 | if self.branches and not matches_branch: |
| 665 | return False |
| 666 | |
| 667 | # refs are ORed |
| 668 | matches_ref = False |
| 669 | for ref in self.refs: |
| 670 | if ref.match(event.ref): |
| 671 | matches_ref = True |
| 672 | if self.refs and not matches_ref: |
| 673 | return False |
| 674 | |
Clark Boylan | b9bcb40 | 2012-06-29 17:44:05 -0700 | [diff] [blame] | 675 | # comment_filters are ORed |
| 676 | matches_comment_filter = False |
| 677 | for comment_filter in self.comment_filters: |
| 678 | if (event.comment is not None and |
| 679 | comment_filter.search(event.comment)): |
| 680 | matches_comment_filter = True |
| 681 | if self.comment_filters and not matches_comment_filter: |
| 682 | return False |
| 683 | |
Antoine Musso | b4e809e | 2012-12-06 16:58:06 +0100 | [diff] [blame] | 684 | # We better have an account provided by Gerrit to do |
| 685 | # email filtering. |
| 686 | if event.account is not None: |
| 687 | # email_filters are ORed |
| 688 | matches_email_filter = False |
| 689 | for email_filter in self.email_filters: |
| 690 | account_email = event.account.get('email') |
| 691 | if (account_email is not None and |
| 692 | email_filter.search(account_email)): |
| 693 | matches_email_filter = True |
| 694 | if self.email_filters and not matches_email_filter: |
| 695 | return False |
| 696 | |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 697 | # approvals are ANDed |
| 698 | for category, value in self.approvals.items(): |
| 699 | matches_approval = False |
| 700 | for eapproval in event.approvals: |
| 701 | if (normalizeCategory(eapproval['description']) == category and |
| 702 | int(eapproval['value']) == int(value)): |
| 703 | matches_approval = True |
James E. Blair | 1e8dd89 | 2012-05-30 09:15:05 -0700 | [diff] [blame] | 704 | if not matches_approval: |
| 705 | return False |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 706 | return True |