James E. Blair | dbfe1cd | 2015-02-07 11:41:19 -0800 | [diff] [blame] | 1 | # Copyright 2012-2015 Hewlett-Packard Development Company, L.P. |
James E. Blair | 4795838 | 2013-01-10 17:26:02 -0800 | [diff] [blame] | 2 | # Copyright 2013 OpenStack Foundation |
Antoine Musso | 80edd5a | 2013-02-13 15:37:53 +0100 | [diff] [blame] | 3 | # Copyright 2013 Antoine "hashar" Musso |
| 4 | # Copyright 2013 Wikimedia Foundation Inc. |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 5 | # |
| 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 7 | # not use this file except in compliance with the License. You may obtain |
| 8 | # a copy of the License at |
| 9 | # |
| 10 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | # |
| 12 | # Unless required by applicable law or agreed to in writing, software |
| 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 15 | # License for the specific language governing permissions and limitations |
| 16 | # under the License. |
| 17 | |
James E. Blair | 71e9412 | 2012-12-24 17:53:08 -0800 | [diff] [blame] | 18 | import extras |
James E. Blair | 8dbd56a | 2012-12-22 10:55:10 -0800 | [diff] [blame] | 19 | import json |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 20 | import logging |
Zhongyue Luo | 1c860d7 | 2012-07-19 11:03:56 +0800 | [diff] [blame] | 21 | import os |
James E. Blair | 5d5bc2b | 2012-07-06 10:24:01 -0700 | [diff] [blame] | 22 | import pickle |
Monty Taylor | b934c1a | 2017-06-16 19:31:47 -0500 | [diff] [blame] | 23 | import queue |
James E. Blair | 8b2a147 | 2017-02-19 15:33:55 -0800 | [diff] [blame] | 24 | import socket |
James E. Blair | 36658cf | 2013-12-06 17:53:48 -0800 | [diff] [blame] | 25 | import sys |
Zhongyue Luo | 1c860d7 | 2012-07-19 11:03:56 +0800 | [diff] [blame] | 26 | import threading |
James E. Blair | 71e9412 | 2012-12-24 17:53:08 -0800 | [diff] [blame] | 27 | import time |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 28 | |
Joshua Hesketh | 0aa7e8b | 2016-07-14 00:12:25 +1000 | [diff] [blame] | 29 | from zuul import configloader |
Morgan Fainberg | 9c4700a | 2016-05-30 14:25:19 -0700 | [diff] [blame] | 30 | from zuul import model |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 31 | from zuul import exceptions |
Sergey Lukjanov | 5ba961b | 2013-12-27 01:21:04 +0400 | [diff] [blame] | 32 | from zuul import version as zuul_version |
Tristan Cacqueray | 91601d7 | 2017-06-15 06:00:12 +0000 | [diff] [blame] | 33 | from zuul.lib.config import get_default |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 34 | |
James E. Blair | 1e8dd89 | 2012-05-30 09:15:05 -0700 | [diff] [blame] | 35 | |
James E. Blair | 468c851 | 2013-12-06 13:27:19 -0800 | [diff] [blame] | 36 | class ManagementEvent(object): |
| 37 | """An event that should be processed within the main queue run loop""" |
| 38 | def __init__(self): |
| 39 | self._wait_event = threading.Event() |
Morgan Fainberg | 1b9bd78 | 2016-05-30 14:03:30 -0700 | [diff] [blame] | 40 | self._exc_info = None |
James E. Blair | 468c851 | 2013-12-06 13:27:19 -0800 | [diff] [blame] | 41 | |
Morgan Fainberg | 1b9bd78 | 2016-05-30 14:03:30 -0700 | [diff] [blame] | 42 | def exception(self, exc_info): |
| 43 | self._exc_info = exc_info |
James E. Blair | 36658cf | 2013-12-06 17:53:48 -0800 | [diff] [blame] | 44 | self._wait_event.set() |
| 45 | |
| 46 | def done(self): |
James E. Blair | 468c851 | 2013-12-06 13:27:19 -0800 | [diff] [blame] | 47 | self._wait_event.set() |
| 48 | |
| 49 | def wait(self, timeout=None): |
| 50 | self._wait_event.wait(timeout) |
Morgan Fainberg | 1b9bd78 | 2016-05-30 14:03:30 -0700 | [diff] [blame] | 51 | if self._exc_info: |
Thomas Bechtold | 7f68ec4 | 2017-06-30 14:24:52 +0200 | [diff] [blame] | 52 | # sys.exc_info returns (type, value, traceback) |
| 53 | type_, exception_instance, traceback = self._exc_info |
| 54 | raise exception_instance.with_traceback(traceback) |
James E. Blair | 468c851 | 2013-12-06 13:27:19 -0800 | [diff] [blame] | 55 | return self._wait_event.is_set() |
| 56 | |
| 57 | |
| 58 | class ReconfigureEvent(ManagementEvent): |
| 59 | """Reconfigure the scheduler. The layout will be (re-)loaded from |
| 60 | the path specified in the configuration. |
| 61 | |
| 62 | :arg ConfigParser config: the new configuration |
| 63 | """ |
| 64 | def __init__(self, config): |
| 65 | super(ReconfigureEvent, self).__init__() |
| 66 | self.config = config |
| 67 | |
| 68 | |
James E. Blair | 646322f | 2017-01-27 15:50:34 -0800 | [diff] [blame] | 69 | class TenantReconfigureEvent(ManagementEvent): |
| 70 | """Reconfigure the given tenant. The layout will be (re-)loaded from |
| 71 | the path specified in the configuration. |
| 72 | |
| 73 | :arg Tenant tenant: the tenant to reconfigure |
James E. Blair | a615c36 | 2017-10-02 17:34:42 -0700 | [diff] [blame^] | 74 | :arg Project project: if supplied, clear the cached configuration |
| 75 | from this project first |
James E. Blair | 646322f | 2017-01-27 15:50:34 -0800 | [diff] [blame] | 76 | """ |
James E. Blair | a615c36 | 2017-10-02 17:34:42 -0700 | [diff] [blame^] | 77 | def __init__(self, tenant, project): |
James E. Blair | 646322f | 2017-01-27 15:50:34 -0800 | [diff] [blame] | 78 | super(TenantReconfigureEvent, self).__init__() |
| 79 | self.tenant = tenant |
James E. Blair | a615c36 | 2017-10-02 17:34:42 -0700 | [diff] [blame^] | 80 | self.project = project |
James E. Blair | 646322f | 2017-01-27 15:50:34 -0800 | [diff] [blame] | 81 | |
| 82 | |
James E. Blair | 36658cf | 2013-12-06 17:53:48 -0800 | [diff] [blame] | 83 | class PromoteEvent(ManagementEvent): |
| 84 | """Promote one or more changes to the head of the queue. |
| 85 | |
Paul Belanger | baca313 | 2016-11-04 12:49:54 -0400 | [diff] [blame] | 86 | :arg str tenant_name: the name of the tenant |
James E. Blair | 36658cf | 2013-12-06 17:53:48 -0800 | [diff] [blame] | 87 | :arg str pipeline_name: the name of the pipeline |
| 88 | :arg list change_ids: a list of strings of change ids in the form |
| 89 | 1234,1 |
| 90 | """ |
| 91 | |
Paul Belanger | baca313 | 2016-11-04 12:49:54 -0400 | [diff] [blame] | 92 | def __init__(self, tenant_name, pipeline_name, change_ids): |
James E. Blair | 36658cf | 2013-12-06 17:53:48 -0800 | [diff] [blame] | 93 | super(PromoteEvent, self).__init__() |
Paul Belanger | baca313 | 2016-11-04 12:49:54 -0400 | [diff] [blame] | 94 | self.tenant_name = tenant_name |
James E. Blair | 36658cf | 2013-12-06 17:53:48 -0800 | [diff] [blame] | 95 | self.pipeline_name = pipeline_name |
| 96 | self.change_ids = change_ids |
| 97 | |
| 98 | |
James E. Blair | d27a96d | 2014-07-10 13:25:13 -0700 | [diff] [blame] | 99 | class EnqueueEvent(ManagementEvent): |
| 100 | """Enqueue a change into a pipeline |
| 101 | |
| 102 | :arg TriggerEvent trigger_event: a TriggerEvent describing the |
| 103 | trigger, pipeline, and change to enqueue |
| 104 | """ |
| 105 | |
| 106 | def __init__(self, trigger_event): |
| 107 | super(EnqueueEvent, self).__init__() |
| 108 | self.trigger_event = trigger_event |
| 109 | |
| 110 | |
James E. Blair | a84f0e4 | 2014-02-06 07:09:22 -0800 | [diff] [blame] | 111 | class ResultEvent(object): |
| 112 | """An event that needs to modify the pipeline state due to a |
| 113 | result from an external system.""" |
| 114 | |
| 115 | pass |
| 116 | |
| 117 | |
| 118 | class BuildStartedEvent(ResultEvent): |
| 119 | """A build has started. |
| 120 | |
| 121 | :arg Build build: The build which has started. |
| 122 | """ |
| 123 | |
| 124 | def __init__(self, build): |
| 125 | self.build = build |
| 126 | |
| 127 | |
| 128 | class BuildCompletedEvent(ResultEvent): |
| 129 | """A build has completed |
| 130 | |
| 131 | :arg Build build: The build which has completed. |
| 132 | """ |
| 133 | |
| 134 | def __init__(self, build): |
| 135 | self.build = build |
| 136 | |
| 137 | |
James E. Blair | 4076e2b | 2014-01-28 12:42:20 -0800 | [diff] [blame] | 138 | class MergeCompletedEvent(ResultEvent): |
| 139 | """A remote merge operation has completed |
| 140 | |
| 141 | :arg BuildSet build_set: The build_set which is ready. |
James E. Blair | 4076e2b | 2014-01-28 12:42:20 -0800 | [diff] [blame] | 142 | :arg bool merged: Whether the merge succeeded (changes with refs). |
| 143 | :arg bool updated: Whether the repo was updated (changes without refs). |
| 144 | :arg str commit: The SHA of the merged commit (changes with refs). |
James E. Blair | 1960d68 | 2017-04-28 15:44:14 -0700 | [diff] [blame] | 145 | :arg dict repo_state: The starting repo state before the merge. |
James E. Blair | 4076e2b | 2014-01-28 12:42:20 -0800 | [diff] [blame] | 146 | """ |
| 147 | |
Tobias Henkel | 34ee088 | 2017-07-31 22:26:12 +0200 | [diff] [blame] | 148 | def __init__(self, build_set, merged, updated, commit, |
James E. Blair | 1960d68 | 2017-04-28 15:44:14 -0700 | [diff] [blame] | 149 | files, repo_state): |
James E. Blair | 4076e2b | 2014-01-28 12:42:20 -0800 | [diff] [blame] | 150 | self.build_set = build_set |
James E. Blair | 4076e2b | 2014-01-28 12:42:20 -0800 | [diff] [blame] | 151 | self.merged = merged |
| 152 | self.updated = updated |
| 153 | self.commit = commit |
James E. Blair | 8b1dc3f | 2016-07-05 16:49:00 -0700 | [diff] [blame] | 154 | self.files = files |
James E. Blair | 1960d68 | 2017-04-28 15:44:14 -0700 | [diff] [blame] | 155 | self.repo_state = repo_state |
James E. Blair | 4076e2b | 2014-01-28 12:42:20 -0800 | [diff] [blame] | 156 | |
| 157 | |
James E. Blair | 8d69239 | 2016-04-08 17:47:58 -0700 | [diff] [blame] | 158 | class NodesProvisionedEvent(ResultEvent): |
| 159 | """Nodes have been provisioned for a build_set |
| 160 | |
| 161 | :arg BuildSet build_set: The build_set which has nodes. |
| 162 | :arg list of Node objects nodes: The provisioned nodes |
| 163 | """ |
| 164 | |
| 165 | def __init__(self, request): |
| 166 | self.request = request |
| 167 | |
| 168 | |
Maru Newby | 3fe5f85 | 2015-01-13 04:22:14 +0000 | [diff] [blame] | 169 | def toList(item): |
| 170 | if not item: |
| 171 | return [] |
| 172 | if isinstance(item, list): |
| 173 | return item |
| 174 | return [item] |
| 175 | |
| 176 | |
James E. Blair | e9d45c3 | 2012-05-31 09:56:45 -0700 | [diff] [blame] | 177 | class Scheduler(threading.Thread): |
James E. Blair | e4de4f4 | 2017-01-19 10:35:24 -0800 | [diff] [blame] | 178 | """The engine of Zuul. |
| 179 | |
| 180 | The Scheduler is reponsible for recieving events and dispatching |
| 181 | them to appropriate components (including pipeline managers, |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 182 | mergers and executors). |
James E. Blair | e4de4f4 | 2017-01-19 10:35:24 -0800 | [diff] [blame] | 183 | |
| 184 | It runs a single threaded main loop which processes events |
| 185 | received one at a time and takes action as appropriate. Other |
| 186 | parts of Zuul may run in their own thread, but synchronization is |
| 187 | performed within the scheduler to reduce or eliminate the need for |
| 188 | locking in most circumstances. |
| 189 | |
| 190 | The main daemon will have one instance of the Scheduler class |
| 191 | running which will persist for the life of the process. The |
| 192 | Scheduler instance is supplied to other Zuul components so that |
| 193 | they can submit events or otherwise communicate with other |
| 194 | components. |
| 195 | |
| 196 | """ |
| 197 | |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 198 | log = logging.getLogger("zuul.Scheduler") |
| 199 | |
James E. Blair | e4d229c | 2016-05-25 15:25:41 -0700 | [diff] [blame] | 200 | def __init__(self, config, testonly=False): |
James E. Blair | e9d45c3 | 2012-05-31 09:56:45 -0700 | [diff] [blame] | 201 | threading.Thread.__init__(self) |
James E. Blair | 8a6f0c2 | 2013-07-01 12:31:34 -0400 | [diff] [blame] | 202 | self.daemon = True |
James E. Blair | 8b2a147 | 2017-02-19 15:33:55 -0800 | [diff] [blame] | 203 | self.hostname = socket.gethostname() |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 204 | self.wake_event = threading.Event() |
James E. Blair | cdccd97 | 2013-07-01 12:10:22 -0700 | [diff] [blame] | 205 | self.layout_lock = threading.Lock() |
James E. Blair | a84f0e4 | 2014-02-06 07:09:22 -0800 | [diff] [blame] | 206 | self.run_handler_lock = threading.Lock() |
James E. Blair | 5d5bc2b | 2012-07-06 10:24:01 -0700 | [diff] [blame] | 207 | self._pause = False |
James E. Blair | 5d5bc2b | 2012-07-06 10:24:01 -0700 | [diff] [blame] | 208 | self._exit = False |
James E. Blair | b0fcae4 | 2012-07-17 11:12:10 -0700 | [diff] [blame] | 209 | self._stopped = False |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 210 | self.executor = None |
James E. Blair | 4076e2b | 2014-01-28 12:42:20 -0800 | [diff] [blame] | 211 | self.merger = None |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 212 | self.connections = None |
James E. Blair | 552b54f | 2016-07-22 13:55:32 -0700 | [diff] [blame] | 213 | self.statsd = extras.try_import('statsd.statsd') |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 214 | # TODO(jeblair): fix this |
Joshua Hesketh | 352264b | 2015-08-11 23:42:08 +1000 | [diff] [blame] | 215 | # Despite triggers being part of the pipeline, there is one trigger set |
| 216 | # per scheduler. The pipeline handles the trigger filters but since |
| 217 | # the events are handled by the scheduler itself it needs to handle |
| 218 | # the loading of the triggers. |
| 219 | # self.triggers['connection_name'] = triggerObject |
James E. Blair | 6c358e7 | 2013-07-29 17:06:47 -0700 | [diff] [blame] | 220 | self.triggers = dict() |
Joshua Hesketh | 352264b | 2015-08-11 23:42:08 +1000 | [diff] [blame] | 221 | self.config = config |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 222 | |
Monty Taylor | b934c1a | 2017-06-16 19:31:47 -0500 | [diff] [blame] | 223 | self.trigger_event_queue = queue.Queue() |
| 224 | self.result_event_queue = queue.Queue() |
| 225 | self.management_event_queue = queue.Queue() |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 226 | self.abide = model.Abide() |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 227 | |
James E. Blair | e4d229c | 2016-05-25 15:25:41 -0700 | [diff] [blame] | 228 | if not testonly: |
| 229 | time_dir = self._get_time_database_dir() |
| 230 | self.time_database = model.TimeDataBase(time_dir) |
James E. Blair | ce8a213 | 2016-05-19 15:21:52 -0700 | [diff] [blame] | 231 | |
Jeremy Stanley | 98b38de | 2015-06-04 21:20:43 +0000 | [diff] [blame] | 232 | self.zuul_version = zuul_version.version_info.release_string() |
Sergey Lukjanov | 5d0438d | 2013-12-24 03:36:39 +0400 | [diff] [blame] | 233 | self.last_reconfigured = None |
Jesse Keating | 71a47ff | 2017-06-06 11:36:43 -0700 | [diff] [blame] | 234 | self.tenant_last_reconfigured = {} |
David Shrewsbury | ffab07a | 2017-07-24 12:45:07 -0400 | [diff] [blame] | 235 | self.autohold_requests = {} |
Sergey Lukjanov | 5ba961b | 2013-12-27 01:21:04 +0400 | [diff] [blame] | 236 | |
James E. Blair | b0fcae4 | 2012-07-17 11:12:10 -0700 | [diff] [blame] | 237 | def stop(self): |
| 238 | self._stopped = True |
Joshua Hesketh | 352264b | 2015-08-11 23:42:08 +1000 | [diff] [blame] | 239 | self.stopConnections() |
James E. Blair | b0fcae4 | 2012-07-17 11:12:10 -0700 | [diff] [blame] | 240 | self.wake_event.set() |
| 241 | |
Jan Hruban | 7083edd | 2015-08-21 14:00:54 +0200 | [diff] [blame] | 242 | def registerConnections(self, connections, webapp, load=True): |
Joshua Hesketh | 9a25675 | 2016-04-04 13:38:51 +1000 | [diff] [blame] | 243 | # load: whether or not to trigger the onLoad for the connection. This |
| 244 | # is useful for not doing a full load during layout validation. |
Joshua Hesketh | 352264b | 2015-08-11 23:42:08 +1000 | [diff] [blame] | 245 | self.connections = connections |
Jan Hruban | 7083edd | 2015-08-21 14:00:54 +0200 | [diff] [blame] | 246 | self.connections.registerWebapp(webapp) |
Joshua Hesketh | 0aa7e8b | 2016-07-14 00:12:25 +1000 | [diff] [blame] | 247 | self.connections.registerScheduler(self, load) |
Joshua Hesketh | 352264b | 2015-08-11 23:42:08 +1000 | [diff] [blame] | 248 | |
| 249 | def stopConnections(self): |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 250 | self.connections.stop() |
James E. Blair | 14abdf4 | 2015-12-09 16:11:53 -0800 | [diff] [blame] | 251 | |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 252 | def setExecutor(self, executor): |
| 253 | self.executor = executor |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 254 | |
James E. Blair | 4076e2b | 2014-01-28 12:42:20 -0800 | [diff] [blame] | 255 | def setMerger(self, merger): |
| 256 | self.merger = merger |
| 257 | |
James E. Blair | 8d69239 | 2016-04-08 17:47:58 -0700 | [diff] [blame] | 258 | def setNodepool(self, nodepool): |
| 259 | self.nodepool = nodepool |
| 260 | |
James E. Blair | dce6cea | 2016-12-20 16:45:32 -0800 | [diff] [blame] | 261 | def setZooKeeper(self, zk): |
| 262 | self.zk = zk |
| 263 | |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 264 | def addEvent(self, event): |
| 265 | self.log.debug("Adding trigger event: %s" % event) |
| 266 | self.trigger_event_queue.put(event) |
| 267 | self.wake_event.set() |
James E. Blair | f62d428 | 2012-12-31 17:01:50 -0800 | [diff] [blame] | 268 | self.log.debug("Done adding trigger event: %s" % event) |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 269 | |
James E. Blair | 11700c3 | 2012-07-05 17:50:05 -0700 | [diff] [blame] | 270 | def onBuildStarted(self, build): |
| 271 | self.log.debug("Adding start event for build: %s" % build) |
James E. Blair | 71e9412 | 2012-12-24 17:53:08 -0800 | [diff] [blame] | 272 | build.start_time = time.time() |
James E. Blair | a84f0e4 | 2014-02-06 07:09:22 -0800 | [diff] [blame] | 273 | event = BuildStartedEvent(build) |
| 274 | self.result_event_queue.put(event) |
James E. Blair | 11700c3 | 2012-07-05 17:50:05 -0700 | [diff] [blame] | 275 | self.wake_event.set() |
James E. Blair | f62d428 | 2012-12-31 17:01:50 -0800 | [diff] [blame] | 276 | self.log.debug("Done adding start event for build: %s" % build) |
James E. Blair | 11700c3 | 2012-07-05 17:50:05 -0700 | [diff] [blame] | 277 | |
James E. Blair | 196f61a | 2017-06-30 15:42:29 -0700 | [diff] [blame] | 278 | def onBuildCompleted(self, build, result, result_data): |
James E. Blair | f035866 | 2015-07-20 15:19:12 -0700 | [diff] [blame] | 279 | self.log.debug("Adding complete event for build: %s result: %s" % ( |
| 280 | build, result)) |
James E. Blair | 71e9412 | 2012-12-24 17:53:08 -0800 | [diff] [blame] | 281 | build.end_time = time.time() |
James E. Blair | 196f61a | 2017-06-30 15:42:29 -0700 | [diff] [blame] | 282 | build.result_data = result_data |
James E. Blair | f035866 | 2015-07-20 15:19:12 -0700 | [diff] [blame] | 283 | # Note, as soon as the result is set, other threads may act |
| 284 | # upon this, even though the event hasn't been fully |
| 285 | # processed. Ensure that any other data from the event (eg, |
| 286 | # timing) is recorded before setting the result. |
| 287 | build.result = result |
James E. Blair | 23ec1ba | 2013-01-04 18:06:10 -0800 | [diff] [blame] | 288 | try: |
James E. Blair | 552b54f | 2016-07-22 13:55:32 -0700 | [diff] [blame] | 289 | if self.statsd and build.pipeline: |
James E. Blair | 66eeebf | 2013-07-27 17:44:32 -0700 | [diff] [blame] | 290 | jobname = build.job.name.replace('.', '_') |
Timothy Chavez | b233208 | 2015-08-07 20:08:04 -0500 | [diff] [blame] | 291 | key = 'zuul.pipeline.%s.all_jobs' % build.pipeline.name |
James E. Blair | 552b54f | 2016-07-22 13:55:32 -0700 | [diff] [blame] | 292 | self.statsd.incr(key) |
Timothy Chavez | b233208 | 2015-08-07 20:08:04 -0500 | [diff] [blame] | 293 | for label in build.node_labels: |
| 294 | # Jenkins includes the node name in its list of labels, so |
| 295 | # we filter it out here, since that is not statistically |
| 296 | # interesting. |
| 297 | if label == build.node_name: |
| 298 | continue |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 299 | dt = int((build.start_time - build.execute_time) * 1000) |
James E. Blair | 50aacbc | 2015-11-17 14:09:59 -0800 | [diff] [blame] | 300 | key = 'zuul.pipeline.%s.label.%s.wait_time' % ( |
| 301 | build.pipeline.name, label) |
James E. Blair | 552b54f | 2016-07-22 13:55:32 -0700 | [diff] [blame] | 302 | self.statsd.timing(key, dt) |
James E. Blair | 66eeebf | 2013-07-27 17:44:32 -0700 | [diff] [blame] | 303 | key = 'zuul.pipeline.%s.job.%s.%s' % (build.pipeline.name, |
| 304 | jobname, build.result) |
James E. Blair | 23ec1ba | 2013-01-04 18:06:10 -0800 | [diff] [blame] | 305 | if build.result in ['SUCCESS', 'FAILURE'] and build.start_time: |
| 306 | dt = int((build.end_time - build.start_time) * 1000) |
James E. Blair | 552b54f | 2016-07-22 13:55:32 -0700 | [diff] [blame] | 307 | self.statsd.timing(key, dt) |
| 308 | self.statsd.incr(key) |
James E. Blair | 50aacbc | 2015-11-17 14:09:59 -0800 | [diff] [blame] | 309 | |
| 310 | key = 'zuul.pipeline.%s.job.%s.wait_time' % ( |
| 311 | build.pipeline.name, jobname) |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 312 | dt = int((build.start_time - build.execute_time) * 1000) |
James E. Blair | 552b54f | 2016-07-22 13:55:32 -0700 | [diff] [blame] | 313 | self.statsd.timing(key, dt) |
James E. Blair | 23ec1ba | 2013-01-04 18:06:10 -0800 | [diff] [blame] | 314 | except: |
| 315 | self.log.exception("Exception reporting runtime stats") |
James E. Blair | a84f0e4 | 2014-02-06 07:09:22 -0800 | [diff] [blame] | 316 | event = BuildCompletedEvent(build) |
| 317 | self.result_event_queue.put(event) |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 318 | self.wake_event.set() |
James E. Blair | f62d428 | 2012-12-31 17:01:50 -0800 | [diff] [blame] | 319 | self.log.debug("Done adding complete event for build: %s" % build) |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 320 | |
Tobias Henkel | 34ee088 | 2017-07-31 22:26:12 +0200 | [diff] [blame] | 321 | def onMergeCompleted(self, build_set, merged, updated, |
James E. Blair | 1960d68 | 2017-04-28 15:44:14 -0700 | [diff] [blame] | 322 | commit, files, repo_state): |
James E. Blair | 4076e2b | 2014-01-28 12:42:20 -0800 | [diff] [blame] | 323 | self.log.debug("Adding merge complete event for build set: %s" % |
| 324 | build_set) |
Tobias Henkel | 34ee088 | 2017-07-31 22:26:12 +0200 | [diff] [blame] | 325 | event = MergeCompletedEvent(build_set, merged, |
James E. Blair | 1960d68 | 2017-04-28 15:44:14 -0700 | [diff] [blame] | 326 | updated, commit, files, repo_state) |
James E. Blair | 4076e2b | 2014-01-28 12:42:20 -0800 | [diff] [blame] | 327 | self.result_event_queue.put(event) |
| 328 | self.wake_event.set() |
| 329 | |
James E. Blair | 8d69239 | 2016-04-08 17:47:58 -0700 | [diff] [blame] | 330 | def onNodesProvisioned(self, req): |
| 331 | self.log.debug("Adding nodes provisioned event for build set: %s" % |
| 332 | req.build_set) |
| 333 | event = NodesProvisionedEvent(req) |
| 334 | self.result_event_queue.put(event) |
| 335 | self.wake_event.set() |
| 336 | |
James E. Blair | a615c36 | 2017-10-02 17:34:42 -0700 | [diff] [blame^] | 337 | def reconfigureTenant(self, tenant, project): |
| 338 | self.log.debug("Submitting tenant reconfiguration event for " |
| 339 | "%s due to project %s", tenant.name, project) |
| 340 | event = TenantReconfigureEvent(tenant, project) |
James E. Blair | 646322f | 2017-01-27 15:50:34 -0800 | [diff] [blame] | 341 | self.management_event_queue.put(event) |
| 342 | self.wake_event.set() |
| 343 | |
James E. Blair | e9d45c3 | 2012-05-31 09:56:45 -0700 | [diff] [blame] | 344 | def reconfigure(self, config): |
James E. Blair | a615c36 | 2017-10-02 17:34:42 -0700 | [diff] [blame^] | 345 | self.log.debug("Submitting reconfiguration event") |
James E. Blair | 468c851 | 2013-12-06 13:27:19 -0800 | [diff] [blame] | 346 | event = ReconfigureEvent(config) |
| 347 | self.management_event_queue.put(event) |
James E. Blair | e9d45c3 | 2012-05-31 09:56:45 -0700 | [diff] [blame] | 348 | self.wake_event.set() |
| 349 | self.log.debug("Waiting for reconfiguration") |
James E. Blair | 468c851 | 2013-12-06 13:27:19 -0800 | [diff] [blame] | 350 | event.wait() |
James E. Blair | e9d45c3 | 2012-05-31 09:56:45 -0700 | [diff] [blame] | 351 | self.log.debug("Reconfiguration complete") |
Sergey Lukjanov | 5d0438d | 2013-12-24 03:36:39 +0400 | [diff] [blame] | 352 | self.last_reconfigured = int(time.time()) |
James E. Blair | 646322f | 2017-01-27 15:50:34 -0800 | [diff] [blame] | 353 | # TODOv3(jeblair): reconfigure time should be per-tenant |
James E. Blair | e9d45c3 | 2012-05-31 09:56:45 -0700 | [diff] [blame] | 354 | |
David Shrewsbury | 36b2adf | 2017-07-31 15:40:13 -0400 | [diff] [blame] | 355 | def autohold(self, tenant_name, project_name, job_name, reason, count): |
David Shrewsbury | ffab07a | 2017-07-24 12:45:07 -0400 | [diff] [blame] | 356 | key = (tenant_name, project_name, job_name) |
| 357 | if count == 0 and key in self.autohold_requests: |
| 358 | self.log.debug("Removing autohold for %s", key) |
| 359 | del self.autohold_requests[key] |
| 360 | else: |
| 361 | self.log.debug("Autohold requested for %s", key) |
David Shrewsbury | 36b2adf | 2017-07-31 15:40:13 -0400 | [diff] [blame] | 362 | self.autohold_requests[key] = (count, reason) |
David Shrewsbury | ffab07a | 2017-07-24 12:45:07 -0400 | [diff] [blame] | 363 | |
Paul Belanger | baca313 | 2016-11-04 12:49:54 -0400 | [diff] [blame] | 364 | def promote(self, tenant_name, pipeline_name, change_ids): |
| 365 | event = PromoteEvent(tenant_name, pipeline_name, change_ids) |
James E. Blair | 36658cf | 2013-12-06 17:53:48 -0800 | [diff] [blame] | 366 | self.management_event_queue.put(event) |
| 367 | self.wake_event.set() |
| 368 | self.log.debug("Waiting for promotion") |
| 369 | event.wait() |
| 370 | self.log.debug("Promotion complete") |
| 371 | |
James E. Blair | d27a96d | 2014-07-10 13:25:13 -0700 | [diff] [blame] | 372 | def enqueue(self, trigger_event): |
| 373 | event = EnqueueEvent(trigger_event) |
| 374 | self.management_event_queue.put(event) |
| 375 | self.wake_event.set() |
| 376 | self.log.debug("Waiting for enqueue") |
| 377 | event.wait() |
| 378 | self.log.debug("Enqueue complete") |
| 379 | |
James E. Blair | 5d5bc2b | 2012-07-06 10:24:01 -0700 | [diff] [blame] | 380 | def exit(self): |
| 381 | self.log.debug("Prepare to exit") |
| 382 | self._pause = True |
| 383 | self._exit = True |
| 384 | self.wake_event.set() |
| 385 | self.log.debug("Waiting for exit") |
| 386 | |
| 387 | def _get_queue_pickle_file(self): |
James E. Blair | d1de946 | 2017-06-23 20:53:09 +0100 | [diff] [blame] | 388 | state_dir = get_default(self.config, 'scheduler', 'state_dir', |
Tristan Cacqueray | 91601d7 | 2017-06-15 06:00:12 +0000 | [diff] [blame] | 389 | '/var/lib/zuul', expand_user=True) |
James E. Blair | 5d5bc2b | 2012-07-06 10:24:01 -0700 | [diff] [blame] | 390 | return os.path.join(state_dir, 'queue.pickle') |
| 391 | |
James E. Blair | ce8a213 | 2016-05-19 15:21:52 -0700 | [diff] [blame] | 392 | def _get_time_database_dir(self): |
James E. Blair | d1de946 | 2017-06-23 20:53:09 +0100 | [diff] [blame] | 393 | state_dir = get_default(self.config, 'scheduler', 'state_dir', |
Tristan Cacqueray | 91601d7 | 2017-06-15 06:00:12 +0000 | [diff] [blame] | 394 | '/var/lib/zuul', expand_user=True) |
James E. Blair | ce8a213 | 2016-05-19 15:21:52 -0700 | [diff] [blame] | 395 | d = os.path.join(state_dir, 'times') |
| 396 | if not os.path.exists(d): |
| 397 | os.mkdir(d) |
| 398 | return d |
| 399 | |
Ricardo Carrillo Cruz | 22994f9 | 2016-12-02 11:41:58 +0000 | [diff] [blame] | 400 | def _get_project_key_dir(self): |
James E. Blair | d1de946 | 2017-06-23 20:53:09 +0100 | [diff] [blame] | 401 | state_dir = get_default(self.config, 'scheduler', 'state_dir', |
Tristan Cacqueray | 91601d7 | 2017-06-15 06:00:12 +0000 | [diff] [blame] | 402 | '/var/lib/zuul', expand_user=True) |
Ricardo Carrillo Cruz | 22994f9 | 2016-12-02 11:41:58 +0000 | [diff] [blame] | 403 | key_dir = os.path.join(state_dir, 'keys') |
| 404 | if not os.path.exists(key_dir): |
| 405 | os.mkdir(key_dir, 0o700) |
| 406 | st = os.stat(key_dir) |
| 407 | mode = st.st_mode & 0o777 |
| 408 | if mode != 0o700: |
| 409 | raise Exception("Project key directory %s must be mode 0700; " |
| 410 | "current mode is %o" % (key_dir, mode)) |
| 411 | return key_dir |
| 412 | |
James E. Blair | 5d5bc2b | 2012-07-06 10:24:01 -0700 | [diff] [blame] | 413 | def _save_queue(self): |
| 414 | pickle_file = self._get_queue_pickle_file() |
| 415 | events = [] |
| 416 | while not self.trigger_event_queue.empty(): |
| 417 | events.append(self.trigger_event_queue.get()) |
| 418 | self.log.debug("Queue length is %s" % len(events)) |
| 419 | if events: |
| 420 | self.log.debug("Saving queue") |
| 421 | pickle.dump(events, open(pickle_file, 'wb')) |
| 422 | |
| 423 | def _load_queue(self): |
| 424 | pickle_file = self._get_queue_pickle_file() |
| 425 | if os.path.exists(pickle_file): |
| 426 | self.log.debug("Loading queue") |
| 427 | events = pickle.load(open(pickle_file, 'rb')) |
| 428 | self.log.debug("Queue length is %s" % len(events)) |
| 429 | for event in events: |
| 430 | self.trigger_event_queue.put(event) |
| 431 | else: |
| 432 | self.log.debug("No queue file found") |
| 433 | |
| 434 | def _delete_queue(self): |
| 435 | pickle_file = self._get_queue_pickle_file() |
| 436 | if os.path.exists(pickle_file): |
| 437 | self.log.debug("Deleting saved queue") |
| 438 | os.unlink(pickle_file) |
| 439 | |
| 440 | def resume(self): |
| 441 | try: |
| 442 | self._load_queue() |
| 443 | except: |
| 444 | self.log.exception("Unable to load queue") |
| 445 | try: |
| 446 | self._delete_queue() |
| 447 | except: |
| 448 | self.log.exception("Unable to delete saved queue") |
| 449 | self.log.debug("Resuming queue processing") |
| 450 | self.wake_event.set() |
| 451 | |
| 452 | def _doPauseEvent(self): |
| 453 | if self._exit: |
| 454 | self.log.debug("Exiting") |
| 455 | self._save_queue() |
| 456 | os._exit(0) |
James E. Blair | cdccd97 | 2013-07-01 12:10:22 -0700 | [diff] [blame] | 457 | |
James E. Blair | 468c851 | 2013-12-06 13:27:19 -0800 | [diff] [blame] | 458 | def _doReconfigureEvent(self, event): |
| 459 | # This is called in the scheduler loop after another thread submits |
| 460 | # a request |
James E. Blair | cdccd97 | 2013-07-01 12:10:22 -0700 | [diff] [blame] | 461 | self.layout_lock.acquire() |
James E. Blair | 468c851 | 2013-12-06 13:27:19 -0800 | [diff] [blame] | 462 | self.config = event.config |
James E. Blair | cdccd97 | 2013-07-01 12:10:22 -0700 | [diff] [blame] | 463 | try: |
James E. Blair | a615c36 | 2017-10-02 17:34:42 -0700 | [diff] [blame^] | 464 | self.log.debug("Full reconfiguration beginning") |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 465 | loader = configloader.ConfigLoader() |
| 466 | abide = loader.loadConfig( |
James E. Blair | 3984036 | 2017-06-23 20:34:02 +0100 | [diff] [blame] | 467 | self.config.get('scheduler', 'tenant_config'), |
Ricardo Carrillo Cruz | 22994f9 | 2016-12-02 11:41:58 +0000 | [diff] [blame] | 468 | self._get_project_key_dir(), |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 469 | self, self.merger, self.connections) |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 470 | for tenant in abide.tenants.values(): |
| 471 | self._reconfigureTenant(tenant) |
| 472 | self.abide = abide |
James E. Blair | cdccd97 | 2013-07-01 12:10:22 -0700 | [diff] [blame] | 473 | finally: |
| 474 | self.layout_lock.release() |
James E. Blair | a615c36 | 2017-10-02 17:34:42 -0700 | [diff] [blame^] | 475 | self.log.debug("Full reconfiguration complete") |
James E. Blair | e9d45c3 | 2012-05-31 09:56:45 -0700 | [diff] [blame] | 476 | |
James E. Blair | 646322f | 2017-01-27 15:50:34 -0800 | [diff] [blame] | 477 | def _doTenantReconfigureEvent(self, event): |
| 478 | # This is called in the scheduler loop after another thread submits |
| 479 | # a request |
| 480 | self.layout_lock.acquire() |
| 481 | try: |
James E. Blair | a615c36 | 2017-10-02 17:34:42 -0700 | [diff] [blame^] | 482 | self.log.debug("Tenant reconfiguration beginning") |
| 483 | # If a change landed to a project, clear out the cached |
| 484 | # config before reconfiguring. |
| 485 | if event.project: |
| 486 | event.project.unparsed_config = None |
James E. Blair | 646322f | 2017-01-27 15:50:34 -0800 | [diff] [blame] | 487 | loader = configloader.ConfigLoader() |
| 488 | abide = loader.reloadTenant( |
James E. Blair | 3984036 | 2017-06-23 20:34:02 +0100 | [diff] [blame] | 489 | self.config.get('scheduler', 'tenant_config'), |
Ricardo Carrillo Cruz | 22994f9 | 2016-12-02 11:41:58 +0000 | [diff] [blame] | 490 | self._get_project_key_dir(), |
James E. Blair | 646322f | 2017-01-27 15:50:34 -0800 | [diff] [blame] | 491 | self, self.merger, self.connections, |
| 492 | self.abide, event.tenant) |
| 493 | tenant = abide.tenants[event.tenant.name] |
| 494 | self._reconfigureTenant(tenant) |
| 495 | self.abide = abide |
| 496 | finally: |
| 497 | self.layout_lock.release() |
James E. Blair | a615c36 | 2017-10-02 17:34:42 -0700 | [diff] [blame^] | 498 | self.log.debug("Tenant reconfiguration complete") |
James E. Blair | 646322f | 2017-01-27 15:50:34 -0800 | [diff] [blame] | 499 | |
James E. Blair | aa30de4 | 2017-04-25 10:56:59 -0700 | [diff] [blame] | 500 | def _reenqueueGetProject(self, tenant, item): |
| 501 | project = item.change.project |
James E. Blair | 6053de4 | 2017-04-05 11:27:11 -0700 | [diff] [blame] | 502 | # Attempt to get the same project as the one passed in. If |
| 503 | # the project is now found on a different connection, return |
| 504 | # the new version of the project. If it is no longer |
| 505 | # available (due to a connection being removed), return None. |
James E. Blair | aa30de4 | 2017-04-25 10:56:59 -0700 | [diff] [blame] | 506 | (trusted, new_project) = tenant.getProject(project.canonical_name) |
James E. Blair | 6053de4 | 2017-04-05 11:27:11 -0700 | [diff] [blame] | 507 | if new_project: |
| 508 | return new_project |
James E. Blair | aa30de4 | 2017-04-25 10:56:59 -0700 | [diff] [blame] | 509 | # If this is a non-live item we may be looking at a |
| 510 | # "foreign" project, ie, one which is not defined in the |
| 511 | # config but is constructed ad-hoc to satisfy a |
| 512 | # cross-repo-dependency. Find the corresponding live item |
| 513 | # and use its source. |
| 514 | child = item |
| 515 | while child and not child.live: |
| 516 | # This assumes that the queue does not branch behind this |
| 517 | # item, which is currently true for non-live items; if |
| 518 | # that changes, this traversal will need to be more |
| 519 | # complex. |
| 520 | if child.items_behind: |
| 521 | child = child.items_behind[0] |
| 522 | else: |
| 523 | child = None |
| 524 | if child is item: |
| 525 | return None |
| 526 | if child and child.live: |
| 527 | (child_trusted, child_project) = tenant.getProject( |
| 528 | child.change.project.canonical_name) |
| 529 | if child_project: |
| 530 | source = child_project.source |
| 531 | new_project = source.getProject(project.name) |
| 532 | return new_project |
| 533 | return None |
James E. Blair | 6053de4 | 2017-04-05 11:27:11 -0700 | [diff] [blame] | 534 | |
James E. Blair | 552b54f | 2016-07-22 13:55:32 -0700 | [diff] [blame] | 535 | def _reenqueueTenant(self, old_tenant, tenant): |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 536 | for name, new_pipeline in tenant.layout.pipelines.items(): |
| 537 | old_pipeline = old_tenant.layout.pipelines.get(name) |
| 538 | if not old_pipeline: |
| 539 | self.log.warning("No old pipeline matching %s found " |
| 540 | "when reconfiguring" % name) |
| 541 | continue |
| 542 | self.log.debug("Re-enqueueing changes for pipeline %s" % name) |
| 543 | items_to_remove = [] |
| 544 | builds_to_cancel = [] |
| 545 | last_head = None |
| 546 | for shared_queue in old_pipeline.queues: |
| 547 | for item in shared_queue.queue: |
| 548 | if not item.item_ahead: |
| 549 | last_head = item |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 550 | item.pipeline = None |
| 551 | item.queue = None |
James E. Blair | 6053de4 | 2017-04-05 11:27:11 -0700 | [diff] [blame] | 552 | item.change.project = self._reenqueueGetProject( |
James E. Blair | aa30de4 | 2017-04-25 10:56:59 -0700 | [diff] [blame] | 553 | tenant, item) |
| 554 | item.item_ahead = None |
| 555 | item.items_behind = [] |
James E. Blair | 027ba99 | 2017-09-20 13:48:32 -0700 | [diff] [blame] | 556 | reenqueued = False |
| 557 | if item.change.project: |
| 558 | try: |
| 559 | reenqueued = new_pipeline.manager.reEnqueueItem( |
| 560 | item, last_head) |
| 561 | except Exception: |
| 562 | self.log.exception( |
| 563 | "Exception while re-enqueing item %s", |
| 564 | item) |
| 565 | if reenqueued: |
James E. Blair | 3b5ff3b | 2016-07-21 10:08:24 -0700 | [diff] [blame] | 566 | for build in item.current_build_set.getBuilds(): |
Fredrik Medley | f8aec83 | 2015-09-28 13:40:20 +0200 | [diff] [blame] | 567 | new_job = item.getJob(build.job.name) |
| 568 | if new_job: |
| 569 | build.job = new_job |
James E. Blair | 3b5ff3b | 2016-07-21 10:08:24 -0700 | [diff] [blame] | 570 | else: |
| 571 | item.removeBuild(build) |
| 572 | builds_to_cancel.append(build) |
| 573 | else: |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 574 | items_to_remove.append(item) |
| 575 | for item in items_to_remove: |
James E. Blair | b5a8f0b | 2017-07-07 17:01:18 -0700 | [diff] [blame] | 576 | self.log.warning( |
| 577 | "Removing item %s during reconfiguration" % (item,)) |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 578 | for build in item.current_build_set.getBuilds(): |
| 579 | builds_to_cancel.append(build) |
| 580 | for build in builds_to_cancel: |
| 581 | self.log.warning( |
| 582 | "Canceling build %s during reconfiguration" % (build,)) |
| 583 | try: |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 584 | self.executor.cancel(build) |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 585 | except Exception: |
| 586 | self.log.exception( |
| 587 | "Exception while canceling build %s " |
Tobias Henkel | 9a0e194 | 2017-03-20 16:16:02 +0100 | [diff] [blame] | 588 | "for change %s" % (build, build.build_set.item.change)) |
Tobias Henkel | fb91a49 | 2017-02-15 07:29:43 +0100 | [diff] [blame] | 589 | finally: |
Tobias Henkel | 9a0e194 | 2017-03-20 16:16:02 +0100 | [diff] [blame] | 590 | tenant.semaphore_handler.release( |
| 591 | build.build_set.item, build.job) |
James E. Blair | 552b54f | 2016-07-22 13:55:32 -0700 | [diff] [blame] | 592 | |
| 593 | def _reconfigureTenant(self, tenant): |
| 594 | # This is called from _doReconfigureEvent while holding the |
| 595 | # layout lock |
| 596 | old_tenant = self.abide.tenants.get(tenant.name) |
Tobias Henkel | 9a0e194 | 2017-03-20 16:16:02 +0100 | [diff] [blame] | 597 | |
James E. Blair | 552b54f | 2016-07-22 13:55:32 -0700 | [diff] [blame] | 598 | if old_tenant: |
Tobias Henkel | 9a0e194 | 2017-03-20 16:16:02 +0100 | [diff] [blame] | 599 | # Copy over semaphore handler so we don't loose the currently |
| 600 | # held semaphores. |
| 601 | tenant.semaphore_handler = old_tenant.semaphore_handler |
| 602 | |
James E. Blair | 552b54f | 2016-07-22 13:55:32 -0700 | [diff] [blame] | 603 | self._reenqueueTenant(old_tenant, tenant) |
Tobias Henkel | 9a0e194 | 2017-03-20 16:16:02 +0100 | [diff] [blame] | 604 | |
James E. Blair | 25796c2 | 2017-09-08 09:34:37 -0700 | [diff] [blame] | 605 | # TODOv3(jeblair): update for tenants |
| 606 | # self.maintainConnectionCache() |
James E. Blair | e511d2f | 2016-12-08 15:22:26 -0800 | [diff] [blame] | 607 | self.connections.reconfigureDrivers(tenant) |
Tobias Henkel | 9a0e194 | 2017-03-20 16:16:02 +0100 | [diff] [blame] | 608 | |
James E. Blair | e511d2f | 2016-12-08 15:22:26 -0800 | [diff] [blame] | 609 | # TODOv3(jeblair): remove postconfig calls? |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 610 | for pipeline in tenant.layout.pipelines.values(): |
James E. Blair | 552b54f | 2016-07-22 13:55:32 -0700 | [diff] [blame] | 611 | for trigger in pipeline.triggers: |
| 612 | trigger.postConfig(pipeline) |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 613 | for reporter in pipeline.actions: |
| 614 | reporter.postConfig() |
Jesse Keating | 71a47ff | 2017-06-06 11:36:43 -0700 | [diff] [blame] | 615 | self.tenant_last_reconfigured[tenant.name] = int(time.time()) |
James E. Blair | 552b54f | 2016-07-22 13:55:32 -0700 | [diff] [blame] | 616 | if self.statsd: |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 617 | try: |
James E. Blair | 552b54f | 2016-07-22 13:55:32 -0700 | [diff] [blame] | 618 | for pipeline in tenant.layout.pipelines.values(): |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 619 | items = len(pipeline.getAllItems()) |
| 620 | # stats.gauges.zuul.pipeline.NAME.current_changes |
| 621 | key = 'zuul.pipeline.%s' % pipeline.name |
James E. Blair | 552b54f | 2016-07-22 13:55:32 -0700 | [diff] [blame] | 622 | self.statsd.gauge(key + '.current_changes', items) |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 623 | except Exception: |
| 624 | self.log.exception("Exception reporting initial " |
| 625 | "pipeline stats:") |
| 626 | |
James E. Blair | 36658cf | 2013-12-06 17:53:48 -0800 | [diff] [blame] | 627 | def _doPromoteEvent(self, event): |
Paul Belanger | baca313 | 2016-11-04 12:49:54 -0400 | [diff] [blame] | 628 | tenant = self.abide.tenants.get(event.tenant_name) |
| 629 | pipeline = tenant.layout.pipelines[event.pipeline_name] |
James E. Blair | 36658cf | 2013-12-06 17:53:48 -0800 | [diff] [blame] | 630 | change_ids = [c.split(',') for c in event.change_ids] |
| 631 | items_to_enqueue = [] |
| 632 | change_queue = None |
| 633 | for shared_queue in pipeline.queues: |
| 634 | if change_queue: |
| 635 | break |
| 636 | for item in shared_queue.queue: |
| 637 | if (item.change.number == change_ids[0][0] and |
Joshua Hesketh | 29d99b7 | 2014-08-19 16:27:42 +1000 | [diff] [blame] | 638 | item.change.patchset == change_ids[0][1]): |
James E. Blair | 36658cf | 2013-12-06 17:53:48 -0800 | [diff] [blame] | 639 | change_queue = shared_queue |
| 640 | break |
| 641 | if not change_queue: |
| 642 | raise Exception("Unable to find shared change queue for %s" % |
| 643 | event.change_ids[0]) |
| 644 | for number, patchset in change_ids: |
| 645 | found = False |
| 646 | for item in change_queue.queue: |
| 647 | if (item.change.number == number and |
Joshua Hesketh | 29d99b7 | 2014-08-19 16:27:42 +1000 | [diff] [blame] | 648 | item.change.patchset == patchset): |
James E. Blair | 36658cf | 2013-12-06 17:53:48 -0800 | [diff] [blame] | 649 | found = True |
| 650 | items_to_enqueue.append(item) |
| 651 | break |
| 652 | if not found: |
| 653 | raise Exception("Unable to find %s,%s in queue %s" % |
| 654 | (number, patchset, change_queue)) |
| 655 | for item in change_queue.queue[:]: |
| 656 | if item not in items_to_enqueue: |
| 657 | items_to_enqueue.append(item) |
| 658 | pipeline.manager.cancelJobs(item) |
| 659 | pipeline.manager.dequeueItem(item) |
| 660 | for item in items_to_enqueue: |
Sean Dague | f39b9ca | 2014-01-10 21:34:35 -0500 | [diff] [blame] | 661 | pipeline.manager.addChange( |
| 662 | item.change, |
| 663 | enqueue_time=item.enqueue_time, |
James E. Blair | f9ab884 | 2014-07-10 13:12:07 -0700 | [diff] [blame] | 664 | quiet=True, |
| 665 | ignore_requirements=True) |
James E. Blair | 36658cf | 2013-12-06 17:53:48 -0800 | [diff] [blame] | 666 | |
James E. Blair | d27a96d | 2014-07-10 13:25:13 -0700 | [diff] [blame] | 667 | def _doEnqueueEvent(self, event): |
Paul Belanger | baca313 | 2016-11-04 12:49:54 -0400 | [diff] [blame] | 668 | tenant = self.abide.tenants.get(event.tenant_name) |
James E. Blair | 0ffa010 | 2017-03-30 13:11:33 -0700 | [diff] [blame] | 669 | (trusted, project) = tenant.getProject(event.project_name) |
Paul Belanger | baca313 | 2016-11-04 12:49:54 -0400 | [diff] [blame] | 670 | pipeline = tenant.layout.pipelines[event.forced_pipeline] |
James E. Blair | 6053de4 | 2017-04-05 11:27:11 -0700 | [diff] [blame] | 671 | change = project.source.getChange(event, project) |
James E. Blair | d27a96d | 2014-07-10 13:25:13 -0700 | [diff] [blame] | 672 | self.log.debug("Event %s for change %s was directly assigned " |
| 673 | "to pipeline %s" % (event, change, self)) |
James E. Blair | d27a96d | 2014-07-10 13:25:13 -0700 | [diff] [blame] | 674 | pipeline.manager.addChange(change, ignore_requirements=True) |
| 675 | |
James E. Blair | e9d45c3 | 2012-05-31 09:56:45 -0700 | [diff] [blame] | 676 | def _areAllBuildsComplete(self): |
| 677 | self.log.debug("Checking if all builds are complete") |
James E. Blair | 8b1dc3f | 2016-07-05 16:49:00 -0700 | [diff] [blame] | 678 | if self.merger.areMergesOutstanding(): |
| 679 | self.log.debug("Waiting on merger") |
| 680 | return False |
James E. Blair | e9d45c3 | 2012-05-31 09:56:45 -0700 | [diff] [blame] | 681 | waiting = False |
Paul Belanger | debd7a7 | 2016-11-11 19:56:15 -0500 | [diff] [blame] | 682 | for tenant in self.abide.tenants.values(): |
| 683 | for pipeline in tenant.layout.pipelines.values(): |
| 684 | for item in pipeline.getAllItems(): |
| 685 | for build in item.current_build_set.getBuilds(): |
| 686 | if build.result is None: |
| 687 | self.log.debug("%s waiting on %s" % |
| 688 | (pipeline.manager, build)) |
| 689 | waiting = True |
James E. Blair | e9d45c3 | 2012-05-31 09:56:45 -0700 | [diff] [blame] | 690 | if not waiting: |
| 691 | self.log.debug("All builds are complete") |
| 692 | return True |
James E. Blair | e9d45c3 | 2012-05-31 09:56:45 -0700 | [diff] [blame] | 693 | return False |
| 694 | |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 695 | def run(self): |
James E. Blair | 552b54f | 2016-07-22 13:55:32 -0700 | [diff] [blame] | 696 | if self.statsd: |
James E. Blair | 71e9412 | 2012-12-24 17:53:08 -0800 | [diff] [blame] | 697 | self.log.debug("Statsd enabled") |
| 698 | else: |
| 699 | self.log.debug("Statsd disabled because python statsd " |
| 700 | "package not found") |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 701 | while True: |
| 702 | self.log.debug("Run handler sleeping") |
| 703 | self.wake_event.wait() |
| 704 | self.wake_event.clear() |
James E. Blair | b0fcae4 | 2012-07-17 11:12:10 -0700 | [diff] [blame] | 705 | if self._stopped: |
James E. Blair | 4076e2b | 2014-01-28 12:42:20 -0800 | [diff] [blame] | 706 | self.log.debug("Run handler stopping") |
James E. Blair | b0fcae4 | 2012-07-17 11:12:10 -0700 | [diff] [blame] | 707 | return |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 708 | self.log.debug("Run handler awake") |
James E. Blair | a84f0e4 | 2014-02-06 07:09:22 -0800 | [diff] [blame] | 709 | self.run_handler_lock.acquire() |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 710 | try: |
James E. Blair | a84f0e4 | 2014-02-06 07:09:22 -0800 | [diff] [blame] | 711 | while not self.management_event_queue.empty(): |
James E. Blair | 468c851 | 2013-12-06 13:27:19 -0800 | [diff] [blame] | 712 | self.process_management_queue() |
James E. Blair | cdccd97 | 2013-07-01 12:10:22 -0700 | [diff] [blame] | 713 | |
James E. Blair | 263fba9 | 2013-02-27 13:07:19 -0800 | [diff] [blame] | 714 | # Give result events priority -- they let us stop builds, |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 715 | # whereas trigger events cause us to execute builds. |
James E. Blair | a84f0e4 | 2014-02-06 07:09:22 -0800 | [diff] [blame] | 716 | while not self.result_event_queue.empty(): |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 717 | self.process_result_queue() |
James E. Blair | a84f0e4 | 2014-02-06 07:09:22 -0800 | [diff] [blame] | 718 | |
| 719 | if not self._pause: |
| 720 | while not self.trigger_event_queue.empty(): |
James E. Blair | 263fba9 | 2013-02-27 13:07:19 -0800 | [diff] [blame] | 721 | self.process_event_queue() |
James E. Blair | e9d45c3 | 2012-05-31 09:56:45 -0700 | [diff] [blame] | 722 | |
James E. Blair | 5d5bc2b | 2012-07-06 10:24:01 -0700 | [diff] [blame] | 723 | if self._pause and self._areAllBuildsComplete(): |
| 724 | self._doPauseEvent() |
James E. Blair | e9d45c3 | 2012-05-31 09:56:45 -0700 | [diff] [blame] | 725 | |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 726 | for tenant in self.abide.tenants.values(): |
| 727 | for pipeline in tenant.layout.pipelines.values(): |
| 728 | while pipeline.manager.processQueue(): |
| 729 | pass |
James E. Blair | 0e933c5 | 2013-07-11 10:18:52 -0700 | [diff] [blame] | 730 | |
James E. Blair | a84f0e4 | 2014-02-06 07:09:22 -0800 | [diff] [blame] | 731 | except Exception: |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 732 | self.log.exception("Exception in run handler:") |
James E. Blair | a84f0e4 | 2014-02-06 07:09:22 -0800 | [diff] [blame] | 733 | # There may still be more events to process |
| 734 | self.wake_event.set() |
| 735 | finally: |
| 736 | self.run_handler_lock.release() |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 737 | |
Joshua Hesketh | 4bd7da3 | 2016-02-17 20:58:47 +1100 | [diff] [blame] | 738 | def maintainConnectionCache(self): |
James E. Blair | 25796c2 | 2017-09-08 09:34:37 -0700 | [diff] [blame] | 739 | # TODOv3(jeblair): update for tenants |
James E. Blair | 0e933c5 | 2013-07-11 10:18:52 -0700 | [diff] [blame] | 740 | relevant = set() |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 741 | for tenant in self.abide.tenants.values(): |
| 742 | for pipeline in tenant.layout.pipelines.values(): |
Joshua Hesketh | dc7820c | 2016-03-11 13:14:28 +1100 | [diff] [blame] | 743 | self.log.debug("Gather relevant cache items for: %s" % |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 744 | pipeline) |
Joshua Hesketh | dc7820c | 2016-03-11 13:14:28 +1100 | [diff] [blame] | 745 | |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 746 | for item in pipeline.getAllItems(): |
| 747 | relevant.add(item.change) |
| 748 | relevant.update(item.change.getRelatedChanges()) |
James E. Blair | 25796c2 | 2017-09-08 09:34:37 -0700 | [diff] [blame] | 749 | for connection in self.connections.values(): |
Joshua Hesketh | 4bd7da3 | 2016-02-17 20:58:47 +1100 | [diff] [blame] | 750 | connection.maintainCache(relevant) |
| 751 | self.log.debug( |
| 752 | "End maintain connection cache for: %s" % connection) |
| 753 | self.log.debug("Connection cache size: %s" % len(relevant)) |
James E. Blair | 0e933c5 | 2013-07-11 10:18:52 -0700 | [diff] [blame] | 754 | |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 755 | def process_event_queue(self): |
| 756 | self.log.debug("Fetching trigger event") |
| 757 | event = self.trigger_event_queue.get() |
| 758 | self.log.debug("Processing trigger event %s" % event) |
James E. Blair | a84f0e4 | 2014-02-06 07:09:22 -0800 | [diff] [blame] | 759 | try: |
James E. Blair | aa30de4 | 2017-04-25 10:56:59 -0700 | [diff] [blame] | 760 | full_project_name = ('/'.join([event.project_hostname, |
| 761 | event.project_name])) |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 762 | for tenant in self.abide.tenants.values(): |
James E. Blair | aa30de4 | 2017-04-25 10:56:59 -0700 | [diff] [blame] | 763 | (trusted, project) = tenant.getProject(full_project_name) |
| 764 | if project is None: |
| 765 | continue |
| 766 | try: |
| 767 | change = project.source.getChange(event) |
| 768 | except exceptions.ChangeNotFound as e: |
| 769 | self.log.debug("Unable to get change %s from " |
| 770 | "source %s", |
| 771 | e.change, project.source) |
| 772 | continue |
James E. Blair | 72facdc | 2017-08-17 10:29:12 -0700 | [diff] [blame] | 773 | if ((event.branch_updated and |
| 774 | hasattr(change, 'files') and |
| 775 | change.updatesConfig()) or |
| 776 | event.branch_created or |
| 777 | event.branch_deleted): |
| 778 | # The change that just landed updates the config |
| 779 | # or a branch was just created or deleted. Clear |
| 780 | # out cached data for this project and perform a |
| 781 | # reconfiguration. |
James E. Blair | a615c36 | 2017-10-02 17:34:42 -0700 | [diff] [blame^] | 782 | self.reconfigureTenant(tenant, change.project) |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 783 | for pipeline in tenant.layout.pipelines.values(): |
Jan Hruban | 324ca5b | 2015-11-05 19:28:54 +0100 | [diff] [blame] | 784 | if event.isPatchsetCreated(): |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 785 | pipeline.manager.removeOldVersionsOfChange(change) |
Jan Hruban | 324ca5b | 2015-11-05 19:28:54 +0100 | [diff] [blame] | 786 | elif event.isChangeAbandoned(): |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 787 | pipeline.manager.removeAbandonedChange(change) |
| 788 | if pipeline.manager.eventMatches(event, change): |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 789 | pipeline.manager.addChange(change) |
James E. Blair | a84f0e4 | 2014-02-06 07:09:22 -0800 | [diff] [blame] | 790 | finally: |
James E. Blair | ff79197 | 2013-01-09 11:45:43 -0800 | [diff] [blame] | 791 | self.trigger_event_queue.task_done() |
James E. Blair | 1e8dd89 | 2012-05-30 09:15:05 -0700 | [diff] [blame] | 792 | |
James E. Blair | 468c851 | 2013-12-06 13:27:19 -0800 | [diff] [blame] | 793 | def process_management_queue(self): |
| 794 | self.log.debug("Fetching management event") |
| 795 | event = self.management_event_queue.get() |
| 796 | self.log.debug("Processing management event %s" % event) |
James E. Blair | 36658cf | 2013-12-06 17:53:48 -0800 | [diff] [blame] | 797 | try: |
| 798 | if isinstance(event, ReconfigureEvent): |
| 799 | self._doReconfigureEvent(event) |
James E. Blair | 21603e6 | 2017-02-20 16:23:05 -0500 | [diff] [blame] | 800 | elif isinstance(event, TenantReconfigureEvent): |
James E. Blair | 646322f | 2017-01-27 15:50:34 -0800 | [diff] [blame] | 801 | self._doTenantReconfigureEvent(event) |
James E. Blair | 36658cf | 2013-12-06 17:53:48 -0800 | [diff] [blame] | 802 | elif isinstance(event, PromoteEvent): |
| 803 | self._doPromoteEvent(event) |
James E. Blair | d27a96d | 2014-07-10 13:25:13 -0700 | [diff] [blame] | 804 | elif isinstance(event, EnqueueEvent): |
| 805 | self._doEnqueueEvent(event.trigger_event) |
James E. Blair | 36658cf | 2013-12-06 17:53:48 -0800 | [diff] [blame] | 806 | else: |
| 807 | self.log.error("Unable to handle event %s" % event) |
| 808 | event.done() |
Morgan Fainberg | 1b9bd78 | 2016-05-30 14:03:30 -0700 | [diff] [blame] | 809 | except Exception: |
James E. Blair | 59424ea | 2017-07-11 09:52:58 -0700 | [diff] [blame] | 810 | self.log.exception("Exception in management event:") |
Morgan Fainberg | 1b9bd78 | 2016-05-30 14:03:30 -0700 | [diff] [blame] | 811 | event.exception(sys.exc_info()) |
James E. Blair | 468c851 | 2013-12-06 13:27:19 -0800 | [diff] [blame] | 812 | self.management_event_queue.task_done() |
| 813 | |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 814 | def process_result_queue(self): |
| 815 | self.log.debug("Fetching result event") |
James E. Blair | a84f0e4 | 2014-02-06 07:09:22 -0800 | [diff] [blame] | 816 | event = self.result_event_queue.get() |
| 817 | self.log.debug("Processing result event %s" % event) |
| 818 | try: |
| 819 | if isinstance(event, BuildStartedEvent): |
| 820 | self._doBuildStartedEvent(event) |
| 821 | elif isinstance(event, BuildCompletedEvent): |
| 822 | self._doBuildCompletedEvent(event) |
James E. Blair | 4076e2b | 2014-01-28 12:42:20 -0800 | [diff] [blame] | 823 | elif isinstance(event, MergeCompletedEvent): |
| 824 | self._doMergeCompletedEvent(event) |
James E. Blair | 8d69239 | 2016-04-08 17:47:58 -0700 | [diff] [blame] | 825 | elif isinstance(event, NodesProvisionedEvent): |
| 826 | self._doNodesProvisionedEvent(event) |
James E. Blair | a84f0e4 | 2014-02-06 07:09:22 -0800 | [diff] [blame] | 827 | else: |
| 828 | self.log.error("Unable to handle event %s" % event) |
| 829 | finally: |
| 830 | self.result_event_queue.task_done() |
| 831 | |
| 832 | def _doBuildStartedEvent(self, event): |
James E. Blair | 4076e2b | 2014-01-28 12:42:20 -0800 | [diff] [blame] | 833 | build = event.build |
| 834 | if build.build_set is not build.build_set.item.current_build_set: |
| 835 | self.log.warning("Build %s is not in the current build set" % |
| 836 | (build,)) |
| 837 | return |
| 838 | pipeline = build.build_set.item.pipeline |
| 839 | if not pipeline: |
| 840 | self.log.warning("Build %s is not associated with a pipeline" % |
| 841 | (build,)) |
| 842 | return |
James E. Blair | ce8a213 | 2016-05-19 15:21:52 -0700 | [diff] [blame] | 843 | try: |
| 844 | build.estimated_time = float(self.time_database.getEstimatedTime( |
James E. Blair | ae0f23c | 2017-09-13 10:55:15 -0600 | [diff] [blame] | 845 | build)) |
James E. Blair | ce8a213 | 2016-05-19 15:21:52 -0700 | [diff] [blame] | 846 | except Exception: |
| 847 | self.log.exception("Exception estimating build time:") |
James E. Blair | 4076e2b | 2014-01-28 12:42:20 -0800 | [diff] [blame] | 848 | pipeline.manager.onBuildStarted(event.build) |
James E. Blair | a84f0e4 | 2014-02-06 07:09:22 -0800 | [diff] [blame] | 849 | |
| 850 | def _doBuildCompletedEvent(self, event): |
James E. Blair | 4076e2b | 2014-01-28 12:42:20 -0800 | [diff] [blame] | 851 | build = event.build |
James E. Blair | e18d460 | 2017-01-05 11:17:28 -0800 | [diff] [blame] | 852 | |
| 853 | # Regardless of any other conditions which might cause us not |
| 854 | # to pass this on to the pipeline manager, make sure we return |
| 855 | # the nodes to nodepool. |
| 856 | try: |
| 857 | nodeset = build.build_set.getJobNodeSet(build.job.name) |
David Shrewsbury | ffab07a | 2017-07-24 12:45:07 -0400 | [diff] [blame] | 858 | autohold_key = (build.pipeline.layout.tenant.name, |
| 859 | build.build_set.item.change.project.canonical_name, |
| 860 | build.job.name) |
| 861 | |
| 862 | try: |
| 863 | self.nodepool.holdNodeSet(nodeset, autohold_key) |
| 864 | except Exception: |
| 865 | self.log.exception("Unable to process autohold for %s", |
| 866 | autohold_key) |
| 867 | |
James E. Blair | 1511bc3 | 2017-01-18 09:25:31 -0800 | [diff] [blame] | 868 | self.nodepool.returnNodeSet(nodeset) |
James E. Blair | e18d460 | 2017-01-05 11:17:28 -0800 | [diff] [blame] | 869 | except Exception: |
| 870 | self.log.exception("Unable to return nodeset %s" % (nodeset,)) |
| 871 | |
James E. Blair | 4076e2b | 2014-01-28 12:42:20 -0800 | [diff] [blame] | 872 | if build.build_set is not build.build_set.item.current_build_set: |
James E. Blair | e18d460 | 2017-01-05 11:17:28 -0800 | [diff] [blame] | 873 | self.log.debug("Build %s is not in the current build set" % |
| 874 | (build,)) |
James E. Blair | 4076e2b | 2014-01-28 12:42:20 -0800 | [diff] [blame] | 875 | return |
| 876 | pipeline = build.build_set.item.pipeline |
| 877 | if not pipeline: |
| 878 | self.log.warning("Build %s is not associated with a pipeline" % |
| 879 | (build,)) |
| 880 | return |
James E. Blair | ce8a213 | 2016-05-19 15:21:52 -0700 | [diff] [blame] | 881 | if build.end_time and build.start_time and build.result: |
| 882 | duration = build.end_time - build.start_time |
Paul Belanger | 87e4ab0 | 2016-06-08 14:17:20 -0400 | [diff] [blame] | 883 | try: |
James E. Blair | ae0f23c | 2017-09-13 10:55:15 -0600 | [diff] [blame] | 884 | self.time_database.update(build, duration, build.result) |
Paul Belanger | 87e4ab0 | 2016-06-08 14:17:20 -0400 | [diff] [blame] | 885 | except Exception: |
| 886 | self.log.exception("Exception recording build time:") |
James E. Blair | 4076e2b | 2014-01-28 12:42:20 -0800 | [diff] [blame] | 887 | pipeline.manager.onBuildCompleted(event.build) |
| 888 | |
| 889 | def _doMergeCompletedEvent(self, event): |
| 890 | build_set = event.build_set |
| 891 | if build_set is not build_set.item.current_build_set: |
| 892 | self.log.warning("Build set %s is not current" % (build_set,)) |
| 893 | return |
| 894 | pipeline = build_set.item.pipeline |
| 895 | if not pipeline: |
| 896 | self.log.warning("Build set %s is not associated with a pipeline" % |
| 897 | (build_set,)) |
| 898 | return |
| 899 | pipeline.manager.onMergeCompleted(event) |
James E. Blair | ee74361 | 2012-05-29 14:49:32 -0700 | [diff] [blame] | 900 | |
James E. Blair | 8d69239 | 2016-04-08 17:47:58 -0700 | [diff] [blame] | 901 | def _doNodesProvisionedEvent(self, event): |
| 902 | request = event.request |
| 903 | build_set = request.build_set |
James E. Blair | a38c28e | 2017-01-04 10:33:20 -0800 | [diff] [blame] | 904 | |
James E. Blair | 6ab79e0 | 2017-01-06 10:10:17 -0800 | [diff] [blame] | 905 | self.nodepool.acceptNodes(request) |
James E. Blair | cbbce0d | 2017-05-19 07:28:29 -0700 | [diff] [blame] | 906 | if request.canceled: |
| 907 | return |
James E. Blair | a38c28e | 2017-01-04 10:33:20 -0800 | [diff] [blame] | 908 | |
James E. Blair | 8d69239 | 2016-04-08 17:47:58 -0700 | [diff] [blame] | 909 | if build_set is not build_set.item.current_build_set: |
| 910 | self.log.warning("Build set %s is not current" % (build_set,)) |
James E. Blair | 6ab79e0 | 2017-01-06 10:10:17 -0800 | [diff] [blame] | 911 | if request.fulfilled: |
James E. Blair | 1511bc3 | 2017-01-18 09:25:31 -0800 | [diff] [blame] | 912 | self.nodepool.returnNodeSet(request.nodeset) |
James E. Blair | 8d69239 | 2016-04-08 17:47:58 -0700 | [diff] [blame] | 913 | return |
| 914 | pipeline = build_set.item.pipeline |
| 915 | if not pipeline: |
| 916 | self.log.warning("Build set %s is not associated with a pipeline" % |
| 917 | (build_set,)) |
James E. Blair | 6ab79e0 | 2017-01-06 10:10:17 -0800 | [diff] [blame] | 918 | if request.fulfilled: |
James E. Blair | 1511bc3 | 2017-01-18 09:25:31 -0800 | [diff] [blame] | 919 | self.nodepool.returnNodeSet(request.nodeset) |
James E. Blair | 8d69239 | 2016-04-08 17:47:58 -0700 | [diff] [blame] | 920 | return |
| 921 | pipeline.manager.onNodesProvisioned(event) |
| 922 | |
Paul Belanger | 6349d15 | 2016-10-30 16:21:17 -0400 | [diff] [blame] | 923 | def formatStatusJSON(self, tenant_name): |
James E. Blair | 59fdbac | 2015-12-07 17:08:06 -0800 | [diff] [blame] | 924 | # TODOv3(jeblair): use tenants |
James E. Blair | 8dbd56a | 2012-12-22 10:55:10 -0800 | [diff] [blame] | 925 | data = {} |
Sergey Lukjanov | 5ba961b | 2013-12-27 01:21:04 +0400 | [diff] [blame] | 926 | |
| 927 | data['zuul_version'] = self.zuul_version |
Tobias Henkel | b4407fc | 2017-07-07 13:52:56 +0200 | [diff] [blame] | 928 | websocket_url = get_default(self.config, 'web', 'websocket_url', None) |
Sergey Lukjanov | 5ba961b | 2013-12-27 01:21:04 +0400 | [diff] [blame] | 929 | |
James E. Blair | 8dbd56a | 2012-12-22 10:55:10 -0800 | [diff] [blame] | 930 | if self._pause: |
| 931 | ret = '<p><b>Queue only mode:</b> preparing to ' |
James E. Blair | 8dbd56a | 2012-12-22 10:55:10 -0800 | [diff] [blame] | 932 | if self._exit: |
| 933 | ret += 'exit' |
| 934 | ret += ', queue length: %s' % self.trigger_event_queue.qsize() |
| 935 | ret += '</p>' |
| 936 | data['message'] = ret |
| 937 | |
James E. Blair | fb682cc | 2013-02-26 15:23:27 -0800 | [diff] [blame] | 938 | data['trigger_event_queue'] = {} |
| 939 | data['trigger_event_queue']['length'] = \ |
| 940 | self.trigger_event_queue.qsize() |
| 941 | data['result_event_queue'] = {} |
| 942 | data['result_event_queue']['length'] = \ |
| 943 | self.result_event_queue.qsize() |
| 944 | |
Sergey Lukjanov | 5d0438d | 2013-12-24 03:36:39 +0400 | [diff] [blame] | 945 | if self.last_reconfigured: |
| 946 | data['last_reconfigured'] = self.last_reconfigured * 1000 |
| 947 | |
James E. Blair | 8dbd56a | 2012-12-22 10:55:10 -0800 | [diff] [blame] | 948 | pipelines = [] |
| 949 | data['pipelines'] = pipelines |
Paul Belanger | 6349d15 | 2016-10-30 16:21:17 -0400 | [diff] [blame] | 950 | tenant = self.abide.tenants.get(tenant_name) |
| 951 | for pipeline in tenant.layout.pipelines.values(): |
Tobias Henkel | b4407fc | 2017-07-07 13:52:56 +0200 | [diff] [blame] | 952 | pipelines.append(pipeline.formatStatusJSON(websocket_url)) |
James E. Blair | 8dbd56a | 2012-12-22 10:55:10 -0800 | [diff] [blame] | 953 | return json.dumps(data) |