James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 1 | # Copyright 2014 OpenStack Foundation |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 4 | # not use this file except in compliance with the License. You may obtain |
| 5 | # a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 12 | # License for the specific language governing permissions and limitations |
| 13 | # under the License. |
| 14 | |
| 15 | import collections |
| 16 | import json |
| 17 | import logging |
James E. Blair | 8293847 | 2016-01-11 14:38:13 -0800 | [diff] [blame] | 18 | import os |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 19 | import shutil |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 20 | import signal |
James E. Blair | 1730297 | 2016-08-10 16:11:42 -0700 | [diff] [blame] | 21 | import socket |
James E. Blair | 8293847 | 2016-01-11 14:38:13 -0800 | [diff] [blame] | 22 | import subprocess |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 23 | import tempfile |
| 24 | import threading |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 25 | import time |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 26 | import traceback |
James E. Blair | a92cbc8 | 2017-01-23 14:56:49 -0800 | [diff] [blame] | 27 | import yaml |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 28 | |
| 29 | import gear |
James E. Blair | e29f80a | 2017-02-22 22:27:11 -0500 | [diff] [blame] | 30 | import git |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 31 | |
James E. Blair | 29b9c96 | 2017-02-13 16:17:22 -0800 | [diff] [blame] | 32 | import zuul.merger.merger |
Monty Taylor | c231d93 | 2017-02-03 09:57:15 -0600 | [diff] [blame] | 33 | import zuul.ansible.action |
Monty Taylor | 825bca5 | 2017-02-20 06:58:46 -0500 | [diff] [blame] | 34 | import zuul.ansible.callback |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 35 | import zuul.ansible.library |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 36 | from zuul.lib import commandsocket |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 37 | |
James E. Blair | 29b9c96 | 2017-02-13 16:17:22 -0800 | [diff] [blame] | 38 | COMMANDS = ['stop', 'pause', 'unpause', 'graceful', 'verbose', |
| 39 | 'unverbose'] |
| 40 | |
| 41 | |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 42 | class Watchdog(object): |
| 43 | def __init__(self, timeout, function, args): |
| 44 | self.timeout = timeout |
| 45 | self.function = function |
| 46 | self.args = args |
| 47 | self.thread = threading.Thread(target=self._run) |
| 48 | self.thread.daemon = True |
| 49 | self.timed_out = None |
| 50 | |
| 51 | def _run(self): |
| 52 | while self._running and time.time() < self.end: |
| 53 | time.sleep(10) |
| 54 | if self._running: |
| 55 | self.timed_out = True |
| 56 | self.function(*self.args) |
| 57 | self.timed_out = False |
| 58 | |
| 59 | def start(self): |
| 60 | self._running = True |
| 61 | self.end = time.time() + self.timeout |
| 62 | self.thread.start() |
| 63 | |
| 64 | def stop(self): |
| 65 | self._running = False |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 66 | |
James E. Blair | 2316191 | 2016-07-28 15:42:14 -0700 | [diff] [blame] | 67 | # TODOv3(mordred): put git repos in a hierarchy that includes source |
| 68 | # hostname, eg: git.openstack.org/openstack/nova. Also, configure |
| 69 | # sources to have an alias, so that the review.openstack.org source |
| 70 | # repos end up in git.openstack.org. |
| 71 | |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 72 | |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 73 | class JobDirPlaybook(object): |
| 74 | def __init__(self, root): |
| 75 | self.root = root |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 76 | self.trusted = None |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 77 | self.path = None |
| 78 | |
| 79 | |
James E. Blair | 8293847 | 2016-01-11 14:38:13 -0800 | [diff] [blame] | 80 | class JobDir(object): |
James E. Blair | 854f889 | 2017-02-02 11:25:39 -0800 | [diff] [blame] | 81 | def __init__(self, root=None, keep=False): |
Monty Taylor | e20c9f2 | 2017-02-22 17:48:07 -0500 | [diff] [blame] | 82 | # root |
| 83 | # ansible |
| 84 | # trusted.cfg |
| 85 | # untrusted.cfg |
| 86 | # work |
K Jonathan Harker | 2c1a623 | 2017-02-21 14:34:08 -0800 | [diff] [blame] | 87 | # src |
James E. Blair | e81ba63 | 2017-02-23 10:12:55 -0500 | [diff] [blame] | 88 | # logs |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 89 | self.keep = keep |
James E. Blair | 854f889 | 2017-02-02 11:25:39 -0800 | [diff] [blame] | 90 | self.root = tempfile.mkdtemp(dir=root) |
Monty Taylor | e20c9f2 | 2017-02-22 17:48:07 -0500 | [diff] [blame] | 91 | # Work |
| 92 | self.work_root = os.path.join(self.root, 'work') |
| 93 | os.makedirs(self.work_root) |
Monty Taylor | d642d85 | 2017-02-23 14:05:42 -0500 | [diff] [blame] | 94 | self.src_root = os.path.join(self.work_root, 'src') |
| 95 | os.makedirs(self.src_root) |
James E. Blair | e81ba63 | 2017-02-23 10:12:55 -0500 | [diff] [blame] | 96 | self.log_root = os.path.join(self.work_root, 'logs') |
| 97 | os.makedirs(self.log_root) |
Monty Taylor | e20c9f2 | 2017-02-22 17:48:07 -0500 | [diff] [blame] | 98 | # Ansible |
James E. Blair | 8293847 | 2016-01-11 14:38:13 -0800 | [diff] [blame] | 99 | self.ansible_root = os.path.join(self.root, 'ansible') |
| 100 | os.makedirs(self.ansible_root) |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 101 | self.known_hosts = os.path.join(self.ansible_root, 'known_hosts') |
James E. Blair | 8293847 | 2016-01-11 14:38:13 -0800 | [diff] [blame] | 102 | self.inventory = os.path.join(self.ansible_root, 'inventory') |
James E. Blair | a92cbc8 | 2017-01-23 14:56:49 -0800 | [diff] [blame] | 103 | self.vars = os.path.join(self.ansible_root, 'vars.yaml') |
James E. Blair | a7f51ca | 2017-02-07 16:01:26 -0800 | [diff] [blame] | 104 | self.playbooks = [] # The list of candidate playbooks |
| 105 | self.playbook = None # A pointer to the candidate we have chosen |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 106 | self.pre_playbooks = [] |
| 107 | self.post_playbooks = [] |
James E. Blair | 5ac9384 | 2017-01-20 06:47:34 -0800 | [diff] [blame] | 108 | self.roles = [] |
| 109 | self.roles_path = [] |
Monty Taylor | e20c9f2 | 2017-02-22 17:48:07 -0500 | [diff] [blame] | 110 | self.untrusted_config = os.path.join( |
| 111 | self.ansible_root, 'untrusted.cfg') |
| 112 | self.trusted_config = os.path.join(self.ansible_root, 'trusted.cfg') |
James E. Blair | e81ba63 | 2017-02-23 10:12:55 -0500 | [diff] [blame] | 113 | self.ansible_log = os.path.join(self.log_root, 'ansible_log.txt') |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 114 | |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 115 | def addPrePlaybook(self): |
| 116 | count = len(self.pre_playbooks) |
| 117 | root = os.path.join(self.ansible_root, 'pre_playbook_%i' % (count,)) |
| 118 | os.makedirs(root) |
| 119 | playbook = JobDirPlaybook(root) |
| 120 | self.pre_playbooks.append(playbook) |
| 121 | return playbook |
| 122 | |
| 123 | def addPostPlaybook(self): |
| 124 | count = len(self.post_playbooks) |
| 125 | root = os.path.join(self.ansible_root, 'post_playbook_%i' % (count,)) |
| 126 | os.makedirs(root) |
| 127 | playbook = JobDirPlaybook(root) |
| 128 | self.post_playbooks.append(playbook) |
| 129 | return playbook |
| 130 | |
James E. Blair | a7f51ca | 2017-02-07 16:01:26 -0800 | [diff] [blame] | 131 | def addPlaybook(self): |
| 132 | count = len(self.playbooks) |
| 133 | root = os.path.join(self.ansible_root, 'playbook_%i' % (count,)) |
| 134 | os.makedirs(root) |
| 135 | playbook = JobDirPlaybook(root) |
| 136 | self.playbooks.append(playbook) |
| 137 | return playbook |
| 138 | |
James E. Blair | 5ac9384 | 2017-01-20 06:47:34 -0800 | [diff] [blame] | 139 | def addRole(self): |
| 140 | count = len(self.roles) |
| 141 | root = os.path.join(self.ansible_root, 'role_%i' % (count,)) |
| 142 | os.makedirs(root) |
| 143 | self.roles.append(root) |
| 144 | return root |
| 145 | |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 146 | def cleanup(self): |
| 147 | if not self.keep: |
| 148 | shutil.rmtree(self.root) |
| 149 | |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 150 | def __enter__(self): |
James E. Blair | 8293847 | 2016-01-11 14:38:13 -0800 | [diff] [blame] | 151 | return self |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 152 | |
| 153 | def __exit__(self, etype, value, tb): |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 154 | self.cleanup() |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 155 | |
| 156 | |
| 157 | class UpdateTask(object): |
| 158 | def __init__(self, project, url): |
| 159 | self.project = project |
| 160 | self.url = url |
| 161 | self.event = threading.Event() |
| 162 | |
| 163 | def __eq__(self, other): |
| 164 | if other.project == self.project: |
| 165 | return True |
| 166 | return False |
| 167 | |
| 168 | def wait(self): |
| 169 | self.event.wait() |
| 170 | |
| 171 | def setComplete(self): |
| 172 | self.event.set() |
| 173 | |
| 174 | |
| 175 | class DeduplicateQueue(object): |
| 176 | def __init__(self): |
| 177 | self.queue = collections.deque() |
| 178 | self.condition = threading.Condition() |
| 179 | |
| 180 | def qsize(self): |
| 181 | return len(self.queue) |
| 182 | |
| 183 | def put(self, item): |
| 184 | # Returns the original item if added, or an equivalent item if |
| 185 | # already enqueued. |
| 186 | self.condition.acquire() |
| 187 | ret = None |
| 188 | try: |
| 189 | for x in self.queue: |
| 190 | if item == x: |
| 191 | ret = x |
| 192 | if ret is None: |
| 193 | ret = item |
| 194 | self.queue.append(item) |
| 195 | self.condition.notify() |
| 196 | finally: |
| 197 | self.condition.release() |
| 198 | return ret |
| 199 | |
| 200 | def get(self): |
| 201 | self.condition.acquire() |
| 202 | try: |
| 203 | while True: |
| 204 | try: |
| 205 | ret = self.queue.popleft() |
| 206 | return ret |
| 207 | except IndexError: |
| 208 | pass |
| 209 | self.condition.wait() |
| 210 | finally: |
| 211 | self.condition.release() |
| 212 | |
| 213 | |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 214 | class ExecutorServer(object): |
| 215 | log = logging.getLogger("zuul.ExecutorServer") |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 216 | |
James E. Blair | 854f889 | 2017-02-02 11:25:39 -0800 | [diff] [blame] | 217 | def __init__(self, config, connections={}, jobdir_root=None, |
| 218 | keep_jobdir=False): |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 219 | self.config = config |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 220 | self.keep_jobdir = keep_jobdir |
James E. Blair | 854f889 | 2017-02-02 11:25:39 -0800 | [diff] [blame] | 221 | self.jobdir_root = jobdir_root |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 222 | # TODOv3(mordred): make the executor name more unique -- |
James E. Blair | 1730297 | 2016-08-10 16:11:42 -0700 | [diff] [blame] | 223 | # perhaps hostname+pid. |
| 224 | self.hostname = socket.gethostname() |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 225 | self.zuul_url = config.get('merger', 'zuul_url') |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 226 | self.command_map = dict( |
| 227 | stop=self.stop, |
| 228 | pause=self.pause, |
| 229 | unpause=self.unpause, |
| 230 | graceful=self.graceful, |
| 231 | verbose=self.verboseOn, |
| 232 | unverbose=self.verboseOff, |
| 233 | ) |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 234 | |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 235 | if self.config.has_option('executor', 'git_dir'): |
| 236 | self.merge_root = self.config.get('executor', 'git_dir') |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 237 | else: |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 238 | self.merge_root = '/var/lib/zuul/executor-git' |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 239 | |
| 240 | if self.config.has_option('merger', 'git_user_email'): |
| 241 | self.merge_email = self.config.get('merger', 'git_user_email') |
| 242 | else: |
| 243 | self.merge_email = None |
| 244 | |
| 245 | if self.config.has_option('merger', 'git_user_name'): |
| 246 | self.merge_name = self.config.get('merger', 'git_user_name') |
| 247 | else: |
| 248 | self.merge_name = None |
| 249 | |
| 250 | self.connections = connections |
James E. Blair | e29f80a | 2017-02-22 22:27:11 -0500 | [diff] [blame] | 251 | # This merger and its git repos are used to maintain |
| 252 | # up-to-date copies of all the repos that are used by jobs, as |
| 253 | # well as to support the merger:cat functon to supply |
| 254 | # configuration information to Zuul when it starts. |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 255 | self.merger = self._getMerger(self.merge_root) |
| 256 | self.update_queue = DeduplicateQueue() |
| 257 | |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 258 | if self.config.has_option('zuul', 'state_dir'): |
| 259 | state_dir = os.path.expanduser( |
| 260 | self.config.get('zuul', 'state_dir')) |
| 261 | else: |
| 262 | state_dir = '/var/lib/zuul' |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 263 | path = os.path.join(state_dir, 'executor.socket') |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 264 | self.command_socket = commandsocket.CommandSocket(path) |
| 265 | ansible_dir = os.path.join(state_dir, 'ansible') |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 266 | self.library_dir = os.path.join(ansible_dir, 'library') |
| 267 | if not os.path.exists(self.library_dir): |
| 268 | os.makedirs(self.library_dir) |
Monty Taylor | c231d93 | 2017-02-03 09:57:15 -0600 | [diff] [blame] | 269 | self.action_dir = os.path.join(ansible_dir, 'action') |
| 270 | if not os.path.exists(self.action_dir): |
| 271 | os.makedirs(self.action_dir) |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 272 | |
Monty Taylor | 825bca5 | 2017-02-20 06:58:46 -0500 | [diff] [blame] | 273 | self.callback_dir = os.path.join(ansible_dir, 'callback') |
| 274 | if not os.path.exists(self.callback_dir): |
| 275 | os.makedirs(self.callback_dir) |
| 276 | |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 277 | library_path = os.path.dirname(os.path.abspath( |
| 278 | zuul.ansible.library.__file__)) |
| 279 | for fn in os.listdir(library_path): |
| 280 | shutil.copy(os.path.join(library_path, fn), self.library_dir) |
Monty Taylor | 825bca5 | 2017-02-20 06:58:46 -0500 | [diff] [blame] | 281 | |
Monty Taylor | c231d93 | 2017-02-03 09:57:15 -0600 | [diff] [blame] | 282 | action_path = os.path.dirname(os.path.abspath( |
| 283 | zuul.ansible.action.__file__)) |
| 284 | for fn in os.listdir(action_path): |
| 285 | shutil.copy(os.path.join(action_path, fn), self.action_dir) |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 286 | |
Monty Taylor | 825bca5 | 2017-02-20 06:58:46 -0500 | [diff] [blame] | 287 | callback_path = os.path.dirname(os.path.abspath( |
| 288 | zuul.ansible.callback.__file__)) |
| 289 | for fn in os.listdir(callback_path): |
| 290 | shutil.copy(os.path.join(callback_path, fn), self.callback_dir) |
| 291 | |
Joshua Hesketh | 50c2178 | 2016-10-13 21:34:14 +1100 | [diff] [blame] | 292 | self.job_workers = {} |
| 293 | |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 294 | def _getMerger(self, root): |
| 295 | return zuul.merger.merger.Merger(root, self.connections, |
| 296 | self.merge_email, self.merge_name) |
| 297 | |
| 298 | def start(self): |
| 299 | self._running = True |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 300 | self._command_running = True |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 301 | server = self.config.get('gearman', 'server') |
| 302 | if self.config.has_option('gearman', 'port'): |
| 303 | port = self.config.get('gearman', 'port') |
| 304 | else: |
| 305 | port = 4730 |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 306 | self.worker = gear.Worker('Zuul Executor Server') |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 307 | self.worker.addServer(server, port) |
| 308 | self.log.debug("Waiting for server") |
| 309 | self.worker.waitForServer() |
| 310 | self.log.debug("Registering") |
| 311 | self.register() |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 312 | |
| 313 | self.log.debug("Starting command processor") |
| 314 | self.command_socket.start() |
| 315 | self.command_thread = threading.Thread(target=self.runCommand) |
| 316 | self.command_thread.daemon = True |
| 317 | self.command_thread.start() |
| 318 | |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 319 | self.log.debug("Starting worker") |
| 320 | self.update_thread = threading.Thread(target=self._updateLoop) |
| 321 | self.update_thread.daemon = True |
| 322 | self.update_thread.start() |
| 323 | self.thread = threading.Thread(target=self.run) |
| 324 | self.thread.daemon = True |
| 325 | self.thread.start() |
| 326 | |
| 327 | def register(self): |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 328 | self.worker.registerFunction("executor:execute") |
| 329 | self.worker.registerFunction("executor:stop:%s" % self.hostname) |
James E. Blair | 8b1dc3f | 2016-07-05 16:49:00 -0700 | [diff] [blame] | 330 | self.worker.registerFunction("merger:merge") |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 331 | self.worker.registerFunction("merger:cat") |
| 332 | |
| 333 | def stop(self): |
| 334 | self.log.debug("Stopping") |
| 335 | self._running = False |
| 336 | self.worker.shutdown() |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 337 | self._command_running = False |
| 338 | self.command_socket.stop() |
James E. Blair | 29b9c96 | 2017-02-13 16:17:22 -0800 | [diff] [blame] | 339 | self.update_queue.put(None) |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 340 | self.log.debug("Stopped") |
| 341 | |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 342 | def pause(self): |
| 343 | # TODOv3: implement |
| 344 | pass |
| 345 | |
| 346 | def unpause(self): |
| 347 | # TODOv3: implement |
| 348 | pass |
| 349 | |
| 350 | def graceful(self): |
| 351 | # TODOv3: implement |
| 352 | pass |
| 353 | |
| 354 | def verboseOn(self): |
| 355 | # TODOv3: implement |
| 356 | pass |
| 357 | |
| 358 | def verboseOff(self): |
| 359 | # TODOv3: implement |
| 360 | pass |
| 361 | |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 362 | def join(self): |
| 363 | self.update_thread.join() |
| 364 | self.thread.join() |
| 365 | |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 366 | def runCommand(self): |
| 367 | while self._command_running: |
| 368 | try: |
| 369 | command = self.command_socket.get() |
Joshua Hesketh | 39ee7ce | 2016-12-09 12:11:39 +1100 | [diff] [blame] | 370 | if command != '_stop': |
| 371 | self.command_map[command]() |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 372 | except Exception: |
| 373 | self.log.exception("Exception while processing command") |
| 374 | |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 375 | def _updateLoop(self): |
| 376 | while self._running: |
| 377 | try: |
| 378 | self._innerUpdateLoop() |
| 379 | except: |
| 380 | self.log.exception("Exception in update thread:") |
| 381 | |
| 382 | def _innerUpdateLoop(self): |
James E. Blair | e29f80a | 2017-02-22 22:27:11 -0500 | [diff] [blame] | 383 | # Inside of a loop that keeps the main repositories up to date |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 384 | task = self.update_queue.get() |
James E. Blair | 29b9c96 | 2017-02-13 16:17:22 -0800 | [diff] [blame] | 385 | if task is None: |
| 386 | # We are asked to stop |
| 387 | return |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 388 | self.log.info("Updating repo %s from %s" % (task.project, task.url)) |
| 389 | self.merger.updateRepo(task.project, task.url) |
| 390 | self.log.debug("Finished updating repo %s from %s" % |
| 391 | (task.project, task.url)) |
| 392 | task.setComplete() |
| 393 | |
| 394 | def update(self, project, url): |
James E. Blair | e29f80a | 2017-02-22 22:27:11 -0500 | [diff] [blame] | 395 | # Update a repository in the main merger |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 396 | task = UpdateTask(project, url) |
| 397 | task = self.update_queue.put(task) |
| 398 | return task |
| 399 | |
| 400 | def run(self): |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 401 | self.log.debug("Starting executor listener") |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 402 | while self._running: |
| 403 | try: |
| 404 | job = self.worker.getJob() |
| 405 | try: |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 406 | if job.name == 'executor:execute': |
| 407 | self.log.debug("Got execute job: %s" % job.unique) |
| 408 | self.executeJob(job) |
| 409 | elif job.name.startswith('executor:stop'): |
James E. Blair | 1730297 | 2016-08-10 16:11:42 -0700 | [diff] [blame] | 410 | self.log.debug("Got stop job: %s" % job.unique) |
| 411 | self.stopJob(job) |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 412 | elif job.name == 'merger:cat': |
| 413 | self.log.debug("Got cat job: %s" % job.unique) |
| 414 | self.cat(job) |
James E. Blair | 8b1dc3f | 2016-07-05 16:49:00 -0700 | [diff] [blame] | 415 | elif job.name == 'merger:merge': |
| 416 | self.log.debug("Got merge job: %s" % job.unique) |
| 417 | self.merge(job) |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 418 | else: |
| 419 | self.log.error("Unable to handle job %s" % job.name) |
| 420 | job.sendWorkFail() |
| 421 | except Exception: |
| 422 | self.log.exception("Exception while running job") |
| 423 | job.sendWorkException(traceback.format_exc()) |
James E. Blair | 29b9c96 | 2017-02-13 16:17:22 -0800 | [diff] [blame] | 424 | except gear.InterruptedError: |
| 425 | pass |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 426 | except Exception: |
| 427 | self.log.exception("Exception while getting job") |
| 428 | |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 429 | def executeJob(self, job): |
Joshua Hesketh | 50c2178 | 2016-10-13 21:34:14 +1100 | [diff] [blame] | 430 | self.job_workers[job.unique] = AnsibleJob(self, job) |
| 431 | self.job_workers[job.unique].run() |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 432 | |
Joshua Hesketh | 50c2178 | 2016-10-13 21:34:14 +1100 | [diff] [blame] | 433 | def finishJob(self, unique): |
| 434 | del(self.job_workers[unique]) |
| 435 | |
| 436 | def stopJob(self, job): |
James E. Blair | caa83ad | 2017-01-27 08:58:07 -0800 | [diff] [blame] | 437 | try: |
| 438 | args = json.loads(job.arguments) |
| 439 | self.log.debug("Stop job with arguments: %s" % (args,)) |
| 440 | unique = args['uuid'] |
| 441 | job_worker = self.job_workers.get(unique) |
| 442 | if not job_worker: |
| 443 | self.log.debug("Unable to find worker for job %s" % (unique,)) |
| 444 | return |
| 445 | try: |
| 446 | job_worker.stop() |
| 447 | except Exception: |
| 448 | self.log.exception("Exception sending stop command " |
| 449 | "to worker:") |
| 450 | finally: |
| 451 | job.sendWorkComplete() |
Joshua Hesketh | 50c2178 | 2016-10-13 21:34:14 +1100 | [diff] [blame] | 452 | |
| 453 | def cat(self, job): |
| 454 | args = json.loads(job.arguments) |
| 455 | task = self.update(args['project'], args['url']) |
| 456 | task.wait() |
| 457 | files = self.merger.getFiles(args['project'], args['url'], |
| 458 | args['branch'], args['files']) |
| 459 | result = dict(updated=True, |
| 460 | files=files, |
| 461 | zuul_url=self.zuul_url) |
| 462 | job.sendWorkComplete(json.dumps(result)) |
| 463 | |
| 464 | def merge(self, job): |
| 465 | args = json.loads(job.arguments) |
| 466 | ret = self.merger.mergeChanges(args['items'], args.get('files')) |
| 467 | result = dict(merged=(ret is not None), |
| 468 | zuul_url=self.zuul_url) |
| 469 | if args.get('files'): |
| 470 | result['commit'], result['files'] = ret |
| 471 | else: |
| 472 | result['commit'] = ret |
| 473 | job.sendWorkComplete(json.dumps(result)) |
| 474 | |
| 475 | |
| 476 | class AnsibleJob(object): |
| 477 | log = logging.getLogger("zuul.AnsibleJob") |
| 478 | |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 479 | RESULT_NORMAL = 1 |
| 480 | RESULT_TIMED_OUT = 2 |
| 481 | RESULT_UNREACHABLE = 3 |
| 482 | RESULT_ABORTED = 4 |
| 483 | |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 484 | def __init__(self, executor_server, job): |
| 485 | self.executor_server = executor_server |
Joshua Hesketh | 50c2178 | 2016-10-13 21:34:14 +1100 | [diff] [blame] | 486 | self.job = job |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 487 | self.jobdir = None |
James E. Blair | caa83ad | 2017-01-27 08:58:07 -0800 | [diff] [blame] | 488 | self.proc = None |
| 489 | self.proc_lock = threading.Lock() |
Joshua Hesketh | 50c2178 | 2016-10-13 21:34:14 +1100 | [diff] [blame] | 490 | self.running = False |
James E. Blair | caa83ad | 2017-01-27 08:58:07 -0800 | [diff] [blame] | 491 | self.aborted = False |
Joshua Hesketh | 50c2178 | 2016-10-13 21:34:14 +1100 | [diff] [blame] | 492 | |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 493 | if self.executor_server.config.has_option( |
| 494 | 'executor', 'private_key_file'): |
| 495 | self.private_key_file = self.executor_server.config.get( |
| 496 | 'executor', 'private_key_file') |
Joshua Hesketh | 50c2178 | 2016-10-13 21:34:14 +1100 | [diff] [blame] | 497 | else: |
| 498 | self.private_key_file = '~/.ssh/id_rsa' |
| 499 | |
| 500 | def run(self): |
| 501 | self.running = True |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 502 | self.thread = threading.Thread(target=self.execute) |
Joshua Hesketh | 50c2178 | 2016-10-13 21:34:14 +1100 | [diff] [blame] | 503 | self.thread.start() |
| 504 | |
| 505 | def stop(self): |
James E. Blair | caa83ad | 2017-01-27 08:58:07 -0800 | [diff] [blame] | 506 | self.aborted = True |
| 507 | self.abortRunningProc() |
Joshua Hesketh | 50c2178 | 2016-10-13 21:34:14 +1100 | [diff] [blame] | 508 | self.thread.join() |
| 509 | |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 510 | def execute(self): |
Joshua Hesketh | 50c2178 | 2016-10-13 21:34:14 +1100 | [diff] [blame] | 511 | try: |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 512 | self.jobdir = JobDir(root=self.executor_server.jobdir_root, |
| 513 | keep=self.executor_server.keep_jobdir) |
| 514 | self._execute() |
James E. Blair | 096c5cd | 2017-02-02 15:33:18 -0800 | [diff] [blame] | 515 | except Exception: |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 516 | self.log.exception("Exception while executing job") |
James E. Blair | 096c5cd | 2017-02-02 15:33:18 -0800 | [diff] [blame] | 517 | self.job.sendWorkException(traceback.format_exc()) |
Joshua Hesketh | 50c2178 | 2016-10-13 21:34:14 +1100 | [diff] [blame] | 518 | finally: |
| 519 | self.running = False |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 520 | try: |
| 521 | self.jobdir.cleanup() |
| 522 | except Exception: |
| 523 | self.log.exception("Error cleaning up jobdir:") |
| 524 | try: |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 525 | self.executor_server.finishJob(self.job.unique) |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 526 | except Exception: |
| 527 | self.log.exception("Error finalizing job thread:") |
Joshua Hesketh | 50c2178 | 2016-10-13 21:34:14 +1100 | [diff] [blame] | 528 | |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 529 | def _execute(self): |
Joshua Hesketh | 50c2178 | 2016-10-13 21:34:14 +1100 | [diff] [blame] | 530 | self.log.debug("Job %s: beginning" % (self.job.unique,)) |
James E. Blair | e47eb77 | 2017-02-02 17:19:40 -0800 | [diff] [blame] | 531 | self.log.debug("Job %s: args: %s" % (self.job.unique, |
| 532 | self.job.arguments,)) |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 533 | self.log.debug("Job %s: job root at %s" % |
| 534 | (self.job.unique, self.jobdir.root)) |
| 535 | args = json.loads(self.job.arguments) |
| 536 | tasks = [] |
| 537 | for project in args['projects']: |
| 538 | self.log.debug("Job %s: updating project %s" % |
| 539 | (self.job.unique, project['name'])) |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 540 | tasks.append(self.executor_server.update( |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 541 | project['name'], project['url'])) |
| 542 | for task in tasks: |
| 543 | task.wait() |
Joshua Hesketh | 50c2178 | 2016-10-13 21:34:14 +1100 | [diff] [blame] | 544 | |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 545 | self.log.debug("Job %s: git updates complete" % (self.job.unique,)) |
James E. Blair | e29f80a | 2017-02-22 22:27:11 -0500 | [diff] [blame] | 546 | for project in args['projects']: |
| 547 | self.log.debug("Cloning %s" % (project['name'],)) |
| 548 | repo = git.Repo.clone_from( |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 549 | os.path.join(self.executor_server.merge_root, |
James E. Blair | e29f80a | 2017-02-22 22:27:11 -0500 | [diff] [blame] | 550 | project['name']), |
Monty Taylor | d642d85 | 2017-02-23 14:05:42 -0500 | [diff] [blame] | 551 | os.path.join(self.jobdir.src_root, |
James E. Blair | e29f80a | 2017-02-22 22:27:11 -0500 | [diff] [blame] | 552 | project['name'])) |
| 553 | repo.remotes.origin.config_writer.set('url', project['url']) |
| 554 | |
| 555 | # Get a merger in order to update the repos involved in this job. |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 556 | merger = self.executor_server._getMerger(self.jobdir.src_root) |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 557 | merge_items = [i for i in args['items'] if i.get('refspec')] |
| 558 | if merge_items: |
| 559 | commit = merger.mergeChanges(merge_items) # noqa |
| 560 | else: |
| 561 | commit = args['items'][-1]['newrev'] # noqa |
James E. Blair | 8293847 | 2016-01-11 14:38:13 -0800 | [diff] [blame] | 562 | |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 563 | # is the playbook in a repo that we have already prepared? |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 564 | self.preparePlaybookRepos(args) |
James E. Blair | c73c73a | 2017-01-20 15:15:15 -0800 | [diff] [blame] | 565 | |
James E. Blair | 5ac9384 | 2017-01-20 06:47:34 -0800 | [diff] [blame] | 566 | self.prepareRoles(args) |
| 567 | |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 568 | # TODOv3: Ansible the ansible thing here. |
| 569 | self.prepareAnsibleFiles(args) |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 570 | |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 571 | data = { |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 572 | 'manager': self.executor_server.hostname, |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 573 | 'url': 'https://server/job/{}/0/'.format(args['job']), |
| 574 | 'worker_name': 'My Worker', |
| 575 | } |
James E. Blair | 1730297 | 2016-08-10 16:11:42 -0700 | [diff] [blame] | 576 | |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 577 | # TODOv3: |
| 578 | # 'name': self.name, |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 579 | # 'manager': self.executor_server.hostname, |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 580 | # 'worker_name': 'My Worker', |
| 581 | # 'worker_hostname': 'localhost', |
| 582 | # 'worker_ips': ['127.0.0.1', '192.168.1.1'], |
| 583 | # 'worker_fqdn': 'zuul.example.org', |
| 584 | # 'worker_program': 'FakeBuilder', |
| 585 | # 'worker_version': 'v1.1', |
| 586 | # 'worker_extra': {'something': 'else'} |
James E. Blair | 1730297 | 2016-08-10 16:11:42 -0700 | [diff] [blame] | 587 | |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 588 | self.job.sendWorkData(json.dumps(data)) |
| 589 | self.job.sendWorkStatus(0, 100) |
James E. Blair | f5dbd00 | 2015-12-23 15:26:17 -0800 | [diff] [blame] | 590 | |
Paul Belanger | 96618ed | 2017-03-01 09:42:33 -0500 | [diff] [blame] | 591 | result = self.runPlaybooks(args) |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 592 | |
| 593 | if result is None: |
| 594 | self.job.sendWorkFail() |
| 595 | return |
| 596 | result = dict(result=result) |
| 597 | self.job.sendWorkComplete(json.dumps(result)) |
| 598 | |
Paul Belanger | 96618ed | 2017-03-01 09:42:33 -0500 | [diff] [blame] | 599 | def runPlaybooks(self, args): |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 600 | result = None |
| 601 | |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 602 | for playbook in self.jobdir.pre_playbooks: |
Paul Belanger | 96618ed | 2017-03-01 09:42:33 -0500 | [diff] [blame] | 603 | # TODOv3(pabelanger): Implement pre-run timeout setting. |
| 604 | pre_status, pre_code = self.runAnsiblePlaybook( |
| 605 | playbook, args['timeout']) |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 606 | if pre_status != self.RESULT_NORMAL or pre_code != 0: |
| 607 | # These should really never fail, so return None and have |
| 608 | # zuul try again |
| 609 | return result |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 610 | |
Paul Belanger | 96618ed | 2017-03-01 09:42:33 -0500 | [diff] [blame] | 611 | job_status, job_code = self.runAnsiblePlaybook( |
| 612 | self.jobdir.playbook, args['timeout']) |
James E. Blair | caa83ad | 2017-01-27 08:58:07 -0800 | [diff] [blame] | 613 | if job_status == self.RESULT_TIMED_OUT: |
| 614 | return 'TIMED_OUT' |
| 615 | if job_status == self.RESULT_ABORTED: |
| 616 | return 'ABORTED' |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 617 | if job_status != self.RESULT_NORMAL: |
| 618 | # The result of the job is indeterminate. Zuul will |
| 619 | # run it again. |
| 620 | return result |
| 621 | |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 622 | success = (job_code == 0) |
| 623 | if success: |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 624 | result = 'SUCCESS' |
| 625 | else: |
| 626 | result = 'FAILURE' |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 627 | |
| 628 | for playbook in self.jobdir.post_playbooks: |
Paul Belanger | 96618ed | 2017-03-01 09:42:33 -0500 | [diff] [blame] | 629 | # TODOv3(pabelanger): Implement post-run timeout setting. |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 630 | post_status, post_code = self.runAnsiblePlaybook( |
Paul Belanger | 96618ed | 2017-03-01 09:42:33 -0500 | [diff] [blame] | 631 | playbook, args['timeout'], success) |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 632 | if post_status != self.RESULT_NORMAL or post_code != 0: |
| 633 | result = 'POST_FAILURE' |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 634 | return result |
James E. Blair | 1730297 | 2016-08-10 16:11:42 -0700 | [diff] [blame] | 635 | |
James E. Blair | 8293847 | 2016-01-11 14:38:13 -0800 | [diff] [blame] | 636 | def getHostList(self, args): |
James E. Blair | ad8dca0 | 2017-02-21 11:48:32 -0500 | [diff] [blame] | 637 | # TODO(clarkb): This prefers v4 because we're not sure if we |
| 638 | # expect v6 to work. If we can determine how to prefer v6 |
| 639 | hosts = [] |
James E. Blair | 34776ee | 2016-08-25 13:53:54 -0700 | [diff] [blame] | 640 | for node in args['nodes']: |
James E. Blair | ad8dca0 | 2017-02-21 11:48:32 -0500 | [diff] [blame] | 641 | ip = node.get('public_ipv4') |
| 642 | if not ip: |
| 643 | ip = node.get('public_ipv6') |
Paul Belanger | 30ba93a | 2017-03-16 16:28:10 -0400 | [diff] [blame] | 644 | hosts.append((node['name'], dict( |
| 645 | ansible_host=ip, |
| 646 | nodepool_az=node.get('az'), |
| 647 | nodepool_provider=node.get('provider'), |
| 648 | nodepool_region=node.get('region')))) |
James E. Blair | 34776ee | 2016-08-25 13:53:54 -0700 | [diff] [blame] | 649 | return hosts |
James E. Blair | 8293847 | 2016-01-11 14:38:13 -0800 | [diff] [blame] | 650 | |
James E. Blair | 5ac9384 | 2017-01-20 06:47:34 -0800 | [diff] [blame] | 651 | def _blockPluginDirs(self, path): |
| 652 | '''Prevent execution of playbooks or roles with plugins |
Monty Taylor | c231d93 | 2017-02-03 09:57:15 -0600 | [diff] [blame] | 653 | |
James E. Blair | 5ac9384 | 2017-01-20 06:47:34 -0800 | [diff] [blame] | 654 | Plugins are loaded from roles and also if there is a plugin |
| 655 | dir adjacent to the playbook. Throw an error if the path |
| 656 | contains a location that would cause a plugin to get loaded. |
| 657 | |
Monty Taylor | c231d93 | 2017-02-03 09:57:15 -0600 | [diff] [blame] | 658 | ''' |
James E. Blair | 5ac9384 | 2017-01-20 06:47:34 -0800 | [diff] [blame] | 659 | for entry in os.listdir(path): |
Monty Taylor | c231d93 | 2017-02-03 09:57:15 -0600 | [diff] [blame] | 660 | if os.path.isdir(entry) and entry.endswith('_plugins'): |
| 661 | raise Exception( |
| 662 | "Ansible plugin dir %s found adjacent to playbook %s in" |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 663 | " non-trusted repo." % (entry, path)) |
Monty Taylor | c231d93 | 2017-02-03 09:57:15 -0600 | [diff] [blame] | 664 | |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 665 | def findPlaybook(self, path, required=False, trusted=False): |
James E. Blair | d130f71 | 2017-01-25 14:56:10 -0800 | [diff] [blame] | 666 | for ext in ['.yaml', '.yml']: |
| 667 | fn = path + ext |
| 668 | if os.path.exists(fn): |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 669 | if not trusted: |
James E. Blair | 5ac9384 | 2017-01-20 06:47:34 -0800 | [diff] [blame] | 670 | playbook_dir = os.path.dirname(os.path.abspath(fn)) |
| 671 | self._blockPluginDirs(playbook_dir) |
James E. Blair | d130f71 | 2017-01-25 14:56:10 -0800 | [diff] [blame] | 672 | return fn |
James E. Blair | a7f51ca | 2017-02-07 16:01:26 -0800 | [diff] [blame] | 673 | if required: |
| 674 | raise Exception("Unable to find playbook %s" % path) |
| 675 | return None |
James E. Blair | d130f71 | 2017-01-25 14:56:10 -0800 | [diff] [blame] | 676 | |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 677 | def preparePlaybookRepos(self, args): |
| 678 | for playbook in args['pre_playbooks']: |
| 679 | jobdir_playbook = self.jobdir.addPrePlaybook() |
James E. Blair | a7f51ca | 2017-02-07 16:01:26 -0800 | [diff] [blame] | 680 | self.preparePlaybookRepo(jobdir_playbook, playbook, |
James E. Blair | 6541c1c | 2017-02-15 16:14:56 -0800 | [diff] [blame] | 681 | args, required=True) |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 682 | |
James E. Blair | a7f51ca | 2017-02-07 16:01:26 -0800 | [diff] [blame] | 683 | for playbook in args['playbooks']: |
| 684 | jobdir_playbook = self.jobdir.addPlaybook() |
| 685 | self.preparePlaybookRepo(jobdir_playbook, playbook, |
James E. Blair | 6541c1c | 2017-02-15 16:14:56 -0800 | [diff] [blame] | 686 | args, required=False) |
James E. Blair | a7f51ca | 2017-02-07 16:01:26 -0800 | [diff] [blame] | 687 | if jobdir_playbook.path is not None: |
| 688 | self.jobdir.playbook = jobdir_playbook |
| 689 | break |
| 690 | if self.jobdir.playbook is None: |
| 691 | raise Exception("No valid playbook found") |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 692 | |
| 693 | for playbook in args['post_playbooks']: |
| 694 | jobdir_playbook = self.jobdir.addPostPlaybook() |
James E. Blair | a7f51ca | 2017-02-07 16:01:26 -0800 | [diff] [blame] | 695 | self.preparePlaybookRepo(jobdir_playbook, playbook, |
James E. Blair | 6541c1c | 2017-02-15 16:14:56 -0800 | [diff] [blame] | 696 | args, required=True) |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 697 | |
James E. Blair | 6541c1c | 2017-02-15 16:14:56 -0800 | [diff] [blame] | 698 | def preparePlaybookRepo(self, jobdir_playbook, playbook, args, required): |
James E. Blair | a7f51ca | 2017-02-07 16:01:26 -0800 | [diff] [blame] | 699 | self.log.debug("Prepare playbook repo for %s" % (playbook,)) |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 700 | # Check out the playbook repo if needed and set the path to |
James E. Blair | c73c73a | 2017-01-20 15:15:15 -0800 | [diff] [blame] | 701 | # the playbook that should be run. |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 702 | jobdir_playbook.trusted = playbook['trusted'] |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 703 | source = self.executor_server.connections.getSource( |
Joshua Hesketh | 50c2178 | 2016-10-13 21:34:14 +1100 | [diff] [blame] | 704 | playbook['connection']) |
James E. Blair | c73c73a | 2017-01-20 15:15:15 -0800 | [diff] [blame] | 705 | project = source.getProject(playbook['project']) |
| 706 | # TODO(jeblair): construct the url in the merger itself |
| 707 | url = source.getGitUrl(project) |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 708 | if not playbook['trusted']: |
James E. Blair | c73c73a | 2017-01-20 15:15:15 -0800 | [diff] [blame] | 709 | # This is a project repo, so it is safe to use the already |
| 710 | # checked out version (from speculative merging) of the |
| 711 | # playbook |
| 712 | for i in args['items']: |
| 713 | if (i['connection_name'] == playbook['connection'] and |
| 714 | i['project'] == playbook['project']): |
| 715 | # We already have this repo prepared |
Monty Taylor | d642d85 | 2017-02-23 14:05:42 -0500 | [diff] [blame] | 716 | path = os.path.join(self.jobdir.src_root, |
James E. Blair | c73c73a | 2017-01-20 15:15:15 -0800 | [diff] [blame] | 717 | project.name, |
| 718 | playbook['path']) |
Monty Taylor | c231d93 | 2017-02-03 09:57:15 -0600 | [diff] [blame] | 719 | jobdir_playbook.path = self.findPlaybook( |
| 720 | path, |
James E. Blair | 6541c1c | 2017-02-15 16:14:56 -0800 | [diff] [blame] | 721 | required=required, |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 722 | trusted=playbook['trusted']) |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 723 | return |
James E. Blair | c73c73a | 2017-01-20 15:15:15 -0800 | [diff] [blame] | 724 | # The playbook repo is either a config repo, or it isn't in |
| 725 | # the stack of changes we are testing, so check out the branch |
| 726 | # tip into a dedicated space. |
| 727 | |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 728 | merger = self.executor_server._getMerger(jobdir_playbook.root) |
James E. Blair | c73c73a | 2017-01-20 15:15:15 -0800 | [diff] [blame] | 729 | merger.checkoutBranch(project.name, url, playbook['branch']) |
| 730 | |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 731 | path = os.path.join(jobdir_playbook.root, |
James E. Blair | c73c73a | 2017-01-20 15:15:15 -0800 | [diff] [blame] | 732 | project.name, |
| 733 | playbook['path']) |
Monty Taylor | c231d93 | 2017-02-03 09:57:15 -0600 | [diff] [blame] | 734 | jobdir_playbook.path = self.findPlaybook( |
| 735 | path, |
James E. Blair | 6541c1c | 2017-02-15 16:14:56 -0800 | [diff] [blame] | 736 | required=required, |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 737 | trusted=playbook['trusted']) |
James E. Blair | c73c73a | 2017-01-20 15:15:15 -0800 | [diff] [blame] | 738 | |
James E. Blair | 5ac9384 | 2017-01-20 06:47:34 -0800 | [diff] [blame] | 739 | def prepareRoles(self, args): |
| 740 | for role in args['roles']: |
| 741 | if role['type'] == 'zuul': |
| 742 | root = self.jobdir.addRole() |
| 743 | self.prepareZuulRole(args, role, root) |
| 744 | |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 745 | def findRole(self, path, trusted=False): |
James E. Blair | 5ac9384 | 2017-01-20 06:47:34 -0800 | [diff] [blame] | 746 | d = os.path.join(path, 'tasks') |
| 747 | if os.path.isdir(d): |
| 748 | # This is a bare role |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 749 | if not trusted: |
James E. Blair | 5ac9384 | 2017-01-20 06:47:34 -0800 | [diff] [blame] | 750 | self._blockPluginDirs(path) |
| 751 | # None signifies that the repo is a bare role |
| 752 | return None |
| 753 | d = os.path.join(path, 'roles') |
| 754 | if os.path.isdir(d): |
| 755 | # This repo has a collection of roles |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 756 | if not trusted: |
James E. Blair | 5ac9384 | 2017-01-20 06:47:34 -0800 | [diff] [blame] | 757 | for entry in os.listdir(d): |
| 758 | self._blockPluginDirs(os.path.join(d, entry)) |
| 759 | return d |
| 760 | # We assume the repository itself is a collection of roles |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 761 | if not trusted: |
James E. Blair | 5ac9384 | 2017-01-20 06:47:34 -0800 | [diff] [blame] | 762 | for entry in os.listdir(path): |
| 763 | self._blockPluginDirs(os.path.join(path, entry)) |
| 764 | return path |
| 765 | |
| 766 | def prepareZuulRole(self, args, role, root): |
| 767 | self.log.debug("Prepare zuul role for %s" % (role,)) |
| 768 | # Check out the role repo if needed |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 769 | source = self.executor_server.connections.getSource( |
James E. Blair | 5ac9384 | 2017-01-20 06:47:34 -0800 | [diff] [blame] | 770 | role['connection']) |
| 771 | project = source.getProject(role['project']) |
| 772 | # TODO(jeblair): construct the url in the merger itself |
| 773 | url = source.getGitUrl(project) |
| 774 | role_repo = None |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 775 | if not role['trusted']: |
James E. Blair | 5ac9384 | 2017-01-20 06:47:34 -0800 | [diff] [blame] | 776 | # This is a project repo, so it is safe to use the already |
| 777 | # checked out version (from speculative merging) of the |
| 778 | # role |
| 779 | |
| 780 | for i in args['items']: |
| 781 | if (i['connection_name'] == role['connection'] and |
| 782 | i['project'] == role['project']): |
| 783 | # We already have this repo prepared; |
| 784 | # copy it into location. |
| 785 | |
Monty Taylor | d642d85 | 2017-02-23 14:05:42 -0500 | [diff] [blame] | 786 | path = os.path.join(self.jobdir.src_root, |
James E. Blair | 5ac9384 | 2017-01-20 06:47:34 -0800 | [diff] [blame] | 787 | project.name) |
| 788 | link = os.path.join(root, role['name']) |
| 789 | os.symlink(path, link) |
| 790 | role_repo = link |
| 791 | break |
| 792 | |
| 793 | # The role repo is either a config repo, or it isn't in |
| 794 | # the stack of changes we are testing, so check out the branch |
| 795 | # tip into a dedicated space. |
| 796 | |
| 797 | if not role_repo: |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 798 | merger = self.executor_server._getMerger(root) |
James E. Blair | 5ac9384 | 2017-01-20 06:47:34 -0800 | [diff] [blame] | 799 | merger.checkoutBranch(project.name, url, 'master') |
| 800 | role_repo = os.path.join(root, project.name) |
| 801 | |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 802 | role_path = self.findRole(role_repo, trusted=role['trusted']) |
James E. Blair | 5ac9384 | 2017-01-20 06:47:34 -0800 | [diff] [blame] | 803 | if role_path is None: |
| 804 | # In the case of a bare role, add the containing directory |
| 805 | role_path = root |
| 806 | self.jobdir.roles_path.append(role_path) |
| 807 | |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 808 | def prepareAnsibleFiles(self, args): |
| 809 | with open(self.jobdir.inventory, 'w') as inventory: |
James E. Blair | 8293847 | 2016-01-11 14:38:13 -0800 | [diff] [blame] | 810 | for host_name, host_vars in self.getHostList(args): |
| 811 | inventory.write(host_name) |
James E. Blair | 8293847 | 2016-01-11 14:38:13 -0800 | [diff] [blame] | 812 | for k, v in host_vars.items(): |
Paul Belanger | 30ba93a | 2017-03-16 16:28:10 -0400 | [diff] [blame] | 813 | inventory.write(' %s=%s' % (k, v)) |
James E. Blair | 8293847 | 2016-01-11 14:38:13 -0800 | [diff] [blame] | 814 | inventory.write('\n') |
James E. Blair | 6c0978c | 2017-02-21 15:03:24 -0500 | [diff] [blame] | 815 | if 'ansible_host' in host_vars: |
| 816 | os.system("ssh-keyscan %s >> %s" % ( |
| 817 | host_vars['ansible_host'], |
| 818 | self.jobdir.known_hosts)) |
| 819 | |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 820 | with open(self.jobdir.vars, 'w') as vars_yaml: |
James E. Blair | 490cf04 | 2017-02-24 23:07:21 -0500 | [diff] [blame] | 821 | zuul_vars = dict(args['vars']) |
Paul Belanger | e2b8d49 | 2017-03-22 18:57:40 -0400 | [diff] [blame^] | 822 | zuul_vars['zuul']['executor'] = dict( |
| 823 | hostname=self.executor_server.hostname, |
| 824 | src_root=self.jobdir.src_root, |
| 825 | log_root=self.jobdir.log_root) |
James E. Blair | a92cbc8 | 2017-01-23 14:56:49 -0800 | [diff] [blame] | 826 | vars_yaml.write( |
| 827 | yaml.safe_dump(zuul_vars, default_flow_style=False)) |
Monty Taylor | e20c9f2 | 2017-02-22 17:48:07 -0500 | [diff] [blame] | 828 | self.writeAnsibleConfig(self.jobdir.untrusted_config) |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 829 | self.writeAnsibleConfig(self.jobdir.trusted_config, trusted=True) |
Monty Taylor | c231d93 | 2017-02-03 09:57:15 -0600 | [diff] [blame] | 830 | |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 831 | def writeAnsibleConfig(self, config_path, trusted=False): |
Monty Taylor | c231d93 | 2017-02-03 09:57:15 -0600 | [diff] [blame] | 832 | with open(config_path, 'w') as config: |
James E. Blair | 8293847 | 2016-01-11 14:38:13 -0800 | [diff] [blame] | 833 | config.write('[defaults]\n') |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 834 | config.write('hostfile = %s\n' % self.jobdir.inventory) |
| 835 | config.write('local_tmp = %s/.ansible/local_tmp\n' % |
| 836 | self.jobdir.root) |
| 837 | config.write('remote_tmp = %s/.ansible/remote_tmp\n' % |
| 838 | self.jobdir.root) |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 839 | config.write('private_key_file = %s\n' % self.private_key_file) |
| 840 | config.write('retry_files_enabled = False\n') |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 841 | config.write('log_path = %s\n' % self.jobdir.ansible_log) |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 842 | config.write('gathering = explicit\n') |
Joshua Hesketh | 50c2178 | 2016-10-13 21:34:14 +1100 | [diff] [blame] | 843 | config.write('library = %s\n' |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 844 | % self.executor_server.library_dir) |
James E. Blair | 5ac9384 | 2017-01-20 06:47:34 -0800 | [diff] [blame] | 845 | if self.jobdir.roles_path: |
| 846 | config.write('roles_path = %s\n' % |
| 847 | ':'.join(self.jobdir.roles_path)) |
Monty Taylor | 825bca5 | 2017-02-20 06:58:46 -0500 | [diff] [blame] | 848 | config.write('callback_plugins = %s\n' |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 849 | % self.executor_server.callback_dir) |
Monty Taylor | 825bca5 | 2017-02-20 06:58:46 -0500 | [diff] [blame] | 850 | config.write('stdout_callback = zuul_stream\n') |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 851 | # bump the timeout because busy nodes may take more than |
| 852 | # 10s to respond |
| 853 | config.write('timeout = 30\n') |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 854 | if not trusted: |
Monty Taylor | c231d93 | 2017-02-03 09:57:15 -0600 | [diff] [blame] | 855 | config.write('action_plugins = %s\n' |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 856 | % self.executor_server.action_dir) |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 857 | |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 858 | # On trusted jobs, we want to prevent the printing of args, |
| 859 | # since trusted jobs might have access to secrets that they may |
Monty Taylor | 40728e3 | 2017-02-20 07:06:58 -0500 | [diff] [blame] | 860 | # need to pass to a task or a role. On the other hand, there |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 861 | # should be no sensitive data in untrusted jobs, and printing |
Monty Taylor | 40728e3 | 2017-02-20 07:06:58 -0500 | [diff] [blame] | 862 | # the args could be useful for debugging. |
| 863 | config.write('display_args_to_stdout = %s\n' % |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 864 | str(not trusted)) |
Monty Taylor | 40728e3 | 2017-02-20 07:06:58 -0500 | [diff] [blame] | 865 | |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 866 | config.write('[ssh_connection]\n') |
Joshua Hesketh | 3f7def3 | 2016-11-21 17:36:44 +1100 | [diff] [blame] | 867 | # NB: when setting pipelining = True, keep_remote_files |
| 868 | # must be False (the default). Otherwise it apparently |
| 869 | # will override the pipelining option and effectively |
| 870 | # disable it. Pipelining has a side effect of running the |
| 871 | # command without a tty (ie, without the -tt argument to |
| 872 | # ssh). We require this behavior so that if a job runs a |
| 873 | # command which expects interactive input on a tty (such |
| 874 | # as sudo) it does not hang. |
| 875 | config.write('pipelining = True\n') |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 876 | ssh_args = "-o ControlMaster=auto -o ControlPersist=60s " \ |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 877 | "-o UserKnownHostsFile=%s" % self.jobdir.known_hosts |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 878 | config.write('ssh_args = %s\n' % ssh_args) |
| 879 | |
James E. Blair | caa83ad | 2017-01-27 08:58:07 -0800 | [diff] [blame] | 880 | def _ansibleTimeout(self, msg): |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 881 | self.log.warning(msg) |
James E. Blair | caa83ad | 2017-01-27 08:58:07 -0800 | [diff] [blame] | 882 | self.abortRunningProc() |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 883 | |
James E. Blair | caa83ad | 2017-01-27 08:58:07 -0800 | [diff] [blame] | 884 | def abortRunningProc(self): |
| 885 | with self.proc_lock: |
| 886 | if not self.proc: |
| 887 | self.log.debug("Abort: no process is running") |
| 888 | return |
| 889 | self.log.debug("Abort: sending kill signal to job " |
| 890 | "process group") |
| 891 | try: |
| 892 | pgid = os.getpgid(self.proc.pid) |
| 893 | os.killpg(pgid, signal.SIGKILL) |
| 894 | except Exception: |
Monty Taylor | e20c9f2 | 2017-02-22 17:48:07 -0500 | [diff] [blame] | 895 | self.log.exception("Exception while killing ansible process:") |
James E. Blair | 8293847 | 2016-01-11 14:38:13 -0800 | [diff] [blame] | 896 | |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 897 | def runAnsible(self, cmd, timeout, trusted=False): |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 898 | env_copy = os.environ.copy() |
| 899 | env_copy['LOGNAME'] = 'zuul' |
| 900 | |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 901 | if trusted: |
Monty Taylor | e20c9f2 | 2017-02-22 17:48:07 -0500 | [diff] [blame] | 902 | env_copy['ANSIBLE_CONFIG'] = self.jobdir.trusted_config |
Monty Taylor | c231d93 | 2017-02-03 09:57:15 -0600 | [diff] [blame] | 903 | else: |
Monty Taylor | e20c9f2 | 2017-02-22 17:48:07 -0500 | [diff] [blame] | 904 | env_copy['ANSIBLE_CONFIG'] = self.jobdir.untrusted_config |
Monty Taylor | c231d93 | 2017-02-03 09:57:15 -0600 | [diff] [blame] | 905 | |
James E. Blair | caa83ad | 2017-01-27 08:58:07 -0800 | [diff] [blame] | 906 | with self.proc_lock: |
| 907 | if self.aborted: |
| 908 | return (self.RESULT_ABORTED, None) |
| 909 | self.log.debug("Ansible command: %s" % (cmd,)) |
| 910 | self.proc = subprocess.Popen( |
| 911 | cmd, |
Monty Taylor | e20c9f2 | 2017-02-22 17:48:07 -0500 | [diff] [blame] | 912 | cwd=self.jobdir.work_root, |
James E. Blair | caa83ad | 2017-01-27 08:58:07 -0800 | [diff] [blame] | 913 | stdout=subprocess.PIPE, |
| 914 | stderr=subprocess.STDOUT, |
| 915 | preexec_fn=os.setsid, |
| 916 | env=env_copy, |
| 917 | ) |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 918 | |
| 919 | ret = None |
Paul Belanger | 96618ed | 2017-03-01 09:42:33 -0500 | [diff] [blame] | 920 | if timeout: |
| 921 | watchdog = Watchdog(timeout, self._ansibleTimeout, |
| 922 | ("Ansible timeout exceeded",)) |
| 923 | watchdog.start() |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 924 | try: |
James E. Blair | caa83ad | 2017-01-27 08:58:07 -0800 | [diff] [blame] | 925 | for line in iter(self.proc.stdout.readline, b''): |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 926 | line = line[:1024].rstrip() |
| 927 | self.log.debug("Ansible output: %s" % (line,)) |
James E. Blair | caa83ad | 2017-01-27 08:58:07 -0800 | [diff] [blame] | 928 | ret = self.proc.wait() |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 929 | finally: |
Paul Belanger | 96618ed | 2017-03-01 09:42:33 -0500 | [diff] [blame] | 930 | if timeout: |
| 931 | watchdog.stop() |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 932 | self.log.debug("Ansible exit code: %s" % (ret,)) |
| 933 | |
James E. Blair | caa83ad | 2017-01-27 08:58:07 -0800 | [diff] [blame] | 934 | with self.proc_lock: |
| 935 | self.proc = None |
| 936 | |
Paul Belanger | 96618ed | 2017-03-01 09:42:33 -0500 | [diff] [blame] | 937 | if timeout and watchdog.timed_out: |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 938 | return (self.RESULT_TIMED_OUT, None) |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 939 | if ret == 3: |
| 940 | # AnsibleHostUnreachable: We had a network issue connecting to |
| 941 | # our zuul-worker. |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 942 | return (self.RESULT_UNREACHABLE, None) |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 943 | elif ret == -9: |
| 944 | # Received abort request. |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 945 | return (self.RESULT_ABORTED, None) |
James E. Blair | 414cb67 | 2016-10-05 13:48:14 -0700 | [diff] [blame] | 946 | |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 947 | return (self.RESULT_NORMAL, ret) |
| 948 | |
Paul Belanger | 96618ed | 2017-03-01 09:42:33 -0500 | [diff] [blame] | 949 | def runAnsiblePlaybook(self, playbook, timeout, success=None): |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 950 | env_copy = os.environ.copy() |
| 951 | env_copy['LOGNAME'] = 'zuul' |
| 952 | |
| 953 | if False: # TODOv3: self.options['verbose']: |
| 954 | verbose = '-vvv' |
| 955 | else: |
| 956 | verbose = '-v' |
| 957 | |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 958 | cmd = ['ansible-playbook', playbook.path] |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 959 | |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 960 | if success is not None: |
| 961 | cmd.extend(['-e', 'success=%s' % str(bool(success))]) |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 962 | |
James E. Blair | 66b274e | 2017-01-31 14:47:52 -0800 | [diff] [blame] | 963 | cmd.extend(['-e@%s' % self.jobdir.vars, verbose]) |
James E. Blair | 412fba8 | 2017-01-26 15:00:50 -0800 | [diff] [blame] | 964 | |
Monty Taylor | c231d93 | 2017-02-03 09:57:15 -0600 | [diff] [blame] | 965 | return self.runAnsible( |
Monty Taylor | e6562aa | 2017-02-20 07:37:39 -0500 | [diff] [blame] | 966 | cmd=cmd, timeout=timeout, trusted=playbook.trusted) |