James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright 2012 Hewlett-Packard Development Company, L.P. |
| 3 | # Copyright 2013-2014 OpenStack Foundation |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 6 | # not use this file except in compliance with the License. You may obtain |
| 7 | # a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 14 | # License for the specific language governing permissions and limitations |
| 15 | # under the License. |
| 16 | |
| 17 | import argparse |
| 18 | import daemon |
| 19 | import extras |
| 20 | |
| 21 | # as of python-daemon 1.6 it doesn't bundle pidlockfile anymore |
| 22 | # instead it depends on lockfile-0.9.1 which uses pidfile. |
| 23 | pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile']) |
| 24 | |
James E. Blair | 119acf3 | 2016-04-18 15:34:36 -0700 | [diff] [blame] | 25 | import logging |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 26 | import os |
David Shrewsbury | eb85647 | 2017-04-13 14:23:04 -0400 | [diff] [blame] | 27 | import pwd |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 28 | import socket |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 29 | import sys |
| 30 | import signal |
David Shrewsbury | eb85647 | 2017-04-13 14:23:04 -0400 | [diff] [blame] | 31 | import tempfile |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 32 | |
| 33 | import zuul.cmd |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 34 | import zuul.executor.server |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 35 | |
James E. Blair | a6a5004 | 2016-06-01 15:33:54 -0700 | [diff] [blame] | 36 | # No zuul imports that pull in paramiko here; it must not be |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 37 | # imported until after the daemonization. |
| 38 | # https://github.com/paramiko/paramiko/issues/59 |
| 39 | # Similar situation with gear and statsd. |
| 40 | |
| 41 | |
Tobias Henkel | cec2966 | 2017-06-02 12:27:47 +0200 | [diff] [blame] | 42 | DEFAULT_FINGER_PORT = 79 |
David Shrewsbury | eb85647 | 2017-04-13 14:23:04 -0400 | [diff] [blame] | 43 | |
| 44 | |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 45 | class Executor(zuul.cmd.ZuulApp): |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 46 | |
| 47 | def parse_arguments(self): |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 48 | parser = argparse.ArgumentParser(description='Zuul executor.') |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 49 | parser.add_argument('-c', dest='config', |
| 50 | help='specify the config file') |
| 51 | parser.add_argument('-d', dest='nodaemon', action='store_true', |
| 52 | help='do not run as a daemon') |
| 53 | parser.add_argument('--version', dest='version', action='version', |
| 54 | version=self._get_version(), |
| 55 | help='show zuul version') |
James E. Blair | f87c5ce | 2016-05-25 08:43:37 -0700 | [diff] [blame] | 56 | parser.add_argument('--keep-jobdir', dest='keep_jobdir', |
| 57 | action='store_true', |
| 58 | help='keep local jobdirs after run completes') |
James E. Blair | a6a5004 | 2016-06-01 15:33:54 -0700 | [diff] [blame] | 59 | parser.add_argument('command', |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 60 | choices=zuul.executor.server.COMMANDS, |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 61 | nargs='?') |
| 62 | |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 63 | self.args = parser.parse_args() |
| 64 | |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 65 | def send_command(self, cmd): |
| 66 | if self.config.has_option('zuul', 'state_dir'): |
| 67 | state_dir = os.path.expanduser( |
| 68 | self.config.get('zuul', 'state_dir')) |
| 69 | else: |
| 70 | state_dir = '/var/lib/zuul' |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 71 | path = os.path.join(state_dir, 'executor.socket') |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 72 | s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) |
| 73 | s.connect(path) |
| 74 | s.sendall('%s\n' % cmd) |
James E. Blair | 119acf3 | 2016-04-18 15:34:36 -0700 | [diff] [blame] | 75 | |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 76 | def exit_handler(self): |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 77 | self.executor.stop() |
| 78 | self.executor.join() |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 79 | |
David Shrewsbury | eb85647 | 2017-04-13 14:23:04 -0400 | [diff] [blame] | 80 | def start_log_streamer(self): |
| 81 | pipe_read, pipe_write = os.pipe() |
| 82 | child_pid = os.fork() |
| 83 | if child_pid == 0: |
| 84 | os.close(pipe_write) |
| 85 | import zuul.lib.log_streamer |
| 86 | |
| 87 | self.log.info("Starting log streamer") |
| 88 | streamer = zuul.lib.log_streamer.LogStreamer( |
Tobias Henkel | cec2966 | 2017-06-02 12:27:47 +0200 | [diff] [blame] | 89 | self.user, '0.0.0.0', self.finger_port, self.jobroot_dir) |
David Shrewsbury | eb85647 | 2017-04-13 14:23:04 -0400 | [diff] [blame] | 90 | |
| 91 | # Keep running until the parent dies: |
| 92 | pipe_read = os.fdopen(pipe_read) |
| 93 | pipe_read.read() |
| 94 | self.log.info("Stopping log streamer") |
| 95 | streamer.stop() |
| 96 | os._exit(0) |
| 97 | else: |
| 98 | os.close(pipe_read) |
| 99 | self.log_streamer_pid = child_pid |
| 100 | |
| 101 | def change_privs(self): |
| 102 | ''' |
| 103 | Drop our privileges to the zuul user. |
| 104 | ''' |
| 105 | if os.getuid() != 0: |
| 106 | return |
David Shrewsbury | e737420 | 2017-04-27 12:28:45 -0400 | [diff] [blame] | 107 | pw = pwd.getpwnam(self.user) |
David Shrewsbury | eb85647 | 2017-04-13 14:23:04 -0400 | [diff] [blame] | 108 | os.setgroups([]) |
| 109 | os.setgid(pw.pw_gid) |
| 110 | os.setuid(pw.pw_uid) |
| 111 | os.umask(0o022) |
| 112 | |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 113 | def main(self, daemon=True): |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 114 | # See comment at top of file about zuul imports |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 115 | |
David Shrewsbury | e737420 | 2017-04-27 12:28:45 -0400 | [diff] [blame] | 116 | if self.config.has_option('executor', 'user'): |
| 117 | self.user = self.config.get('executor', 'user') |
| 118 | else: |
| 119 | self.user = 'zuul' |
| 120 | |
David Shrewsbury | eb85647 | 2017-04-13 14:23:04 -0400 | [diff] [blame] | 121 | if self.config.has_option('zuul', 'jobroot_dir'): |
| 122 | self.jobroot_dir = os.path.expanduser( |
| 123 | self.config.get('zuul', 'jobroot_dir')) |
| 124 | else: |
David Shrewsbury | e737420 | 2017-04-27 12:28:45 -0400 | [diff] [blame] | 125 | self.jobroot_dir = tempfile.gettempdir() |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 126 | |
David Shrewsbury | eb85647 | 2017-04-13 14:23:04 -0400 | [diff] [blame] | 127 | self.setup_logging('executor', 'log_config') |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 128 | self.log = logging.getLogger("zuul.Executor") |
James E. Blair | 119acf3 | 2016-04-18 15:34:36 -0700 | [diff] [blame] | 129 | |
Tobias Henkel | cec2966 | 2017-06-02 12:27:47 +0200 | [diff] [blame] | 130 | if self.config.has_option('executor', 'finger_port'): |
| 131 | self.finger_port = int(self.config.get('executor', 'finger_port')) |
| 132 | else: |
| 133 | self.finger_port = DEFAULT_FINGER_PORT |
| 134 | |
David Shrewsbury | eb85647 | 2017-04-13 14:23:04 -0400 | [diff] [blame] | 135 | self.start_log_streamer() |
| 136 | self.change_privs() |
David Shrewsbury | d678429 | 2017-04-13 12:08:29 -0400 | [diff] [blame] | 137 | |
Paul Belanger | 9cd1286 | 2017-03-15 13:03:59 -0400 | [diff] [blame] | 138 | ExecutorServer = zuul.executor.server.ExecutorServer |
| 139 | self.executor = ExecutorServer(self.config, self.connections, |
David Shrewsbury | eb85647 | 2017-04-13 14:23:04 -0400 | [diff] [blame] | 140 | jobdir_root=self.jobroot_dir, |
Paul Belanger | 9cd1286 | 2017-03-15 13:03:59 -0400 | [diff] [blame] | 141 | keep_jobdir=self.args.keep_jobdir) |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 142 | self.executor.start() |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 143 | |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 144 | signal.signal(signal.SIGUSR2, zuul.cmd.stack_dump_handler) |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 145 | if daemon: |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 146 | self.executor.join() |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 147 | else: |
| 148 | while True: |
| 149 | try: |
| 150 | signal.pause() |
| 151 | except KeyboardInterrupt: |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 152 | print("Ctrl + C: asking executor to exit nicely...\n") |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 153 | self.exit_handler() |
| 154 | sys.exit(0) |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 155 | |
| 156 | |
| 157 | def main(): |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 158 | server = Executor() |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 159 | server.parse_arguments() |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 160 | server.read_config() |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 161 | |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 162 | if server.args.command in zuul.executor.server.COMMANDS: |
James E. Blair | a6a5004 | 2016-06-01 15:33:54 -0700 | [diff] [blame] | 163 | server.send_command(server.args.command) |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 164 | sys.exit(0) |
| 165 | |
Tristan Cacqueray | bad3b12 | 2017-05-21 03:07:08 +0000 | [diff] [blame] | 166 | server.configure_connections(source_only=True) |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 167 | |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 168 | if server.config.has_option('executor', 'pidfile'): |
| 169 | pid_fn = os.path.expanduser(server.config.get('executor', 'pidfile')) |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 170 | else: |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 171 | pid_fn = '/var/run/zuul-executor/zuul-executor.pid' |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 172 | pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10) |
| 173 | |
| 174 | if server.args.nodaemon: |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 175 | server.main(False) |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 176 | else: |
| 177 | with daemon.DaemonContext(pidfile=pid): |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 178 | server.main(True) |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 179 | |
| 180 | |
| 181 | if __name__ == "__main__": |
| 182 | sys.path.insert(0, '.') |
| 183 | main() |