James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright 2012 Hewlett-Packard Development Company, L.P. |
| 3 | # Copyright 2013-2014 OpenStack Foundation |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 6 | # not use this file except in compliance with the License. You may obtain |
| 7 | # a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 14 | # License for the specific language governing permissions and limitations |
| 15 | # under the License. |
| 16 | |
| 17 | import argparse |
| 18 | import daemon |
| 19 | import extras |
| 20 | |
| 21 | # as of python-daemon 1.6 it doesn't bundle pidlockfile anymore |
| 22 | # instead it depends on lockfile-0.9.1 which uses pidfile. |
| 23 | pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile']) |
| 24 | |
James E. Blair | 119acf3 | 2016-04-18 15:34:36 -0700 | [diff] [blame] | 25 | import logging |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 26 | import os |
David Shrewsbury | eb85647 | 2017-04-13 14:23:04 -0400 | [diff] [blame] | 27 | import pwd |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 28 | import socket |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 29 | import sys |
| 30 | import signal |
David Shrewsbury | eb85647 | 2017-04-13 14:23:04 -0400 | [diff] [blame] | 31 | import tempfile |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 32 | |
| 33 | import zuul.cmd |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 34 | import zuul.executor.server |
Tristan Cacqueray | 91601d7 | 2017-06-15 06:00:12 +0000 | [diff] [blame] | 35 | from zuul.lib.config import get_default |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 36 | |
James E. Blair | a6a5004 | 2016-06-01 15:33:54 -0700 | [diff] [blame] | 37 | # No zuul imports that pull in paramiko here; it must not be |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 38 | # imported until after the daemonization. |
| 39 | # https://github.com/paramiko/paramiko/issues/59 |
| 40 | # Similar situation with gear and statsd. |
| 41 | |
| 42 | |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 43 | class Executor(zuul.cmd.ZuulApp): |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 44 | |
| 45 | def parse_arguments(self): |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 46 | parser = argparse.ArgumentParser(description='Zuul executor.') |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 47 | parser.add_argument('-c', dest='config', |
| 48 | help='specify the config file') |
| 49 | parser.add_argument('-d', dest='nodaemon', action='store_true', |
| 50 | help='do not run as a daemon') |
| 51 | parser.add_argument('--version', dest='version', action='version', |
| 52 | version=self._get_version(), |
| 53 | help='show zuul version') |
James E. Blair | f87c5ce | 2016-05-25 08:43:37 -0700 | [diff] [blame] | 54 | parser.add_argument('--keep-jobdir', dest='keep_jobdir', |
| 55 | action='store_true', |
| 56 | help='keep local jobdirs after run completes') |
James E. Blair | a6a5004 | 2016-06-01 15:33:54 -0700 | [diff] [blame] | 57 | parser.add_argument('command', |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 58 | choices=zuul.executor.server.COMMANDS, |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 59 | nargs='?') |
| 60 | |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 61 | self.args = parser.parse_args() |
| 62 | |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 63 | def send_command(self, cmd): |
James E. Blair | 01d733e | 2017-06-23 20:47:51 +0100 | [diff] [blame] | 64 | state_dir = get_default(self.config, 'executor', 'state_dir', |
Tristan Cacqueray | 91601d7 | 2017-06-15 06:00:12 +0000 | [diff] [blame] | 65 | '/var/lib/zuul', expand_user=True) |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 66 | path = os.path.join(state_dir, 'executor.socket') |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 67 | s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) |
| 68 | s.connect(path) |
James E. Blair | 04e6030 | 2017-06-08 14:32:03 -0700 | [diff] [blame] | 69 | cmd = '%s\n' % cmd |
| 70 | s.sendall(cmd.encode('utf8')) |
James E. Blair | 119acf3 | 2016-04-18 15:34:36 -0700 | [diff] [blame] | 71 | |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 72 | def exit_handler(self): |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 73 | self.executor.stop() |
| 74 | self.executor.join() |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 75 | |
David Shrewsbury | eb85647 | 2017-04-13 14:23:04 -0400 | [diff] [blame] | 76 | def start_log_streamer(self): |
| 77 | pipe_read, pipe_write = os.pipe() |
| 78 | child_pid = os.fork() |
| 79 | if child_pid == 0: |
| 80 | os.close(pipe_write) |
| 81 | import zuul.lib.log_streamer |
| 82 | |
| 83 | self.log.info("Starting log streamer") |
| 84 | streamer = zuul.lib.log_streamer.LogStreamer( |
James E. Blair | 7e6e0a1 | 2017-07-25 11:04:42 -0700 | [diff] [blame^] | 85 | self.user, '0.0.0.0', self.finger_port, self.job_dir) |
David Shrewsbury | eb85647 | 2017-04-13 14:23:04 -0400 | [diff] [blame] | 86 | |
| 87 | # Keep running until the parent dies: |
| 88 | pipe_read = os.fdopen(pipe_read) |
| 89 | pipe_read.read() |
| 90 | self.log.info("Stopping log streamer") |
| 91 | streamer.stop() |
| 92 | os._exit(0) |
| 93 | else: |
| 94 | os.close(pipe_read) |
| 95 | self.log_streamer_pid = child_pid |
| 96 | |
| 97 | def change_privs(self): |
| 98 | ''' |
| 99 | Drop our privileges to the zuul user. |
| 100 | ''' |
| 101 | if os.getuid() != 0: |
| 102 | return |
David Shrewsbury | e737420 | 2017-04-27 12:28:45 -0400 | [diff] [blame] | 103 | pw = pwd.getpwnam(self.user) |
David Shrewsbury | eb85647 | 2017-04-13 14:23:04 -0400 | [diff] [blame] | 104 | os.setgroups([]) |
| 105 | os.setgid(pw.pw_gid) |
| 106 | os.setuid(pw.pw_uid) |
| 107 | os.umask(0o022) |
| 108 | |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 109 | def main(self, daemon=True): |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 110 | # See comment at top of file about zuul imports |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 111 | |
Tristan Cacqueray | 91601d7 | 2017-06-15 06:00:12 +0000 | [diff] [blame] | 112 | self.user = get_default(self.config, 'executor', 'user', 'zuul') |
David Shrewsbury | e737420 | 2017-04-27 12:28:45 -0400 | [diff] [blame] | 113 | |
James E. Blair | 7e6e0a1 | 2017-07-25 11:04:42 -0700 | [diff] [blame^] | 114 | if self.config.has_option('executor', 'job_dir'): |
| 115 | self.job_dir = os.path.expanduser( |
| 116 | self.config.get('executor', 'job_dir')) |
| 117 | if not os.path.isdir(self.job_dir): |
| 118 | print("Invalid job_dir: {job_dir}".format( |
| 119 | job_dir=self.job_dir)) |
Monty Taylor | 50e6b6b | 2017-05-31 09:00:32 -0500 | [diff] [blame] | 120 | sys.exit(1) |
David Shrewsbury | eb85647 | 2017-04-13 14:23:04 -0400 | [diff] [blame] | 121 | else: |
James E. Blair | 7e6e0a1 | 2017-07-25 11:04:42 -0700 | [diff] [blame^] | 122 | self.job_dir = tempfile.gettempdir() |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 123 | |
David Shrewsbury | eb85647 | 2017-04-13 14:23:04 -0400 | [diff] [blame] | 124 | self.setup_logging('executor', 'log_config') |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 125 | self.log = logging.getLogger("zuul.Executor") |
James E. Blair | 119acf3 | 2016-04-18 15:34:36 -0700 | [diff] [blame] | 126 | |
Monty Taylor | 0dbe159 | 2017-06-11 10:57:27 -0500 | [diff] [blame] | 127 | self.finger_port = int( |
| 128 | get_default(self.config, 'executor', 'finger_port', |
| 129 | zuul.executor.server.DEFAULT_FINGER_PORT) |
| 130 | ) |
Tobias Henkel | cec2966 | 2017-06-02 12:27:47 +0200 | [diff] [blame] | 131 | |
David Shrewsbury | eb85647 | 2017-04-13 14:23:04 -0400 | [diff] [blame] | 132 | self.start_log_streamer() |
| 133 | self.change_privs() |
David Shrewsbury | d678429 | 2017-04-13 12:08:29 -0400 | [diff] [blame] | 134 | |
Paul Belanger | 9cd1286 | 2017-03-15 13:03:59 -0400 | [diff] [blame] | 135 | ExecutorServer = zuul.executor.server.ExecutorServer |
| 136 | self.executor = ExecutorServer(self.config, self.connections, |
James E. Blair | 7e6e0a1 | 2017-07-25 11:04:42 -0700 | [diff] [blame^] | 137 | jobdir_root=self.job_dir, |
Monty Taylor | 0dbe159 | 2017-06-11 10:57:27 -0500 | [diff] [blame] | 138 | keep_jobdir=self.args.keep_jobdir, |
| 139 | log_streaming_port=self.finger_port) |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 140 | self.executor.start() |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 141 | |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 142 | signal.signal(signal.SIGUSR2, zuul.cmd.stack_dump_handler) |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 143 | if daemon: |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 144 | self.executor.join() |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 145 | else: |
| 146 | while True: |
| 147 | try: |
| 148 | signal.pause() |
| 149 | except KeyboardInterrupt: |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 150 | print("Ctrl + C: asking executor to exit nicely...\n") |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 151 | self.exit_handler() |
| 152 | sys.exit(0) |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 153 | |
| 154 | |
| 155 | def main(): |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 156 | server = Executor() |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 157 | server.parse_arguments() |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 158 | server.read_config() |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 159 | |
Paul Belanger | 174a827 | 2017-03-14 13:20:10 -0400 | [diff] [blame] | 160 | if server.args.command in zuul.executor.server.COMMANDS: |
James E. Blair | a6a5004 | 2016-06-01 15:33:54 -0700 | [diff] [blame] | 161 | server.send_command(server.args.command) |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 162 | sys.exit(0) |
| 163 | |
Tristan Cacqueray | bad3b12 | 2017-05-21 03:07:08 +0000 | [diff] [blame] | 164 | server.configure_connections(source_only=True) |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 165 | |
Tristan Cacqueray | 91601d7 | 2017-06-15 06:00:12 +0000 | [diff] [blame] | 166 | pid_fn = get_default(server.config, 'executor', 'pidfile', |
| 167 | '/var/run/zuul-executor/zuul-executor.pid', |
| 168 | expand_user=True) |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 169 | pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10) |
| 170 | |
| 171 | if server.args.nodaemon: |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 172 | server.main(False) |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 173 | else: |
| 174 | with daemon.DaemonContext(pidfile=pid): |
James E. Blair | c4b2041 | 2016-05-27 16:45:26 -0700 | [diff] [blame] | 175 | server.main(True) |
James E. Blair | 9f36519 | 2016-04-18 10:34:48 -0700 | [diff] [blame] | 176 | |
| 177 | |
| 178 | if __name__ == "__main__": |
| 179 | sys.path.insert(0, '.') |
| 180 | main() |