| #!/usr/bin/env python |
| # Copyright 2012 Hewlett-Packard Development Company, L.P. |
| # Copyright 2013 OpenStack Foundation |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| # not use this file except in compliance with the License. You may obtain |
| # a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations |
| # under the License. |
| |
| import argparse |
| import ConfigParser |
| import daemon |
| import extras |
| |
| # as of python-daemon 1.6 it doesn't bundle pidlockfile anymore |
| # instead it depends on lockfile-0.9.1 which uses pidfile. |
| pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile']) |
| |
| import logging |
| import logging.config |
| import os |
| import sys |
| import signal |
| import traceback |
| |
| # No zuul imports here because they pull in paramiko which must not be |
| # imported until after the daemonization. |
| # https://github.com/paramiko/paramiko/issues/59 |
| # Similar situation with gear and statsd. |
| |
| |
| def stack_dump_handler(signum, frame): |
| signal.signal(signal.SIGUSR2, signal.SIG_IGN) |
| log_str = "" |
| for thread_id, stack_frame in sys._current_frames().items(): |
| log_str += "Thread: %s\n" % thread_id |
| log_str += "".join(traceback.format_stack(stack_frame)) |
| log = logging.getLogger("zuul.stack_dump") |
| log.debug(log_str) |
| signal.signal(signal.SIGUSR2, stack_dump_handler) |
| |
| |
| class Server(object): |
| def __init__(self): |
| self.args = None |
| self.config = None |
| self.gear_server_pid = None |
| |
| def parse_arguments(self): |
| parser = argparse.ArgumentParser(description='Project gating system.') |
| parser.add_argument('-c', dest='config', |
| help='specify the config file') |
| parser.add_argument('-l', dest='layout', |
| help='specify the layout file') |
| parser.add_argument('-d', dest='nodaemon', action='store_true', |
| help='do not run as a daemon') |
| parser.add_argument('-t', dest='validate', nargs='?', const=True, |
| metavar='JOB_LIST', |
| help='validate layout file syntax (optionally ' |
| 'providing the path to a file with a list of ' |
| 'available job names)') |
| parser.add_argument('--version', dest='version', action='version', |
| version=self._get_version(), |
| help='show zuul version') |
| self.args = parser.parse_args() |
| |
| def _get_version(self): |
| from zuul.version import version_info as zuul_version_info |
| return "Zuul version: %s" % zuul_version_info.version_string() |
| |
| def read_config(self): |
| self.config = ConfigParser.ConfigParser() |
| if self.args.config: |
| locations = [self.args.config] |
| else: |
| locations = ['/etc/zuul/zuul.conf', |
| '~/zuul.conf'] |
| for fp in locations: |
| if os.path.exists(os.path.expanduser(fp)): |
| self.config.read(os.path.expanduser(fp)) |
| return |
| raise Exception("Unable to locate config file in %s" % locations) |
| |
| def setup_logging(self, section, parameter): |
| if self.config.has_option(section, parameter): |
| fp = os.path.expanduser(self.config.get(section, parameter)) |
| if not os.path.exists(fp): |
| raise Exception("Unable to read logging config file at %s" % |
| fp) |
| logging.config.fileConfig(fp) |
| else: |
| logging.basicConfig(level=logging.DEBUG) |
| |
| def reconfigure_handler(self, signum, frame): |
| signal.signal(signal.SIGHUP, signal.SIG_IGN) |
| self.read_config() |
| self.setup_logging('zuul', 'log_config') |
| self.sched.reconfigure(self.config) |
| signal.signal(signal.SIGHUP, self.reconfigure_handler) |
| |
| def exit_handler(self, signum, frame): |
| signal.signal(signal.SIGUSR1, signal.SIG_IGN) |
| self.sched.exit() |
| self.sched.join() |
| self.stop_gear_server() |
| |
| def term_handler(self, signum, frame): |
| self.stop_gear_server() |
| os._exit(0) |
| |
| def test_config(self, job_list_path): |
| # See comment at top of file about zuul imports |
| import zuul.scheduler |
| import zuul.launcher.gearman |
| import zuul.trigger.gerrit |
| |
| logging.basicConfig(level=logging.DEBUG) |
| self.sched = zuul.scheduler.Scheduler() |
| self.sched.registerReporter(None, 'gerrit') |
| self.sched.registerReporter(None, 'smtp') |
| self.sched.registerTrigger(None, 'gerrit') |
| self.sched.registerTrigger(None, 'timer') |
| layout = self.sched.testConfig(self.config.get('zuul', |
| 'layout_config')) |
| if not job_list_path: |
| return False |
| |
| failure = False |
| path = os.path.expanduser(job_list_path) |
| if not os.path.exists(path): |
| raise Exception("Unable to find job list: %s" % path) |
| jobs = set() |
| jobs.add('noop') |
| for line in open(path): |
| v = line.strip() |
| if v: |
| jobs.add(v) |
| for job in sorted(layout.jobs): |
| if job not in jobs: |
| print "Job %s not defined" % job |
| failure = True |
| return failure |
| |
| def start_gear_server(self): |
| pipe_read, pipe_write = os.pipe() |
| child_pid = os.fork() |
| if child_pid == 0: |
| os.close(pipe_write) |
| self.setup_logging('gearman_server', 'log_config') |
| import gear |
| statsd_host = os.environ.get('STATSD_HOST') |
| statsd_port = int(os.environ.get('STATSD_PORT', 8125)) |
| gear.Server(4730, |
| statsd_host=statsd_host, |
| statsd_port=statsd_port, |
| statsd_prefix='zuul.geard') |
| |
| # Keep running until the parent dies: |
| pipe_read = os.fdopen(pipe_read) |
| pipe_read.read() |
| os._exit(0) |
| else: |
| os.close(pipe_read) |
| self.gear_server_pid = child_pid |
| self.gear_pipe_write = pipe_write |
| |
| def stop_gear_server(self): |
| if self.gear_server_pid: |
| os.kill(self.gear_server_pid, signal.SIGKILL) |
| |
| def main(self): |
| # See comment at top of file about zuul imports |
| import zuul.scheduler |
| import zuul.launcher.gearman |
| import zuul.merger.client |
| import zuul.lib.swift |
| import zuul.reporter.gerrit |
| import zuul.reporter.smtp |
| import zuul.trigger.gerrit |
| import zuul.trigger.timer |
| import zuul.webapp |
| import zuul.rpclistener |
| |
| if (self.config.has_option('gearman_server', 'start') and |
| self.config.getboolean('gearman_server', 'start')): |
| self.start_gear_server() |
| |
| self.setup_logging('zuul', 'log_config') |
| self.log = logging.getLogger("zuul.Server") |
| |
| self.sched = zuul.scheduler.Scheduler() |
| self.swift = zuul.lib.swift.Swift(self.config) |
| |
| gearman = zuul.launcher.gearman.Gearman(self.config, self.sched, |
| self.swift) |
| merger = zuul.merger.client.MergeClient(self.config, self.sched) |
| gerrit = zuul.trigger.gerrit.Gerrit(self.config, self.sched) |
| timer = zuul.trigger.timer.Timer(self.config, self.sched) |
| webapp = zuul.webapp.WebApp(self.sched) |
| rpc = zuul.rpclistener.RPCListener(self.config, self.sched) |
| gerrit_reporter = zuul.reporter.gerrit.Reporter(gerrit) |
| smtp_reporter = zuul.reporter.smtp.Reporter( |
| self.config.get('smtp', 'default_from') |
| if self.config.has_option('smtp', 'default_from') else 'zuul', |
| self.config.get('smtp', 'default_to') |
| if self.config.has_option('smtp', 'default_to') else 'zuul', |
| self.config.get('smtp', 'server') |
| if self.config.has_option('smtp', 'server') else 'localhost', |
| self.config.get('smtp', 'port') |
| if self.config.has_option('smtp', 'port') else 25 |
| ) |
| |
| self.sched.setLauncher(gearman) |
| self.sched.setMerger(merger) |
| self.sched.registerTrigger(gerrit) |
| self.sched.registerTrigger(timer) |
| self.sched.registerReporter(gerrit_reporter) |
| self.sched.registerReporter(smtp_reporter) |
| |
| self.log.info('Starting scheduler') |
| self.sched.start() |
| self.sched.reconfigure(self.config) |
| self.sched.resume() |
| self.log.info('Starting Webapp') |
| webapp.start() |
| self.log.info('Starting RPC') |
| rpc.start() |
| |
| signal.signal(signal.SIGHUP, self.reconfigure_handler) |
| signal.signal(signal.SIGUSR1, self.exit_handler) |
| signal.signal(signal.SIGUSR2, stack_dump_handler) |
| signal.signal(signal.SIGTERM, self.term_handler) |
| while True: |
| try: |
| signal.pause() |
| except KeyboardInterrupt: |
| print "Ctrl + C: asking scheduler to exit nicely...\n" |
| self.exit_handler(signal.SIGINT, None) |
| |
| |
| def main(): |
| server = Server() |
| server.parse_arguments() |
| |
| server.read_config() |
| |
| if server.args.layout: |
| server.config.set('zuul', 'layout_config', server.args.layout) |
| |
| if server.args.validate: |
| path = server.args.validate |
| if path is True: |
| path = None |
| sys.exit(server.test_config(path)) |
| |
| if server.config.has_option('zuul', 'pidfile'): |
| pid_fn = os.path.expanduser(server.config.get('zuul', 'pidfile')) |
| else: |
| pid_fn = '/var/run/zuul/zuul.pid' |
| pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10) |
| |
| if server.args.nodaemon: |
| server.main() |
| else: |
| with daemon.DaemonContext(pidfile=pid): |
| server.main() |
| |
| |
| if __name__ == "__main__": |
| sys.path.insert(0, '.') |
| main() |