Merge "Normalize daemon process handling" into feature/zuulv3
diff --git a/tests/unit/test_scheduler_cmd.py b/tests/unit/test_scheduler_cmd.py
deleted file mode 100644
index ee6200f..0000000
--- a/tests/unit/test_scheduler_cmd.py
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env python
-
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import os
-
-import testtools
-import zuul.cmd.scheduler
-
-from tests import base
-
-
-class TestSchedulerCmdArguments(testtools.TestCase):
-
- def setUp(self):
- super(TestSchedulerCmdArguments, self).setUp()
- self.app = zuul.cmd.scheduler.Scheduler()
-
- def test_test_config(self):
- conf_path = os.path.join(base.FIXTURE_DIR, 'zuul.conf')
- self.app.parse_arguments(['-t', '-c', conf_path])
- self.assertTrue(self.app.args.validate)
- self.app.read_config()
- self.assertEqual(0, self.app.test_config())
diff --git a/zuul/cmd/__init__.py b/zuul/cmd/__init__.py
index 86f7f12..e150f9c 100755
--- a/zuul/cmd/__init__.py
+++ b/zuul/cmd/__init__.py
@@ -14,7 +14,9 @@
# License for the specific language governing permissions and limitations
# under the License.
+import argparse
import configparser
+import daemon
import extras
import io
import logging
@@ -28,8 +30,13 @@
yappi = extras.try_import('yappi')
objgraph = extras.try_import('objgraph')
+# as of python-daemon 1.6 it doesn't bundle pidlockfile anymore
+# instead it depends on lockfile-0.9.1 which uses pidfile.
+pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
+
from zuul.ansible import logconfig
import zuul.lib.connections
+from zuul.lib.config import get_default
# Do not import modules that will pull in paramiko which must not be
# imported until after the daemonization.
@@ -87,6 +94,8 @@
class ZuulApp(object):
+ app_name = None # type: str
+ app_description = None # type: str
def __init__(self):
self.args = None
@@ -97,7 +106,21 @@
from zuul.version import version_info as zuul_version_info
return "Zuul version: %s" % zuul_version_info.release_string()
- def read_config(self):
+ def createParser(self):
+ parser = argparse.ArgumentParser(description=self.app_description)
+ parser.add_argument('-c', dest='config',
+ help='specify the config file')
+ parser.add_argument('--version', dest='version', action='version',
+ version=self._get_version(),
+ help='show zuul version')
+ return parser
+
+ def parseArguments(self, args=None):
+ parser = self.createParser()
+ self.args = parser.parse_args(args)
+ return parser
+
+ def readConfig(self):
self.config = configparser.ConfigParser()
if self.args.config:
locations = [self.args.config]
@@ -130,3 +153,34 @@
def configure_connections(self, source_only=False):
self.connections = zuul.lib.connections.ConnectionRegistry()
self.connections.configure(self.config, source_only)
+
+
+class ZuulDaemonApp(ZuulApp):
+ def createParser(self):
+ parser = super(ZuulDaemonApp, self).createParser()
+ parser.add_argument('-d', dest='nodaemon', action='store_true',
+ help='do not run as a daemon')
+ return parser
+
+ def getPidFile(self):
+ pid_fn = get_default(self.config, self.app_name, 'pidfile',
+ '/var/run/zuul/%s.pid' % self.app_name,
+ expand_user=True)
+ return pid_fn
+
+ def main(self):
+ self.parseArguments()
+ self.readConfig()
+
+ pid_fn = self.getPidFile()
+ pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
+
+ if self.args.nodaemon:
+ self.run()
+ else:
+ # Exercise the pidfile before we do anything else (including
+ # logging or daemonizing)
+ with daemon.DaemonContext(pidfile=pid):
+ pass
+ with daemon.DaemonContext(pidfile=pid):
+ self.run()
diff --git a/zuul/cmd/client.py b/zuul/cmd/client.py
index 7a26a62..ebf59b9 100755
--- a/zuul/cmd/client.py
+++ b/zuul/cmd/client.py
@@ -30,18 +30,14 @@
class Client(zuul.cmd.ZuulApp):
+ app_name = 'zuul'
+ app_description = 'Zuul RPC client.'
log = logging.getLogger("zuul.Client")
- def parse_arguments(self):
- parser = argparse.ArgumentParser(
- description='Zuul Project Gating System Client.')
- parser.add_argument('-c', dest='config',
- help='specify the config file')
+ def createParser(self):
+ parser = super(Client, self).createParser()
parser.add_argument('-v', dest='verbose', action='store_true',
help='verbose output')
- parser.add_argument('--version', dest='version', action='version',
- version=self._get_version(),
- help='show zuul version')
subparsers = parser.add_subparsers(title='commands',
description='valid commands',
@@ -133,7 +129,10 @@
# TODO: add filters such as queue, project, changeid etc
show_running_jobs.set_defaults(func=self.show_running_jobs)
- self.args = parser.parse_args()
+ return parser
+
+ def parseArguments(self, args=None):
+ parser = super(Client, self).parseArguments()
if not getattr(self.args, 'func', None):
parser.print_help()
sys.exit(1)
@@ -156,8 +155,8 @@
logging.basicConfig(level=logging.DEBUG)
def main(self):
- self.parse_arguments()
- self.read_config()
+ self.parseArguments()
+ self.readConfig()
self.setup_logging()
self.server = self.config.get('gearman', 'server')
@@ -363,10 +362,8 @@
def main():
- client = Client()
- client.main()
+ Client().main()
if __name__ == "__main__":
- sys.path.insert(0, '.')
main()
diff --git a/zuul/cmd/executor.py b/zuul/cmd/executor.py
index 979989d..aef8c95 100755
--- a/zuul/cmd/executor.py
+++ b/zuul/cmd/executor.py
@@ -14,14 +14,6 @@
# License for the specific language governing permissions and limitations
# under the License.
-import argparse
-import daemon
-import extras
-
-# as of python-daemon 1.6 it doesn't bundle pidlockfile anymore
-# instead it depends on lockfile-0.9.1 which uses pidfile.
-pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
-
import grp
import logging
import os
@@ -41,25 +33,24 @@
# Similar situation with gear and statsd.
-class Executor(zuul.cmd.ZuulApp):
+class Executor(zuul.cmd.ZuulDaemonApp):
+ app_name = 'executor'
+ app_description = 'A standalone Zuul executor.'
- def parse_arguments(self):
- parser = argparse.ArgumentParser(description='Zuul executor.')
- parser.add_argument('-c', dest='config',
- help='specify the config file')
- parser.add_argument('-d', dest='nodaemon', action='store_true',
- help='do not run as a daemon')
- parser.add_argument('--version', dest='version', action='version',
- version=self._get_version(),
- help='show zuul version')
+ def createParser(self):
+ parser = super(Executor, self).createParser()
parser.add_argument('--keep-jobdir', dest='keep_jobdir',
action='store_true',
help='keep local jobdirs after run completes')
parser.add_argument('command',
choices=zuul.executor.server.COMMANDS,
nargs='?')
+ return parser
- self.args = parser.parse_args()
+ def parseArguments(self, args=None):
+ super(Executor, self).parseArguments()
+ if self.args.command:
+ self.args.nodaemon = True
def send_command(self, cmd):
state_dir = get_default(self.config, 'executor', 'state_dir',
@@ -111,8 +102,12 @@
os.chdir(pw.pw_dir)
os.umask(0o022)
- def main(self, daemon=True):
- # See comment at top of file about zuul imports
+ def run(self):
+ if self.args.command in zuul.executor.server.COMMANDS:
+ self.send_command(self.args.command)
+ sys.exit(0)
+
+ self.configure_connections(source_only=True)
self.user = get_default(self.config, 'executor', 'user', 'zuul')
@@ -145,9 +140,8 @@
self.executor.start()
signal.signal(signal.SIGUSR2, zuul.cmd.stack_dump_handler)
- if daemon:
- self.executor.join()
- else:
+
+ if self.args.nodaemon:
while True:
try:
signal.pause()
@@ -155,31 +149,13 @@
print("Ctrl + C: asking executor to exit nicely...\n")
self.exit_handler()
sys.exit(0)
+ else:
+ self.executor.join()
def main():
- server = Executor()
- server.parse_arguments()
- server.read_config()
-
- if server.args.command in zuul.executor.server.COMMANDS:
- server.send_command(server.args.command)
- sys.exit(0)
-
- server.configure_connections(source_only=True)
-
- pid_fn = get_default(server.config, 'executor', 'pidfile',
- '/var/run/zuul-executor/zuul-executor.pid',
- expand_user=True)
- pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
-
- if server.args.nodaemon:
- server.main(False)
- else:
- with daemon.DaemonContext(pidfile=pid):
- server.main(True)
+ Executor().main()
if __name__ == "__main__":
- sys.path.insert(0, '.')
main()
diff --git a/zuul/cmd/merger.py b/zuul/cmd/merger.py
index 9771fff..56b6b44 100755
--- a/zuul/cmd/merger.py
+++ b/zuul/cmd/merger.py
@@ -14,19 +14,9 @@
# License for the specific language governing permissions and limitations
# under the License.
-import argparse
-import daemon
-import extras
-
-# as of python-daemon 1.6 it doesn't bundle pidlockfile anymore
-# instead it depends on lockfile-0.9.1 which uses pidfile.
-pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
-
-import sys
import signal
import zuul.cmd
-from zuul.lib.config import get_default
# No zuul imports here because they pull in paramiko which must not be
# imported until after the daemonization.
@@ -34,28 +24,21 @@
# Similar situation with gear and statsd.
-class Merger(zuul.cmd.ZuulApp):
-
- def parse_arguments(self):
- parser = argparse.ArgumentParser(description='Zuul merge worker.')
- parser.add_argument('-c', dest='config',
- help='specify the config file')
- parser.add_argument('-d', dest='nodaemon', action='store_true',
- help='do not run as a daemon')
- parser.add_argument('--version', dest='version', action='version',
- version=self._get_version(),
- help='show zuul version')
- self.args = parser.parse_args()
+class Merger(zuul.cmd.ZuulDaemonApp):
+ app_name = 'merger'
+ app_description = 'A standalone Zuul merger.'
def exit_handler(self, signum, frame):
signal.signal(signal.SIGUSR1, signal.SIG_IGN)
self.merger.stop()
self.merger.join()
- def main(self):
+ def run(self):
# See comment at top of file about zuul imports
import zuul.merger.server
+ self.configure_connections(source_only=True)
+
self.setup_logging('merger', 'log_config')
self.merger = zuul.merger.server.MergeServer(self.config,
@@ -73,24 +56,8 @@
def main():
- server = Merger()
- server.parse_arguments()
-
- server.read_config()
- server.configure_connections(source_only=True)
-
- pid_fn = get_default(server.config, 'merger', 'pidfile',
- '/var/run/zuul-merger/zuul-merger.pid',
- expand_user=True)
- pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
-
- if server.args.nodaemon:
- server.main()
- else:
- with daemon.DaemonContext(pidfile=pid):
- server.main()
+ Merger().main()
if __name__ == "__main__":
- sys.path.insert(0, '.')
main()
diff --git a/zuul/cmd/scheduler.py b/zuul/cmd/scheduler.py
index 2d71f4d..bfcbef8 100755
--- a/zuul/cmd/scheduler.py
+++ b/zuul/cmd/scheduler.py
@@ -14,14 +14,6 @@
# License for the specific language governing permissions and limitations
# under the License.
-import argparse
-import daemon
-import extras
-
-# as of python-daemon 1.6 it doesn't bundle pidlockfile anymore
-# instead it depends on lockfile-0.9.1 which uses pidfile.
-pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
-
import logging
import os
import sys
@@ -37,25 +29,14 @@
# Similar situation with gear and statsd.
-class Scheduler(zuul.cmd.ZuulApp):
+class Scheduler(zuul.cmd.ZuulDaemonApp):
+ app_name = 'scheduler'
+ app_description = 'The main zuul process.'
+
def __init__(self):
super(Scheduler, self).__init__()
self.gear_server_pid = None
- def parse_arguments(self, args=None):
- parser = argparse.ArgumentParser(description='Project gating system.')
- parser.add_argument('-c', dest='config',
- help='specify the config file')
- parser.add_argument('-d', dest='nodaemon', action='store_true',
- help='do not run as a daemon')
- parser.add_argument('-t', dest='validate', action='store_true',
- help='validate config file syntax (Does not'
- 'validate config repo validity)')
- parser.add_argument('--version', dest='version', action='version',
- version=self._get_version(),
- help='show zuul version')
- self.args = parser.parse_args(args)
-
def reconfigure_handler(self, signum, frame):
signal.signal(signal.SIGHUP, signal.SIG_IGN)
self.log.debug("Reconfiguration triggered")
@@ -77,20 +58,6 @@
self.stop_gear_server()
os._exit(0)
- def test_config(self):
- # See comment at top of file about zuul imports
- import zuul.scheduler
- import zuul.executor.client
-
- logging.basicConfig(level=logging.DEBUG)
- try:
- self.sched = zuul.scheduler.Scheduler(self.config,
- testonly=True)
- except Exception as e:
- self.log.error("%s" % e)
- return -1
- return 0
-
def start_gear_server(self):
pipe_read, pipe_write = os.pipe()
child_pid = os.fork()
@@ -134,7 +101,7 @@
if self.gear_server_pid:
os.kill(self.gear_server_pid, signal.SIGKILL)
- def main(self):
+ def run(self):
# See comment at top of file about zuul imports
import zuul.scheduler
import zuul.executor.client
@@ -206,26 +173,8 @@
def main():
- scheduler = Scheduler()
- scheduler.parse_arguments()
-
- scheduler.read_config()
-
- if scheduler.args.validate:
- sys.exit(scheduler.test_config())
-
- pid_fn = get_default(scheduler.config, 'scheduler', 'pidfile',
- '/var/run/zuul-scheduler/zuul-scheduler.pid',
- expand_user=True)
- pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
-
- if scheduler.args.nodaemon:
- scheduler.main()
- else:
- with daemon.DaemonContext(pidfile=pid):
- scheduler.main()
+ Scheduler().main()
if __name__ == "__main__":
- sys.path.insert(0, '.')
main()
diff --git a/zuul/cmd/web.py b/zuul/cmd/web.py
index 9432656..6e5489f 100755
--- a/zuul/cmd/web.py
+++ b/zuul/cmd/web.py
@@ -13,10 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
-import argparse
import asyncio
-import daemon
-import extras
import logging
import signal
import sys
@@ -27,28 +24,15 @@
from zuul.lib.config import get_default
-# as of python-daemon 1.6 it doesn't bundle pidlockfile anymore
-# instead it depends on lockfile-0.9.1 which uses pidfile.
-pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
-
-class WebServer(zuul.cmd.ZuulApp):
-
- def parse_arguments(self):
- parser = argparse.ArgumentParser(description='Zuul Web Server.')
- parser.add_argument('-c', dest='config',
- help='specify the config file')
- parser.add_argument('-d', dest='nodaemon', action='store_true',
- help='do not run as a daemon')
- parser.add_argument('--version', dest='version', action='version',
- version=self._get_version(),
- help='show zuul version')
- self.args = parser.parse_args()
+class WebServer(zuul.cmd.ZuulDaemonApp):
+ app_name = 'web'
+ app_description = 'A standalone Zuul web server.'
def exit_handler(self, signum, frame):
self.web.stop()
- def _main(self):
+ def _run(self):
params = dict()
params['listen_address'] = get_default(self.config,
@@ -91,28 +75,19 @@
loop.close()
self.log.info("Zuul Web Server stopped")
- def main(self):
+ def run(self):
self.setup_logging('web', 'log_config')
self.log = logging.getLogger("zuul.WebServer")
try:
- self._main()
+ self._run()
except Exception:
self.log.exception("Exception from WebServer:")
def main():
- server = WebServer()
- server.parse_arguments()
- server.read_config()
+ WebServer().main()
- pid_fn = get_default(server.config, 'web', 'pidfile',
- '/var/run/zuul-web/zuul-web.pid', expand_user=True)
- pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
-
- if server.args.nodaemon:
- server.main()
- else:
- with daemon.DaemonContext(pidfile=pid):
- server.main()
+if __name__ == "__main__":
+ main()