SIGUSR2 logs stack traces for active threads.
* doc/source/zuul.rst: Document SIGUSR2 behavior.
* zuul/cmd/server.py: When SIGUSR2 is received log stack traces for all
active running threads. This is useful for debugging deadlock
situations. Note that this makes use of sys._current_frames which may
not play nice with all implementations of Python.
*tests/test_stack_dump.py: Test the stack dump signal handler with a new
test file, class, and test method.
* requirements.txt: Add argparse to requirements list so that py26 tests
can pass when zuul.cmd.server is imported.
Change-Id: I8ad8155b16f324e832c191f0a619ff89ef804a87
diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst
index d1cbbc5..73ebf71 100644
--- a/doc/source/zuul.rst
+++ b/doc/source/zuul.rst
@@ -653,3 +653,7 @@
http://zuul.openstack.org/ gate 'reverify no bug'
./tools/zuul-changes.py --review-host=review.openstack.org \
http://zuul.openstack.org/ check 'recheck no bug'
+
+If you send a SIGUSR2 to the zuul-server process, Zuul will dump a stack
+trace for each running thread into its debug log. This is useful for
+tracking down deadlock or otherwise slow threads.
diff --git a/requirements.txt b/requirements.txt
index 25b69bd..9760a79 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,7 @@
d2to1>=0.2.10,<0.3
pbr>=0.5,<0.6
+argparse
PyYAML
python-jenkins
Paste
diff --git a/tests/test_stack_dump.py b/tests/test_stack_dump.py
new file mode 100644
index 0000000..cc8cf8f
--- /dev/null
+++ b/tests/test_stack_dump.py
@@ -0,0 +1,34 @@
+# Copyright 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import fixtures
+import logging
+import signal
+import testtools
+
+import zuul.cmd.server
+
+
+class TestStackDump(testtools.TestCase):
+ def setUp(self):
+ super(TestStackDump, self).setUp()
+ self.log_fixture = self.useFixture(
+ fixtures.FakeLogger(level=logging.DEBUG))
+
+ def test_stack_dump_logs(self):
+ "Test that stack dumps end up in logs."
+
+ zuul.cmd.server.stack_dump_handler(signal.SIGUSR2, None)
+ self.assertIn("Thread", self.log_fixture.output)
+ self.assertIn("test_stack_dump_logs", self.log_fixture.output)
diff --git a/zuul/cmd/server.py b/zuul/cmd/server.py
index 4842f7a..75f7dd9 100755
--- a/zuul/cmd/server.py
+++ b/zuul/cmd/server.py
@@ -23,10 +23,12 @@
# instead it depends on lockfile-0.9.1 which uses pidfile.
pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
+import logging
import logging.config
import os
import sys
import signal
+import traceback
import gear
@@ -35,6 +37,17 @@
# https://github.com/paramiko/paramiko/issues/59
+def stack_dump_handler(signum, frame):
+ signal.signal(signal.SIGUSR2, signal.SIG_IGN)
+ log_str = ""
+ for thread_id, stack_frame in sys._current_frames().items():
+ log_str += "Thread: %s\n" % thread_id
+ log_str += "".join(traceback.format_stack(stack_frame))
+ log = logging.getLogger("zuul.stack_dump")
+ log.debug(log_str)
+ signal.signal(signal.SIGUSR2, stack_dump_handler)
+
+
class Server(object):
def __init__(self):
self.args = None
@@ -179,6 +192,7 @@
signal.signal(signal.SIGHUP, self.reconfigure_handler)
signal.signal(signal.SIGUSR1, self.exit_handler)
+ signal.signal(signal.SIGUSR2, stack_dump_handler)
signal.signal(signal.SIGTERM, self.term_handler)
while True:
try: