SIGUSR2 logs stack traces for active threads.

* doc/source/zuul.rst: Document SIGUSR2 behavior.

* zuul/cmd/server.py: When SIGUSR2 is received log stack traces for all
active running threads. This is useful for debugging deadlock
situations. Note that this makes use of sys._current_frames which may
not play nice with all implementations of Python.

*tests/test_stack_dump.py: Test the stack dump signal handler with a new
test file, class, and test method.

* requirements.txt: Add argparse to requirements list so that py26 tests
can pass when zuul.cmd.server is imported.

Change-Id: I8ad8155b16f324e832c191f0a619ff89ef804a87

doc/source/zuul.rst

@@ -653,3 +653,7 @@ recheck changes for the gate and check pipelines respectively::
       http://zuul.openstack.org/ gate 'reverify no bug'
   ./tools/zuul-changes.py --review-host=review.openstack.org \
       http://zuul.openstack.org/ check 'recheck no bug'
+
+If you send a SIGUSR2 to the zuul-server process, Zuul will dump a stack
+trace for each running thread into its debug log. This is useful for
+tracking down deadlock or otherwise slow threads.

requirements.txt

@@ -1,6 +1,7 @@
 d2to1>=0.2.10,<0.3
 pbr>=0.5,<0.6
 
+argparse
 PyYAML
 python-jenkins
 Paste

tests/test_stack_dump.py

@@ -0,0 +1,34 @@
+# Copyright 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import fixtures
+import logging
+import signal
+import testtools
+
+import zuul.cmd.server
+
+
+class TestStackDump(testtools.TestCase):
+    def setUp(self):
+        super(TestStackDump, self).setUp()
+        self.log_fixture = self.useFixture(
+            fixtures.FakeLogger(level=logging.DEBUG))
+
+    def test_stack_dump_logs(self):
+        "Test that stack dumps end up in logs."
+
+        zuul.cmd.server.stack_dump_handler(signal.SIGUSR2, None)
+        self.assertIn("Thread", self.log_fixture.output)
+        self.assertIn("test_stack_dump_logs", self.log_fixture.output)

zuul/cmd/server.py

@@ -23,10 +23,12 @@ import extras
 # instead it depends on lockfile-0.9.1 which uses pidfile.
 pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
 
+import logging
 import logging.config
 import os
 import sys
 import signal
+import traceback
 
 import gear
 
@@ -35,6 +37,17 @@ import gear
 # https://github.com/paramiko/paramiko/issues/59
 
 
+def stack_dump_handler(signum, frame):
+    signal.signal(signal.SIGUSR2, signal.SIG_IGN)
+    log_str = ""
+    for thread_id, stack_frame in sys._current_frames().items():
+        log_str += "Thread: %s\n" % thread_id
+        log_str += "".join(traceback.format_stack(stack_frame))
+    log = logging.getLogger("zuul.stack_dump")
+    log.debug(log_str)
+    signal.signal(signal.SIGUSR2, stack_dump_handler)
+
+
 class Server(object):
     def __init__(self):
         self.args = None
@@ -179,6 +192,7 @@ class Server(object):
 
         signal.signal(signal.SIGHUP, self.reconfigure_handler)
         signal.signal(signal.SIGUSR1, self.exit_handler)
+        signal.signal(signal.SIGUSR2, stack_dump_handler)
         signal.signal(signal.SIGTERM, self.term_handler)
         while True:
             try: