Add toggleable yappi profiling to zuul

Add a bit more functionality to the SIGUSR2 handler in zuul. When that
handler is run toggle the state of the yappi profiler and when we turn
the profiler off log the profiling data. Do this in addition to the
normal thread dumps.

Change-Id: I127cfb924f0576072d26bde0b4204f8636195d97

zuul/cmd/__init__.py

@@ -15,6 +15,8 @@
 # under the License.
 
 import ConfigParser
+import cStringIO
+import extras
 import logging
 import logging.config
 import os
@@ -22,6 +24,8 @@ import signal
 import sys
 import traceback
 
+yappi = extras.try_import('yappi')
+
 # No zuul imports here because they pull in paramiko which must not be
 # imported until after the daemonization.
 # https://github.com/paramiko/paramiko/issues/59
@@ -36,6 +40,17 @@ def stack_dump_handler(signum, frame):
         log_str += "".join(traceback.format_stack(stack_frame))
     log = logging.getLogger("zuul.stack_dump")
     log.debug(log_str)
+    if yappi:
+        if not yappi.is_running():
+            yappi.start()
+        else:
+            yappi.stop()
+            yappi_out = cStringIO.StringIO()
+            yappi.get_func_stats().print_all(out=yappi_out)
+            yappi.get_thread_stats().print_all(out=yappi_out)
+            log.debug(yappi_out.getvalue())
+            yappi_out.close()
+            yappi.clear_stats()
     signal.signal(signal.SIGUSR2, stack_dump_handler)