Add config option to turn eventlet debug on/off

By default, this will be turned off.  This will cause eventlet to not
print stack traces to stderr which can be very annoying on production
systems.  It is still recommended to turn it on for development or
debuging purposes.

DocImpact
Change-Id: I5e5b902d3d9ed85f784549e53f2ee2fc87cbe2e5

doc/source/deployment_guide.rst

@@ -235,6 +235,7 @@ disable_fallocate    false       Disable "fast fail" fallocate checks if the
                                  underlying filesystem does not support it.
 log_custom_handlers  None        Comma-separated list of functions to call
                                  to setup custom log handlers.
+eventlet_debug       false       If true, turn on debug logging for eventlet
 ===================  ==========  =============================================
 
 [object-server]
@@ -344,6 +345,7 @@ disable_fallocate    false       Disable "fast fail" fallocate checks if the
                                  underlying filesystem does not support it.
 log_custom_handlers  None        Comma-separated list of functions to call
                                  to setup custom log handlers.
+eventlet_debug       false       If true, turn on debug logging for eventlet
 ===================  ==========  ============================================
 
 [container-server]
@@ -446,6 +448,7 @@ disable_fallocate    false       Disable "fast fail" fallocate checks if the
                                  underlying filesystem does not support it.
 log_custom_handlers  None        Comma-separated list of functions to call
                                  to setup custom log handlers.
+eventlet_debug       false       If true, turn on debug logging for eventlet
 ===================  ==========  =============================================
 
 [account-server]
@@ -545,6 +548,8 @@ cors_allow_origin                              This is a list of hosts that
 log_custom_handlers           None             Comma separated list of functions
                                                to call to setup custom log
                                                handlers.
+eventlet_debug                false            If true, turn on debug logging
+                                               for eventlet
 ============================  ===============  =============================
 
 [proxy-server]

doc/source/development_saio.rst

@@ -294,6 +294,7 @@ Sample configuration files are provided with all defaults in line-by-line commen
         bind_port = 8080
         user = <your-user-name>
         log_facility = LOG_LOCAL1
+        eventlet_debug = true
 
         [pipeline:main]
         pipeline = healthcheck cache tempauth proxy-logging proxy-server
@@ -337,6 +338,7 @@ Sample configuration files are provided with all defaults in line-by-line commen
         user = <your-user-name>
         log_facility = LOG_LOCAL2
         recon_cache_path = /var/cache/swift
+        eventlet_debug = true
 
         [pipeline:main]
         pipeline = recon account-server
@@ -364,6 +366,7 @@ Sample configuration files are provided with all defaults in line-by-line commen
         user = <your-user-name>
         log_facility = LOG_LOCAL3
         recon_cache_path = /var/cache/swift2
+        eventlet_debug = true
 
         [pipeline:main]
         pipeline = recon account-server
@@ -391,6 +394,7 @@ Sample configuration files are provided with all defaults in line-by-line commen
         user = <your-user-name>
         log_facility = LOG_LOCAL4
         recon_cache_path = /var/cache/swift3
+        eventlet_debug = true
 
         [pipeline:main]
         pipeline = recon account-server
@@ -418,6 +422,7 @@ Sample configuration files are provided with all defaults in line-by-line commen
         user = <your-user-name>
         log_facility = LOG_LOCAL5
         recon_cache_path = /var/cache/swift4
+        eventlet_debug = true
 
         [pipeline:main]
         pipeline = recon account-server
@@ -445,6 +450,7 @@ Sample configuration files are provided with all defaults in line-by-line commen
         user = <your-user-name>
         log_facility = LOG_LOCAL2
         recon_cache_path = /var/cache/swift
+        eventlet_debug = true
 
         [pipeline:main]
         pipeline = recon container-server
@@ -474,6 +480,7 @@ Sample configuration files are provided with all defaults in line-by-line commen
         user = <your-user-name>
         log_facility = LOG_LOCAL3
         recon_cache_path = /var/cache/swift2
+        eventlet_debug = true
 
         [pipeline:main]
         pipeline = recon container-server
@@ -503,6 +510,7 @@ Sample configuration files are provided with all defaults in line-by-line commen
         user = <your-user-name>
         log_facility = LOG_LOCAL4
         recon_cache_path = /var/cache/swift3
+        eventlet_debug = true
 
         [pipeline:main]
         pipeline = recon container-server
@@ -532,6 +540,7 @@ Sample configuration files are provided with all defaults in line-by-line commen
         user = <your-user-name>
         log_facility = LOG_LOCAL5
         recon_cache_path = /var/cache/swift4
+        eventlet_debug = true
 
         [pipeline:main]
         pipeline = recon container-server
@@ -562,6 +571,7 @@ Sample configuration files are provided with all defaults in line-by-line commen
         user = <your-user-name>
         log_facility = LOG_LOCAL2
         recon_cache_path = /var/cache/swift
+        eventlet_debug = true
 
         [pipeline:main]
         pipeline = recon object-server
@@ -589,6 +599,7 @@ Sample configuration files are provided with all defaults in line-by-line commen
         user = <your-user-name>
         log_facility = LOG_LOCAL3
         recon_cache_path = /var/cache/swift2
+        eventlet_debug = true
 
         [pipeline:main]
         pipeline = recon object-server
@@ -616,6 +627,7 @@ Sample configuration files are provided with all defaults in line-by-line commen
         user = <your-user-name>
         log_facility = LOG_LOCAL4
         recon_cache_path = /var/cache/swift3
+        eventlet_debug = true
 
         [pipeline:main]
         pipeline = recon object-server
@@ -643,6 +655,7 @@ Sample configuration files are provided with all defaults in line-by-line commen
         user = <your-user-name>
         log_facility = LOG_LOCAL5
         recon_cache_path = /var/cache/swift4
+        eventlet_debug = true
 
         [pipeline:main]
         pipeline = recon object-server

etc/account-server.conf-sample

@@ -28,6 +28,7 @@
 # If you don't mind the extra disk space usage in overhead, you can turn this
 # on to preallocate disk space with SQLite databases to decrease fragmentation.
 # db_preallocation = off
+# eventlet_debug = false
 
 [pipeline:main]
 pipeline = healthcheck recon account-server

etc/container-server.conf-sample

@@ -31,6 +31,7 @@
 # If you don't mind the extra disk space usage in overhead, you can turn this
 # on to preallocate disk space with SQLite databases to decrease fragmentation.
 # db_preallocation = off
+# eventlet_debug = false
 
 [pipeline:main]
 pipeline = healthcheck recon container-server

etc/object-server.conf-sample

@@ -26,6 +26,7 @@
 # log_statsd_port = 8125
 # log_statsd_default_sample_rate = 1
 # log_statsd_metric_prefix =
+# eventlet_debug = false
 
 [pipeline:main]
 pipeline = healthcheck recon object-server

etc/proxy-server.conf-sample

@@ -29,6 +29,7 @@
 # log_statsd_metric_prefix =
 # Use a comma separated list of full url (http://foo.bar:1234,https://foo.bar)
 # cors_allow_origin =
+# eventlet_debug = false
 
 [pipeline:main]
 pipeline = catch_errors healthcheck cache ratelimit tempauth proxy-logging proxy-server

swift/common/daemon.py

@@ -18,6 +18,8 @@ import sys
 import signal
 from re import sub
 
+import eventlet.debug
+
 from swift.common import utils
 
 
@@ -89,6 +91,10 @@ def run_daemon(klass, conf_file, section_name='', once=False, **kwargs):
     if utils.config_true_value(conf.get('disable_fallocate', 'no')):
         utils.disable_fallocate()
 
+    # By default, disable eventlet printing stacktraces
+    eventlet_debug = utils.config_true_value(conf.get('eventlet_debug', 'no'))
+    eventlet.debug.hub_exceptions(eventlet_debug)
+
     try:
         klass(conf).run(once=once, **kwargs)
     except KeyboardInterrupt:

swift/common/wsgi.py

@@ -24,6 +24,7 @@ from itertools import chain
 from StringIO import StringIO
 
 import eventlet
+import eventlet.debug
 from eventlet import greenio, GreenPool, sleep, wsgi, listen
 from paste.deploy import loadapp, appconfig
 from eventlet.green import socket, ssl
@@ -134,6 +135,8 @@ def run_wsgi(conf_file, app_section, *args, **kwargs):
         wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60)
         eventlet.hubs.use_hub('poll')
         eventlet.patcher.monkey_patch(all=False, socket=True)
+        eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no'))
+        eventlet.debug.hub_exceptions(eventlet_debug)
         app = loadapp('config:%s' % conf_file,
                       global_conf={'log_name': log_name})
         pool = GreenPool(size=1024)