Browse Source

Merge "Profile Oslo Service processes"

Zuul 2 months ago
parent
commit
fdcfd0fcff

+ 37
- 0
doc/source/user/usage.rst View File

@@ -179,3 +179,40 @@ logging options by sending a SIGHUP.
179 179
             logging.setup(cfg.CONF, 'foo')
180 180
 
181 181
 
182
+Profiling
183
+~~~~~~~~~
184
+
185
+Processes spawned through oslo_service.service can be profiled (function
186
+calltrace) through eventlet_backdoor module. Service has to configure
187
+backdoor_port option to enable it's workers to listen on TCP ports.
188
+Then user can send "prof()" command to capture worker processes function
189
+calltrace.
190
+
191
+1) To start profiling send "prof()" command on processes listening port
192
+
193
+2) To stop profiling and capture "pstat" calltrace to a file, send prof
194
+   command with filename as argument i.e "prof(filename)"
195
+   on worker processes listening port. Stats file (in pstat format) with
196
+   user provided filename by adding .prof as suffix will be generated
197
+   in temp directory.
198
+
199
+For example, to profile neutron server process (which is listening on
200
+port 8002 configured through backdoor_port option),
201
+
202
+.. code-block:: bash
203
+
204
+    $ echo "prof()" | nc localhost 8002
205
+    $ neutron net-create n1; neutron port-create --name p1 n1;
206
+    $ neutron port-delete p1; neutron port-delete p1
207
+    $ echo "prof('neutron')" | nc localhost 8002
208
+
209
+
210
+This will generate "/tmp/neutron.prof" as stats file. Later user can print
211
+the stats from the trace file like below
212
+
213
+.. code-block:: python
214
+
215
+    import pstats
216
+
217
+    stats = pstats.Stats('/tmp/neutron.prof')
218
+    stats.print_stats()

+ 1
- 0
lower-constraints.txt View File

@@ -72,3 +72,4 @@ traceback2==1.4.0
72 72
 unittest2==1.1.0
73 73
 WebOb==1.7.1
74 74
 wrapt==1.7.0
75
+Yappi==0.98

+ 27
- 0
oslo_service/eventlet_backdoor.py View File

@@ -22,10 +22,12 @@ import logging
22 22
 import os
23 23
 import pprint
24 24
 import sys
25
+import tempfile
25 26
 import traceback
26 27
 
27 28
 import eventlet.backdoor
28 29
 import greenlet
30
+import yappi
29 31
 
30 32
 from eventlet.green import socket
31 33
 from oslo_service._i18n import _
@@ -89,6 +91,30 @@ def _find_objects(t):
89 91
     return [o for o in gc.get_objects() if isinstance(o, t)]
90 92
 
91 93
 
94
+def _capture_profile(fname=''):
95
+    if not fname:
96
+        yappi.set_clock_type('cpu')
97
+        # We need to set context to greenlet to profile greenlets
98
+        # https://bitbucket.org/sumerc/yappi/pull-requests/3
99
+        yappi.set_context_id_callback(
100
+            lambda: id(greenlet.getcurrent()))
101
+        yappi.set_context_name_callback(
102
+            lambda: greenlet.getcurrent().__class__.__name__)
103
+        yappi.start()
104
+    else:
105
+        yappi.stop()
106
+        stats = yappi.get_func_stats()
107
+        # User should provide filename. This file with a suffix .prof
108
+        # will be created in temp directory.
109
+        try:
110
+            stats_file = os.path.join(tempfile.gettempdir(), fname + '.prof')
111
+            stats.save(stats_file, "pstat")
112
+        except Exception as e:
113
+            print("Error while saving the trace stats ", str(e))
114
+        finally:
115
+            yappi.clear_stats()
116
+
117
+
92 118
 def _print_greenthreads(simple=True):
93 119
     for i, gt in enumerate(_find_objects(greenlet.greenlet)):
94 120
         print(i, gt)
@@ -175,6 +201,7 @@ def _initialize_if_enabled(conf):
175 201
         'fo': _find_objects,
176 202
         'pgt': _print_greenthreads,
177 203
         'pnt': _print_nativethreads,
204
+        'prof': _capture_profile,
178 205
     }
179 206
 
180 207
     if conf.backdoor_port is None and conf.backdoor_socket is None:

+ 5
- 0
releasenotes/notes/profile-worker-5d3fd0f0251d62b8.yaml View File

@@ -0,0 +1,5 @@
1
+---
2
+features:
3
+  - |
4
+    Add support for profiling (capture function calltrace) service's worker
5
+    processes.

+ 1
- 0
requirements.txt View File

@@ -17,3 +17,4 @@ oslo.i18n>=3.15.3 # Apache-2.0
17 17
 PasteDeploy>=1.5.0 # MIT
18 18
 Routes>=2.3.1 # MIT
19 19
 Paste>=2.0.2 # MIT
20
+Yappi>=0.98 # MIT

Loading…
Cancel
Save