Browse Source

Cache database and message queue connection objects

Recently in the gate we have seen a trace on some work-in-progress
patches:

  OperationalError: (pymysql.err.OperationalError)
                    (1040, u'Too many connections')

and at least one operator has reported that the number of database
connections increased significantly going from Mitaka to Newton.

It was suspected that the increase was caused by creating new oslo.db
transaction context managers on-the-fly when switching database
connections for cells. Comparing the dstat --tcp output of runs of the
gate-tempest-dsvm-neutron-full-ubuntu-xenial job with and without
caching of the database connections showed a difference of 445 active
TCP connections and 1495 active TCP connections, respectively [1].

This adds caching of the oslo.db transaction context managers and the
oslo.messaging transports to avoid creating a large number of objects
that are not being garbage-collected as expected.

Closes-Bug: #1691545

[1] https://docs.google.com/spreadsheets/d/1DIfFfX3kaA_SRoCM-aO7BN4IBEShChXLztOBFeKryt4/edit?usp=sharing

 Conflicts:
	nova/context.py
	nova/tests/unit/test_context.py

NOTE(melwitt): Conflicts caused by the fact that the set_target_cell
function doesn't exist in Ocata and message queue connections were
not stored on the context in Ocata.

Change-Id: I17e0eb836dd87aac5859f506e7d771d42753d31a
(cherry picked from commit 47fa88d947)
tags/15.0.5
melanie witt 2 years ago
parent
commit
f4159d1755

+ 22
- 2
nova/context.py View File

@@ -34,6 +34,10 @@ from nova import policy
34 34
 from nova import utils
35 35
 
36 36
 LOG = logging.getLogger(__name__)
37
+# TODO(melwitt): This cache should be cleared whenever WSGIService receives a
38
+# SIGHUP and periodically based on an expiration time. Currently, none of the
39
+# cell caches are purged, so neither is this one, for now.
40
+CELL_CACHE = {}
37 41
 
38 42
 
39 43
 class _ContextAuthPlugin(plugin.BaseAuthPlugin):
@@ -366,11 +370,27 @@ def target_cell(context, cell_mapping):
366 370
     :param context: The RequestContext to add connection information
367 371
     :param cell_mapping: A objects.CellMapping object
368 372
     """
373
+    global CELL_CACHE
374
+
369 375
     original_db_connection = context.db_connection
370 376
     # avoid circular import
371 377
     from nova import db
372
-    db_connection_string = cell_mapping.database_connection
373
-    context.db_connection = db.create_context_manager(db_connection_string)
378
+
379
+    # Synchronize access to the cache by multiple API workers.
380
+    @utils.synchronized(cell_mapping.uuid)
381
+    def get_or_set_cached_cell_and_set_connections():
382
+        try:
383
+            cell_db_conn = CELL_CACHE[cell_mapping.uuid]
384
+        except KeyError:
385
+            db_connection_string = cell_mapping.database_connection
386
+            context.db_connection = db.create_context_manager(
387
+                db_connection_string)
388
+            CELL_CACHE[cell_mapping.uuid] = context.db_connection
389
+        else:
390
+            context.db_connection = cell_db_conn
391
+
392
+    get_or_set_cached_cell_and_set_connections()
393
+
374 394
     try:
375 395
         yield context
376 396
     finally:

+ 1
- 0
nova/test.py View File

@@ -233,6 +233,7 @@ class TestCase(testtools.TestCase):
233 233
         # NOTE(danms): Reset the cached list of cells
234 234
         from nova.compute import api
235 235
         api.CELLS = []
236
+        context.CELL_CACHE = {}
236 237
 
237 238
         self.cell_mappings = {}
238 239
         self.host_mappings = {}

+ 21
- 1
nova/tests/unit/test_context.py View File

@@ -20,6 +20,7 @@ from nova import context
20 20
 from nova import exception
21 21
 from nova import objects
22 22
 from nova import test
23
+from nova.tests import uuidsentinel as uuids
23 24
 
24 25
 
25 26
 class ContextTestCase(test.NoDBTestCase):
@@ -298,7 +299,9 @@ class ContextTestCase(test.NoDBTestCase):
298 299
                                       roles=['admin', 'weasel'])
299 300
         # Verify the existing db_connection, if any, is restored
300 301
         ctxt.db_connection = mock.sentinel.db_conn
301
-        mapping = objects.CellMapping(database_connection='fake://')
302
+        mapping = objects.CellMapping(database_connection='fake://',
303
+                                      transport_url='fake://',
304
+                                      uuid=uuids.cell)
302 305
         with context.target_cell(ctxt, mapping):
303 306
             self.assertEqual(ctxt.db_connection, mock.sentinel.cm)
304 307
         self.assertEqual(mock.sentinel.db_conn, ctxt.db_connection)
@@ -308,3 +311,20 @@ class ContextTestCase(test.NoDBTestCase):
308 311
         self.assertIsNone(ctxt.user_id)
309 312
         self.assertIsNone(ctxt.project_id)
310 313
         self.assertFalse(ctxt.is_admin)
314
+
315
+    @mock.patch('nova.db.create_context_manager')
316
+    def test_target_cell_caching(self, mock_create_cm):
317
+        mock_create_cm.return_value = mock.sentinel.db_conn_obj
318
+        ctxt = context.get_context()
319
+        mapping = objects.CellMapping(database_connection='fake://db',
320
+                                      transport_url='fake://mq',
321
+                                      uuid=uuids.cell)
322
+        # First call should create new connection objects.
323
+        with context.target_cell(ctxt, mapping):
324
+            self.assertEqual(mock.sentinel.db_conn_obj, ctxt.db_connection)
325
+        mock_create_cm.assert_called_once_with('fake://db')
326
+        # Second call should use cached objects.
327
+        mock_create_cm.reset_mock()
328
+        with context.target_cell(ctxt, mapping):
329
+            self.assertEqual(mock.sentinel.db_conn_obj, ctxt.db_connection)
330
+        mock_create_cm.assert_not_called()

+ 10
- 0
releasenotes/notes/bug-1691545-1acd6512effbdffb.yaml View File

@@ -0,0 +1,10 @@
1
+---
2
+fixes:
3
+  - |
4
+    Fixes `bug 1691545`_ in which there was a significant increase in database
5
+    connections because of the way connections to cell databases were being
6
+    established. With this fix, objects related to database connections are
7
+    cached in the API service and reused to prevent new connections being
8
+    established for every communication with cell databases.
9
+
10
+    .. _bug 1691545: https://bugs.launchpad.net/nova/+bug/1691545

Loading…
Cancel
Save