From a0e3057b9a9fdbc23a3a9f700136aad6e8799cb3 Mon Sep 17 00:00:00 2001 From: Ilya Etingof Date: Tue, 27 Aug 2019 14:29:15 +0200 Subject: [PATCH] Do not reuse sqlite connection Prior to this patch, `PersistentDict` has been reusing sqlite connection for all subsequent calls. This proved to fail in MT environments. With this change, `PersistentDict` reopens sqlite connection from each object and closes it down in hope to make it MT-safe. Change-Id: Ic3e4618f44236c86e59c62bebfd817b40b5235aa --- sushy_tools/emulator/memoize.py | 42 ++++++++++--------- .../tests/unit/emulator/test_memoize.py | 5 --- 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/sushy_tools/emulator/memoize.py b/sushy_tools/emulator/memoize.py index cb9ae5c3..88158a35 100644 --- a/sushy_tools/emulator/memoize.py +++ b/sushy_tools/emulator/memoize.py @@ -19,6 +19,7 @@ try: except ImportError: import collections +import contextlib from functools import wraps import os import pickle @@ -73,18 +74,16 @@ def memoize(permanent_cache=None): class PersistentDict(MutableMapping): DBPATH = os.path.join(tempfile.gettempdir(), 'sushy-emulator') - _connection = None + _dbpath = None def make_permanent(self, dbpath, dbfile): dbpath = dbpath or self.DBPATH if not os.path.exists(dbpath): os.makedirs(dbpath) - dbpath = os.path.join(dbpath, dbfile) + '.sqlite' - self._connection = sqlite3.connect(dbpath) + self._dbpath = os.path.join(dbpath, dbfile) + '.sqlite' - with self.connection as connection: - cursor = connection.cursor() + with self.connection() as cursor: cursor.execute( 'create table if not exists cache ' '(key blob primary key not null, value blob not null)' @@ -100,18 +99,25 @@ class PersistentDict(MutableMapping): def decode(blob): return pickle.loads(blob) - @property + @contextlib.contextmanager def connection(self): - if not self._connection: + if not self._dbpath: raise TypeError('Dict is not yet persistent') - return self._connection + connection = sqlite3.connect(self._dbpath) + + try: + yield connection.cursor() + + connection.commit() + + finally: + connection.close() def __getitem__(self, key): key = self.encode(key) - with self.connection as connection: - cursor = connection.cursor() + with self.connection() as cursor: cursor.execute( 'select value from cache where key=?', (key,) @@ -127,8 +133,7 @@ class PersistentDict(MutableMapping): key = self.encode(key) value = self.encode(value) - with self.connection as connection: - cursor = connection.cursor() + with self.connection() as cursor: cursor.execute( 'insert or replace into cache values (?, ?)', (key, value) @@ -137,9 +142,7 @@ class PersistentDict(MutableMapping): def __delitem__(self, key): key = self.encode(key) - with self.connection as connection: - cursor = connection.cursor() - + with self.connection() as cursor: cursor.execute( 'select count(*) from cache where key=?', (key,) @@ -154,8 +157,7 @@ class PersistentDict(MutableMapping): ) def __iter__(self): - with self.connection as connection: - cursor = connection.cursor() + with self.connection() as cursor: cursor.execute( 'select key from cache' ) @@ -165,9 +167,9 @@ class PersistentDict(MutableMapping): yield self.decode(r[0]) def __len__(self): - with self.connection as connection: - cursor = connection.cursor() + with self.connection() as cursor: cursor.execute( 'select count(*) from cache' ) - return cursor.fetchone()[0] + count = cursor.fetchone()[0] + return count diff --git a/sushy_tools/tests/unit/emulator/test_memoize.py b/sushy_tools/tests/unit/emulator/test_memoize.py index 5c654802..2d9cd20a 100644 --- a/sushy_tools/tests/unit/emulator/test_memoize.py +++ b/sushy_tools/tests/unit/emulator/test_memoize.py @@ -116,7 +116,6 @@ class PersistentDictTestCase(base.BaseTestCase): mock_pickle.dumps.return_value = 'pickled-key' mock_connection = mock_sqlite3.connect.return_value - mock_connection = mock_connection.__enter__.return_value mock_cursor = mock_connection.cursor.return_value mock_cursor.fetchone.return_value = ['pickled-value'] @@ -136,7 +135,6 @@ class PersistentDictTestCase(base.BaseTestCase): 'pickled-key', 'pickled-value'] mock_connection = mock_sqlite3.connect.return_value - mock_connection = mock_connection.__enter__.return_value mock_cursor = mock_connection.cursor.return_value pd[1] = 2 @@ -153,7 +151,6 @@ class PersistentDictTestCase(base.BaseTestCase): mock_pickle.dumps.return_value = 'pickled-key' mock_connection = mock_sqlite3.connect.return_value - mock_connection = mock_connection.__enter__.return_value mock_cursor = mock_connection.cursor.return_value del pd[1] @@ -169,7 +166,6 @@ class PersistentDictTestCase(base.BaseTestCase): mock_pickle.dumps.return_value = 'pickled-key' mock_connection = mock_sqlite3.connect.return_value - mock_connection = mock_connection.__enter__.return_value mock_cursor = mock_connection.cursor.return_value mock_cursor.fetchall.return_value = [['pickled-key']] @@ -186,7 +182,6 @@ class PersistentDictTestCase(base.BaseTestCase): pd.make_permanent('/', 'file') mock_connection = mock_sqlite3.connect.return_value - mock_connection = mock_connection.__enter__.return_value mock_cursor = mock_connection.cursor.return_value expected = 1