openstack/cinderlib
Code Issues Proposed changes

Add key-value storage in persistence plugins

Browse Source
This commit is contained in:
Gorka Eguileor
2018-05-17 20:26:37 +02:00
parent d70ebd933d
commit 467ba49b32
8 changed files with 191 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
cinderlib/__init__.py
View File

@@ -12,6 +12,7 @@ DEFAULT_USER_ID = objects.DEFAULT_USER_ID
Volume = objects.Volume
Snapshot = objects.Snapshot
Connection = objects.Connection
KeyValue = objects.KeyValue
load = serialization.load
json = serialization.json

9
cinderlib/objects.py
View File

@@ -45,6 +45,15 @@ DEFAULT_USER_ID = 'cinderlib'
volume_cmd.objects.register_all()
class KeyValue(object):
def __init__(self, key=None, value=None):
self.key = key
self.value = value
def __eq__(self, other):
return (self.key, self.value) == (other.key, other.value)
class Object(object):
"""Base class for our resource representation objects."""
DEFAULT_FIELDS_VALUES = {}

9
cinderlib/persistence/base.py
View File

@@ -43,6 +43,9 @@ class PersistenceDriverBase(object):
def get_connections(self, connection_id=None, volume_id=None):
raise NotImplemented()
def get_key_values(self, key):
raise NotImplemented()
def set_volume(self, volume):
self.reset_change_tracker(volume)
@@ -52,6 +55,9 @@ class PersistenceDriverBase(object):
def set_connection(self, connection):
self.reset_change_tracker(connection)
def set_key_value(self, key, value):
raise NotImplemented()
def delete_volume(self, volume):
self._set_deleted(volume)
self.reset_change_tracker(volume)
@@ -64,6 +70,9 @@ class PersistenceDriverBase(object):
self._set_deleted(connection)
self.reset_change_tracker(connection)
def delete_key_value(self, key):
raise NotImplemented()
def _set_deleted(self, resource):
resource._ovo.deleted = True
resource._ovo.deleted_at = timeutils.utcnow()

51
cinderlib/persistence/dbms.py
View File

@@ -21,6 +21,7 @@ from cinder.cmd import volume as volume_cmd
from cinder.db import api as db_api
from cinder.db import migration
from cinder.db.sqlalchemy import api as sqla_api
from cinder.db.sqlalchemy import models
from cinder import objects as cinder_objs
from oslo_log import log
@@ -31,6 +32,12 @@ from cinderlib.persistence import base as persistence_base
LOG = log.getLogger(__name__)
class KeyValue(models.BASE, models.models.ModelBase, objects.KeyValue):
__tablename__ = 'cinderlib_persistence_key_value'
key = models.Column(models.String(255), primary_key=True)
value = models.Column(models.Text)
class DBPersistence(persistence_base.PersistenceDriverBase):
def __init__(self, connection, sqlite_synchronous=True,
soft_deletes=False):
@@ -47,8 +54,13 @@ class DBPersistence(persistence_base.PersistenceDriverBase):
lazy=True)
migration.db_sync()
self._create_key_value_table()
super(DBPersistence, self).__init__()
def _create_key_value_table(self):
models.BASE.metadata.create_all(sqla_api.get_engine(),
tables=[KeyValue.__table__])
@property
def db(self):
return self.db_instance
@@ -86,6 +98,17 @@ class DBPersistence(persistence_base.PersistenceDriverBase):
for ovo in ovos.objects]
return result
def _get_kv(self, key=None, session=None):
session = session or sqla_api.get_session()
query = session.query(KeyValue)
if key is None:
return query.all()
res = query.filter_by(key=key).first()
return [res] if res else []
def get_key_values(self, key=None):
return self._get_kv(key)
def set_volume(self, volume):
changed = self.get_changed_fields(volume)
if not changed:
@@ -132,15 +155,22 @@ class DBPersistence(persistence_base.PersistenceDriverBase):
changed)
super(DBPersistence, self).set_connection(connection)
def set_key_value(self, key_value):
session = sqla_api.get_session()
with session.begin():
kv = self._get_kv(key_value.key, session)
kv = kv[0] if kv else KeyValue(key=key_value.key)
kv.value = key_value.value
session.add(kv)
def delete_volume(self, volume):
if self.soft_deletes:
LOG.debug('soft deleting volume %s', volume.id)
self.db.volume_destroy(objects.CONTEXT, volume.id)
else:
LOG.debug('hard deleting volume %s', volume.id)
sqla_api.model_query(objects.CONTEXT,
sqla_api.models.Volume
).filter_by(id=volume.id).delete()
query = sqla_api.model_query(objects.CONTEXT, models.Volume)
query.filter_by(id=volume.id).delete()
super(DBPersistence, self).delete_volume(volume)
def delete_snapshot(self, snapshot):
@@ -149,9 +179,8 @@ class DBPersistence(persistence_base.PersistenceDriverBase):
self.db.snapshot_destroy(objects.CONTEXT, snapshot.id)
else:
LOG.debug('hard deleting snapshot %s', snapshot.id)
sqla_api.model_query(objects.CONTEXT,
sqla_api.models.Snapshot
).filter_by(id=snapshot.id).delete()
query = sqla_api.model_query(objects.CONTEXT, models.Snapshot)
query.filter_by(id=snapshot.id).delete()
super(DBPersistence, self).delete_snapshot(snapshot)
def delete_connection(self, connection):
@@ -160,11 +189,15 @@ class DBPersistence(persistence_base.PersistenceDriverBase):
self.db.attachment_destroy(objects.CONTEXT, connection.id)
else:
LOG.debug('hard deleting connection %s', connection.id)
sqla_api.model_query(objects.CONTEXT,
sqla_api.models.VolumeAttachment
).filter_by(id=connection.id).delete()
query = sqla_api.model_query(objects.CONTEXT,
models.VolumeAttachment)
query.filter_by(id=connection.id).delete()
super(DBPersistence, self).delete_connection(connection)
def delete_key_value(self, key_value):
query = sqla_api.get_session().query(KeyValue)
query.filter_by(key=key_value.key).delete()
class MemoryDBPersistence(DBPersistence):
def __init__(self):

16
cinderlib/persistence/memory.py
View File

@@ -13,6 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from cinderlib import objects
from cinderlib.persistence import base as persistence_base
@@ -20,6 +21,7 @@ class MemoryPersistence(persistence_base.PersistenceDriverBase):
volumes = {}
snapshots = {}
connections = {}
key_values = {}
def __init__(self):
# Create fake DB for drivers
@@ -66,6 +68,14 @@ class MemoryPersistence(persistence_base.PersistenceDriverBase):
result = self._filter_by(result, 'volume_id', volume_id)
return result
def get_key_values(self, key=None):
try:
result = ([self.key_values[key]] if key
else self.key_values.values())
except KeyError:
return []
return result
def set_volume(self, volume):
self.volumes[volume.id] = volume
super(MemoryPersistence, self).set_volume(volume)
@@ -78,6 +88,9 @@ class MemoryPersistence(persistence_base.PersistenceDriverBase):
self.connections[connection.id] = connection
super(MemoryPersistence, self).set_connection(connection)
def set_key_value(self, key_value):
self.key_values[key_value.key] = key_value
def delete_volume(self, volume):
self.volumes.pop(volume.id, None)
super(MemoryPersistence, self).delete_volume(volume)
@@ -89,3 +102,6 @@ class MemoryPersistence(persistence_base.PersistenceDriverBase):
def delete_connection(self, connection):
self.connections.pop(connection.id, None)
super(MemoryPersistence, self).delete_connection(connection)
def delete_key_value(self, key_value):
self.key_values.pop(key_value.key, None)

95
tests/unit/persistence/base.py
View File

@@ -16,6 +16,7 @@
import unittest2
from cinder.cmd import volume as volume_cmd
from cinder.db.sqlalchemy import models
from oslo_versionedobjects import fields
import cinderlib
@@ -45,8 +46,8 @@ class BasePersistenceTest(unittest2.TestCase):
cinderlib.Backend.backends = {}
super(BasePersistenceTest, self).tearDown()
def sorted(self, resources):
return sorted(resources, key=lambda x: x.id)
def sorted(self, resources, key='id'):
return sorted(resources, key=lambda x: getattr(x, key))
def create_n_volumes(self, n):
return self.create_volumes([{'size': i, 'name': 'disk%s' % i}
@@ -79,7 +80,18 @@ class BasePersistenceTest(unittest2.TestCase):
self.persistence.set_connection(conn)
return self.sorted(conns)
def create_key_values(self):
kvs = []
for i in range(2):
kv = cinderlib.KeyValue(key='key%i' % i, value='value%i' % i)
kvs.append(kv)
self.persistence.set_key_value(kv)
return kvs
def _convert_to_dict(self, obj):
if isinstance(obj, models.BASE):
return dict(obj)
if not isinstance(obj, cinderlib.objects.Object):
return obj
@@ -172,6 +184,19 @@ class BasePersistenceTest(unittest2.TestCase):
volume_id=vols[0].id)
self.assertListEqualObj([], res)
def test_delete_volume(self):
vols = self.create_n_volumes(2)
self.persistence.delete_volume(vols[0])
res = self.persistence.get_volumes()
self.assertListEqualObj([vols[1]], res)
def test_delete_volume_not_found(self):
vols = self.create_n_volumes(2)
fake_vol = cinderlib.Volume(backend_or_vol=self.backend)
self.persistence.delete_volume(fake_vol)
res = self.persistence.get_volumes()
self.assertListEqualObj(vols, self.sorted(res))
def test_set_snapshot(self):
raise NotImplemented('Test class must implement this method')
@@ -237,6 +262,19 @@ class BasePersistenceTest(unittest2.TestCase):
volume_id=snaps[0].volume.id)
self.assertListEqualObj([], res)
def test_delete_snapshot(self):
snaps = self.create_snapshots()
self.persistence.delete_snapshot(snaps[0])
res = self.persistence.get_snapshots()
self.assertListEqualObj([snaps[1]], res)
def test_delete_snapshot_not_found(self):
snaps = self.create_snapshots()
fake_snap = cinderlib.Snapshot(snaps[0].volume)
self.persistence.delete_snapshot(fake_snap)
res = self.persistence.get_snapshots()
self.assertListEqualObj(snaps, self.sorted(res))
def test_set_connection(self):
raise NotImplemented('Test class must implement this method')
@@ -283,3 +321,56 @@ class BasePersistenceTest(unittest2.TestCase):
res = self.persistence.get_connections(volume_id=conns[0].volume.id,
connection_id=conns[1].id)
self.assertListEqualObj([], res)
def test_delete_connection(self):
conns = self.create_connections()
self.persistence.delete_connection(conns[1])
res = self.persistence.get_connections()
self.assertListEqualObj([conns[0]], res)
def test_delete_connection_not_found(self):
conns = self.create_connections()
fake_conn = cinderlib.Connection(self.backend,
volume=conns[0].volume)
self.persistence.delete_connection(fake_conn)
res = self.persistence.get_connections()
self.assertListEqualObj(conns, self.sorted(res))
def test_set_key_values(self):
raise NotImplemented('Test class must implement this method')
def assertKVsEqual(self, expected, actual):
if len(expected) == len(actual):
for (key, value), actual in zip(expected, actual):
self.assertEqual(key, actual.key)
self.assertEqual(value, actual.value)
return
assert False, '%s is not equal to %s' % (expected, actual)
def get_key_values_all(self):
kvs = self.create_key_values()
res = self.persistence.get_key_values()
self.assertListEqual(kvs, self.sorted(res, 'key'))
def test_get_key_values_by_key(self):
kvs = self.create_key_values()
res = self.persistence.get_key_values(key=kvs[1].key)
self.assertListEqual([kvs[1]], res)
def test_get_key_values_by_key_not_found(self):
self.create_key_values()
res = self.persistence.get_key_values(key='fake-uuid')
self.assertListEqual([], res)
def test_delete_key_value(self):
kvs = self.create_key_values()
self.persistence.delete_key_value(kvs[1])
res = self.persistence.get_key_values()
self.assertListEqual([kvs[0]], res)
def test_delete_key_not_found(self):
kvs = self.create_key_values()
fake_key = cinderlib.KeyValue('fake-key')
self.persistence.delete_key_value(fake_key)
res = self.persistence.get_key_values()
self.assertListEqual(kvs, self.sorted(res, 'key'))

12
tests/unit/persistence/test_dbms.py
View File

@@ -20,6 +20,7 @@ from cinder import objects as cinder_ovos
from oslo_db import api as oslo_db_api
import cinderlib
from cinderlib.persistence import dbms
from tests.unit.persistence import base
@@ -32,6 +33,7 @@ class TestMemoryDBPersistence(base.BasePersistenceTest):
sqla_api.models.VolumeAttachment).delete()
sqla_api.model_query(self.context,
sqla_api.models.Volume).delete()
sqla_api.get_session().query(dbms.KeyValue).delete()
super(TestMemoryDBPersistence, self).tearDown()
def test_db(self):
@@ -89,6 +91,16 @@ class TestMemoryDBPersistence(base.BasePersistenceTest):
self.assertEqualObj(conn, cl_conn)
def test_set_key_values(self):
res = sqla_api.get_session().query(dbms.KeyValue).all()
self.assertListEqual([], res)
expected = [dbms.KeyValue(key='key', value='value')]
self.persistence.set_key_value(expected[0])
actual = sqla_api.get_session().query(dbms.KeyValue).all()
self.assertListEqualObj(expected, actual)
class TestDBPersistence(TestMemoryDBPersistence):
CONNECTION = 'sqlite:///' + tempfile.NamedTemporaryFile().name

9
tests/unit/persistence/test_memory.py
View File

@@ -14,6 +14,7 @@
# under the License.
import cinderlib
from cinderlib.persistence import memory
from tests.unit.persistence import base
@@ -25,6 +26,7 @@ class TestMemoryPersistence(base.BasePersistenceTest):
self.persistence.volumes = {}
self.persistence.snapshots = {}
self.persistence.connections = {}
self.persistence.key_values = {}
super(TestMemoryPersistence, self).tearDown()
def test_db(self):
@@ -55,3 +57,10 @@ class TestMemoryPersistence(base.BasePersistenceTest):
self.persistence.set_connection(conn)
self.assertDictEqual({conn.id: conn}, self.persistence.connections)
def test_set_key_values(self):
self.assertDictEqual({}, self.persistence.key_values)
expected = [cinderlib.KeyValue('key', 'value')]
self.persistence.set_key_value(expected[0])
self.assertTrue('key' in self.persistence.key_values)
self.assertEqual(expected, self.persistence.key_values.values())