Add key-value storage in persistence plugins

cinderlib/__init__.py

@@ -12,6 +12,7 @@ DEFAULT_USER_ID = objects.DEFAULT_USER_ID
 Volume = objects.Volume
 Snapshot = objects.Snapshot
 Connection = objects.Connection
+KeyValue = objects.KeyValue
 
 load = serialization.load
 json = serialization.json

cinderlib/objects.py

@@ -45,6 +45,15 @@ DEFAULT_USER_ID = 'cinderlib'
 volume_cmd.objects.register_all()
 
 
+class KeyValue(object):
+    def __init__(self, key=None, value=None):
+        self.key = key
+        self.value = value
+
+    def __eq__(self, other):
+        return (self.key, self.value) == (other.key, other.value)
+
+
 class Object(object):
     """Base class for our resource representation objects."""
     DEFAULT_FIELDS_VALUES = {}

cinderlib/persistence/base.py

@@ -43,6 +43,9 @@ class PersistenceDriverBase(object):
     def get_connections(self, connection_id=None, volume_id=None):
         raise NotImplemented()
 
+    def get_key_values(self, key):
+        raise NotImplemented()
+
     def set_volume(self, volume):
         self.reset_change_tracker(volume)
 
@@ -52,6 +55,9 @@ class PersistenceDriverBase(object):
     def set_connection(self, connection):
         self.reset_change_tracker(connection)
 
+    def set_key_value(self, key, value):
+        raise NotImplemented()
+
     def delete_volume(self, volume):
         self._set_deleted(volume)
         self.reset_change_tracker(volume)
@@ -64,6 +70,9 @@ class PersistenceDriverBase(object):
         self._set_deleted(connection)
         self.reset_change_tracker(connection)
 
+    def delete_key_value(self, key):
+        raise NotImplemented()
+
     def _set_deleted(self, resource):
         resource._ovo.deleted = True
         resource._ovo.deleted_at = timeutils.utcnow()

cinderlib/persistence/dbms.py

@@ -21,6 +21,7 @@ from cinder.cmd import volume as volume_cmd
 from cinder.db import api as db_api
 from cinder.db import migration
 from cinder.db.sqlalchemy import api as sqla_api
+from cinder.db.sqlalchemy import models
 from cinder import objects as cinder_objs
 from oslo_log import log
 
@@ -31,6 +32,12 @@ from cinderlib.persistence import base as persistence_base
 LOG = log.getLogger(__name__)
 
 
+class KeyValue(models.BASE, models.models.ModelBase, objects.KeyValue):
+    __tablename__ = 'cinderlib_persistence_key_value'
+    key = models.Column(models.String(255), primary_key=True)
+    value = models.Column(models.Text)
+
+
 class DBPersistence(persistence_base.PersistenceDriverBase):
     def __init__(self, connection, sqlite_synchronous=True,
                  soft_deletes=False):
@@ -47,8 +54,13 @@ class DBPersistence(persistence_base.PersistenceDriverBase):
             lazy=True)
 
         migration.db_sync()
+        self._create_key_value_table()
         super(DBPersistence, self).__init__()
 
+    def _create_key_value_table(self):
+        models.BASE.metadata.create_all(sqla_api.get_engine(),
+                                        tables=[KeyValue.__table__])
+
     @property
     def db(self):
         return self.db_instance
@@ -86,6 +98,17 @@ class DBPersistence(persistence_base.PersistenceDriverBase):
                   for ovo in ovos.objects]
         return result
 
+    def _get_kv(self, key=None, session=None):
+        session = session or sqla_api.get_session()
+        query = session.query(KeyValue)
+        if key is None:
+            return query.all()
+        res = query.filter_by(key=key).first()
+        return [res] if res else []
+
+    def get_key_values(self, key=None):
+        return self._get_kv(key)
+
     def set_volume(self, volume):
         changed = self.get_changed_fields(volume)
         if not changed:
@@ -132,15 +155,22 @@ class DBPersistence(persistence_base.PersistenceDriverBase):
                                              changed)
         super(DBPersistence, self).set_connection(connection)
 
+    def set_key_value(self, key_value):
+        session = sqla_api.get_session()
+        with session.begin():
+            kv = self._get_kv(key_value.key, session)
+            kv = kv[0] if kv else KeyValue(key=key_value.key)
+            kv.value = key_value.value
+            session.add(kv)
+
     def delete_volume(self, volume):
         if self.soft_deletes:
             LOG.debug('soft deleting volume %s', volume.id)
             self.db.volume_destroy(objects.CONTEXT, volume.id)
         else:
             LOG.debug('hard deleting volume %s', volume.id)
-            sqla_api.model_query(objects.CONTEXT,
+            query = sqla_api.model_query(objects.CONTEXT, models.Volume)
-                                 sqla_api.models.Volume
+            query.filter_by(id=volume.id).delete()
-                                 ).filter_by(id=volume.id).delete()
         super(DBPersistence, self).delete_volume(volume)
 
     def delete_snapshot(self, snapshot):
@@ -149,9 +179,8 @@ class DBPersistence(persistence_base.PersistenceDriverBase):
             self.db.snapshot_destroy(objects.CONTEXT, snapshot.id)
         else:
             LOG.debug('hard deleting snapshot %s', snapshot.id)
-            sqla_api.model_query(objects.CONTEXT,
+            query = sqla_api.model_query(objects.CONTEXT, models.Snapshot)
-                                 sqla_api.models.Snapshot
+            query.filter_by(id=snapshot.id).delete()
-                                 ).filter_by(id=snapshot.id).delete()
         super(DBPersistence, self).delete_snapshot(snapshot)
 
     def delete_connection(self, connection):
@@ -160,11 +189,15 @@ class DBPersistence(persistence_base.PersistenceDriverBase):
             self.db.attachment_destroy(objects.CONTEXT, connection.id)
         else:
             LOG.debug('hard deleting connection %s', connection.id)
-            sqla_api.model_query(objects.CONTEXT,
+            query = sqla_api.model_query(objects.CONTEXT,
-                                 sqla_api.models.VolumeAttachment
+                                         models.VolumeAttachment)
-                                 ).filter_by(id=connection.id).delete()
+            query.filter_by(id=connection.id).delete()
         super(DBPersistence, self).delete_connection(connection)
 
+    def delete_key_value(self, key_value):
+        query = sqla_api.get_session().query(KeyValue)
+        query.filter_by(key=key_value.key).delete()
+
 
 class MemoryDBPersistence(DBPersistence):
     def __init__(self):

cinderlib/persistence/memory.py

@@ -13,6 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from cinderlib import objects
 from cinderlib.persistence import base as persistence_base
 
 
@@ -20,6 +21,7 @@ class MemoryPersistence(persistence_base.PersistenceDriverBase):
     volumes = {}
     snapshots = {}
     connections = {}
+    key_values = {}
 
     def __init__(self):
         # Create fake DB for drivers
@@ -66,6 +68,14 @@ class MemoryPersistence(persistence_base.PersistenceDriverBase):
         result = self._filter_by(result, 'volume_id', volume_id)
         return result
 
+    def get_key_values(self, key=None):
+        try:
+            result = ([self.key_values[key]] if key
+                      else self.key_values.values())
+        except KeyError:
+            return []
+        return result
+
     def set_volume(self, volume):
         self.volumes[volume.id] = volume
         super(MemoryPersistence, self).set_volume(volume)
@@ -78,6 +88,9 @@ class MemoryPersistence(persistence_base.PersistenceDriverBase):
         self.connections[connection.id] = connection
         super(MemoryPersistence, self).set_connection(connection)
 
+    def set_key_value(self, key_value):
+        self.key_values[key_value.key] = key_value
+
     def delete_volume(self, volume):
         self.volumes.pop(volume.id, None)
         super(MemoryPersistence, self).delete_volume(volume)
@@ -89,3 +102,6 @@ class MemoryPersistence(persistence_base.PersistenceDriverBase):
     def delete_connection(self, connection):
         self.connections.pop(connection.id, None)
         super(MemoryPersistence, self).delete_connection(connection)
+
+    def delete_key_value(self, key_value):
+        self.key_values.pop(key_value.key, None)

tests/unit/persistence/base.py

@@ -16,6 +16,7 @@
 import unittest2
 
 from cinder.cmd import volume as volume_cmd
+from cinder.db.sqlalchemy import models
 from oslo_versionedobjects import fields
 
 import cinderlib
@@ -45,8 +46,8 @@ class BasePersistenceTest(unittest2.TestCase):
         cinderlib.Backend.backends = {}
         super(BasePersistenceTest, self).tearDown()
 
-    def sorted(self, resources):
+    def sorted(self, resources, key='id'):
-        return sorted(resources, key=lambda x: x.id)
+        return sorted(resources, key=lambda x: getattr(x, key))
 
     def create_n_volumes(self, n):
         return self.create_volumes([{'size': i, 'name': 'disk%s' % i}
@@ -79,7 +80,18 @@ class BasePersistenceTest(unittest2.TestCase):
             self.persistence.set_connection(conn)
         return self.sorted(conns)
 
+    def create_key_values(self):
+        kvs = []
+        for i in range(2):
+            kv = cinderlib.KeyValue(key='key%i' % i, value='value%i' % i)
+            kvs.append(kv)
+            self.persistence.set_key_value(kv)
+        return kvs
+
     def _convert_to_dict(self, obj):
+        if isinstance(obj, models.BASE):
+            return dict(obj)
+
         if not isinstance(obj, cinderlib.objects.Object):
             return obj
 
@@ -172,6 +184,19 @@ class BasePersistenceTest(unittest2.TestCase):
                                            volume_id=vols[0].id)
         self.assertListEqualObj([], res)
 
+    def test_delete_volume(self):
+        vols = self.create_n_volumes(2)
+        self.persistence.delete_volume(vols[0])
+        res = self.persistence.get_volumes()
+        self.assertListEqualObj([vols[1]], res)
+
+    def test_delete_volume_not_found(self):
+        vols = self.create_n_volumes(2)
+        fake_vol = cinderlib.Volume(backend_or_vol=self.backend)
+        self.persistence.delete_volume(fake_vol)
+        res = self.persistence.get_volumes()
+        self.assertListEqualObj(vols, self.sorted(res))
+
     def test_set_snapshot(self):
         raise NotImplemented('Test class must implement this method')
 
@@ -237,6 +262,19 @@ class BasePersistenceTest(unittest2.TestCase):
                                              volume_id=snaps[0].volume.id)
         self.assertListEqualObj([], res)
 
+    def test_delete_snapshot(self):
+        snaps = self.create_snapshots()
+        self.persistence.delete_snapshot(snaps[0])
+        res = self.persistence.get_snapshots()
+        self.assertListEqualObj([snaps[1]], res)
+
+    def test_delete_snapshot_not_found(self):
+        snaps = self.create_snapshots()
+        fake_snap = cinderlib.Snapshot(snaps[0].volume)
+        self.persistence.delete_snapshot(fake_snap)
+        res = self.persistence.get_snapshots()
+        self.assertListEqualObj(snaps, self.sorted(res))
+
     def test_set_connection(self):
         raise NotImplemented('Test class must implement this method')
 
@@ -283,3 +321,56 @@ class BasePersistenceTest(unittest2.TestCase):
         res = self.persistence.get_connections(volume_id=conns[0].volume.id,
                                                connection_id=conns[1].id)
         self.assertListEqualObj([], res)
+
+    def test_delete_connection(self):
+        conns = self.create_connections()
+        self.persistence.delete_connection(conns[1])
+        res = self.persistence.get_connections()
+        self.assertListEqualObj([conns[0]], res)
+
+    def test_delete_connection_not_found(self):
+        conns = self.create_connections()
+        fake_conn = cinderlib.Connection(self.backend,
+                                         volume=conns[0].volume)
+        self.persistence.delete_connection(fake_conn)
+        res = self.persistence.get_connections()
+        self.assertListEqualObj(conns, self.sorted(res))
+
+    def test_set_key_values(self):
+        raise NotImplemented('Test class must implement this method')
+
+    def assertKVsEqual(self, expected, actual):
+        if len(expected) == len(actual):
+            for (key, value), actual in zip(expected, actual):
+                self.assertEqual(key, actual.key)
+                self.assertEqual(value, actual.value)
+            return
+        assert False, '%s is not equal to %s' % (expected, actual)
+
+    def get_key_values_all(self):
+        kvs = self.create_key_values()
+        res = self.persistence.get_key_values()
+        self.assertListEqual(kvs, self.sorted(res, 'key'))
+
+    def test_get_key_values_by_key(self):
+        kvs = self.create_key_values()
+        res = self.persistence.get_key_values(key=kvs[1].key)
+        self.assertListEqual([kvs[1]], res)
+
+    def test_get_key_values_by_key_not_found(self):
+        self.create_key_values()
+        res = self.persistence.get_key_values(key='fake-uuid')
+        self.assertListEqual([], res)
+
+    def test_delete_key_value(self):
+        kvs = self.create_key_values()
+        self.persistence.delete_key_value(kvs[1])
+        res = self.persistence.get_key_values()
+        self.assertListEqual([kvs[0]], res)
+
+    def test_delete_key_not_found(self):
+        kvs = self.create_key_values()
+        fake_key = cinderlib.KeyValue('fake-key')
+        self.persistence.delete_key_value(fake_key)
+        res = self.persistence.get_key_values()
+        self.assertListEqual(kvs, self.sorted(res, 'key'))

tests/unit/persistence/test_dbms.py

@@ -20,6 +20,7 @@ from cinder import objects as cinder_ovos
 from oslo_db import api as oslo_db_api
 
 import cinderlib
+from cinderlib.persistence import dbms
 from tests.unit.persistence import base
 
 
@@ -32,6 +33,7 @@ class TestMemoryDBPersistence(base.BasePersistenceTest):
                              sqla_api.models.VolumeAttachment).delete()
         sqla_api.model_query(self.context,
                              sqla_api.models.Volume).delete()
+        sqla_api.get_session().query(dbms.KeyValue).delete()
         super(TestMemoryDBPersistence, self).tearDown()
 
     def test_db(self):
@@ -89,6 +91,16 @@ class TestMemoryDBPersistence(base.BasePersistenceTest):
 
         self.assertEqualObj(conn, cl_conn)
 
+    def test_set_key_values(self):
+        res = sqla_api.get_session().query(dbms.KeyValue).all()
+        self.assertListEqual([], res)
+
+        expected = [dbms.KeyValue(key='key', value='value')]
+        self.persistence.set_key_value(expected[0])
+
+        actual = sqla_api.get_session().query(dbms.KeyValue).all()
+        self.assertListEqualObj(expected, actual)
+
 
 class TestDBPersistence(TestMemoryDBPersistence):
     CONNECTION = 'sqlite:///' + tempfile.NamedTemporaryFile().name

tests/unit/persistence/test_memory.py

@@ -14,6 +14,7 @@
 #    under the License.
 
 import cinderlib
+from cinderlib.persistence import memory
 from tests.unit.persistence import base
 
 
@@ -25,6 +26,7 @@ class TestMemoryPersistence(base.BasePersistenceTest):
         self.persistence.volumes = {}
         self.persistence.snapshots = {}
         self.persistence.connections = {}
+        self.persistence.key_values = {}
         super(TestMemoryPersistence, self).tearDown()
 
     def test_db(self):
@@ -55,3 +57,10 @@ class TestMemoryPersistence(base.BasePersistenceTest):
 
         self.persistence.set_connection(conn)
         self.assertDictEqual({conn.id: conn}, self.persistence.connections)
+
+    def test_set_key_values(self):
+        self.assertDictEqual({}, self.persistence.key_values)
+        expected = [cinderlib.KeyValue('key', 'value')]
+        self.persistence.set_key_value(expected[0])
+        self.assertTrue('key' in self.persistence.key_values)
+        self.assertEqual(expected, self.persistence.key_values.values())