Add db table for Glance Metadata

This commit implements the blueprint
https://blueprints.launchpad.net/cinder/+spec/retain-glance-metadata-for-billing

It creates the new table volume_glance_metadata in the cinder
database, provides the CRUD methods for it, and populates the table
when a volume or snapshot is created from a Glance image.

Patch set 2: remove superflous line

Patch set 3: Fix incorrect column types in sqlalchemy/models.py

Patch set 4: Define exception class GlanceMetadataExists

Change-Id: I8f98f6eaae005a33bfd49cea783774407b7aa120

cinder/db/api.py

@@ -375,6 +375,55 @@ def volume_type_extra_specs_update_or_create(context, volume_type_id,
 ###################
 
 
+def volume_glance_metadata_create(context, volume_id, key, value):
+    """Update the Glance metadata for the specified volume."""
+    return IMPL.volume_glance_metadata_create(context, volume_id,
+                                              key, value)
+
+
+def volume_glance_metadata_get(context, volume_id):
+    """Return the glance metadata for a volume."""
+    return IMPL.volume_glance_metadata_get(context, volume_id)
+
+
+def volume_snapshot_glance_metadata_get(context, snapshot_id):
+    """Return the Glance metadata for the specified snapshot."""
+    return IMPL.volume_snapshot_glance_metadata_get(context, snapshot_id)
+
+
+def volume_glance_metadata_copy_to_snapshot(context, snapshot_id, volume_id):
+    """
+    Update the Glance metadata for a snapshot by copying all of the key:value
+    pairs from the originating volume. This is so that a volume created from
+    the snapshot will retain the original metadata.
+    """
+    return IMPL.volume_glance_metadata_copy_to_snapshot(context, snapshot_id,
+                                                        volume_id)
+
+
+def volume_glance_metadata_copy_to_volume(context, volume_id, snapshot_id):
+    """
+    Update the Glance metadata from a volume (created from a snapshot) by
+    copying all of the key:value pairs from the originating snapshot. This is
+    so that the Glance metadata from the original volume is retained.
+    """
+    return IMPL.volume_glance_metadata_copy_to_volume(context, volume_id,
+                                                      snapshot_id)
+
+
+def volume_glance_metadata_delete_by_volume(context, volume_id):
+    """Delete the glance metadata for a volume."""
+    return IMPL.volume_glance_metadata_delete_by_volume(context, volume_id)
+
+
+def volume_glance_metadata_delete_by_snapshot(context, snapshot_id):
+    """Delete the glance metadata for a snapshot."""
+    return IMPL.volume_glance_metadata_delete_by_snapshot(context, snapshot_id)
+
+
+###################
+
+
 def sm_backend_conf_create(context, values):
     """Create a new SM Backend Config entry."""
     return IMPL.sm_backend_conf_create(context, values)

cinder/db/sqlalchemy/api.py

@@ -142,6 +142,20 @@ def require_volume_exists(f):
     return wrapper
 
 
+def require_snapshot_exists(f):
+    """Decorator to require the specified snapshot to exist.
+
+    Requires the wrapped function to use context and snapshot_id as
+    their first two arguments.
+    """
+
+    def wrapper(context, snapshot_id, *args, **kwargs):
+        db.api.snapshot_get(context, snapshot_id)
+        return f(context, snapshot_id, *args, **kwargs)
+    wrapper.__name__ = f.__name__
+    return wrapper
+
+
 def model_query(context, *args, **kwargs):
     """Query helper that accounts for context's `read_deleted` field.
 
@@ -1439,6 +1453,134 @@ def volume_type_extra_specs_update_or_create(context, volume_type_id,
 ####################
 
 
+@require_context
+@require_volume_exists
+def volume_glance_metadata_get(context, volume_id, session=None):
+    """Return the Glance metadata for the specified volume."""
+    if not session:
+        session = get_session()
+
+    return session.query(models.VolumeGlanceMetadata).\
+                         filter_by(volume_id=volume_id).\
+                         filter_by(deleted=False).all()
+
+
+@require_context
+@require_snapshot_exists
+def volume_snapshot_glance_metadata_get(context, snapshot_id, session=None):
+    """Return the Glance metadata for the specified snapshot."""
+    if not session:
+        session = get_session()
+
+    return session.query(models.VolumeGlanceMetadata).\
+                         filter_by(snapshot_id=snapshot_id).\
+                         filter_by(deleted=False).all()
+
+
+@require_context
+@require_volume_exists
+def volume_glance_metadata_create(context, volume_id, key, value,
+                                  session=None):
+    """
+    Update the Glance metadata for a volume by adding a new key:value pair.
+    This API does not support changing the value of a key once it has been
+    created.
+    """
+    if session is None:
+        session = get_session()
+
+    with session.begin():
+        rows = session.query(models.VolumeGlanceMetadata).\
+                filter_by(volume_id=volume_id).\
+                filter_by(key=key).\
+                filter_by(deleted=False).all()
+
+        if len(rows) > 0:
+            raise exception.GlanceMetadataExists(key=key,
+                                                 volume_id=volume_id)
+
+        vol_glance_metadata = models.VolumeGlanceMetadata()
+        vol_glance_metadata.volume_id = volume_id
+        vol_glance_metadata.key = key
+        vol_glance_metadata.value = value
+
+        vol_glance_metadata.save(session=session)
+
+    return
+
+
+@require_context
+@require_snapshot_exists
+def volume_glance_metadata_copy_to_snapshot(context, snapshot_id, volume_id,
+                                            session=None):
+    """
+    Update the Glance metadata for a snapshot by copying all of the key:value
+    pairs from the originating volume. This is so that a volume created from
+    the snapshot will retain the original metadata.
+    """
+    if session is None:
+        session = get_session()
+
+    metadata = volume_glance_metadata_get(context, volume_id, session=session)
+    with session.begin():
+        for meta in metadata:
+            vol_glance_metadata = models.VolumeGlanceMetadata()
+            vol_glance_metadata.snapshot_id = snapshot_id
+            vol_glance_metadata.key = meta['key']
+            vol_glance_metadata.value = meta['value']
+
+            vol_glance_metadata.save(session=session)
+
+
+@require_context
+@require_volume_exists
+def volume_glance_metadata_copy_to_volume(context, volume_id, snapshot_id,
+                                          session=None):
+    """
+    Update the Glance metadata from a volume (created from a snapshot) by
+    copying all of the key:value pairs from the originating snapshot. This is
+    so that the Glance metadata from the original volume is retained.
+    """
+    if session is None:
+        session = get_session()
+
+    metadata = volume_snapshot_glance_metadata_get(context, snapshot_id,
+                                                session=session)
+    with session.begin():
+        for meta in metadata:
+            vol_glance_metadata = models.VolumeGlanceMetadata()
+            vol_glance_metadata.volume_id = volume_id
+            vol_glance_metadata.key = meta['key']
+            vol_glance_metadata.value = meta['value']
+
+            vol_glance_metadata.save(session=session)
+
+
+@require_context
+def volume_glance_metadata_delete_by_volume(context, volume_id):
+    session = get_session()
+    session.query(models.VolumeGlanceMetadata).\
+        filter_by(volume_id=volume_id).\
+        filter_by(deleted=False).\
+        update({'deleted': True,
+                'deleted_at': timeutils.utcnow(),
+                'updated_at': literal_column('updated_at')})
+
+
+@require_context
+def volume_glance_metadata_delete_by_snapshot(context, snapshot_id):
+    session = get_session()
+    session.query(models.VolumeGlanceMetadata).\
+        filter_by(snapshot_id=snapshot_id).\
+        filter_by(deleted=False).\
+        update({'deleted': True,
+                'deleted_at': timeutils.utcnow(),
+                'updated_at': literal_column('updated_at')})
+
+
+####################
+
+
 @require_admin_context
 def sm_backend_conf_create(context, values):
     backend_conf = models.SMBackendConf()

cinder/db/sqlalchemy/migrate_repo/versions/003_glance_metadata.py

@@ -0,0 +1,74 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2012 OpenStack LLC.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from sqlalchemy import Column, DateTime, Text, Boolean
+from sqlalchemy import MetaData, Integer, String, Table, ForeignKey
+
+from cinder.openstack.common import log as logging
+
+LOG = logging.getLogger(__name__)
+
+
+def upgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+
+    # Just for the ForeignKey and column creation to succeed, these are not the
+    # actual definitions of tables .
+    #
+    volumes = Table('volumes', meta,
+           Column('id', Integer(), primary_key=True, nullable=False),
+           mysql_engine='InnoDB'
+           )
+    snapshots = Table('snapshots', meta,
+           Column('id', Integer(), primary_key=True, nullable=False),
+           mysql_engine='InnoDB'
+           )
+    # Create new table
+    volume_glance_metadata = Table('volume_glance_metadata', meta,
+            Column('created_at', DateTime(timezone=False)),
+            Column('updated_at', DateTime(timezone=False)),
+            Column('deleted_at', DateTime(timezone=False)),
+            Column('deleted', Boolean(create_constraint=True, name=None)),
+            Column('id', Integer(), primary_key=True, nullable=False),
+            Column('volume_id', String(length=36), ForeignKey('volumes.id')),
+            Column('snapshot_id', String(length=36),
+                   ForeignKey('snapshots.id')),
+            Column('key', String(255)),
+            Column('value', Text),
+            mysql_engine='InnoDB'
+    )
+
+    try:
+        volume_glance_metadata.create()
+    except Exception:
+        LOG.exception("Exception while creating table "
+                      "'volume_glance_metedata'")
+        meta.drop_all(tables=[volume_glance_metadata])
+        raise
+
+
+def downgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+
+    volume_glance_metadata = Table('volume_glance_metadata',
+                                   meta, autoload=True)
+    try:
+        volume_glance_metadata.drop()
+    except Exception:
+        LOG.error(_("volume_glance_metadata table not dropped"))
+        raise

cinder/db/sqlalchemy/models.py

@@ -21,7 +21,7 @@
 SQLAlchemy models for cinder data.
 """
 
-from sqlalchemy import Column, Integer, String, schema
+from sqlalchemy import Column, Integer, String, Text, schema
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import ForeignKey, DateTime, Boolean
@@ -205,6 +205,16 @@ class VolumeTypeExtraSpecs(BASE, CinderBase):
     )
 
 
+class VolumeGlanceMetadata(BASE, CinderBase):
+    """Glance metadata for a bootable volume"""
+    __tablename__ = 'volume_glance_metadata'
+    id = Column(Integer, primary_key=True, nullable=False)
+    volume_id = Column(String(36), ForeignKey('volumes.id'))
+    snapshot_id = Column(String(36), ForeignKey('snapshots.id'))
+    key = Column(String(255))
+    value = Column(Text)
+
+
 class Quota(BASE, CinderBase):
     """Represents a single quota override for a project.
 
@@ -378,6 +388,7 @@ def register_models():
               VolumeMetadata,
               VolumeTypeExtraSpecs,
               VolumeTypes,
+              VolumeGlanceMetadata,
               )
     engine = create_engine(FLAGS.sql_connection, echo=False)
     for model in models:

cinder/exception.py

@@ -490,3 +490,8 @@ class NfsNoSharesMounted(NotFound):
 
 class NfsNoSuitableShareFound(NotFound):
     message = _("There is no share which can host %(volume_size)sG")
+
+
+class GlanceMetadataExists(Invalid):
+    message = _("Glance metadata cannot be updated, key %(key)s"
+                " exists for volume id %(volume_id)s")

cinder/tests/test_volume_glance_metadata.py

@@ -0,0 +1,114 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright (c) 2011 Zadara Storage Inc.
+# Copyright (c) 2011 OpenStack LLC.
+# Copyright 2011 University of Southern California
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+Unit Tests for volume types extra specs code
+"""
+
+from cinder import context
+from cinder import db
+from cinder import exception
+from cinder import test
+
+
+class VolumeGlanceMetadataTestCase(test.TestCase):
+
+    def setUp(self):
+        super(VolumeGlanceMetadataTestCase, self).setUp()
+        self.context = context.get_admin_context()
+
+    def tearDown(self):
+        super(VolumeGlanceMetadataTestCase, self).tearDown()
+
+    def test_vol_glance_metadata_bad_vol_id(self):
+        ctxt = context.get_admin_context()
+        self.assertRaises(exception.VolumeNotFound,
+                          db.volume_glance_metadata_create,
+                          ctxt, 1, 'key1', 'value1')
+        self.assertRaises(exception.VolumeNotFound,
+                          db.volume_glance_metadata_get, ctxt, 1)
+        db.volume_glance_metadata_delete_by_volume(ctxt, 10)
+
+    def test_vol_update_glance_metadata(self):
+        ctxt = context.get_admin_context()
+        db.volume_create(ctxt, {'id': 1})
+        db.volume_create(ctxt, {'id': 2})
+        vol_metadata = db.volume_glance_metadata_create(ctxt, 1, 'key1',
+                                                        'value1')
+        vol_metadata = db.volume_glance_metadata_create(ctxt, 2, 'key1',
+                                                        'value1')
+        vol_metadata = db.volume_glance_metadata_create(ctxt, 2, 'key2',
+                                                        'value2')
+
+        expected_metadata_1 = {'volume_id': '1',
+                              'key': 'key1',
+                              'value': 'value1'}
+
+        metadata = db.volume_glance_metadata_get(ctxt, 1)
+        self.assertEqual(len(metadata), 1)
+        for key, value in expected_metadata_1.items():
+            self.assertEqual(metadata[0][key], value)
+
+        expected_metadata_2 = ({'volume_id': '2',
+                                'key': 'key1',
+                                'value': 'value1'},
+                               {'volume_id': '2',
+                                'key': 'key2',
+                                'value': 'value2'})
+
+        metadata = db.volume_glance_metadata_get(ctxt, 2)
+        self.assertEqual(len(metadata), 2)
+        for expected, meta in zip(expected_metadata_2, metadata):
+            for key, value in expected.iteritems():
+                self.assertEqual(meta[key], value)
+
+        self.assertRaises(exception.GlanceMetadataExists,
+                          db.volume_glance_metadata_create,
+                          ctxt, 1, 'key1', 'value1a')
+
+        metadata = db.volume_glance_metadata_get(ctxt, 1)
+        self.assertEqual(len(metadata), 1)
+        for key, value in expected_metadata_1.items():
+            self.assertEqual(metadata[0][key], value)
+
+    def test_vol_delete_glance_metadata(self):
+        ctxt = context.get_admin_context()
+        db.volume_create(ctxt, {'id': 1})
+        db.volume_glance_metadata_delete_by_volume(ctxt, 1)
+        vol_metadata = db.volume_glance_metadata_create(ctxt, 1, 'key1',
+                                                        'value1')
+        db.volume_glance_metadata_delete_by_volume(ctxt, 1)
+        metadata = db.volume_glance_metadata_get(ctxt, 1)
+        self.assertEqual(len(metadata), 0)
+        db.volume_glance_metadata_delete_by_volume(ctxt, 1)
+        metadata = db.volume_glance_metadata_get(ctxt, 1)
+        self.assertEqual(len(metadata), 0)
+
+    def test_vol_glance_metadata_copy_to_snapshot(self):
+        ctxt = context.get_admin_context()
+        db.volume_create(ctxt, {'id': 1})
+        db.snapshot_create(ctxt, {'id': 100, 'volume_id': 1})
+        vol_meta = db.volume_glance_metadata_create(ctxt, 1, 'key1',
+                                                    'value1')
+        db.volume_glance_metadata_copy_to_snapshot(ctxt, 100, 1)
+
+        expected_meta = {'snapshot_id': '100',
+                          'key': 'key1',
+                          'value': 'value1'}
+
+        for meta in db.volume_snapshot_glance_metadata_get(ctxt, 100):
+            for (key, value) in expected_meta.items():
+                self.assertEquals(meta[key], value)

cinder/volume/manager.py

@@ -134,6 +134,7 @@ class VolumeManager(manager.SchedulerDependentManager):
 
         status = 'available'
         model_update = False
+        image_meta = None
 
         try:
             vol_name = volume_ref['name']
@@ -153,6 +154,7 @@ class VolumeManager(manager.SchedulerDependentManager):
                                glance.get_remote_image_service(context,
                                                                image_id)
                 image_location = image_service.get_location(context, image_id)
+                image_meta = image_service.show(context, image_id)
                 cloned = self.driver.clone_image(volume_ref, image_location)
                 if not cloned:
                     model_update = self.driver.create_volume(volume_ref)
@@ -171,6 +173,11 @@ class VolumeManager(manager.SchedulerDependentManager):
                 self.db.volume_update(context,
                                       volume_ref['id'], {'status': 'error'})
 
+        if snapshot_id:
+            # Copy any Glance metadata from the original volume
+            self.db.volume_glance_metadata_copy_to_volume(context,
+                                               volume_ref['id'], snapshot_id)
+
         now = timeutils.utcnow()
         self.db.volume_update(context,
                               volume_ref['id'], {'status': status,
@@ -179,6 +186,23 @@ class VolumeManager(manager.SchedulerDependentManager):
         self._reset_stats()
 
         if image_id and not cloned:
+            if image_meta:
+                # Copy all of the Glance image properties to the
+                # volume_glance_metadata table for future reference.
+                self.db.volume_glance_metadata_create(context,
+                                                      volume_ref['id'],
+                                                      'image_id', image_id)
+                name = image_meta.get('name', None)
+                if name:
+                    self.db.volume_glance_metadata_create(context,
+                                                          volume_ref['id'],
+                                                          'image_name', name)
+                image_properties = image_meta.get('properties', {})
+                for key, value in image_properties.items():
+                    self.db.volume_glance_metadata_create(context,
+                                                          volume_ref['id'],
+                                                          key, value)
+
             #copy the image onto the volume.
             self._copy_image_to_volume(context, volume_ref, image_id)
         self._notify_about_volume_usage(context, volume_ref, "create.end")
@@ -222,6 +246,7 @@ class VolumeManager(manager.SchedulerDependentManager):
             reservations = None
             LOG.exception(_("Failed to update usages deleting volume"))
 
+        self.db.volume_glance_metadata_delete_by_volume(context, volume_id)
         self.db.volume_destroy(context, volume_id)
         LOG.debug(_("volume %s: deleted successfully"), volume_ref['name'])
         self._notify_about_volume_usage(context, volume_ref, "delete.end")
@@ -255,6 +280,8 @@ class VolumeManager(manager.SchedulerDependentManager):
         self.db.snapshot_update(context,
                                 snapshot_ref['id'], {'status': 'available',
                                                      'progress': '100%'})
+        self.db.volume_glance_metadata_copy_to_snapshot(context,
+                                                snapshot_ref['id'], volume_id)
         LOG.debug(_("snapshot %s: created successfully"), snapshot_ref['name'])
         return snapshot_id
 
@@ -278,6 +305,7 @@ class VolumeManager(manager.SchedulerDependentManager):
                                         snapshot_ref['id'],
                                         {'status': 'error_deleting'})
 
+        self.db.volume_glance_metadata_delete_by_snapshot(context, snapshot_id)
         self.db.snapshot_destroy(context, snapshot_id)
         LOG.debug(_("snapshot %s: deleted successfully"), snapshot_ref['name'])
         return True