Add the availability_zone to the volume.usage notifications

The volume.usage notifications are used to enable traffic based billing
on volumes. The availability_zone of the instance is an important piece of
information that can enable more specialized billing of customers.

This makes the volume.usage events consistent with the volume.create.start/end
snapshot.create.start/end, etc. emitted from cinder, which contain the
availability_zone field.

Change-Id: I9b6b03174cd03235d0e6cf01e34a6da13bd9bb70

nova/compute/utils.py

@@ -297,6 +297,7 @@ def usage_volume_info(vol_usage):
           volume_id=vol_usage['volume_id'],
           tenant_id=vol_usage['project_id'],
           user_id=vol_usage['user_id'],
+          availability_zone=vol_usage['availability_zone'],
           instance_id=vol_usage['instance_uuid'],
           last_refreshed=null_safe_str(last_refreshed_time),
           reads=vol_usage['tot_reads'] + vol_usage['curr_reads'],

nova/conductor/manager.py

@@ -314,6 +314,7 @@ class ConductorManager(manager.Manager):
         self.db.vol_usage_update(context, vol_id, rd_req, rd_bytes, wr_req,
                                  wr_bytes, instance['uuid'],
                                  instance['project_id'], instance['user_id'],
+                                 instance['availability_zone'],
                                  last_refreshed, update_totals)
 
     @rpc_common.client_exceptions(exception.ComputeHostNotFound,

nova/db/api.py

@@ -1461,12 +1461,13 @@ def vol_get_usage_by_time(context, begin):
 
 
 def vol_usage_update(context, id, rd_req, rd_bytes, wr_req, wr_bytes,
-                     instance_id, project_id, user_id,
+                     instance_id, project_id, user_id, availability_zone,
                      last_refreshed=None, update_totals=False):
     """Update cached volume usage for a volume
        Creates new record if needed."""
     return IMPL.vol_usage_update(context, id, rd_req, rd_bytes, wr_req,
                                  wr_bytes, instance_id, project_id, user_id,
+                                 availability_zone,
                                  last_refreshed=last_refreshed,
                                  update_totals=update_totals)
 

nova/db/sqlalchemy/api.py

@@ -4288,8 +4288,8 @@ def vol_get_usage_by_time(context, begin):
 
 @require_context
 def vol_usage_update(context, id, rd_req, rd_bytes, wr_req, wr_bytes,
-                     instance_id, project_id, user_id, last_refreshed=None,
+                     instance_id, project_id, user_id, availability_zone,
-                     update_totals=False, session=None):
+                     last_refreshed=None, update_totals=False, session=None):
     if not session:
         session = get_session()
 
@@ -4308,7 +4308,8 @@ def vol_usage_update(context, id, rd_req, rd_bytes, wr_req, wr_bytes,
                       'curr_write_bytes': wr_bytes,
                       'instance_uuid': instance_id,
                       'project_id': project_id,
-                      'user_id': user_id}
+                      'user_id': user_id,
+                      'availability_zone': availability_zone}
         else:
             values = {'tot_last_refreshed': last_refreshed,
                       'tot_reads': models.VolumeUsage.tot_reads + rd_req,
@@ -4323,7 +4324,8 @@ def vol_usage_update(context, id, rd_req, rd_bytes, wr_req, wr_bytes,
                       'curr_write_bytes': 0,
                       'instance_uuid': instance_id,
                       'project_id': project_id,
-                      'user_id': user_id}
+                      'user_id': user_id,
+                      'availability_zone': availability_zone}
 
         rows = model_query(context, models.VolumeUsage,
                            session=session, read_deleted="yes").\
@@ -4340,6 +4342,7 @@ def vol_usage_update(context, id, rd_req, rd_bytes, wr_req, wr_bytes,
         vol_usage.instance_uuid = instance_id
         vol_usage.project_id = project_id
         vol_usage.user_id = user_id
+        vol_usage.availability_zone = availability_zone
 
         if not update_totals:
             vol_usage.curr_reads = rd_req

nova/db/sqlalchemy/migrate_repo/versions/176_add_availability_zone_to_volume_usage_cache.py

@@ -0,0 +1,55 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 OpenStack Foundation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+#
+# This migration adds the availability_zone to the volume_usage_cache table.
+#
+# This table keeps a cache of the volume usage. Every minute (or what ever
+# is configured for volume_usage_poll_interval value) one row in this table
+# gets updated for each attached volume. After this patch is applied, for
+# any currently attached volumes, the value will immediately be null, but
+# will get updated to the correct value on the next tick of the volume
+# usage poll.
+#
+# The volume usage poll function is the only code to access this table. The
+# sequence of operation in that function is to first update the field with
+# the correct value and then to pass the updated data to the consumer.
+#
+# Hence this new column does not need to be pre-populated.
+#
+
+from sqlalchemy import MetaData, String, Table, Column
+
+from nova.openstack.common import log as logging
+
+LOG = logging.getLogger(__name__)
+
+
+def upgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+
+    volume_usage_cache = Table('volume_usage_cache', meta, autoload=True)
+    availability_zone = Column('availability_zone', String(255))
+    volume_usage_cache.create_column(availability_zone)
+
+
+def downgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+
+    volume_usage_cache = Table('volume_usage_cache', meta, autoload=True)
+    volume_usage_cache.drop_column('availability_zone')

nova/db/sqlalchemy/models.py

@@ -893,6 +893,7 @@ class VolumeUsage(BASE, NovaBase):
     instance_uuid = Column(String(36))
     project_id = Column(String(36))
     user_id = Column(String(36))
+    availability_zone = Column(String(255))
     tot_last_refreshed = Column(DateTime)
     tot_reads = Column(BigInteger, default=0)
     tot_read_bytes = Column(BigInteger, default=0)

nova/tests/compute/test_compute.py

@@ -454,6 +454,7 @@ class ComputeVolumeTestCase(BaseTestCase):
                       'instance_uuid': 'fake_instance_uuid',
                       'project_id': 'fake_project_id',
                       'user_id': 'fake_user_id',
+                      'availability_zone': 'fake-az',
                       'tot_reads': 11,
                       'curr_reads': 22,
                       'tot_read_bytes': 33,
@@ -483,6 +484,7 @@ class ComputeVolumeTestCase(BaseTestCase):
         self.assertEquals(payload['read_bytes'], 77)
         self.assertEquals(payload['writes'], 121)
         self.assertEquals(payload['write_bytes'], 165)
+        self.assertEquals(payload['availability_zone'], 'fake-az')
 
     def test_detach_volume_usage(self):
         # Test that detach volume update the volume usage cache table correctly

nova/tests/conductor/test_conductor.py

@@ -82,6 +82,7 @@ class _BaseTestCase(object):
         inst['ephemeral_gb'] = 0
         inst['architecture'] = 'x86_64'
         inst['os_type'] = 'Linux'
+        inst['availability_zone'] = 'fake-az'
         inst.update(params)
         return db.instance_create(self.context, inst)
 
@@ -374,17 +375,18 @@ class _BaseTestCase(object):
 
     def test_vol_usage_update(self):
         self.mox.StubOutWithMock(db, 'vol_usage_update')
+        inst = self._create_fake_instance({
+                'project_id': 'fake-project_id',
+                'user_id': 'fake-user_id',
+                })
         db.vol_usage_update(self.context, 'fake-vol', 'rd-req', 'rd-bytes',
-                            'wr-req', 'wr-bytes', 'fake-id',
+                            'wr-req', 'wr-bytes', inst['uuid'],
-                            'fake-project_id', 'fake-user_id', 'fake-refr',
+                            'fake-project_id', 'fake-user_id', 'fake-az',
-                            'fake-bool')
+                            'fake-refr', 'fake-bool')
         self.mox.ReplayAll()
         self.conductor.vol_usage_update(self.context, 'fake-vol', 'rd-req',
                                         'rd-bytes', 'wr-req', 'wr-bytes',
-                                        {'uuid': 'fake-id',
+                                        inst, 'fake-refr', 'fake-bool')
-                                         'project_id': 'fake-project_id',
-                                         'user_id': 'fake-user_id'},
-                                        'fake-refr', 'fake-bool')
 
     def test_compute_node_create(self):
         self.mox.StubOutWithMock(db, 'compute_node_create')

nova/tests/test_db_api.py

@@ -2695,17 +2695,20 @@ class VolumeUsageDBApiTestCase(test.TestCase):
                                         wr_req=30, wr_bytes=40,
                                         instance_id='fake-instance-uuid1',
                                         project_id='fake-project-uuid1',
-                                        user_id='fake-user-uuid1')
+                                        user_id='fake-user-uuid1',
+                                        availability_zone='fake-az')
         vol_usage = db.vol_usage_update(ctxt, 2, rd_req=100, rd_bytes=200,
                                         wr_req=300, wr_bytes=400,
                                         instance_id='fake-instance-uuid2',
                                         project_id='fake-project-uuid2',
-                                        user_id='fake-user-uuid2')
+                                        user_id='fake-user-uuid2',
+                                        availability_zone='fake-az')
         vol_usage = db.vol_usage_update(ctxt, 1, rd_req=1000, rd_bytes=2000,
                                         wr_req=3000, wr_bytes=4000,
                                         instance_id='fake-instance-uuid1',
                                         project_id='fake-project-uuid1',
                                         user_id='fake-user-uuid1',
+                                        availability_zone='fake-az',
                                         last_refreshed=refreshed_time)
 
         vol_usages = db.vol_get_usage_by_time(ctxt, start_time)
@@ -2723,6 +2726,7 @@ class VolumeUsageDBApiTestCase(test.TestCase):
                                'project_id': 'fake-project-uuid',
                                'user_id': 'fake-user-uuid',
                                'instance_uuid': 'fake-instance-uuid',
+                               'availability_zone': 'fake-az',
                                'tot_reads': 600,
                                'tot_read_bytes': 800,
                                'tot_writes': 1000,
@@ -2736,23 +2740,27 @@ class VolumeUsageDBApiTestCase(test.TestCase):
                                         wr_req=300, wr_bytes=400,
                                         instance_id='fake-instance-uuid',
                                         project_id='fake-project-uuid',
-                                        user_id='fake-user-uuid')
+                                        user_id='fake-user-uuid',
+                                        availability_zone='fake-az')
         vol_usage = db.vol_usage_update(ctxt, 1, rd_req=200, rd_bytes=300,
                                         wr_req=400, wr_bytes=500,
                                         instance_id='fake-instance-uuid',
                                         project_id='fake-project-uuid',
                                         user_id='fake-user-uuid',
+                                        availability_zone='fake-az',
                                         update_totals=True)
         vol_usage = db.vol_usage_update(ctxt, 1, rd_req=300, rd_bytes=400,
                                         wr_req=500, wr_bytes=600,
                                         instance_id='fake-instance-uuid',
                                         project_id='fake-project-uuid',
+                                        availability_zone='fake-az',
                                         user_id='fake-user-uuid')
         vol_usage = db.vol_usage_update(ctxt, 1, rd_req=400, rd_bytes=500,
                                         wr_req=600, wr_bytes=700,
                                         instance_id='fake-instance-uuid',
                                         project_id='fake-project-uuid',
                                         user_id='fake-user-uuid',
+                                        availability_zone='fake-az',
                                         update_totals=True)
 
         vol_usages = db.vol_get_usage_by_time(ctxt, start_time)

nova/tests/test_migrations.py

@@ -1269,6 +1269,22 @@ class TestNovaMigrations(BaseMigrationTestCase, CommonTestsMixIn):
         self.assertFalse('user_id' in rows[0])
         self.assertEqual(rows[0]['instance_id'], None)
 
+    def _check_176(self, engine, data):
+        volume_usage_cache = get_table(engine, 'volume_usage_cache')
+        # Get the record
+        rows = volume_usage_cache.select().execute().fetchall()
+        self.assertEqual(len(rows), 1)
+
+        self.assertEqual(rows[0]['availability_zone'], None)
+
+    def _post_downgrade_176(self, engine):
+        volume_usage_cache = get_table(engine, 'volume_usage_cache')
+        # Get the record
+        rows = volume_usage_cache.select().execute().fetchall()
+        self.assertEqual(len(rows), 1)
+
+        self.assertFalse('availability_zone' in rows[0])
+
 
 class TestBaremetalMigrations(BaseMigrationTestCase, CommonTestsMixIn):
     """Test sqlalchemy-migrate migrations."""