Refactor instance usage notifications for compute manager

This patch makes compute manager call notify_usage_exists() via the
conductor instead of directly, avoiding the direct database access
required to fetch bandwidth information. I initially just added
a notify_bw_usage() method to conductor, which just handled the
bandwidth part (which is the database hit). However, the metadata
payload part may be depended upon by custom stuff for periodic
notifications, so copying all that stuff as well made it a bunch
of duplicated stuff. Thus, I decided to just make it a direct proxy
for the compute_utils method.

Note that this is a bit of a departure from the previous pattern of
just proxying db interfaces in conductor, but I think the simplicity
gain is worth it.

Related to blueprint no-db-compute

Change-Id: I09f2652d4903f4d6036c6e3d74db063952341856

nova/compute/manager.py

@@ -1362,7 +1362,7 @@ class ComputeManager(manager.SchedulerDependentManager):
             # to point to the new one... we have to override it.
             orig_image_ref_url = glance.generate_image_url(orig_image_ref)
             extra_usage_info = {'image_ref_url': orig_image_ref_url}
-            compute_utils.notify_usage_exists(context, instance,
+            self.conductor_api.notify_usage_exists(context, instance,
                     current_period=True, system_metadata=orig_sys_metadata,
                     extra_usage_info=extra_usage_info)
 
@@ -1826,7 +1826,7 @@ class ComputeManager(manager.SchedulerDependentManager):
 
         # NOTE(comstud): A revert_resize is essentially a resize back to
         # the old size, so we need to send a usage event here.
-        compute_utils.notify_usage_exists(
+        self.conductor_api.notify_usage_exists(
                 context, instance, current_period=True)
 
         with self._error_out_instance_on_exception(context, instance['uuid'],
@@ -1994,7 +1994,7 @@ class ComputeManager(manager.SchedulerDependentManager):
 
         with self._error_out_instance_on_exception(context, instance['uuid'],
                                                    reservations):
-            compute_utils.notify_usage_exists(
+            self.conductor_api.notify_usage_exists(
                     context, instance, current_period=True)
             self._notify_about_instance_usage(
                     context, instance, "resize.prep.start")
@@ -3158,7 +3158,7 @@ class ComputeManager(manager.SchedulerDependentManager):
                                               self.host, num_instances)
                 for instance in instances:
                     try:
-                        compute_utils.notify_usage_exists(
+                        self.conductor_api.notify_usage_exists(
                             context, instance,
                             ignore_missing_network_data=False)
                         successes += 1

nova/conductor/api.py

@@ -313,6 +313,13 @@ class LocalAPI(object):
                                                begin, end, host,
                                                errors, message)
 
+    def notify_usage_exists(self, context, instance, current_period=False,
+                            ignore_missing_network_data=True,
+                            system_metadata=None, extra_usage_info=None):
+        return self._manager.notify_usage_exists(
+            context, instance, current_period, ignore_missing_network_data,
+            system_metadata, extra_usage_info)
+
 
 class API(object):
     """Conductor API that does updates via RPC to the ConductorManager."""
@@ -603,3 +610,10 @@ class API(object):
         return self.conductor_rpcapi.task_log_end_task(context, task_name,
                                                        begin, end, host,
                                                        errors, message)
+
+    def notify_usage_exists(self, context, instance, current_period=False,
+                            ignore_missing_network_data=True,
+                            system_metadata=None, extra_usage_info=None):
+        return self.conductor_rpcapi.notify_usage_exists(
+            context, instance, current_period, ignore_missing_network_data,
+            system_metadata, extra_usage_info)

nova/conductor/manager.py

@@ -14,6 +14,7 @@
 
 """Handles database requests from other nova services."""
 
+from nova.compute import utils as compute_utils
 from nova import exception
 from nova import manager
 from nova import notifications
@@ -44,7 +45,7 @@ datetime_fields = ['launched_at', 'terminated_at']
 class ConductorManager(manager.SchedulerDependentManager):
     """Mission: TBD."""
 
-    RPC_API_VERSION = '1.38'
+    RPC_API_VERSION = '1.39'
 
     def __init__(self, *args, **kwargs):
         super(ConductorManager, self).__init__(service_name='conductor',
@@ -343,3 +344,10 @@ class ConductorManager(manager.SchedulerDependentManager):
         result = self.db.task_log_end_task(context.elevated(), task_name,
                                            begin, end, host, errors, message)
         return jsonutils.to_primitive(result)
+
+    def notify_usage_exists(self, context, instance, current_period=False,
+                            ignore_missing_network_data=True,
+                            system_metadata=None, extra_usage_info=None):
+        compute_utils.notify_usage_exists(context, instance, current_period,
+                                          ignore_missing_network_data,
+                                          system_metadata, extra_usage_info)

nova/conductor/rpcapi.py

@@ -71,6 +71,7 @@ class ConductorAPI(nova.openstack.common.rpc.proxy.RpcProxy):
     1.36 - Added instance_fault_create
     1.37 - Added task_log_get, task_log_begin_task, task_log_end_task
     1.38 - Added service name to instance_update
+    1.39 - Added notify_usage_exists
     """
 
     BASE_RPC_API_VERSION = '1.0'
@@ -361,3 +362,16 @@ class ConductorAPI(nova.openstack.common.rpc.proxy.RpcProxy):
                             begin=begin, end=end, host=host, errors=errors,
                             message=message)
         return self.call(context, msg, version='1.37')
+
+    def notify_usage_exists(self, context, instance, current_period=False,
+                            ignore_missing_network_data=True,
+                            system_metadata=None, extra_usage_info=None):
+        instance_p = jsonutils.to_primitive(instance)
+        system_metadata_p = jsonutils.to_primitive(system_metadata)
+        extra_usage_info_p = jsonutils.to_primitive(extra_usage_info)
+        msg = self.make_msg('notify_usage_exists', instance=instance_p,
+                  current_period=current_period,
+                  ignore_missing_network_data=ignore_missing_network_data,
+                  system_metadata=system_metadata_p,
+                  extra_usage_info=extra_usage_info_p)
+        return self.call(context, msg, version='1.39')

nova/tests/compute/test_compute.py

@@ -3473,6 +3473,26 @@ class ComputeTestCase(BaseTestCase):
         result = self.compute._get_instances_on_driver(fake_context)
         self.assertEqual(driver_instances, result)
 
+    def test_instance_usage_audit(self):
+        instances = [{'uuid': 'foo'}]
+        self.flags(instance_usage_audit=True)
+        self.stubs.Set(compute_utils, 'has_audit_been_run',
+                       lambda *a, **k: False)
+        self.stubs.Set(self.compute.conductor_api,
+                       'instance_get_active_by_window_joined',
+                       lambda *a, **k: instances)
+        self.stubs.Set(compute_utils, 'start_instance_usage_audit',
+                       lambda *a, **k: None)
+        self.stubs.Set(compute_utils, 'finish_instance_usage_audit',
+                       lambda *a, **k: None)
+
+        self.mox.StubOutWithMock(self.compute.conductor_api,
+                                 'notify_usage_exists')
+        self.compute.conductor_api.notify_usage_exists(
+            self.context, instances[0], ignore_missing_network_data=False)
+        self.mox.ReplayAll()
+        self.compute._instance_usage_audit(self.context)
+
 
 class ComputeAPITestCase(BaseTestCase):
 

nova/tests/conductor/test_conductor.py

@@ -17,6 +17,7 @@
 import mox
 
 from nova.compute import instance_types
+from nova.compute import utils as compute_utils
 from nova.compute import vm_states
 from nova import conductor
 from nova.conductor import api as conductor_api
@@ -26,6 +27,7 @@ from nova import context
 from nova import db
 from nova.db.sqlalchemy import models
 from nova import exception as exc
+from nova import notifications
 from nova.openstack.common import jsonutils
 from nova.openstack.common.rpc import common as rpc_common
 from nova.openstack.common import timeutils
@@ -463,6 +465,36 @@ class _BaseTestCase(object):
             self.context, 'task', 'begin', 'end', 'host', 'errors', 'message')
         self.assertEqual(result, 'result')
 
+    def test_notify_usage_exists(self):
+        info = {
+            'audit_period_beginning': 'start',
+            'audit_period_ending': 'end',
+            'bandwidth': 'bw_usage',
+            'image_meta': {},
+            'extra': 'info',
+            }
+        instance = {
+            'system_metadata': [],
+            }
+
+        self.mox.StubOutWithMock(notifications, 'audit_period_bounds')
+        self.mox.StubOutWithMock(notifications, 'bandwidth_usage')
+        self.mox.StubOutWithMock(compute_utils, 'notify_about_instance_usage')
+
+        notifications.audit_period_bounds(False).AndReturn(('start', 'end'))
+        notifications.bandwidth_usage(instance, 'start', True).AndReturn(
+            'bw_usage')
+        compute_utils.notify_about_instance_usage(self.context, instance,
+                                                  'exists',
+                                                  system_metadata={},
+                                                  extra_usage_info=info)
+
+        self.mox.ReplayAll()
+
+        self.conductor.notify_usage_exists(self.context, instance,
+                                           system_metadata={},
+                                           extra_usage_info=dict(extra='info'))
+
 
 class ConductorTestCase(_BaseTestCase, test.TestCase):
     """Conductor Manager Tests."""