Adds more usage data to Nova's usage notifications.

Adds in bandwidth, state and IP data on standard notifications,
and new notifications on add/remove IP.

These were missing before, and are needed to meet spec.
This fixes bug 849117

Change-Id: Ie586ff3a91a56e5f5eff8abc6905ba6a0b624451

bin/instance-usage-audit

@@ -24,14 +24,13 @@
    system consuming usage notification feeds to calculate instance usage
    for each tenant.
 
-   Time periods are specified like so:
+   Time periods are specified as 'hour', 'month', 'day' or 'year'
-   <number>[mdy]
 
-   1m = previous month. If the script is run April 1, it will generate usages
+   hour = previous hour. If run at 9:07am, will generate usage for 8-9am.
-        for March 1 thry March 31.
+   month = previous month. If the script is run April 1, it will generate
-   3m = 3 previous months.
+           usages for March 1 thry March 31.
-   90d = previous 90 days.
+   day = previous day. if run on July 4th, it generates usages for July 3rd.
-   1y = previous year. If run on Jan 1, it generates usages for
+   year = previous year. If run on Jan 1, it generates usages for
         Jan 1 thru Dec 31 of the previous year.
 """
 
@@ -59,57 +58,20 @@ from nova import flags
 from nova import log as logging
 from nova import utils
 
-from nova.notifier import api as notifier_api
+from nova.compute.utils import notify_usage_exists
 
 FLAGS = flags.FLAGS
-flags.DEFINE_string('instance_usage_audit_period', '1m',
-                    'time period to generate instance usages for.')
-
-
-def time_period(period):
-    today = datetime.date.today()
-    unit = period[-1]
-    if unit not in 'mdy':
-        raise ValueError('Time period must be m, d, or y')
-    n = int(period[:-1])
-    if unit == 'm':
-        year = today.year - (n // 12)
-        n = n % 12
-        if n >= today.month:
-            year -= 1
-            month = 12 + (today.month - n)
-        else:
-            month = today.month - n
-        begin = datetime.datetime(day=1, month=month, year=year)
-        end = datetime.datetime(day=1, month=today.month, year=today.year)
-
-    elif unit == 'y':
-        begin = datetime.datetime(day=1, month=1, year=today.year - n)
-        end = datetime.datetime(day=1, month=1, year=today.year)
-
-    elif unit == 'd':
-        b = today - datetime.timedelta(days=n)
-        begin = datetime.datetime(day=b.day, month=b.month, year=b.year)
-        end = datetime.datetime(day=today.day,
-                               month=today.month,
-                               year=today.year)
-
-    return (begin, end)
 
 if __name__ == '__main__':
+    admin_context = context.get_admin_context()
     utils.default_flagfile()
     flags.FLAGS(sys.argv)
     logging.setup()
-    begin, end = time_period(FLAGS.instance_usage_audit_period)
+    begin, end = utils.current_audit_period()
     print "Creating usages for %s until %s" % (str(begin), str(end))
-    ctxt = context.get_admin_context()
+    instances = db.instance_get_active_by_window_joined(admin_context,
-    instances = db.instance_get_active_by_window_joined(ctxt, begin, end)
+                                                        begin,
+                                                        end)
     print "%s instances" % len(instances)
     for instance_ref in instances:
-        usage_info = utils.usage_from_instance(instance_ref,
+        notify_usage_exists(instance_ref)
-                              audit_period_begining=str(begin),
-                              audit_period_ending=str(end))
-        notifier_api.notify('compute.%s' % FLAGS.host,
-                            'compute.instance.exists',
-                            notifier_api.INFO,
-                            usage_info)

nova/exception.py

@@ -853,3 +853,7 @@ class InstanceTypeDiskTooSmall(NovaException):
 
 class InsufficientFreeMemory(NovaException):
     message = _("Insufficient free memory on compute node to start %(uuid)s.")
+
+
+class CouldNotFetchMetrics(NovaException):
+    message = _("Could not fetch bandwidth/cpu/disk metrics for this host.")

nova/flags.py

@@ -406,6 +406,10 @@ DEFINE_list('zone_capabilities',
                  'Key/Multi-value list representng capabilities of this zone')
 DEFINE_string('build_plan_encryption_key', None,
         '128bit (hex) encryption key for scheduler build plans.')
+DEFINE_string('instance_usage_audit_period', 'month',
+              'time period to generate instance usages for.')
+DEFINE_integer('bandwith_poll_interval', 600,
+               'interval to pull bandwidth usage info')
 
 DEFINE_bool('start_guests_on_host_boot', False,
             'Whether to restart guests when the host reboots')

nova/tests/test_compute.py

@@ -33,6 +33,7 @@ from nova.scheduler import driver as scheduler_driver
 from nova import rpc
 from nova import test
 from nova import utils
+import nova
 
 from nova.compute import instance_types
 from nova.compute import manager as compute_manager
@@ -448,6 +449,48 @@ class ComputeTestCase(test.TestCase):
         self.assert_(console)
         self.compute.terminate_instance(self.context, instance_id)
 
+    def test_add_fixed_ip_usage_notification(self):
+        def dummy(*args, **kwargs):
+            pass
+
+        self.stubs.Set(nova.network.API, 'add_fixed_ip_to_instance',
+                       dummy)
+        self.stubs.Set(nova.compute.manager.ComputeManager,
+                       'inject_network_info', dummy)
+        self.stubs.Set(nova.compute.manager.ComputeManager,
+                       'reset_network', dummy)
+
+        instance_id = self._create_instance()
+
+        self.assertEquals(len(test_notifier.NOTIFICATIONS), 0)
+        self.compute.add_fixed_ip_to_instance(self.context,
+                                              instance_id,
+                                              1)
+
+        self.assertEquals(len(test_notifier.NOTIFICATIONS), 1)
+        self.compute.terminate_instance(self.context, instance_id)
+
+    def test_remove_fixed_ip_usage_notification(self):
+        def dummy(*args, **kwargs):
+            pass
+
+        self.stubs.Set(nova.network.API, 'remove_fixed_ip_from_instance',
+                       dummy)
+        self.stubs.Set(nova.compute.manager.ComputeManager,
+                       'inject_network_info', dummy)
+        self.stubs.Set(nova.compute.manager.ComputeManager,
+                       'reset_network', dummy)
+
+        instance_id = self._create_instance()
+
+        self.assertEquals(len(test_notifier.NOTIFICATIONS), 0)
+        self.compute.remove_fixed_ip_from_instance(self.context,
+                                              instance_id,
+                                              1)
+
+        self.assertEquals(len(test_notifier.NOTIFICATIONS), 1)
+        self.compute.terminate_instance(self.context, instance_id)
+
     def test_run_instance_usage_notification(self):
         """Ensure run instance generates apropriate usage notification"""
         instance_id = self._create_instance()
@@ -457,7 +500,7 @@ class ComputeTestCase(test.TestCase):
         self.assertEquals(msg['priority'], 'INFO')
         self.assertEquals(msg['event_type'], 'compute.instance.create')
         payload = msg['payload']
-        self.assertEquals(payload['project_id'], self.project_id)
+        self.assertEquals(payload['tenant_id'], self.project_id)
         self.assertEquals(payload['user_id'], self.user_id)
         self.assertEquals(payload['instance_id'], instance_id)
         self.assertEquals(payload['instance_type'], 'm1.tiny')
@@ -476,12 +519,16 @@ class ComputeTestCase(test.TestCase):
         test_notifier.NOTIFICATIONS = []
         self.compute.terminate_instance(self.context, instance_id)
 
-        self.assertEquals(len(test_notifier.NOTIFICATIONS), 1)
+        self.assertEquals(len(test_notifier.NOTIFICATIONS), 2)
         msg = test_notifier.NOTIFICATIONS[0]
         self.assertEquals(msg['priority'], 'INFO')
+        self.assertEquals(msg['event_type'], 'compute.instance.exists')
+
+        msg = test_notifier.NOTIFICATIONS[1]
+        self.assertEquals(msg['priority'], 'INFO')
         self.assertEquals(msg['event_type'], 'compute.instance.delete')
         payload = msg['payload']
-        self.assertEquals(payload['project_id'], self.project_id)
+        self.assertEquals(payload['tenant_id'], self.project_id)
         self.assertEquals(payload['user_id'], self.user_id)
         self.assertEquals(payload['instance_id'], instance_id)
         self.assertEquals(payload['instance_type'], 'm1.tiny')
@@ -564,7 +611,7 @@ class ComputeTestCase(test.TestCase):
         self.assertEquals(msg['priority'], 'INFO')
         self.assertEquals(msg['event_type'], 'compute.instance.resize.prep')
         payload = msg['payload']
-        self.assertEquals(payload['project_id'], self.project_id)
+        self.assertEquals(payload['tenant_id'], self.project_id)
         self.assertEquals(payload['user_id'], self.user_id)
         self.assertEquals(payload['instance_id'], instance_id)
         self.assertEquals(payload['instance_type'], 'm1.tiny')

nova/tests/test_compute_utils.py

@@ -0,0 +1,99 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Tests For misc util methods used with compute.
+"""
+
+from datetime import datetime
+from datetime import timedelta
+
+from nova import db
+from nova import exception
+from nova import flags
+from nova import context
+from nova import test
+from nova import log as logging
+from nova import utils
+import nova.image.fake
+from nova.compute import utils as compute_utils
+from nova.compute import instance_types
+from nova.notifier import test_notifier
+
+
+LOG = logging.getLogger('nova.tests.compute_utils')
+FLAGS = flags.FLAGS
+flags.DECLARE('stub_network', 'nova.compute.manager')
+
+
+class UsageInfoTestCase(test.TestCase):
+
+    def setUp(self):
+        super(UsageInfoTestCase, self).setUp()
+        self.flags(connection_type='fake',
+                   stub_network=True,
+                   notification_driver='nova.notifier.test_notifier',
+                   network_manager='nova.network.manager.FlatManager')
+        self.compute = utils.import_object(FLAGS.compute_manager)
+        self.user_id = 'fake'
+        self.project_id = 'fake'
+        self.context = context.RequestContext(self.user_id, self.project_id)
+        test_notifier.NOTIFICATIONS = []
+
+        def fake_show(meh, context, id):
+            return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1}}
+
+        self.stubs.Set(nova.image.fake._FakeImageService, 'show', fake_show)
+
+    def _create_instance(self, params={}):
+        """Create a test instance"""
+        inst = {}
+        inst['image_ref'] = 1
+        inst['reservation_id'] = 'r-fakeres'
+        inst['launch_time'] = '10'
+        inst['user_id'] = self.user_id
+        inst['project_id'] = self.project_id
+        type_id = instance_types.get_instance_type_by_name('m1.tiny')['id']
+        inst['instance_type_id'] = type_id
+        inst['ami_launch_index'] = 0
+        inst.update(params)
+        return db.instance_create(self.context, inst)['id']
+
+    def test_notify_usage_exists(self):
+        """Ensure 'exists' notification generates apropriate usage data."""
+        instance_id = self._create_instance()
+        instance = db.instance_get(self.context, instance_id)
+        compute_utils.notify_usage_exists(instance)
+        self.assertEquals(len(test_notifier.NOTIFICATIONS), 1)
+        msg = test_notifier.NOTIFICATIONS[0]
+        self.assertEquals(msg['priority'], 'INFO')
+        self.assertEquals(msg['event_type'], 'compute.instance.exists')
+        payload = msg['payload']
+        self.assertEquals(payload['tenant_id'], self.project_id)
+        self.assertEquals(payload['user_id'], self.user_id)
+        self.assertEquals(payload['instance_id'], instance_id)
+        self.assertEquals(payload['instance_type'], 'm1.tiny')
+        type_id = instance_types.get_instance_type_by_name('m1.tiny')['id']
+        self.assertEquals(str(payload['instance_type_id']), str(type_id))
+        for attr in ('display_name', 'created_at', 'launched_at',
+                     'state', 'state_description', 'fixed_ips',
+                     'bandwidth', 'audit_period_begining',
+                     'audit_period_ending'):
+            self.assertTrue(attr in payload,
+                            msg="Key %s not in payload" % attr)
+        self.assertEquals(payload['image_ref'], '1')
+        self.compute.terminate_instance(self.context, instance_id)

nova/utils.py

@@ -294,9 +294,46 @@ EASIER_PASSWORD_SYMBOLS = ('23456789'  # Removed: 0, 1
                            'ABCDEFGHJKLMNPQRSTUVWXYZ')  # Removed: I, O
 
 
+def current_audit_period(unit=None):
+    if not unit:
+        unit = FLAGS.instance_usage_audit_period
+    rightnow = utcnow()
+    if unit not in ('month', 'day', 'year', 'hour'):
+        raise ValueError('Time period must be hour, day, month or year')
+    n = 1  # we are currently only using multiples of 1 unit (mdragon)
+    if unit == 'month':
+        year = rightnow.year - (n // 12)
+        n = n % 12
+        if n >= rightnow.month:
+            year -= 1
+            month = 12 + (rightnow.month - n)
+        else:
+            month = rightnow.month - n
+        begin = datetime.datetime(day=1, month=month, year=year)
+        end = datetime.datetime(day=1,
+                                month=rightnow.month,
+                                year=rightnow.year)
+
+    elif unit == 'year':
+        begin = datetime.datetime(day=1, month=1, year=rightnow.year - n)
+        end = datetime.datetime(day=1, month=1, year=rightnow.year)
+
+    elif unit == 'day':
+        b = rightnow - datetime.timedelta(days=n)
+        begin = datetime.datetime(day=b.day, month=b.month, year=b.year)
+        end = datetime.datetime(day=rightnow.day,
+                               month=rightnow.month,
+                               year=rightnow.year)
+    elif unit == 'hour':
+        end = rightnow.replace(minute=0, second=0, microsecond=0)
+        begin = end - datetime.timedelta(hours=n)
+
+    return (begin, end)
+
+
 def usage_from_instance(instance_ref, **kw):
     usage_info = dict(
-          project_id=instance_ref['project_id'],
+          tenant_id=instance_ref['project_id'],
           user_id=instance_ref['user_id'],
           instance_id=instance_ref['id'],
           instance_type=instance_ref['instance_type']['name'],
@@ -305,7 +342,11 @@ def usage_from_instance(instance_ref, **kw):
           created_at=str(instance_ref['created_at']),
           launched_at=str(instance_ref['launched_at']) \
                       if instance_ref['launched_at'] else '',
-          image_ref=instance_ref['image_ref'])
+          image_ref=instance_ref['image_ref'],
+          state=instance_ref['vm_state'],
+          state_description=instance_ref['task_state'] \
+                             if instance_ref['task_state'] else '',
+          fixed_ips=[a.address for a in instance_ref['fixed_ips']])
     usage_info.update(kw)
     return usage_info
 
@@ -324,7 +365,7 @@ def last_octet(address):
     return int(address.split('.')[-1])
 
 
-def  get_my_linklocal(interface):
+def get_my_linklocal(interface):
     try:
         if_str = execute('ip', '-f', 'inet6', '-o', 'addr', 'show', interface)
         condition = '\s+inet6\s+([0-9a-f:]+)/\d+\s+scope\s+link'