Collect more accurate bandwidth data for XenServer

This changes the method used to poll xenserver for bandwidth data.
The reccomended way of collecting such data from xenserver (namely the
RRD files provided by the hosts) do not seem to be reliable, they
will sometimes be correct, often will be signifigantly under (> 10%),
and occasionally will show artifacts, such as phantom 4gb bandwidth
'spikes'.

This patch changes that to use the much simpler method of simply polling the
byte counters on the VIF network devices on the host. (We have old non-nova
code that does that on xenserver, and that method is known to work).

This should also make it much easier for other hypervisors other than
xenserver to implement bandwidth polling, as polling the counters is a rather
more universal method.

Fixes bug 1055737

Change-Id: I6a280d8bbfcc74914f888b11bc09349a270a5f58

nova/tests/test_db_api.py

@@ -478,18 +478,24 @@ class DbApiTestCase(test.TestCase):
                                'start_period': start_period,
                                'bw_in': 100,
                                'bw_out': 200,
+                               'last_ctr_in': 12345,
+                               'last_ctr_out': 67890,
                                'last_refreshed': now},
                               {'uuid': 'fake_uuid2',
                                'mac': 'fake_mac2',
                                'start_period': start_period,
                                'bw_in': 200,
                                'bw_out': 300,
+                               'last_ctr_in': 22345,
+                               'last_ctr_out': 77890,
                                'last_refreshed': now},
                               {'uuid': 'fake_uuid3',
                                'mac': 'fake_mac3',
                                'start_period': start_period,
                                'bw_in': 400,
                                'bw_out': 500,
+                               'last_ctr_in': 32345,
+                               'last_ctr_out': 87890,
                                'last_refreshed': uuid3_refreshed}]
 
         def _compare(bw_usage, expected):
@@ -504,18 +510,19 @@ class DbApiTestCase(test.TestCase):
         # Add 3 entries
         db.bw_usage_update(ctxt, 'fake_uuid1',
                 'fake_mac1', start_period,
-                100, 200)
+                100, 200, 12345, 67890)
         db.bw_usage_update(ctxt, 'fake_uuid2',
                 'fake_mac2', start_period,
-                100, 200)
+                100, 200, 42, 42)
         # Test explicit refreshed time
         db.bw_usage_update(ctxt, 'fake_uuid3',
                 'fake_mac3', start_period,
-                400, 500, last_refreshed=uuid3_refreshed)
+                400, 500, 32345, 87890,
+                last_refreshed=uuid3_refreshed)
         # Update 2nd entry
         db.bw_usage_update(ctxt, 'fake_uuid2',
                 'fake_mac2', start_period,
-                200, 300)
+                200, 300, 22345, 77890)
 
         bw_usages = db.bw_usage_get_by_uuids(ctxt,
                 ['fake_uuid1', 'fake_uuid2', 'fake_uuid3'], start_period)

nova/tests/test_xenapi.py

@@ -1442,11 +1442,23 @@ class XenAPIGenerateLocal(stubs.XenAPITestBase):
         self.assertCalled(instance)
 
 
-class XenAPIBWUsageTestCase(stubs.XenAPITestBase):
+class XenAPIBWCountersTestCase(stubs.XenAPITestBase):
+    FAKE_VMS = {'test1:ref': dict(name_label='test1',
+                                   other_config=dict(nova_uuid='hash'),
+                                   domid='12',
+                                   _vifmap={'0': "a:b:c:d...",
+                                           '1': "e:f:12:q..."}),
+                'test2:ref': dict(name_label='test2',
+                                   other_config=dict(nova_uuid='hash'),
+                                   domid='42',
+                                   _vifmap={'0': "a:3:c:d...",
+                                           '1': "e:f:42:q..."}),
+               }
+
     def setUp(self):
-        super(XenAPIBWUsageTestCase, self).setUp()
+        super(XenAPIBWCountersTestCase, self).setUp()
-        self.stubs.Set(vm_utils, 'compile_metrics',
+        self.stubs.Set(vm_utils, 'list_vms',
-                       XenAPIBWUsageTestCase._fake_compile_metrics)
+                       XenAPIBWCountersTestCase._fake_list_vms)
         self.flags(xenapi_connection_url='test_url',
                    xenapi_connection_password='test_pass',
                    firewall_driver='nova.virt.xenapi.firewall.'
@@ -1454,21 +1466,75 @@ class XenAPIBWUsageTestCase(stubs.XenAPITestBase):
         stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests)
         self.conn = xenapi_conn.XenAPIDriver(False)
 
-    @classmethod
+        def _fake_get_vif_device_map(vm_rec):
-    def _fake_compile_metrics(cls, start_time, stop_time=None):
+            return vm_rec['_vifmap']
-        raise exception.CouldNotFetchMetrics()
 
-    def test_get_all_bw_usage_in_failure_case(self):
+        self.stubs.Set(self.conn._vmops, "_get_vif_device_map",
-        """Test that get_all_bw_usage returns an empty list when metrics
+                                         _fake_get_vif_device_map)
-        compilation failed.  c.f. bug #910045.
+
+    @classmethod
+    def _fake_list_vms(cls, session):
+        return cls.FAKE_VMS.iteritems()
+
+    @classmethod
+    def _fake_fetch_bandwidth_mt(cls, session):
+        return {}
+
+    @classmethod
+    def _fake_fetch_bandwidth(cls, session):
+        return {'42':
+                    {'0': {'bw_in': 21024, 'bw_out': 22048},
+                     '1': {'bw_in': 231337, 'bw_out': 221212121}},
+                '12':
+                    {'0': {'bw_in': 1024, 'bw_out': 2048},
+                     '1': {'bw_in': 31337, 'bw_out': 21212121}},
+                }
+
+    def test_get_all_bw_counters(self):
+        class testinstance(object):
+            def __init__(self, name, uuid):
+                self.name = name
+                self.uuid = uuid
+
+        self.stubs.Set(vm_utils, 'fetch_bandwidth',
+                       XenAPIBWCountersTestCase._fake_fetch_bandwidth)
+        result = self.conn.get_all_bw_counters([testinstance(
+                                                   name='test1',
+                                                   uuid='1-2-3'),
+                                                testinstance(
+                                                   name='test2',
+                                                   uuid='4-5-6')])
+        self.assertEqual(len(result), 4)
+        self.assertIn(dict(uuid='1-2-3',
+                           mac_address="a:b:c:d...",
+                           bw_in=1024,
+                           bw_out=2048), result)
+        self.assertIn(dict(uuid='1-2-3',
+                           mac_address="e:f:12:q...",
+                           bw_in=31337,
+                           bw_out=21212121), result)
+
+        self.assertIn(dict(uuid='4-5-6',
+                           mac_address="a:3:c:d...",
+                           bw_in=21024,
+                           bw_out=22048), result)
+        self.assertIn(dict(uuid='4-5-6',
+                           mac_address="e:f:42:q...",
+                           bw_in=231337,
+                           bw_out=221212121), result)
+
+    def test_get_all_bw_counters_in_failure_case(self):
+        """Test that get_all_bw_conters returns an empty list when
+        no data returned from Xenserver.  c.f. bug #910045.
         """
         class testinstance(object):
             def __init__(self):
                 self.name = "instance-0001"
                 self.uuid = "1-2-3-4-5"
 
-        result = self.conn.get_all_bw_usage([testinstance()],
+        self.stubs.Set(vm_utils, 'fetch_bandwidth',
-                                            timeutils.utcnow())
+                       XenAPIBWCountersTestCase._fake_fetch_bandwidth_mt)
+        result = self.conn.get_all_bw_counters([testinstance()])
         self.assertEqual(result, [])