Fix allocated capacity report on non pool drivers

The allocated_capacity_gb is being calculated by the core Cinder code and then
added to the driver provided stats, but it only works if the driver is
reporting data of pools and it doesn't work if they are reporting the
stats as the whole backend.

When reporting stats as the whole backend, which is the case of the RBD
driver, we will not send allocated_capacity_gb to the manager.

This patch fixes this by adding the stats from the special fixed pool
that is created by the _count_allocated_capacity method for drivers that
don't report pools information on stats from drivers that don't report
pools information.

Closes-Bug: #1712549
Closes-Bug: #1706057
Change-Id: Ia7ff6f05df85b1752b4e0353a734f3071dc8ff39

cinder/tests/unit/volume/test_volume.py

@@ -146,6 +146,8 @@ class VolumeTestCase(base.BaseVolumeTestCase):
         self.assertEqual(opts['backend_availability_zone'],
                          manager.availability_zone)
 
+    @mock.patch('cinder.volume.manager.VolumeManager._append_volume_stats',
+                mock.Mock())
     @mock.patch.object(vol_manager.VolumeManager,
                        'update_service_capabilities')
     def test_report_filter_goodness_function(self, mock_update):
@@ -2729,6 +2731,48 @@ class VolumeTestCase(base.BaseVolumeTestCase):
         self.assertEqual("attaching", volume['status'])
         self.assertEqual("attaching", volume['attach_status'])
 
+    def test__append_volume_stats_with_pools(self):
+        manager = vol_manager.VolumeManager()
+        manager.stats = {'pools': {'pool1': {'allocated_capacity_gb': 20},
+                                   'pool2': {'allocated_capacity_gb': 10}}}
+        vol_stats = {'vendor_name': 'Open Source', 'pools': [
+            {'pool_name': 'pool1', 'provisioned_capacity_gb': 31},
+            {'pool_name': 'pool2', 'provisioned_capacity_gb': 21}]}
+        manager._append_volume_stats(vol_stats)
+
+        expected = {'provisioned_capacity_gb': 30, 'allocated_capacity_gb': 20}
+        expected = {'vendor_name': 'Open Source', 'pools': [
+            {'pool_name': 'pool1', 'provisioned_capacity_gb': 31,
+             'allocated_capacity_gb': 20},
+            {'pool_name': 'pool2', 'provisioned_capacity_gb': 21,
+             'allocated_capacity_gb': 10}]}
+        self.assertDictEqual(expected, vol_stats)
+
+    def test__append_volume_stats_no_pools(self):
+        manager = vol_manager.VolumeManager()
+        manager.stats = {'pools': {'backend': {'allocated_capacity_gb': 20}}}
+        vol_stats = {'provisioned_capacity_gb': 30}
+        manager._append_volume_stats(vol_stats)
+
+        expected = {'provisioned_capacity_gb': 30, 'allocated_capacity_gb': 20}
+        self.assertDictEqual(expected, vol_stats)
+
+    def test__append_volume_stats_no_pools_no_volumes(self):
+        manager = vol_manager.VolumeManager()
+        # This is what gets set on c-vol manager's init_host method
+        manager.stats = {'pools': {}, 'allocated_capacity_gb': 0}
+        vol_stats = {'provisioned_capacity_gb': 30}
+
+        manager._append_volume_stats(vol_stats)
+
+        expected = {'provisioned_capacity_gb': 30, 'allocated_capacity_gb': 0}
+        self.assertDictEqual(expected, vol_stats)
+
+    def test__append_volume_stats_driver_error(self):
+        manager = vol_manager.VolumeManager()
+        self.assertRaises(exception.ProgrammingError,
+                          manager._append_volume_stats, {'pools': 'bad_data'})
+
 
 class VolumeTestCaseLocks(base.BaseVolumeTestCase):
     MOCK_TOOZ = False

cinder/volume/manager.py

@@ -2413,16 +2413,31 @@ class VolumeManager(manager.CleanableManager,
 
     def _append_volume_stats(self, vol_stats):
         pools = vol_stats.get('pools', None)
-        if pools and isinstance(pools, list):
-            for pool in pools:
-                pool_name = pool['pool_name']
-                try:
-                    pool_stats = self.stats['pools'][pool_name]
-                except KeyError:
-                    # Pool not found in volume manager
-                    pool_stats = dict(allocated_capacity_gb=0)
+        if pools:
+            if isinstance(pools, list):
+                for pool in pools:
+                    pool_name = pool['pool_name']
+                    try:
+                        pool_stats = self.stats['pools'][pool_name]
+                    except KeyError:
+                        # Pool not found in volume manager
+                        pool_stats = dict(allocated_capacity_gb=0)
 
-                pool.update(pool_stats)
+                    pool.update(pool_stats)
+            else:
+                raise exception.ProgrammingError(
+                    reason='Pools stats reported by the driver are not '
+                           'reported in a list')
+        # For drivers that are not reporting their stats by pool we will use
+        # the data from the special fixed pool created by
+        # _count_allocated_capacity.
+        elif self.stats.get('pools'):
+            vol_stats.update(next(iter(self.stats['pools'].values())))
+        # This is a special subcase of the above no pool case that happens when
+        # we don't have any volumes yet.
+        else:
+            vol_stats.update(self.stats)
+            vol_stats.pop('pools', None)
 
     def _append_filter_goodness_functions(self, volume_stats):
         """Returns volume_stats updated as needed."""