Fix volume caching in PowerFlex driver

Volume caching stops working after 126th volume created from
the same image due to snapshot limit per volume in PowerFlex.
The driver needs to throw exception "SnapshotLimitReached" to
invalidate cache and start over with new volume snapshots.

Closes-Bug: #1942095
Change-Id: Ic61f23c87de5dad52c06061849e3db970cee4327

cinder/tests/unit/volume/drivers/dell_emc/powerflex/__init__.py

@@ -69,12 +69,14 @@ class TestPowerFlexDriver(test.TestCase):
         Invalid='1',
         BadStatus='2',
         ValidVariant='3',
+        BadStatusWithDetails='4',
     ))
     __RESPONSE_MODE_NAMES = {
         '0': 'Valid',
         '1': 'Invalid',
         '2': 'BadStatus',
         '3': 'ValidVariant',
+        '4': 'BadStatusWithDetails',
     }
 
     BAD_STATUS_RESPONSE = mocks.MockHTTPSResponse(
@@ -175,6 +177,7 @@ class TestPowerFlexDriver(test.TestCase):
         RESPONSE_MODE.Valid: Respond with valid data
         RESPONSE_MODE.Invalid: Respond with invalid data
         RESPONSE_MODE.BadStatus: Response with not-OK status code.
+        RESPONSE_MODE.BadStatusWithDetails: as BadStatus but with "details".
         """
         self.__https_response_mode = mode
 

cinder/tests/unit/volume/drivers/dell_emc/powerflex/test_create_snapshot.py

@@ -24,6 +24,7 @@ from cinder.tests.unit import fake_snapshot
 from cinder.tests.unit import fake_volume
 from cinder.tests.unit.volume.drivers.dell_emc import powerflex
 from cinder.tests.unit.volume.drivers.dell_emc.powerflex import mocks
+from cinder.volume.drivers.dell_emc.powerflex import rest_client
 from cinder.volume.drivers.dell_emc.powerflex import utils as flex_utils
 
 
@@ -84,6 +85,20 @@ class TestCreateSnapShot(powerflex.TestPowerFlexDriver):
                 'instances/System/action/snapshotVolumes':
                     self.BAD_STATUS_RESPONSE,
             },
+            self.RESPONSE_MODE.BadStatusWithDetails: {
+                'instances/System/action/snapshotVolumes':
+                    mocks.MockHTTPSResponse(
+                        {
+                            'errorCode': 0,
+                            'message': 'Error with details',
+                            'details': [
+                                {
+                                    'rc': rest_client.TOO_MANY_SNAPS_ERROR,
+                                },
+                            ],
+                        }, 500
+                    ),
+            },
             self.RESPONSE_MODE.Invalid: {
                 'types/Volume/instances/getByName::' +
                 self.volume_name_2x_enc: None,
@@ -116,3 +131,12 @@ class TestCreateSnapShot(powerflex.TestPowerFlexDriver):
     def test_create_snapshot(self):
         self.set_https_response_mode(self.RESPONSE_MODE.Valid)
         self.driver.create_snapshot(self.snapshot)
+
+    def test_create_snapshot_limit_reached(self):
+        self.set_https_response_mode(
+            self.RESPONSE_MODE.BadStatusWithDetails)
+        self.assertRaises(
+            exception.SnapshotLimitReached,
+            self.driver.create_snapshot,
+            self.snapshot
+        )

cinder/volume/drivers/dell_emc/powerflex/rest_client.py

@@ -35,8 +35,11 @@ VOLUME_MIGRATION_IN_PROGRESS_ERROR = 717
 VOLUME_MIGRATION_ALREADY_ON_DESTINATION_POOL_ERROR = 718
 VOLUME_NOT_FOUND_ERROR = 79
 OLD_VOLUME_NOT_FOUND_ERROR = 78
+TOO_MANY_SNAPS_ERROR = 182
 ILLEGAL_SYNTAX = 0
 
+MAX_SNAPS_IN_VTREE = 126
+
 
 class RestClient(object):
     def __init__(self, configuration, is_primary=True):
@@ -214,6 +217,11 @@ class RestClient(object):
                    {"vol_name": volume_provider_id,
                     "response": response["message"]})
             LOG.error(msg)
+            # check if the volume reached snapshot limit
+            if ("details" in response and
+                    response["details"][0]["rc"] == TOO_MANY_SNAPS_ERROR):
+                raise exception.SnapshotLimitReached(
+                    set_limit=MAX_SNAPS_IN_VTREE)
             raise exception.VolumeBackendAPIException(data=msg)
         return response["volumeIdList"][0]
 

releasenotes/notes/fix-powerflex-volume-cache-da3fa1769ef78ae8.yaml

@@ -0,0 +1,10 @@
+---
+fixes:
+  - |
+    PowerFlex driver `bug #1942095
+    <https://bugs.launchpad.net/cinder/+bug/1942095>`_: Fixed Cinder
+    volume caching mechanism for the driver. Now the driver
+    correctly raises ``exception.SnapshotLimitReached`` when maximum
+    snapshots are created for a given volume and a volume cache is
+    invalidated to allow a new row of fast volume clones.
+