Merge "Add pagination for the list_volumes call"

shade/_normalize.py

@@ -679,11 +679,6 @@ class Normalizer(object):
         :returns: A list of normalized dicts.
         """
         ret = []
-        # With pagination we don't get a top-level container
-        # so we need to explicitly extract the list of volumes
-        if isinstance(volumes, dict):
-            volumes = volumes.get('volumes', [])
-
         for volume in volumes:
             ret.append(self._normalize_volume(volume))
         return ret

shade/openstackcloud.py

@@ -1601,11 +1601,33 @@ class OpenStackCloud(
         :returns: A list of volume ``munch.Munch``.
 
         """
+        def _list(response):
+            # NOTE(rods)The shade Adapter is removing the top-level
+            # container but not with pagination or in a few other
+            # circumstances, so `response` can be a list of Volumes,
+            # or a dict like {'volumes_list': [...], 'volume': [...]}.
+            # We need the type check to work around the issue until
+            # next commit where we'll move the top-level container
+            # removing from the adapter to the related call.
+            if isinstance(response, list):
+                volumes.extend(response)
+            if isinstance(response, dict):
+                volumes.extend(meta.obj_list_to_dict(response['volumes']))
+                endpoint = None
+                if 'volumes_links' in response:
+                    for l in response['volumes_links']:
+                        if 'rel' in l and 'next' == l['rel']:
+                            endpoint = l['href']
+                            break
+                    if endpoint:
+                        _list(self._volume_client.get(endpoint))
+
         if not cache:
             warnings.warn('cache argument to list_volumes is deprecated. Use '
                           'invalidate instead.')
-        return self._normalize_volumes(
-            self._volume_client.get('/volumes/detail'))
+        volumes = []
+        _list(self._volume_client.get('/volumes/detail'))
+        return self._normalize_volumes(volumes)
 
     @_utils.cache_on_arguments()
     def list_volume_types(self, get_extra=True):

shade/tests/functional/test_volume.py

@@ -91,3 +91,23 @@ class TestVolume(base.BaseFunctionalTestCase):
         volume = self.user_cloud.get_volume(volume_name)
         if volume:
             self.user_cloud.delete_volume(volume_name, wait=True)
+
+    def test_list_volumes_pagination(self):
+        '''Test pagination for list volumes functionality'''
+        volumes = []
+        # the number of created volumes needs to be higher than
+        # CONF.osapi_max_limit but not higher than volume quotas for
+        # the test user in the tenant(default quotas is set to 10)
+        num_volumes = 8
+        for i in range(num_volumes):
+            name = self.getUniqueString()
+            self.addCleanup(self.cleanup, name)
+            v = self.user_cloud.create_volume(display_name=name, size=1)
+            volumes.append(v)
+        result = []
+        for i in self.user_cloud.list_volumes():
+            if i['name'] and i['name'].startswith(self.id()):
+                result.append(i['id'])
+        self.assertEqual(
+            sorted([i['id'] for i in volumes]),
+            sorted(result))

shade/tests/unit/test_volume.py

@@ -285,12 +285,35 @@ class TestVolume(base.RequestsMockTestCase):
         self.register_uris([
             dict(method='GET',
                  uri=self.get_mock_url(
-                     'volumev2', 'public', append=['volumes', 'detail']),
+                     'volumev2', 'public',
+                     append=['volumes', 'detail']),
                  json={
-                     'volumes': [vol1, vol2],
+                     'volumes': [vol1],
                      'volumes_links': [
-                         {'href': 'https://volume.example.com/fake_url',
-                          'rel': 'next'}]})])
+                         {'href': self.get_mock_url(
+                             'volumev2', 'public',
+                             append=['volumes', 'detail'],
+                             qs_elements=['marker=01']),
+                          'rel': 'next'}]}),
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     'volumev2', 'public',
+                     append=['volumes', 'detail'],
+                     qs_elements=['marker=01']),
+                 json={
+                     'volumes': [vol2],
+                     'volumes_links': [
+                         {'href': self.get_mock_url(
+                             'volumev2', 'public',
+                             append=['volumes', 'detail'],
+                             qs_elements=['marker=02']),
+                          'rel': 'next'}]}),
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     'volumev2', 'public',
+                     append=['volumes', 'detail'],
+                     qs_elements=['marker=02']),
+                 json={'volumes': []})])
         self.assertEqual(
             [self.cloud._normalize_volume(vol1),
              self.cloud._normalize_volume(vol2)],