Support truncated flag returned by identity service

Create a custom list with flag `truncated` to support corresponding
response from the identity service.

This is wanted by Horizon, that wants to know that the list returned
from keystone is not full and that more strict filters need to be
applied.

The previous attempt in commit c28d408149
was reverted by d20b300589 because it
broke other code. This commit changes the way the flag is added and
verifies that existing code will not break.

Change-Id: Ia86cfd91110adae6d7ab86ff1f152a8f9be27837
Closes-Bug: 1520244

keystoneclient/base.py

@@ -76,6 +76,36 @@ def filter_kwargs(f):
     return func
 
 
+class TruncatedList(list):
+    """List with attribute `truncated`.
+
+    The main purpose of this class is to handle flag `truncated` returned
+    by Identity Service. It subclasses standard Python list and overrides
+    only equality operators.
+
+    :param bool truncated: whether the list is truncated or not.
+    """
+    def __init__(self, collection, truncated=False):
+        super(TruncatedList, self).__init__(collection)
+        self.truncated = truncated
+
+    def __eq__(self, other):
+        """Compare this list with another one.
+
+        Two TruncatedLists are equal if the lists they carry are equal
+        and their attributes `truncated` are equal.
+
+        If another value has not attribute `truncated`, it is assumed to
+        be False.
+        """
+        values_eq = super(TruncatedList, self).__eq__(other)
+        truncated_eq = self.truncated == getattr(other, 'truncated', False)
+        return values_eq and truncated_eq
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+
 class Manager(object):
     """Basic manager type providing common operations.
 
@@ -117,6 +147,8 @@ class Manager(object):
         :param body: data that will be encoded as JSON and passed in POST
             request (GET will be sent by default)
         :param kwargs: Additional arguments will be passed to the request.
+        :returns: list of objects with indication of truncation
+        :rtype: :py:class:`keystoneclient.base.TruncatedList`
         """
         if body:
             resp, body = self.client.post(url, body=body, **kwargs)
@@ -127,6 +159,7 @@ class Manager(object):
             obj_class = self.resource_class
 
         data = body[response_key]
+        truncated = body.get('truncated', False)
         # NOTE(ja): keystone returns values as list as {'values': [ ... ]}
         #           unlike other services which just return the list...
         try:
@@ -134,7 +167,8 @@ class Manager(object):
         except (KeyError, TypeError):
             pass
 
-        return [obj_class(self, res, loaded=True) for res in data if res]
+        objects = [obj_class(self, res, loaded=True) for res in data if res]
+        return TruncatedList(objects, truncated=truncated)
 
     def _get(self, url, response_key, **kwargs):
         """Get an object from collection.

keystoneclient/tests/unit/test_base.py

@@ -180,3 +180,29 @@ class ManagerTest(utils.TestCase):
             management=True)
         put_mock.assert_called_once_with(self.url, management=True, body=None)
         self.assertEqual(rsrc.hi, 1)
+
+
+class TruncatedListTest(utils.TestCase):
+    """Test that TruncatedList will not break existing checks
+
+    A lot of code assumes that the value returned from list() is a python
+    list, not an iterable object. Because of that, they perform various
+    list-specific checks. This code should not be broken.
+    """
+
+    def test_eq(self):
+        # flag `truncated` doesn't affect the check if it's False
+        self.assertEqual([], base.TruncatedList([], truncated=False))
+        self.assertEqual([1, 2, 3], base.TruncatedList([1, 2, 3],
+                                                       truncated=False))
+
+        # flag `truncated` affects the check if it's True
+        self.assertNotEqual([], base.TruncatedList([], truncated=True))
+        self.assertNotEqual([1, 2, 3], base.TruncatedList([1, 2, 3],
+                                                          truncated=True))
+
+        # flag `truncated` affects the equality check
+        self.assertNotEqual(base.TruncatedList([], truncated=True),
+                            base.TruncatedList([], truncated=False))
+        self.assertNotEqual(base.TruncatedList([1, 2, 3], truncated=True),
+                            base.TruncatedList([1, 2, 3], truncated=False))

keystoneclient/tests/unit/v3/utils.py

@@ -192,11 +192,16 @@ class CrudTests(object):
         kwargs.setdefault(uuid.uuid4().hex, uuid.uuid4().hex)
         return kwargs
 
-    def encode(self, entity):
+    def encode(self, entity, truncated=None):
+        encoded = {}
+        if truncated is not None:
+            encoded['truncated'] = truncated
         if isinstance(entity, dict):
-            return {self.key: entity}
+            encoded[self.key] = entity
+            return encoded
         if isinstance(entity, list):
-            return {self.collection_key: entity}
+            encoded[self.collection_key] = entity
+            return encoded
         raise NotImplementedError('Are you sure you want to encode that?')
 
     def stub_entity(self, method, parts=None, entity=None, id=None, **kwargs):
@@ -287,14 +292,22 @@ class CrudTests(object):
 
         self.assertRaises(TypeError, self.manager.list, **filter_kwargs)
 
-    def test_list(self, ref_list=None, expected_path=None,
-                  expected_query=None, **filter_kwargs):
+    def _test_list(self, ref_list=None, expected_path=None,
+                   expected_query=None, truncated=None, **filter_kwargs):
         ref_list = ref_list or [self.new_ref(), self.new_ref()]
         expected_path = self._get_expected_path(expected_path)
 
-        self.requests_mock.get(urlparse.urljoin(self.TEST_URL, expected_path),
-                               json=self.encode(ref_list))
+        # We want to catch all cases: when `truncated` is not returned by the
+        # server, when it's False and when it's True.
+        # Attribute `truncated` of the returned list-like object should exist
+        # in all these cases. It should be False if the server returned a list
+        # without the flag.
+        expected_truncated = False
+        if truncated:
+            expected_truncated = truncated
 
+        self.requests_mock.get(urlparse.urljoin(self.TEST_URL, expected_path),
+                               json=self.encode(ref_list, truncated=truncated))
         returned_list = self.manager.list(**filter_kwargs)
         self.assertEqual(len(ref_list), len(returned_list))
         [self.assertIsInstance(r, self.model) for r in returned_list]
@@ -313,6 +326,20 @@ class CrudTests(object):
         for key in qs_args:
             self.assertIn(key, qs_args_expected)
 
+        self.assertEqual(expected_truncated, returned_list.truncated)
+
+    def test_list(self, ref_list=None, expected_path=None,
+                  expected_query=None, **filter_kwargs):
+        # test simple list, without any truncation
+        self._test_list(ref_list, expected_path, expected_query,
+                        **filter_kwargs)
+        # test when a server returned a list with truncated=False
+        self._test_list(ref_list, expected_path, expected_query,
+                        truncated=False, **filter_kwargs)
+        # test when a server returned a list with truncated=True
+        self._test_list(ref_list, expected_path, expected_query,
+                        truncated=True, **filter_kwargs)
+
     def test_list_params(self):
         ref_list = [self.new_ref()]
         filter_kwargs = {uuid.uuid4().hex: uuid.uuid4().hex}