Deduplicate next-page URL's query params

This avoid HTTP 414 caused by endless addition of query params onto
the next-page's URL that already contains them in case where the number
of pages is high.

Change-Id: I4f89e8e4837bb7c08c841e50070541038a2d2cc2

openstack/resource.py

@@ -1429,6 +1429,17 @@ class Resource(dict):
                     if limit:
                         params['limit'] = limit
                     next_link = uri
+
+        # Parse params from Link (next page URL) into params.
+        # This prevents duplication of query parameters that with large
+        # number of pages result in HTTP 414 error eventually.
+        if next_link:
+            parts = six.moves.urllib.parse.urlparse(next_link)
+            query_params = six.moves.urllib.parse.parse_qs(parts.query)
+            params.update(query_params)
+            next_link = six.moves.urllib.parse.urljoin(next_link,
+                                                       parts.path)
+
         # If we still have no link, and limit was given and is non-zero,
         # and the number of records yielded equals the limit, then the user
         # is playing pagination ball so we should go ahead and try once more.

openstack/tests/unit/test_resource.py

@@ -1534,6 +1534,53 @@ class TestResourceActions(base.TestCase):
         self.assertEqual(2, len(self.session.get.call_args_list))
         self.assertIsInstance(results[0], self.test_class)
 
+    def test_list_response_paginated_with_links_and_query(self):
+        q_limit = 1
+        ids = [1, 2]
+        mock_response = mock.Mock()
+        mock_response.status_code = 200
+        mock_response.links = {}
+        mock_response.json.side_effect = [
+            {
+                "resources": [{"id": ids[0]}],
+                "resources_links": [{
+                    "href": "https://example.com/next-url?limit=%d" % q_limit,
+                    "rel": "next",
+                }]
+            }, {
+                "resources": [{"id": ids[1]}],
+            }, {
+                "resources": [],
+            }]
+
+        self.session.get.return_value = mock_response
+
+        class Test(self.test_class):
+            _query_mapping = resource.QueryParameters("limit")
+
+        results = list(Test.list(self.session, paginated=True, limit=q_limit))
+
+        self.assertEqual(2, len(results))
+        self.assertEqual(ids[0], results[0].id)
+        self.assertEqual(ids[1], results[1].id)
+        self.assertEqual(
+            mock.call('base_path',
+                      headers={'Accept': 'application/json'}, params={
+                          'limit': q_limit,
+                      },
+                      microversion=None),
+            self.session.get.mock_calls[0])
+        self.assertEqual(
+            mock.call('https://example.com/next-url',
+                      headers={'Accept': 'application/json'}, params={
+                          'limit': [str(q_limit)],
+                      },
+                      microversion=None),
+            self.session.get.mock_calls[2])
+
+        self.assertEqual(3, len(self.session.get.call_args_list))
+        self.assertIsInstance(results[0], self.test_class)
+
     def test_list_response_paginated_with_microversions(self):
         class Test(resource.Resource):
             service = self.service_name