Handle RequestURITooLong error in large instance table

Neutron API supports filtering in listing API. If long filter parameters
such as long UUID list are passed, it exceeds URL length limit and
RequestURITooLong is raised. In such case, we need to split filter
parameters into chunks.

This commit add a convenient method to handle the case, which split
a list of long filter parameter into chunks and repeat listing API call.
However, it is not easy to generalize this behavior because filtering
parameters are specific to caller's logic, so this patch only fixes
a problem in servers_update_addresses().

Change-Id: I63ec4bad722351f640797bcbec4a38d5972f135c
Closes-Bug: #1349841

openstack_dashboard/api/neutron.py

@@ -28,6 +28,7 @@ import netaddr
 from django.conf import settings
 from django.utils.datastructures import SortedDict
 from django.utils.translation import ugettext_lazy as _
+from neutronclient.common import exceptions as neutron_exc
 from neutronclient.v2_0 import client as neutron_client
 
 from horizon import messages
@@ -516,6 +517,60 @@ def neutronclient(request):
     return c
 
 
+def list_resources_with_long_filters(list_method,
+                                     filter_attr, filter_values, **params):
+    """List neutron resources with handling RequestURITooLong exception.
+
+    If filter parameters are long, list resources API request leads to
+    414 error (URL is too long). For such case, this method split
+    list parameters specified by a list_field argument into chunks
+    and call the specified list_method repeatedly.
+
+    :param list_method: Method used to retrieve resource list.
+    :param filter_attr: attribute name to be filtered. The value corresponding
+        to this attribute is specified by "filter_values".
+        If you want to specify more attributes for a filter condition,
+        pass them as keyword arguments like "attr2=values2".
+    :param filter_values: values of "filter_attr" to be filtered.
+        If filter_values are too long and the total URI lenght exceed the
+        maximum lenght supported by the neutron server, filter_values will
+        be split into sub lists if filter_values is a list.
+    :param params: parameters to pass a specified listing API call
+        without any changes. You can specify more filter conditions
+        in addition to a pair of filter_attr and filter_values.
+    """
+    try:
+        params[filter_attr] = filter_values
+        return list_method(**params)
+    except neutron_exc.RequestURITooLong as uri_len_exc:
+        # The URI is too long because of too many filter values.
+        # Use the excess attribute of the exception to know how many
+        # filter values can be inserted into a single request.
+
+        # We consider only the filter condition from (filter_attr,
+        # filter_values) and do not consider other filter conditions
+        # which may be specified in **params.
+        if type(filter_values) != list:
+            filter_values = [filter_values]
+
+        # Length of each query filter is:
+        # <key>=<value>& (e.g., id=<uuid>)
+        # The length will be key_len + value_maxlen + 2
+        all_filter_len = sum(len(filter_attr) + len(val) + 2
+                             for val in filter_values)
+        allowed_filter_len = all_filter_len - uri_len_exc.excess
+
+        val_maxlen = max(len(val) for val in filter_values)
+        filter_maxlen = len(filter_attr) + val_maxlen + 2
+        chunk_size = allowed_filter_len / filter_maxlen
+
+        resources = []
+        for i in range(0, len(filter_values), chunk_size):
+            params[filter_attr] = filter_values[i:i + chunk_size]
+            resources.extend(list_method(**params))
+        return resources
+
+
 def network_list(request, **params):
     LOG.debug("network_list(): params=%s", params)
     networks = neutronclient(request).list_networks(**params).get('networks')
@@ -563,6 +618,7 @@ def network_get(request, network_id, expand_subnet=True, **params):
     if expand_subnet:
         network['subnets'] = [subnet_get(request, sid)
                               for sid in network['subnets']]
+
     return Network(network)
 
 
@@ -859,16 +915,19 @@ def servers_update_addresses(request, servers, all_tenants=False):
 
     # Get all (filtered for relevant servers) information from Neutron
     try:
-        ports = port_list(request,
-                          device_id=[instance.id for instance in servers])
+        ports = list_resources_with_long_filters(
+            port_list, 'device_id', [instance.id for instance in servers],
+            request=request)
         fips = FloatingIpManager(request)
         if fips.is_supported():
-            floating_ips = fips.list(all_tenants=all_tenants,
-                                     port_id=[port.id for port in ports])
+            floating_ips = list_resources_with_long_filters(
+                fips.list, 'port_id', [port.id for port in ports],
+                all_tenants=all_tenants)
         else:
             floating_ips = []
-        networks = network_list(request,
-                                id=[port.network_id for port in ports])
+        networks = list_resources_with_long_filters(
+            network_list, 'id', set([port.network_id for port in ports]),
+            request=request)
     except Exception:
         error_message = _('Unable to connect to Neutron.')
         LOG.error(error_message)

openstack_dashboard/test/api_tests/network_tests.py

@@ -286,7 +286,7 @@ class NetworkApiNeutronTests(NetworkApiNeutronTestBase):
                 .AndReturn({'floatingips': assoc_fips})
             self.qclient.list_ports(tenant_id=tenant_id) \
                 .AndReturn({'ports': self.api_ports.list()})
-        self.qclient.list_networks(id=server_network_ids) \
+        self.qclient.list_networks(id=set(server_network_ids)) \
             .AndReturn({'networks': server_networks})
         self.qclient.list_subnets() \
             .AndReturn({'subnets': self.api_subnets.list()})

openstack_dashboard/test/api_tests/neutron_tests.py

@@ -12,8 +12,12 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import uuid
+
 from django.test.utils import override_settings
 
+from neutronclient.common import exceptions as neutron_exc
+
 from openstack_dashboard import api
 from openstack_dashboard import policy
 from openstack_dashboard.test import helpers as test
@@ -392,3 +396,33 @@ class NeutronApiTests(test.APITestCase):
 
     def test_get_router_ha_permission_without_l3_ha_extension(self):
         self._test_get_router_ha_permission_with_policy_check(False)
+
+    def test_list_resources_with_long_filters(self):
+        # In this tests, port_list is called with id=[10 port ID]
+        # filter. It generates about 40*10 char length URI.
+        # Each port ID is converted to "id=<UUID>&" in URI and
+        # it means 40 chars (len(UUID)=36).
+        # If excess lenght is 220, it means 400-220=180 chars
+        # can be sent in the first request.
+        # As a result three API calls with 4, 4, 2 port ID
+        # are expected.
+
+        ports = [{'id': str(uuid.uuid4()),
+                  'name': 'port%s' % i,
+                  'admin_state_up': True}
+                 for i in range(10)]
+        port_ids = [port['id'] for port in ports]
+
+        neutronclient = self.stub_neutronclient()
+        uri_len_exc = neutron_exc.RequestURITooLong(excess=220)
+        neutronclient.list_ports(id=port_ids).AndRaise(uri_len_exc)
+        for i in range(0, 10, 4):
+            neutronclient.list_ports(id=port_ids[i:i + 4]) \
+                .AndReturn({'ports': ports[i:i + 4]})
+        self.mox.ReplayAll()
+
+        ret_val = api.neutron.list_resources_with_long_filters(
+            api.neutron.port_list, 'id', port_ids,
+            request=self.request)
+        self.assertEqual(10, len(ret_val))
+        self.assertEqual(port_ids, [p.id for p in ret_val])