feat(util,Request): Add options to disable csv parsing of query params (#820)

* Add option to to_query_str to encode lists as multiple occurences of
  the same parameter.
* Add option to parse_query_string to not split comma-delimited values.
* Add an option to RequestOptions for toggling csv parsing.

In all cases, the options default to the previous behavior in order to
avoid introducing any breaking changes.

Co-Authored-By: Daniel Schwarz <qwesda@me.com>
Co-Authored-By: Steven Ly <sly@openroad.ca>
Co-Authored-By: Kurt Griffiths <inbox@kgriffs.com>

Closes #749

falcon/request.py

@@ -284,6 +284,7 @@ class Request(object):
                 self._params = parse_query_string(
                     self.query_string,
                     keep_blank_qs_values=self.options.keep_blank_qs_values,
+                    parse_qs_csv=self.options.auto_parse_qs_csv,
                 )
 
             else:
@@ -1153,6 +1154,7 @@ class Request(object):
             extra_params = parse_query_string(
                 body,
                 keep_blank_qs_values=self.options.keep_blank_qs_values,
+                parse_qs_csv=self.options.auto_parse_qs_csv,
             )
 
             self._params.update(extra_params)
@@ -1190,8 +1192,11 @@ class RequestOptions(object):
     """This class is a container for ``Request`` options.
 
     Attributes:
-        keep_blank_qs_values (bool): Set to ``True`` in order to retain
+        keep_blank_qs_values (bool): Set to ``True`` to keep query string
-            blank values in query string parameters (default ``False``).
+            fields even if they do not have a value (default ``False``).
+            For comma-separated values, this option also determines
+            whether or not empty elements in the parsed list are
+            retained.
         auto_parse_form_urlencoded: Set to ``True`` in order to
             automatically consume the request stream and merge the
             results into the request's query string params when the
@@ -1202,18 +1207,29 @@ class RequestOptions(object):
             Note:
                 The character encoding for fields, before
                 percent-encoding non-ASCII bytes, is assumed to be
-                UTF-8. The special `_charset_` field is ignored if present.
+                UTF-8. The special `_charset_` field is ignored if
+                present.
 
                 Falcon expects form-encoded request bodies to be
                 encoded according to the standard W3C algorithm (see
                 also http://goo.gl/6rlcux).
 
+        auto_parse_qs_csv: Set to ``False`` to treat commas in a query
+            string value as literal characters, rather than as a comma-
+            separated list (default ``True``). When this option is
+            enabled, the value will be split on any non-percent-encoded
+            commas. Disable this option when encoding lists as multiple
+            occurrences of the same parameter, and when values may be
+            encoded in alternative formats in which the comma character
+            is significant.
     """
     __slots__ = (
         'keep_blank_qs_values',
         'auto_parse_form_urlencoded',
+        'auto_parse_qs_csv',
     )
 
     def __init__(self):
         self.keep_blank_qs_values = False
         self.auto_parse_form_urlencoded = False
+        self.auto_parse_qs_csv = True

falcon/util/misc.py

@@ -148,7 +148,7 @@ def http_date_to_dt(http_date, obs_date=False):
     raise ValueError('time data %r does not match known formats' % http_date)
 
 
-def to_query_str(params):
+def to_query_str(params, comma_delimited_lists=True):
     """Converts a dictionary of params to a query string.
 
     Args:
@@ -157,6 +157,10 @@ def to_query_str(params):
             something that can be converted into a ``str``. If `params`
             is a ``list``, it will be converted to a comma-delimited string
             of values (e.g., 'thing=1,2,3')
+        comma_delimited_lists (bool, default ``True``):
+            If set to ``False`` encode lists by specifying multiple instances
+            of the parameter (e.g., 'thing=1&thing=2&thing=3')
+
 
     Returns:
         str: A URI query string including the '?' prefix, or an empty string
@@ -175,7 +179,20 @@ def to_query_str(params):
         elif v is False:
             v = 'false'
         elif isinstance(v, list):
-            v = ','.join(map(str, v))
+            if comma_delimited_lists:
+                v = ','.join(map(str, v))
+            else:
+                for list_value in v:
+                    if list_value is True:
+                        list_value = 'true'
+                    elif list_value is False:
+                        list_value = 'false'
+                    else:
+                        list_value = str(list_value)
+
+                    query_str += k + '=' + list_value + '&'
+
+                continue
         else:
             v = str(v)
 

falcon/util/uri.py

@@ -246,11 +246,12 @@ else:
         return decoded_uri.decode('utf-8', 'replace')
 
 
-def parse_query_string(query_string, keep_blank_qs_values=False):
+def parse_query_string(query_string, keep_blank_qs_values=False,
+                       parse_qs_csv=True):
     """Parse a query string into a dict.
 
     Query string parameters are assumed to use standard form-encoding. Only
-    parameters with values are parsed. for example, given 'foo=bar&flag',
+    parameters with values are returned. For example, given 'foo=bar&flag',
     this function would ignore 'flag' unless the `keep_blank_qs_values` option
     is set.
 
@@ -269,8 +270,16 @@ def parse_query_string(query_string, keep_blank_qs_values=False):
 
     Args:
         query_string (str): The query string to parse.
-        keep_blank_qs_values (bool): If set to ``True``, preserves boolean
+        keep_blank_qs_values (bool): Set to ``True`` to return fields even if
-            fields and fields with no content as blank strings.
+            they do not have a value (default ``False``). For comma-separated
+            values, this option also determines whether or not empty elements
+            in the parsed list are retained.
+        parse_qs_csv: Set to ``False`` in order to disable splitting query
+            parameters on ``,`` (default ``True``). Depending on the user agent,
+            encoding lists as multiple occurrences of the same parameter might
+            be preferable. In this case, setting `parse_qs_csv` to ``False``
+            will cause the framework to treat commas as literal characters in
+            each occurring parameter value.
 
     Returns:
         dict: A dictionary of (*name*, *value*) pairs, one per query
@@ -309,7 +318,7 @@ def parse_query_string(query_string, keep_blank_qs_values=False):
                 params[k] = [old_value, decode(v)]
 
         else:
-            if ',' in v:
+            if parse_qs_csv and ',' in v:
                 # NOTE(kgriffs): Falcon supports a more compact form of
                 # lists, in which the elements are comma-separated and
                 # assigned to a single param instance. If it turns out that

tests/test_options.py

@@ -1,16 +1,32 @@
+import ddt
+
 from falcon.request import RequestOptions
 import falcon.testing as testing
 
 
+@ddt.ddt
 class TestRequestOptions(testing.TestBase):
 
-    def test_correct_options(self):
+    def test_option_defaults(self):
         options = RequestOptions()
+
         self.assertFalse(options.keep_blank_qs_values)
-        options.keep_blank_qs_values = True
+        self.assertFalse(options.auto_parse_form_urlencoded)
-        self.assertTrue(options.keep_blank_qs_values)
+        self.assertTrue(options.auto_parse_qs_csv)
-        options.keep_blank_qs_values = False
+
-        self.assertFalse(options.keep_blank_qs_values)
+    @ddt.data(
+        'keep_blank_qs_values',
+        'auto_parse_form_urlencoded',
+        'auto_parse_qs_csv',
+    )
+    def test_options_toggle(self, option_name):
+        options = RequestOptions()
+
+        setattr(options, option_name, True)
+        self.assertTrue(getattr(options, option_name))
+
+        setattr(options, option_name, False)
+        self.assertFalse(getattr(options, option_name))
 
     def test_incorrect_options(self):
         options = RequestOptions()

tests/test_query_params.py

@@ -65,6 +65,60 @@ class _TestQueryParams(testing.TestBase):
         self.assertEqual(req.get_param_as_list('id', int), [23, 42])
         self.assertEqual(req.get_param('q'), u'\u8c46 \u74e3')
 
+    def test_option_auto_parse_qs_csv_simple_false(self):
+        self.api.req_options.auto_parse_qs_csv = False
+
+        query_string = 'id=23,42,,&id=2'
+        self.simulate_request('/', query_string=query_string)
+
+        req = self.resource.req
+
+        self.assertEqual(req.params['id'], [u'23,42,,', u'2'])
+        self.assertIn(req.get_param('id'), [u'23,42,,', u'2'])
+        self.assertEqual(req.get_param_as_list('id'), [u'23,42,,', u'2'])
+
+    def test_option_auto_parse_qs_csv_simple_true(self):
+        self.api.req_options.auto_parse_qs_csv = True
+
+        query_string = 'id=23,42,,&id=2'
+        self.simulate_request('/', query_string=query_string)
+
+        req = self.resource.req
+
+        self.assertEqual(req.params['id'], [u'23', u'42', u'2'])
+        self.assertIn(req.get_param('id'), [u'23', u'42', u'2'])
+        self.assertEqual(req.get_param_as_list('id', int), [23, 42, 2])
+
+    def test_option_auto_parse_qs_csv_complex_false(self):
+        self.api.req_options.auto_parse_qs_csv = False
+
+        encoded_json = '%7B%22msg%22:%22Testing%201,2,3...%22,%22code%22:857%7D'
+        decoded_json = '{"msg":"Testing 1,2,3...","code":857}'
+
+        query_string = ('colors=red,green,blue&limit=1'
+                        '&list-ish1=f,,x&list-ish2=,0&list-ish3=a,,,b'
+                        '&empty1=&empty2=,&empty3=,,'
+                        '&thing=' + encoded_json)
+
+        self.simulate_request('/', query_string=query_string)
+
+        req = self.resource.req
+
+        self.assertIn(req.get_param('colors'), 'red,green,blue')
+        self.assertEqual(req.get_param_as_list('colors'), [u'red,green,blue'])
+
+        self.assertEqual(req.get_param_as_list('limit'), ['1'])
+
+        self.assertEqual(req.get_param_as_list('empty1'), None)
+        self.assertEqual(req.get_param_as_list('empty2'), [u','])
+        self.assertEqual(req.get_param_as_list('empty3'), [u',,'])
+
+        self.assertEqual(req.get_param_as_list('list-ish1'), [u'f,,x'])
+        self.assertEqual(req.get_param_as_list('list-ish2'), [u',0'])
+        self.assertEqual(req.get_param_as_list('list-ish3'), [u'a,,,b'])
+
+        self.assertEqual(req.get_param('thing'), decoded_json)
+
     def test_bad_percentage(self):
         query_string = 'x=%%20%+%&y=peregrine&z=%a%z%zz%1%20e'
         self.simulate_request('/', query_string=query_string)

tests/test_utils.py

@@ -128,6 +128,16 @@ class TestFalconUtils(testtools.TestCase):
             falcon.to_query_str({'things': ['a', 'b']}),
             '?things=a,b')
 
+        expected = ('?things=a&things=b&things=&things=None'
+                    '&things=true&things=false&things=0')
+
+        actual = falcon.to_query_str(
+            {'things': ['a', 'b', '', None, True, False, 0]},
+            comma_delimited_lists=False
+        )
+
+        self.assertEqual(actual, expected)
+
     def test_pack_query_params_several(self):
         garbage_in = {
             'limit': 17,