openstack/deb-python-falcon
Code Issues Proposed changes

feat(util,Request): Add options to disable csv parsing of query params (#820)

Browse Source 
* Add option to to_query_str to encode lists as multiple occurences of
  the same parameter.
* Add option to parse_query_string to not split comma-delimited values.
* Add an option to RequestOptions for toggling csv parsing.

In all cases, the options default to the previous behavior in order to
avoid introducing any breaking changes.

Co-Authored-By: Daniel Schwarz <qwesda@me.com>
Co-Authored-By: Steven Ly <sly@openroad.ca>
Co-Authored-By: Kurt Griffiths <inbox@kgriffs.com>

Closes #749
This commit is contained in:
Kurt Griffiths
2016-07-09 10:34:01 -06:00
committed by GitHub
parent 50b1759ee7
commit 0f64e94a5a
6 changed files with 137 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
falcon/request.py
View File

@@ -284,6 +284,7 @@ class Request(object):
self._params = parse_query_string(
self.query_string,
keep_blank_qs_values=self.options.keep_blank_qs_values,
parse_qs_csv=self.options.auto_parse_qs_csv,
)
else:
@@ -1153,6 +1154,7 @@ class Request(object):
extra_params = parse_query_string(
body,
keep_blank_qs_values=self.options.keep_blank_qs_values,
parse_qs_csv=self.options.auto_parse_qs_csv,
)
self._params.update(extra_params)
@@ -1190,8 +1192,11 @@ class RequestOptions(object):
"""This class is a container for ``Request`` options.
Attributes:
keep_blank_qs_values (bool): Set to ``True`` in order to retain
blank values in query string parameters (default ``False``).
keep_blank_qs_values (bool): Set to ``True`` to keep query string
fields even if they do not have a value (default ``False``).
For comma-separated values, this option also determines
whether or not empty elements in the parsed list are
retained.
auto_parse_form_urlencoded: Set to ``True`` in order to
automatically consume the request stream and merge the
results into the request's query string params when the
@@ -1202,18 +1207,29 @@ class RequestOptions(object):
Note:
The character encoding for fields, before
percent-encoding non-ASCII bytes, is assumed to be
UTF-8. The special `_charset_` field is ignored if present.
UTF-8. The special `_charset_` field is ignored if
present.
Falcon expects form-encoded request bodies to be
encoded according to the standard W3C algorithm (see
also http://goo.gl/6rlcux).
auto_parse_qs_csv: Set to ``False`` to treat commas in a query
string value as literal characters, rather than as a comma-
separated list (default ``True``). When this option is
enabled, the value will be split on any non-percent-encoded
commas. Disable this option when encoding lists as multiple
occurrences of the same parameter, and when values may be
encoded in alternative formats in which the comma character
is significant.
"""
__slots__ = (
'keep_blank_qs_values',
'auto_parse_form_urlencoded',
'auto_parse_qs_csv',
)
def __init__(self):
self.keep_blank_qs_values = False
self.auto_parse_form_urlencoded = False
self.auto_parse_qs_csv = True

21
falcon/util/misc.py
View File

@@ -148,7 +148,7 @@ def http_date_to_dt(http_date, obs_date=False):
raise ValueError('time data %r does not match known formats' % http_date)
def to_query_str(params):
def to_query_str(params, comma_delimited_lists=True):
"""Converts a dictionary of params to a query string.
Args:
@@ -157,6 +157,10 @@ def to_query_str(params):
something that can be converted into a ``str``. If `params`
is a ``list``, it will be converted to a comma-delimited string
of values (e.g., 'thing=1,2,3')
comma_delimited_lists (bool, default ``True``):
If set to ``False`` encode lists by specifying multiple instances
of the parameter (e.g., 'thing=1&thing=2&thing=3')
Returns:
str: A URI query string including the '?' prefix, or an empty string
@@ -175,7 +179,20 @@ def to_query_str(params):
elif v is False:
v = 'false'
elif isinstance(v, list):
v = ','.join(map(str, v))
if comma_delimited_lists:
v = ','.join(map(str, v))
else:
for list_value in v:
if list_value is True:
list_value = 'true'
elif list_value is False:
list_value = 'false'
else:
list_value = str(list_value)
query_str += k + '=' + list_value + '&'
continue
else:
v = str(v)

19
falcon/util/uri.py
View File

@@ -246,11 +246,12 @@ else:
return decoded_uri.decode('utf-8', 'replace')
def parse_query_string(query_string, keep_blank_qs_values=False):
def parse_query_string(query_string, keep_blank_qs_values=False,
parse_qs_csv=True):
"""Parse a query string into a dict.
Query string parameters are assumed to use standard form-encoding. Only
parameters with values are parsed. for example, given 'foo=bar&flag',
parameters with values are returned. For example, given 'foo=bar&flag',
this function would ignore 'flag' unless the `keep_blank_qs_values` option
is set.
@@ -269,8 +270,16 @@ def parse_query_string(query_string, keep_blank_qs_values=False):
Args:
query_string (str): The query string to parse.
keep_blank_qs_values (bool): If set to ``True``, preserves boolean
fields and fields with no content as blank strings.
keep_blank_qs_values (bool): Set to ``True`` to return fields even if
they do not have a value (default ``False``). For comma-separated
values, this option also determines whether or not empty elements
in the parsed list are retained.
parse_qs_csv: Set to ``False`` in order to disable splitting query
parameters on ``,`` (default ``True``). Depending on the user agent,
encoding lists as multiple occurrences of the same parameter might
be preferable. In this case, setting `parse_qs_csv` to ``False``
will cause the framework to treat commas as literal characters in
each occurring parameter value.
Returns:
dict: A dictionary of (*name*, *value*) pairs, one per query
@@ -309,7 +318,7 @@ def parse_query_string(query_string, keep_blank_qs_values=False):
params[k] = [old_value, decode(v)]
else:
if ',' in v:
if parse_qs_csv and ',' in v:
# NOTE(kgriffs): Falcon supports a more compact form of
# lists, in which the elements are comma-separated and
# assigned to a single param instance. If it turns out that

26
tests/test_options.py
View File

@@ -1,16 +1,32 @@
import ddt
from falcon.request import RequestOptions
import falcon.testing as testing
@ddt.ddt
class TestRequestOptions(testing.TestBase):
def test_correct_options(self):
def test_option_defaults(self):
options = RequestOptions()
self.assertFalse(options.keep_blank_qs_values)
options.keep_blank_qs_values = True
self.assertTrue(options.keep_blank_qs_values)
options.keep_blank_qs_values = False
self.assertFalse(options.keep_blank_qs_values)
self.assertFalse(options.auto_parse_form_urlencoded)
self.assertTrue(options.auto_parse_qs_csv)
@ddt.data(
'keep_blank_qs_values',
'auto_parse_form_urlencoded',
'auto_parse_qs_csv',
)
def test_options_toggle(self, option_name):
options = RequestOptions()
setattr(options, option_name, True)
self.assertTrue(getattr(options, option_name))
setattr(options, option_name, False)
self.assertFalse(getattr(options, option_name))
def test_incorrect_options(self):
options = RequestOptions()

54
tests/test_query_params.py
View File

@@ -65,6 +65,60 @@ class _TestQueryParams(testing.TestBase):
self.assertEqual(req.get_param_as_list('id', int), [23, 42])
self.assertEqual(req.get_param('q'), u'\u8c46 \u74e3')
def test_option_auto_parse_qs_csv_simple_false(self):
self.api.req_options.auto_parse_qs_csv = False
query_string = 'id=23,42,,&id=2'
self.simulate_request('/', query_string=query_string)
req = self.resource.req
self.assertEqual(req.params['id'], [u'23,42,,', u'2'])
self.assertIn(req.get_param('id'), [u'23,42,,', u'2'])
self.assertEqual(req.get_param_as_list('id'), [u'23,42,,', u'2'])
def test_option_auto_parse_qs_csv_simple_true(self):
self.api.req_options.auto_parse_qs_csv = True
query_string = 'id=23,42,,&id=2'
self.simulate_request('/', query_string=query_string)
req = self.resource.req
self.assertEqual(req.params['id'], [u'23', u'42', u'2'])
self.assertIn(req.get_param('id'), [u'23', u'42', u'2'])
self.assertEqual(req.get_param_as_list('id', int), [23, 42, 2])
def test_option_auto_parse_qs_csv_complex_false(self):
self.api.req_options.auto_parse_qs_csv = False
encoded_json = '%7B%22msg%22:%22Testing%201,2,3...%22,%22code%22:857%7D'
decoded_json = '{"msg":"Testing 1,2,3...","code":857}'
query_string = ('colors=red,green,blue&limit=1'
'&list-ish1=f,,x&list-ish2=,0&list-ish3=a,,,b'
'&empty1=&empty2=,&empty3=,,'
'&thing=' + encoded_json)
self.simulate_request('/', query_string=query_string)
req = self.resource.req
self.assertIn(req.get_param('colors'), 'red,green,blue')
self.assertEqual(req.get_param_as_list('colors'), [u'red,green,blue'])
self.assertEqual(req.get_param_as_list('limit'), ['1'])
self.assertEqual(req.get_param_as_list('empty1'), None)
self.assertEqual(req.get_param_as_list('empty2'), [u','])
self.assertEqual(req.get_param_as_list('empty3'), [u',,'])
self.assertEqual(req.get_param_as_list('list-ish1'), [u'f,,x'])
self.assertEqual(req.get_param_as_list('list-ish2'), [u',0'])
self.assertEqual(req.get_param_as_list('list-ish3'), [u'a,,,b'])
self.assertEqual(req.get_param('thing'), decoded_json)
def test_bad_percentage(self):
query_string = 'x=%%20%+%&y=peregrine&z=%a%z%zz%1%20e'
self.simulate_request('/', query_string=query_string)

10
tests/test_utils.py
View File

@@ -128,6 +128,16 @@ class TestFalconUtils(testtools.TestCase):
falcon.to_query_str({'things': ['a', 'b']}),
'?things=a,b')
expected = ('?things=a&things=b&things=&things=None'
'&things=true&things=false&things=0')
actual = falcon.to_query_str(
{'things': ['a', 'b', '', None, True, False, 0]},
comma_delimited_lists=False
)
self.assertEqual(actual, expected)
def test_pack_query_params_several(self):
garbage_in = {
'limit': 17,