Fix stable gate
This is a combination of 2 commits. --- py2/3: Stop using stdlib's putrequest(); it only does ASCII Note that this only affects the functest client. See also: https://bugs.python.org/issue36274 and https://bugs.python.org/issue38216 This was previously done just for py3 compatibility, but following https://github.com/python/cpython/commit/bb8071a our stable gates are all broken -- apparently, they're running a 2.7 pre-release? (cherry picked from commitc0ae48ba9a
) (cherry picked from commit2b4d58952c
) --- bufferedhttp: ensure query params are properly quoted Recent versions of py27 [1] have begun raising InvalidURL if you try to include non-ASCII characters in the request path. This was observed recently in the periodic checks of stable/ocata and stable/pike. In particular, we would spin up some in-process servers in test.unit.proxy.test_server.TestSocketObjectVersions and do a container listing with a prefix param that included raw (unquoted) UTF-8. This query string would pass unmolested through the proxy, tripping the InvalidURL error when bufferedhttp called putrequest. More recent versions of Swift would not exhibit this particular failure, as the listing_formats middleware would force a decoding/re-encoding of the query string for account and container requests. However, object requests with errant query strings would likely be able to trip the same error. Swift on py3 should not exhibit this behavior, as we so thoroughly re-write the request line to avoid hitting https://bugs.python.org/issue33973. Now, always parse and re-encode the query string in bufferedhttp. This prevents any errors on object requests and cleans up any callers that might use bufferedhttp directly. [1] Anything after https://github.com/python/cpython/commit/bb8071a; see https://bugs.python.org/issue30458 Depends-On: https://review.opendev.org/684769 Closes-Bug: 1843816 Related-Change: Id3ce37aa0402e2d8dd5784ce329d7cb4fbaf700d Related-Change: Ie648f5c04d4415f3b620fb196fa567ce7575d522 (cherry picked from commit49f62f6ab7
) (cherry picked from commit9cc6d41389
) --- Change-Id: I4eafc5f057df8a3c15560ace255d05602db56ef6
This commit is contained in:
parent
fbfb89bebd
commit
1dd0e1ca4b
|
@ -34,7 +34,7 @@ import socket
|
|||
import eventlet
|
||||
from eventlet.green.httplib import CONTINUE, HTTPConnection, HTTPMessage, \
|
||||
HTTPResponse, HTTPSConnection, _UNKNOWN
|
||||
from six.moves.urllib.parse import quote
|
||||
from six.moves.urllib.parse import quote, parse_qsl, urlencode
|
||||
import six
|
||||
|
||||
if six.PY2:
|
||||
|
@ -233,6 +233,15 @@ def http_connect_raw(ipaddr, port, method, path, headers=None,
|
|||
else:
|
||||
conn = BufferedHTTPConnection('%s:%s' % (ipaddr, port))
|
||||
if query_string:
|
||||
# Round trip to ensure proper quoting
|
||||
if six.PY2:
|
||||
query_string = urlencode(parse_qsl(
|
||||
query_string, keep_blank_values=True))
|
||||
else:
|
||||
query_string = urlencode(
|
||||
parse_qsl(query_string, keep_blank_values=True,
|
||||
encoding='latin1'),
|
||||
encoding='latin1')
|
||||
path += '?' + query_string
|
||||
conn.path = path
|
||||
conn.putrequest(method, path, skip_host=(headers and 'Host' in headers))
|
||||
|
|
|
@ -104,6 +104,91 @@ def listing_items(method):
|
|||
items = []
|
||||
|
||||
|
||||
def putrequest(self, method, url, skip_host=False, skip_accept_encoding=False):
|
||||
'''Send a request to the server.
|
||||
|
||||
This is mostly a regurgitation of CPython's HTTPConnection.putrequest,
|
||||
but fixed up so we can still send arbitrary bytes in the request line
|
||||
on py3. See also: https://bugs.python.org/issue36274
|
||||
|
||||
To use, swap out a HTTP(S)Connection's putrequest with something like::
|
||||
|
||||
conn.putrequest = putrequest.__get__(conn)
|
||||
|
||||
:param method: specifies an HTTP request method, e.g. 'GET'.
|
||||
:param url: specifies the object being requested, e.g. '/index.html'.
|
||||
:param skip_host: if True does not add automatically a 'Host:' header
|
||||
:param skip_accept_encoding: if True does not add automatically an
|
||||
'Accept-Encoding:' header
|
||||
'''
|
||||
# (Mostly) inline the HTTPConnection implementation; just fix it
|
||||
# so we can send non-ascii request lines. For comparison, see
|
||||
# https://github.com/python/cpython/blob/v2.7.16/Lib/httplib.py#L888-L1003
|
||||
# and https://github.com/python/cpython/blob/v3.7.2/
|
||||
# Lib/http/client.py#L1061-L1183
|
||||
if self._HTTPConnection__response \
|
||||
and self._HTTPConnection__response.isclosed():
|
||||
self._HTTPConnection__response = None
|
||||
|
||||
if self._HTTPConnection__state == http_client._CS_IDLE:
|
||||
self._HTTPConnection__state = http_client._CS_REQ_STARTED
|
||||
else:
|
||||
raise http_client.CannotSendRequest(self._HTTPConnection__state)
|
||||
|
||||
self._method = method
|
||||
if not url:
|
||||
url = '/'
|
||||
self._path = url
|
||||
request = '%s %s %s' % (method, url, self._http_vsn_str)
|
||||
if not isinstance(request, bytes):
|
||||
# This choice of encoding is the whole reason we copy/paste from
|
||||
# cpython. When making backend requests, it should never be
|
||||
# necessary; however, we have some functional tests that want
|
||||
# to send non-ascii bytes.
|
||||
# TODO: when https://bugs.python.org/issue36274 is resolved, make
|
||||
# sure we fix up our API to match whatever upstream chooses to do
|
||||
self._output(request.encode('latin1'))
|
||||
else:
|
||||
self._output(request)
|
||||
|
||||
if self._http_vsn == 11:
|
||||
if not skip_host:
|
||||
netloc = ''
|
||||
if url.startswith('http'):
|
||||
nil, netloc, nil, nil, nil = urllib.parse.urlsplit(url)
|
||||
|
||||
if netloc:
|
||||
try:
|
||||
netloc_enc = netloc.encode("ascii")
|
||||
except UnicodeEncodeError:
|
||||
netloc_enc = netloc.encode("idna")
|
||||
self.putheader('Host', netloc_enc)
|
||||
else:
|
||||
if self._tunnel_host:
|
||||
host = self._tunnel_host
|
||||
port = self._tunnel_port
|
||||
else:
|
||||
host = self.host
|
||||
port = self.port
|
||||
|
||||
try:
|
||||
host_enc = host.encode("ascii")
|
||||
except UnicodeEncodeError:
|
||||
host_enc = host.encode("idna")
|
||||
|
||||
if host.find(':') >= 0:
|
||||
host_enc = b'[' + host_enc + b']'
|
||||
|
||||
if port == self.default_port:
|
||||
self.putheader('Host', host_enc)
|
||||
else:
|
||||
host_enc = host_enc.decode("ascii")
|
||||
self.putheader('Host', "%s:%s" % (host_enc, port))
|
||||
|
||||
if not skip_accept_encoding:
|
||||
self.putheader('Accept-Encoding', 'identity')
|
||||
|
||||
|
||||
class Connection(object):
|
||||
def __init__(self, config):
|
||||
for key in 'auth_host auth_port auth_ssl username password'.split():
|
||||
|
@ -125,6 +210,7 @@ class Connection(object):
|
|||
self.storage_host = None
|
||||
self.storage_port = None
|
||||
self.storage_url = None
|
||||
self.connection = None # until you call .http_connect()/.put_start()
|
||||
|
||||
self.conn_class = None
|
||||
|
||||
|
@ -209,6 +295,7 @@ class Connection(object):
|
|||
self.connection = self.conn_class(self.storage_host,
|
||||
port=self.storage_port)
|
||||
# self.connection.set_debuglevel(3)
|
||||
self.connection.putrequest = putrequest.__get__(self.connection)
|
||||
|
||||
def make_path(self, path=None, cfg=None):
|
||||
if path is None:
|
||||
|
@ -338,6 +425,7 @@ class Connection(object):
|
|||
self.connection = self.conn_class(self.storage_host,
|
||||
port=self.storage_port)
|
||||
# self.connection.set_debuglevel(3)
|
||||
self.connection.putrequest = putrequest.__get__(self.connection)
|
||||
self.connection.putrequest('PUT', path)
|
||||
for key, value in headers.items():
|
||||
self.connection.putheader(key, value)
|
||||
|
|
|
@ -59,7 +59,8 @@ class TestBufferedHTTP(unittest.TestCase):
|
|||
fp.flush()
|
||||
self.assertEqual(
|
||||
fp.readline(),
|
||||
'PUT /dev/%s/path/..%%25/?omg&no=%%7f HTTP/1.1\r\n' %
|
||||
'PUT /dev/%s/path/..%%25/?omg=&no=%%7F&%%FF=%%FF'
|
||||
'&no=%%25ff HTTP/1.1\r\n' %
|
||||
expected_par)
|
||||
headers = {}
|
||||
line = fp.readline()
|
||||
|
@ -82,7 +83,7 @@ class TestBufferedHTTP(unittest.TestCase):
|
|||
'PUT', '/path/..%/', {
|
||||
'content-length': 7,
|
||||
'x-header': 'value'},
|
||||
query_string='omg&no=%7f')
|
||||
query_string='omg&no=%7f&\xff=%ff&no=%25ff')
|
||||
conn.send('REQUEST\r\n')
|
||||
self.assertTrue(conn.sock.getsockopt(socket.IPPROTO_TCP,
|
||||
socket.TCP_NODELAY))
|
||||
|
|
Loading…
Reference in New Issue