Chunked readline fixes by schmir, ported from gevent by redbo.
This commit is contained in:
Ryan Williams
@@ -90,39 +90,54 @@ class Input(object):
|
||||
self.position += len(read)
|
||||
return read
|
||||
|
||||
def _chunked_read(self, rfile, length=None):
|
||||
def _chunked_read(self, rfile, length=None, use_readline=False):
|
||||
if self.wfile is not None:
|
||||
## 100 Continue
|
||||
self.wfile.write(self.wfile_line)
|
||||
self.wfile = None
|
||||
self.wfile_line = None
|
||||
|
||||
response = []
|
||||
try:
|
||||
if length is None:
|
||||
if self.chunk_length > self.position:
|
||||
response.append(rfile.read(self.chunk_length - self.position))
|
||||
while self.chunk_length != 0:
|
||||
self.chunk_length = int(rfile.readline(), 16)
|
||||
response.append(rfile.read(self.chunk_length))
|
||||
rfile.readline()
|
||||
if length == 0:
|
||||
return ""
|
||||
|
||||
if length < 0:
|
||||
length = None
|
||||
|
||||
if use_readline:
|
||||
reader = self.rfile.readline
|
||||
else:
|
||||
while length > 0 and self.chunk_length != 0:
|
||||
if self.chunk_length > self.position:
|
||||
response.append(rfile.read(
|
||||
min(self.chunk_length - self.position, length)))
|
||||
last_read = len(response[-1])
|
||||
if last_read == 0:
|
||||
reader = self.rfile.read
|
||||
|
||||
response = []
|
||||
while self.chunk_length != 0:
|
||||
maxreadlen = self.chunk_length - self.position
|
||||
if length is not None and length < maxreadlen:
|
||||
maxreadlen = length
|
||||
|
||||
if maxreadlen > 0:
|
||||
data = reader(maxreadlen)
|
||||
if not data:
|
||||
self.chunk_length = 0
|
||||
raise IOError("unexpected end of file while parsing chunked data")
|
||||
|
||||
datalen = len(data)
|
||||
response.append(data)
|
||||
|
||||
self.position += datalen
|
||||
if self.chunk_length == self.position:
|
||||
rfile.readline()
|
||||
|
||||
if length is not None:
|
||||
length -= datalen
|
||||
if length == 0:
|
||||
break
|
||||
length -= last_read
|
||||
self.position += last_read
|
||||
if self.chunk_length == self.position:
|
||||
rfile.readline()
|
||||
else:
|
||||
self.chunk_length = int(rfile.readline(), 16)
|
||||
self.position = 0
|
||||
if not self.chunk_length:
|
||||
rfile.readline()
|
||||
if use_readline and data[-1] == "\n":
|
||||
break
|
||||
else:
|
||||
self.chunk_length = int(rfile.readline().split(";", 1)[0], 16)
|
||||
self.position = 0
|
||||
if self.chunk_length == 0:
|
||||
rfile.readline()
|
||||
except greenio.SSL.ZeroReturnError:
|
||||
pass
|
||||
return ''.join(response)
|
||||
@@ -133,7 +148,10 @@ class Input(object):
|
||||
return self._do_read(self.rfile.read, length)
|
||||
|
||||
def readline(self, size=None):
|
||||
return self._do_read(self.rfile.readline)
|
||||
if self.chunked_input:
|
||||
return self._chunked_read(self.rfile, size, True)
|
||||
else:
|
||||
return self._do_read(self.rfile.readline, size)
|
||||
|
||||
def readlines(self, hint=None):
|
||||
return self._do_read(self.rfile.readlines, hint)
|
||||
@@ -348,8 +366,9 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler):
|
||||
finally:
|
||||
if hasattr(result, 'close'):
|
||||
result.close()
|
||||
if (self.environ['eventlet.input'].position
|
||||
< self.environ.get('CONTENT_LENGTH', 0)):
|
||||
if (self.environ['eventlet.input'].chunked_input or
|
||||
self.environ['eventlet.input'].position \
|
||||
< self.environ['eventlet.input'].content_length):
|
||||
## Read and discard body if there was no pending 100-continue
|
||||
if not self.environ['eventlet.input'].wfile:
|
||||
while self.environ['eventlet.input'].read(MINIMUM_CHUNK_SIZE):
|
||||
|
||||
@@ -145,7 +145,6 @@ def read_http(sock):
|
||||
if CONTENT_LENGTH in headers:
|
||||
num = int(headers[CONTENT_LENGTH])
|
||||
body = fd.read(num)
|
||||
#print body
|
||||
else:
|
||||
# read until EOF
|
||||
body = fd.read()
|
||||
@@ -845,8 +844,13 @@ class TestHttpd(_TestBase):
|
||||
|
||||
def test_aborted_chunked_post(self):
|
||||
read_content = event.Event()
|
||||
blew_up = [False]
|
||||
def chunk_reader(env, start_response):
|
||||
content = env['wsgi.input'].read(1024)
|
||||
try:
|
||||
content = env['wsgi.input'].read(1024)
|
||||
except IOError:
|
||||
blew_up[0] = True
|
||||
content = 'ok'
|
||||
read_content.send(content)
|
||||
start_response('200 OK', [('Content-Type', 'text/plain')])
|
||||
return [content]
|
||||
@@ -863,7 +867,8 @@ class TestHttpd(_TestBase):
|
||||
sock.close()
|
||||
# the test passes if we successfully get here, and read all the data
|
||||
# in spite of the early close
|
||||
self.assertEqual(read_content.wait(), expected_body)
|
||||
self.assertEqual(read_content.wait(), 'ok')
|
||||
self.assert_(blew_up[0])
|
||||
|
||||
def read_headers(sock):
|
||||
fd = sock.makefile()
|
||||
@@ -908,7 +913,6 @@ class IterableAlreadyHandledTest(_TestBase):
|
||||
|
||||
fd.flush()
|
||||
response_line, headers = read_headers(sock)
|
||||
print headers
|
||||
self.assertEqual(response_line, 'HTTP/1.1 200 OK\r\n')
|
||||
self.assert_('connection' not in headers)
|
||||
fd.write('GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n')
|
||||
@@ -918,6 +922,125 @@ class IterableAlreadyHandledTest(_TestBase):
|
||||
self.assertEqual(headers.get('transfer-encoding'), 'chunked')
|
||||
self.assertEqual(body, '0\r\n\r\n') # Still coming back chunked
|
||||
|
||||
class TestChunkedInput(_TestBase):
|
||||
dirt = ""
|
||||
validator = None
|
||||
def application(self, env, start_response):
|
||||
input = env['wsgi.input']
|
||||
response = []
|
||||
|
||||
pi = env["PATH_INFO"]
|
||||
|
||||
if pi=="/short-read":
|
||||
d=input.read(10)
|
||||
response = [d]
|
||||
elif pi=="/lines":
|
||||
for x in input:
|
||||
response.append(x)
|
||||
elif pi=="/ping":
|
||||
input.read()
|
||||
response.append("pong")
|
||||
else:
|
||||
raise RuntimeError("bad path")
|
||||
|
||||
start_response('200 OK', [('Content-Type', 'text/plain')])
|
||||
return response
|
||||
|
||||
def connect(self):
|
||||
return eventlet.connect(('localhost', self.port))
|
||||
|
||||
def set_site(self):
|
||||
self.site = Site()
|
||||
self.site.application = self.application
|
||||
|
||||
def chunk_encode(self, chunks, dirt=None):
|
||||
if dirt is None:
|
||||
dirt = self.dirt
|
||||
|
||||
b = ""
|
||||
for c in chunks:
|
||||
b += "%x%s\r\n%s\r\n" % (len(c), dirt, c)
|
||||
return b
|
||||
|
||||
def body(self, dirt=None):
|
||||
return self.chunk_encode(["this", " is ", "chunked", "\nline", " 2", "\n", "line3", ""], dirt=dirt)
|
||||
|
||||
def ping(self, fd):
|
||||
fd.sendall("GET /ping HTTP/1.1\r\n\r\n")
|
||||
self.assertEquals(read_http(fd)[-1], "pong")
|
||||
|
||||
def test_short_read_with_content_length(self):
|
||||
body = self.body()
|
||||
req = "POST /short-read HTTP/1.1\r\ntransfer-encoding: Chunked\r\nContent-Length:1000\r\n\r\n" + body
|
||||
|
||||
fd = self.connect()
|
||||
fd.sendall(req)
|
||||
self.assertEquals(read_http(fd)[-1], "this is ch")
|
||||
|
||||
self.ping(fd)
|
||||
|
||||
def test_short_read_with_zero_content_length(self):
|
||||
body = self.body()
|
||||
req = "POST /short-read HTTP/1.1\r\ntransfer-encoding: Chunked\r\nContent-Length:0\r\n\r\n" + body
|
||||
fd = self.connect()
|
||||
fd.sendall(req)
|
||||
self.assertEquals(read_http(fd)[-1], "this is ch")
|
||||
|
||||
self.ping(fd)
|
||||
|
||||
def test_short_read(self):
|
||||
body = self.body()
|
||||
req = "POST /short-read HTTP/1.1\r\ntransfer-encoding: Chunked\r\n\r\n" + body
|
||||
|
||||
fd = self.connect()
|
||||
fd.sendall(req)
|
||||
self.assertEquals(read_http(fd)[-1], "this is ch")
|
||||
|
||||
self.ping(fd)
|
||||
|
||||
def test_dirt(self):
|
||||
body = self.body(dirt="; here is dirt\0bla")
|
||||
req = "POST /ping HTTP/1.1\r\ntransfer-encoding: Chunked\r\n\r\n" + body
|
||||
|
||||
fd = self.connect()
|
||||
fd.sendall(req)
|
||||
self.assertEquals(read_http(fd)[-1], "pong")
|
||||
|
||||
self.ping(fd)
|
||||
|
||||
def test_chunked_readline(self):
|
||||
body = self.body()
|
||||
req = "POST /lines HTTP/1.1\r\nContent-Length: %s\r\ntransfer-encoding: Chunked\r\n\r\n%s" % (len(body), body)
|
||||
|
||||
fd = self.connect()
|
||||
fd.sendall(req)
|
||||
self.assertEquals(read_http(fd)[-1], 'this is chunked\nline 2\nline3')
|
||||
|
||||
def test_close_before_finished(self):
|
||||
import signal
|
||||
|
||||
got_signal = []
|
||||
def handler(*args):
|
||||
got_signal.append(1)
|
||||
raise KeyboardInterrupt()
|
||||
|
||||
signal.signal(signal.SIGALRM, handler)
|
||||
signal.alarm(1)
|
||||
|
||||
try:
|
||||
body = '4\r\nthi'
|
||||
req = "POST /short-read HTTP/1.1\r\ntransfer-encoding: Chunked\r\n\r\n" + body
|
||||
|
||||
fd = self.connect()
|
||||
fd.sendall(req)
|
||||
fd.close()
|
||||
eventlet.sleep(0.0)
|
||||
finally:
|
||||
signal.alarm(0)
|
||||
signal.signal(signal.SIGALRM, signal.SIG_DFL)
|
||||
|
||||
assert not got_signal, "caught alarm signal. infinite loop detected."
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in New Issue
Block a user