wsgi: capitalize_response_headers option

Setting this to False provides compatibility with broken clients
which expect response header names in particular case, such as
ETag by AWS Java SDK.
https://github.com/eventlet/eventlet/issues/80

eventlet/wsgi.py

@@ -203,6 +203,7 @@ class FileObjectForHeaders(object):
 class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler):
     protocol_version = 'HTTP/1.1'
     minimum_chunk_size = MINIMUM_CHUNK_SIZE
+    capitalize_response_headers = True
 
     def setup(self):
         # overriding SocketServer.setup to correctly handle SSL.Connection objects
@@ -378,11 +379,16 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler):
                     # Avoid dangling circular ref
                     exc_info = None
 
-            capitalized_headers = [('-'.join([x.capitalize()
+            # Response headers capitalization
-                                              for x in key.split('-')]), value)
+            # CONTent-TYpe: TExt/PlaiN -> Content-Type: TExt/PlaiN
+            # Per HTTP RFC standard, header name is case-insensitive.
+            # Please, fix your client to ignore header case if possible.
+            if self.capitalize_response_headers:
+                response_headers = [
+                    ('-'.join([x.capitalize() for x in key.split('-')]), value)
                     for key, value in response_headers]
 
-            headers_set[:] = [status, capitalized_headers]
+            headers_set[:] = [status, response_headers]
             return write
 
         try:
@@ -392,9 +398,12 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler):
                     or isinstance(getattr(result, '_obj', None), _AlreadyHandled)):
                     self.close_connection = 1
                     return
+
+                # Set content-length if possible
                 if not headers_sent and hasattr(result, '__len__') and \
                         'Content-Length' not in [h for h, _v in headers_set[1]]:
                     headers_set[1].append(('Content-Length', str(sum(map(len, result)))))
+
                 towrite = []
                 towrite_size = 0
                 just_written_size = 0
@@ -544,7 +553,8 @@ class Server(BaseHTTPServer.HTTPServer):
                  log_format=DEFAULT_LOG_FORMAT,
                  url_length_limit=MAX_REQUEST_LINE,
                  debug=True,
-                 socket_timeout=None):
+                 socket_timeout=None,
+                 capitalize_response_headers=True):
 
         self.outstanding_requests = 0
         self.socket = socket
@@ -566,6 +576,14 @@ class Server(BaseHTTPServer.HTTPServer):
         self.url_length_limit = url_length_limit
         self.debug = debug
         self.socket_timeout = socket_timeout
+        self.capitalize_response_headers = capitalize_response_headers
+
+        if not self.capitalize_response_headers:
+            warnings.warn("""capitalize_response_headers is disabled.
+ Please, make sure you know what you are doing.
+ HTTP headers names are case-insensitive per RFC standard.
+ Most likely, you need to fix HTTP parsing in your client software.""",
+                DeprecationWarning, stacklevel=3)
 
     def get_environ(self):
         d = {
@@ -592,6 +610,7 @@ class Server(BaseHTTPServer.HTTPServer):
         proto = types.InstanceType(self.protocol)
         if self.minimum_chunk_size is not None:
             proto.minimum_chunk_size = self.minimum_chunk_size
+        proto.capitalize_response_headers = self.capitalize_response_headers
         try:
             proto.__init__(sock, address, self)
         except socket.timeout:
@@ -630,7 +649,8 @@ def server(sock, site,
            log_format=DEFAULT_LOG_FORMAT,
            url_length_limit=MAX_REQUEST_LINE,
            debug=True,
-           socket_timeout=None):
+           socket_timeout=None,
+           capitalize_response_headers=True):
     """Start up a WSGI server handling requests from the supplied server
     socket.  This function loops forever.  The *sock* object will be closed after server exits,
     but the underlying file descriptor will remain open, so if you have a dup() of *sock*,
@@ -653,6 +673,7 @@ def server(sock, site,
     :param url_length_limit: A maximum allowed length of the request url. If exceeded, 414 error is returned.
     :param debug: True if the server should send exception tracebacks to the clients on 500 errors.  If False, the server will respond with empty bodies.
     :param socket_timeout: Timeout for client connections' socket operations. Default None means wait forever.
+    :param capitalize_response_headers: Normalize response headers' names to Foo-Bar. Default is True.
     """
     serv = Server(sock, sock.getsockname(),
                   site, log,
@@ -667,6 +688,7 @@ def server(sock, site,
                   url_length_limit=url_length_limit,
                   debug=debug,
                   socket_timeout=socket_timeout,
+                  capitalize_response_headers=capitalize_response_headers,
                   )
     if server_event is not None:
         server_event.send(serv)

tests/wsgi_test.py

@@ -1,4 +1,5 @@
 import cgi
+import collections
 from eventlet import greenthread
 import eventlet
 import errno
@@ -28,6 +29,11 @@ except ImportError:
     from StringIO import StringIO
 
 
+HttpReadResult = collections.namedtuple(
+    'HttpReadResult',
+    'status headers_lower body headers_original')
+
+
 def hello_world(env, start_response):
     if env['PATH_INFO'] == 'notexist':
         start_response('404 Not Found', [('Content-type', 'text/plain')])
@@ -147,7 +153,7 @@ class ConnectionClosed(Exception):
 def read_http(sock):
     fd = sock.makefile()
     try:
-        response_line = fd.readline()
+        response_line = fd.readline().rstrip('\r\n')
     except socket.error as exc:
         if get_errno(exc) == 10053:
             raise ConnectionClosed
@@ -155,7 +161,6 @@ def read_http(sock):
     if not response_line:
         raise ConnectionClosed(response_line)
 
-
     header_lines = []
     while True:
         line = fd.readline()
@@ -164,24 +169,37 @@ def read_http(sock):
         else:
             header_lines.append(line)
 
-    headers = dict()
+    headers_original = {}
+    headers_lower = {}
     for x in header_lines:
         x = x.strip()
         if not x:
             continue
         key, value = x.split(':', 1)
-        assert key.lower() not in headers, "%s header duplicated" % key
+        key = key.rstrip()
-        headers[key.lower()] = value
+        value = value.lstrip()
+        key_lower = key.lower()
+        # FIXME: Duplicate headers are allowed as per HTTP RFC standard,
+        # the client and/or intermediate proxies are supposed to treat them
+        # as a single header with values concatenated using space (' ') delimiter.
+        assert key_lower not in headers_lower, "header duplicated: {0}".format(key)
+        headers_original[key] = value
+        headers_lower[key_lower] = value
 
-
+    content_length_str = headers_lower.get(CONTENT_LENGTH.lower(), '')
-    if CONTENT_LENGTH in headers:
+    if content_length_str:
-        num = int(headers[CONTENT_LENGTH])
+        num = int(content_length_str)
         body = fd.read(num)
     else:
         # read until EOF
         body = fd.read()
 
-    return response_line, headers, body
+    result = HttpReadResult(
+        status=response_line,
+        headers_lower=headers_lower,
+        body=body,
+        headers_original=headers_original)
+    return result
 
 
 class _TestBase(LimitedTestCase):
@@ -341,8 +359,8 @@ class TestHttpd(_TestBase):
 
         # send some junk after the actual request
         fd.write('01234567890123456789')
-        reqline, headers, body = read_http(sock)
+        result = read_http(sock)
-        self.assertEqual(body, 'a is a, body is a=a')
+        self.assertEqual(result.body, 'a is a, body is a=a')
         fd.close()
 
     def test_008_correctresponse(self):
@@ -352,14 +370,14 @@ class TestHttpd(_TestBase):
         fd = sock.makefile('w')
         fd.write('GET / HTTP/1.1\r\nHost: localhost\r\n\r\n')
         fd.flush()
-        response_line_200,_,_ = read_http(sock)
+        result_200 = read_http(sock)
         fd.write('GET /notexist HTTP/1.1\r\nHost: localhost\r\n\r\n')
         fd.flush()
-        response_line_404,_,_ = read_http(sock)
+        result_404 = read_http(sock)
         fd.write('GET / HTTP/1.1\r\nHost: localhost\r\n\r\n')
         fd.flush()
-        response_line_test,_,_ = read_http(sock)
+        result_test = read_http(sock)
-        self.assertEqual(response_line_200,response_line_test)
+        self.assertEqual(result_200.status, result_test.status)
         fd.close()
         sock.close()
 
@@ -496,16 +514,16 @@ class TestHttpd(_TestBase):
         fd = sock.makefile('w')
         fd.write('GET /a HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n')
         fd.flush()
-        response_line, headers, body = read_http(sock)
+        result1 = read_http(sock)
-        self.assert_('content-length' in headers)
+        self.assert_('content-length' in result1.headers_lower)
 
         sock = eventlet.connect(('localhost', self.port))
         fd = sock.makefile('w')
         fd.write('GET /b HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n')
         fd.flush()
-        response_line, headers, body = read_http(sock)
+        result2 = read_http(sock)
-        self.assert_('transfer-encoding' in headers)
+        self.assert_('transfer-encoding' in result2.headers_lower)
-        self.assert_(headers['transfer-encoding'] == 'chunked')
+        self.assert_(result2.headers_lower['transfer-encoding'] == 'chunked')
 
     def test_016_repeated_content_length(self):
         """
@@ -576,15 +594,15 @@ class TestHttpd(_TestBase):
         fd.write('GET / HTTP/1.0\r\nHost: localhost\r\nConnection: keep-alive\r\n\r\n')
         fd.flush()
 
-        response_line, headers, body = read_http(sock)
+        result1 = read_http(sock)
-        self.assert_('connection' in headers)
+        self.assert_('connection' in result1.headers_lower)
-        self.assertEqual('keep-alive', headers['connection'])
+        self.assertEqual('keep-alive', result1.headers_lower['connection'])
         # repeat request to verify connection is actually still open
         fd.write('GET / HTTP/1.0\r\nHost: localhost\r\nConnection: keep-alive\r\n\r\n')
         fd.flush()
-        response_line, headers, body = read_http(sock)
+        result2 = read_http(sock)
-        self.assert_('connection' in headers)
+        self.assert_('connection' in result2.headers_lower)
-        self.assertEqual('keep-alive', headers['connection'])
+        self.assertEqual('keep-alive', result2.headers_lower['connection'])
         sock.close()
 
     def test_019_fieldstorage_compat(self):
@@ -728,9 +746,9 @@ class TestHttpd(_TestBase):
         fd = sock.makefile('rw')
         fd.write('PUT / HTTP/1.1\r\nHost: localhost\r\nContent-length: 1025\r\nExpect: 100-continue\r\n\r\n')
         fd.flush()
-        response_line, headers, body = read_http(sock)
+        result = read_http(sock)
-        self.assert_(response_line.startswith('HTTP/1.1 417 Expectation Failed'))
+        self.assertEquals(result.status, 'HTTP/1.1 417 Expectation Failed')
-        self.assertEquals(body, 'failure')
+        self.assertEquals(result.body, 'failure')
         fd.write('PUT / HTTP/1.1\r\nHost: localhost\r\nContent-length: 7\r\nExpect: 100-continue\r\n\r\ntesting')
         fd.flush()
         header_lines = []
@@ -794,10 +812,10 @@ class TestHttpd(_TestBase):
 
         sock.sendall('GET / HTTP/1.0\r\nHost: localhost\r\nConnection: keep-alive\r\n\r\n')
 
-        response_line, headers, body = read_http(sock)
+        result = read_http(sock)
-        self.assertEqual(headers['connection'], 'close')
+        self.assertEqual(result.headers_lower['connection'], 'close')
-        self.assertNotEqual(headers.get('transfer-encoding'), 'chunked')
+        self.assertNotEqual(result.headers_lower.get('transfer-encoding'), 'chunked')
-        self.assertEquals(body, "thisischunked")
+        self.assertEquals(result.body, "thisischunked")
 
     def test_minimum_chunk_size_parameter_leaves_httpprotocol_class_member_intact(self):
         start_size = wsgi.HttpProtocol.minimum_chunk_size
@@ -819,10 +837,10 @@ class TestHttpd(_TestBase):
 
         sock.sendall('GET / HTTP/1.1\r\nHost: localhost\r\n\r\n')
 
-        response_line, headers, body = read_http(sock)
+        result = read_http(sock)
-        self.assertEqual(response_line, 'HTTP/1.1 200 OK\r\n')
+        self.assertEqual(result.status, 'HTTP/1.1 200 OK')
-        self.assertEqual(headers.get('transfer-encoding'), 'chunked')
+        self.assertEqual(result.headers_lower.get('transfer-encoding'), 'chunked')
-        self.assertEqual(body, '27\r\nThe dwarves of yore made mighty spells,\r\n25\r\nWhile hammers fell like ringing bells\r\n')
+        self.assertEqual(result.body, '27\r\nThe dwarves of yore made mighty spells,\r\n25\r\nWhile hammers fell like ringing bells\r\n')
 
         # verify that socket is closed by server
         self.assertEqual(sock.recv(1), '')
@@ -835,8 +853,8 @@ class TestHttpd(_TestBase):
             ('localhost', self.port))
 
         sock.sendall('GET / HTTP/1.0\r\nHost: localhost\r\nConnection: keep-alive\r\n\r\n')
-        response_line, headers, body = read_http(sock)
+        result = read_http(sock)
-        self.assertEqual(headers['connection'], 'close')
+        self.assertEqual(result.headers_lower['connection'], 'close')
 
     def test_027_keepalive_chunked(self):
         self.site.application = chunked_post
@@ -954,9 +972,8 @@ class TestHttpd(_TestBase):
         fd = sock.makefile('rw')
         fd.write(request)
         fd.flush()
-        response_line, headers, body = read_http(sock)
+        result = read_http(sock)
-        self.assertEquals(response_line,
+        self.assertEquals(result.status, 'HTTP/1.0 400 Header Line Too Long')
-            'HTTP/1.0 400 Header Line Too Long\r\n')
         fd.close()
 
     def test_031_reject_large_headers(self):
@@ -966,9 +983,8 @@ class TestHttpd(_TestBase):
         fd = sock.makefile('rw')
         fd.write(request)
         fd.flush()
-        response_line, headers, body = read_http(sock)
+        result = read_http(sock)
-        self.assertEquals(response_line,
+        self.assertEquals(result.status, 'HTTP/1.0 400 Headers Too Large')
-            'HTTP/1.0 400 Headers Too Large\r\n')
         fd.close()
 
     def test_032_wsgi_input_as_iterable(self):
@@ -994,8 +1010,8 @@ class TestHttpd(_TestBase):
         fd = sock.makefile('rw')
         fd.write(request)
         fd.flush()
-        response_line, headers, body = read_http(sock)
+        result = read_http(sock)
-        self.assertEquals(body, upload_data)
+        self.assertEquals(result.body, upload_data)
         fd.close()
         self.assertEquals(g[0], 1)
 
@@ -1076,10 +1092,10 @@ class TestHttpd(_TestBase):
         fd = sock.makefile('rw')
         fd.write('GET / HTTP/1.1\r\nHost: localhost\r\n\r\n')
         fd.flush()
-        response_line, headers, body = read_http(sock)
+        result = read_http(sock)
-        self.assert_(response_line.startswith('HTTP/1.1 500 Internal Server Error'))
+        self.assertEqual(result.status, 'HTTP/1.1 500 Internal Server Error')
-        self.assertEqual(headers['connection'], 'close')
+        self.assertEqual(result.headers_lower['connection'], 'close')
-        self.assert_('transfer-encoding' not in headers)
+        self.assert_('transfer-encoding' not in result.headers_lower)
 
     def test_unicode_raises_error(self):
         def wsgi_app(environ, start_response):
@@ -1091,10 +1107,10 @@ class TestHttpd(_TestBase):
         fd = sock.makefile('rw')
         fd.write('GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n')
         fd.flush()
-        response_line, headers, body = read_http(sock)
+        result = read_http(sock)
-        self.assert_(response_line.startswith('HTTP/1.1 500 Internal Server Error'))
+        self.assertEqual(result.status, 'HTTP/1.1 500 Internal Server Error')
-        self.assertEqual(headers['connection'], 'close')
+        self.assertEqual(result.headers_lower['connection'], 'close')
-        self.assert_('unicode' in body)
+        self.assert_('unicode' in result.body)
 
     def test_path_info_decoding(self):
         def wsgi_app(environ, start_response):
@@ -1107,10 +1123,10 @@ class TestHttpd(_TestBase):
         fd.write('GET /a*b@%40%233 HTTP/1.1\r\nHost: localhost\r\nConnection: '\
                 'close\r\n\r\n')
         fd.flush()
-        response_line, headers, body = read_http(sock)
+        result = read_http(sock)
-        self.assert_(response_line.startswith('HTTP/1.1 200'))
+        self.assertEqual(result.status, 'HTTP/1.1 200 OK')
-        self.assert_('decoded: /a*b@@#3' in body)
+        self.assert_('decoded: /a*b@@#3' in result.body)
-        self.assert_('raw: /a*b@%40%233' in body)
+        self.assert_('raw: /a*b@%40%233' in result.body)
 
     def test_ipv6(self):
         try:
@@ -1145,11 +1161,11 @@ class TestHttpd(_TestBase):
         fd = sock.makefile('w')
         fd.write('GET / HTTP/1.1\r\nHost: localhost\r\n\r\n')
         fd.flush()
-        response_line, headers, body = read_http(sock)
+        result1 = read_http(sock)
-        self.assert_(response_line.startswith('HTTP/1.1 500 Internal Server Error'))
+        self.assertEqual(result1.status, 'HTTP/1.1 500 Internal Server Error')
-        self.assertEqual(body, '')
+        self.assertEqual(result1.body, '')
-        self.assertEqual(headers['connection'], 'close')
+        self.assertEqual(result1.headers_lower['connection'], 'close')
-        self.assert_('transfer-encoding' not in headers)
+        self.assert_('transfer-encoding' not in result1.headers_lower)
 
         # verify traceback when debugging enabled
         self.spawn_server(debug=True)
@@ -1158,13 +1174,13 @@ class TestHttpd(_TestBase):
         fd = sock.makefile('w')
         fd.write('GET / HTTP/1.1\r\nHost: localhost\r\n\r\n')
         fd.flush()
-        response_line, headers, body = read_http(sock)
+        result2 = read_http(sock)
-        self.assert_(response_line.startswith('HTTP/1.1 500 Internal Server Error'))
+        self.assertEqual(result2.status, 'HTTP/1.1 500 Internal Server Error')
-        self.assert_('intentional crash' in body)
+        self.assert_('intentional crash' in result2.body)
-        self.assert_('RuntimeError' in body)
+        self.assert_('RuntimeError' in result2.body)
-        self.assert_('Traceback' in body)
+        self.assert_('Traceback' in result2.body)
-        self.assertEqual(headers['connection'], 'close')
+        self.assertEqual(result2.headers_lower['connection'], 'close')
-        self.assert_('transfer-encoding' not in headers)
+        self.assert_('transfer-encoding' not in result2.headers_lower)
 
     def test_client_disconnect(self):
         """Issue #95 Server must handle disconnect from client in the middle of response
@@ -1222,6 +1238,25 @@ class TestHttpd(_TestBase):
         except ConnectionClosed:
             pass
 
+    def test_disable_header_name_capitalization(self):
+        # Disable HTTP header name capitalization
+        #
+        # https://github.com/eventlet/eventlet/issues/80
+        random_case_header = ('eTAg', 'TAg-VAluE')
+        def wsgi_app(environ, start_response):
+            start_response('200 oK', [random_case_header])
+            return ['']
+
+        self.spawn_server(site=wsgi_app, capitalize_response_headers=False)
+
+        sock = eventlet.connect(('localhost', self.port))
+        sock.sendall('GET / HTTP/1.1\r\nHost: localhost\r\n\r\n')
+        result = read_http(sock)
+        sock.close()
+        self.assertEqual(result.status, 'HTTP/1.1 200 oK')
+        self.assertEqual(result.headers_lower[random_case_header[0].lower()], random_case_header[1])
+        self.assertEqual(result.headers_original[random_case_header[0]], random_case_header[1])
+
 
 def read_headers(sock):
     fd = sock.makefile()
@@ -1273,10 +1308,10 @@ class IterableAlreadyHandledTest(_TestBase):
         self.assert_('connection' not in headers)
         fd.write('GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n')
         fd.flush()
-        response_line, headers, body = read_http(sock)
+        result = read_http(sock)
-        self.assertEqual(response_line, 'HTTP/1.1 200 OK\r\n')
+        self.assertEqual(result.status, 'HTTP/1.1 200 OK')
-        self.assertEqual(headers.get('transfer-encoding'), 'chunked')
+        self.assertEqual(result.headers_lower.get('transfer-encoding'), 'chunked')
-        self.assertEqual(body, '0\r\n\r\n') # Still coming back chunked
+        self.assertEqual(result.body, '0\r\n\r\n') # Still coming back chunked
 
 
 class ProxiedIterableAlreadyHandledTest(IterableAlreadyHandledTest):
@@ -1356,7 +1391,7 @@ class TestChunkedInput(_TestBase):
 
     def ping(self, fd):
         fd.sendall("GET /ping HTTP/1.1\r\n\r\n")
-        self.assertEquals(read_http(fd)[-1], "pong")
+        self.assertEquals(read_http(fd).body, "pong")
 
     def test_short_read_with_content_length(self):
         body = self.body()
@@ -1364,7 +1399,7 @@ class TestChunkedInput(_TestBase):
 
         fd = self.connect()
         fd.sendall(req)
-        self.assertEquals(read_http(fd)[-1], "this is ch")
+        self.assertEquals(read_http(fd).body, "this is ch")
 
         self.ping(fd)
         fd.close()
@@ -1374,7 +1409,7 @@ class TestChunkedInput(_TestBase):
         req = "POST /short-read HTTP/1.1\r\ntransfer-encoding: Chunked\r\nContent-Length:0\r\n\r\n" + body
         fd = self.connect()
         fd.sendall(req)
-        self.assertEquals(read_http(fd)[-1], "this is ch")
+        self.assertEquals(read_http(fd).body, "this is ch")
 
         self.ping(fd)
         fd.close()
@@ -1385,7 +1420,7 @@ class TestChunkedInput(_TestBase):
 
         fd = self.connect()
         fd.sendall(req)
-        self.assertEquals(read_http(fd)[-1], "this is ch")
+        self.assertEquals(read_http(fd).body, "this is ch")
 
         self.ping(fd)
         fd.close()
@@ -1396,7 +1431,7 @@ class TestChunkedInput(_TestBase):
 
         fd = self.connect()
         fd.sendall(req)
-        self.assertEquals(read_http(fd)[-1], "pong")
+        self.assertEquals(read_http(fd).body, "pong")
 
         self.ping(fd)
         fd.close()
@@ -1407,7 +1442,7 @@ class TestChunkedInput(_TestBase):
 
         fd = self.connect()
         fd.sendall(req)
-        self.assertEquals(read_http(fd)[-1], 'this is chunked\nline 2\nline3')
+        self.assertEquals(read_http(fd).body, 'this is chunked\nline 2\nline3')
         fd.close()
 
     def test_chunked_readline_wsgi_override_minimum_chunk_size(self):