From 4505d21ba03dce307682c6da7229605d3dcd51a2 Mon Sep 17 00:00:00 2001 From: Cyril Roelandt Date: Fri, 10 Jan 2014 21:18:40 +0100 Subject: [PATCH 01/11] Fix relative import --- tests/functional/test_requests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test_requests.py b/tests/functional/test_requests.py index de0e118..09a1a2f 100644 --- a/tests/functional/test_requests.py +++ b/tests/functional/test_requests.py @@ -35,7 +35,7 @@ from sure import within, microseconds, expect from httpretty import HTTPretty, httprettified from httpretty.core import decode_utf8 -from base import FIXTURE_FILE, use_tornado_server +from .base import FIXTURE_FILE, use_tornado_server from tornado import version as tornado_version try: From 303922fa081716c35664e93992b9003214a317bc Mon Sep 17 00:00:00 2001 From: Cyril Roelandt Date: Mon, 13 Jan 2014 00:44:42 +0100 Subject: [PATCH 02/11] Python 3: fix parse_request_body() Return a decoded string rather than bytes. --- httpretty/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/httpretty/core.py b/httpretty/core.py index 5e42c49..05a5ec1 100644 --- a/httpretty/core.py +++ b/httpretty/core.py @@ -204,6 +204,7 @@ class HTTPrettyRequest(BaseHTTPRequestHandler, BaseClass): content_type = self.headers.get('content-type', '') do_parse = PARSING_FUNCTIONS.get(content_type, FALLBACK_FUNCTION) + body = decode_utf8(body) try: return do_parse(body) except: From f8154e1552e0e08642658f6a37ff360d4c4ad598 Mon Sep 17 00:00:00 2001 From: Cyril Roelandt Date: Fri, 10 Jan 2014 21:23:17 +0100 Subject: [PATCH 03/11] Python 3: Pass bytes to sendall() and friends. Thanks to Victor Stinner for this patch. --- httpretty/core.py | 2 +- tests/unit/test_core.py | 42 ++++++++++++++++++++--------------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/httpretty/core.py b/httpretty/core.py index 05a5ec1..9a0aea3 100644 --- a/httpretty/core.py +++ b/httpretty/core.py @@ -344,7 +344,7 @@ class fakesock(object): self._sent_data.append(data) try: - requestline, _ = data.split('\r\n', 1) + requestline, _ = data.split(b'\r\n', 1) method, path, version = parse_requestline(requestline) is_parsing_headers = True except ValueError: diff --git a/tests/unit/test_core.py b/tests/unit/test_core.py index e2551fb..5296406 100644 --- a/tests/unit/test_core.py +++ b/tests/unit/test_core.py @@ -301,16 +301,16 @@ def test_fakesock_socket_real_sendall(old_socket): # Background: the real socket will stop returning bytes after the # first call real_socket = old_socket.return_value - real_socket.recv.side_effect = ['response from server', ""] + real_socket.recv.side_effect = [b'response from server', b""] # Given a fake socket socket = fakesock.socket() # When I call real_sendall with data, some args and kwargs - socket.real_sendall("SOMEDATA", 'some extra args...', foo='bar') + socket.real_sendall(b"SOMEDATA", b'some extra args...', foo=b'bar') # Then it should have called sendall in the real socket - real_socket.sendall.assert_called_once_with("SOMEDATA", 'some extra args...', foo='bar') + real_socket.sendall.assert_called_once_with(b"SOMEDATA", b'some extra args...', foo=b'bar') # And the timeout was set to 0 real_socket.settimeout.assert_called_once_with(0) @@ -322,7 +322,7 @@ def test_fakesock_socket_real_sendall(old_socket): ]) # And the buffer should contain the data from the server - socket.fd.getvalue().should.equal("response from server") + socket.fd.getvalue().should.equal(b"response from server") # And connect was never called real_socket.connect.called.should.be.false @@ -336,17 +336,17 @@ def test_fakesock_socket_real_sendall_continue_eagain(socket, old_socket): # Background: the real socket will stop returning bytes after the # first call real_socket = old_socket.return_value - real_socket.recv.side_effect = [SocketErrorStub(errno.EAGAIN), 'after error', ""] + real_socket.recv.side_effect = [SocketErrorStub(errno.EAGAIN), b'after error', b""] # Given a fake socket socket = fakesock.socket() # When I call real_sendall with data, some args and kwargs - socket.real_sendall("SOMEDATA", 'some extra args...', foo='bar') + socket.real_sendall(b"SOMEDATA", b'some extra args...', foo=b'bar') # Then it should have called sendall in the real socket - real_socket.sendall.assert_called_once_with("SOMEDATA", 'some extra args...', foo='bar') + real_socket.sendall.assert_called_once_with(b"SOMEDATA", b'some extra args...', foo=b'bar') # And the timeout was set to 0 real_socket.settimeout.assert_called_once_with(0) @@ -358,7 +358,7 @@ def test_fakesock_socket_real_sendall_continue_eagain(socket, old_socket): ]) # And the buffer should contain the data from the server - socket.fd.getvalue().should.equal("after error") + socket.fd.getvalue().should.equal(b"after error") # And connect was never called real_socket.connect.called.should.be.false @@ -372,16 +372,16 @@ def test_fakesock_socket_real_sendall_socket_error(socket, old_socket): # Background: the real socket will stop returning bytes after the # first call real_socket = old_socket.return_value - real_socket.recv.side_effect = [SocketErrorStub(42), 'after error', ""] + real_socket.recv.side_effect = [SocketErrorStub(42), b'after error', ""] # Given a fake socket socket = fakesock.socket() # When I call real_sendall with data, some args and kwargs - socket.real_sendall("SOMEDATA", 'some extra args...', foo='bar') + socket.real_sendall(b"SOMEDATA", b'some extra args...', foo=b'bar') # Then it should have called sendall in the real socket - real_socket.sendall.assert_called_once_with("SOMEDATA", 'some extra args...', foo='bar') + real_socket.sendall.assert_called_once_with(b"SOMEDATA", b'some extra args...', foo=b'bar') # And the timeout was set to 0 real_socket.settimeout.assert_called_once_with(0) @@ -390,7 +390,7 @@ def test_fakesock_socket_real_sendall_socket_error(socket, old_socket): real_socket.recv.assert_called_once_with(16) # And the buffer should contain the data from the server - socket.fd.getvalue().should.equal("") + socket.fd.getvalue().should.equal(b"") # And connect was never called real_socket.connect.called.should.be.false @@ -403,7 +403,7 @@ def test_fakesock_socket_real_sendall_when_http(POTENTIAL_HTTP_PORTS, old_socket # Background: the real socket will stop returning bytes after the # first call real_socket = old_socket.return_value - real_socket.recv.side_effect = ['response from foobar :)', ""] + real_socket.recv.side_effect = [b'response from foobar :)', b""] # And the potential http port is 4000 POTENTIAL_HTTP_PORTS.__contains__.side_effect = lambda other: int(other) == 4000 @@ -415,7 +415,7 @@ def test_fakesock_socket_real_sendall_when_http(POTENTIAL_HTTP_PORTS, old_socket socket.connect(('foobar.com', 4000)) # And send some data - socket.real_sendall("SOMEDATA") + socket.real_sendall(b"SOMEDATA") # Then connect should have been called real_socket.connect.assert_called_once_with(('foobar.com', 4000)) @@ -430,7 +430,7 @@ def test_fakesock_socket_real_sendall_when_http(POTENTIAL_HTTP_PORTS, old_socket ]) # And the buffer should contain the data from the server - socket.fd.getvalue().should.equal("response from foobar :)") + socket.fd.getvalue().should.equal(b"response from foobar :)") @patch('httpretty.core.old_socket') @@ -456,7 +456,7 @@ def test_fakesock_socket_sendall_with_valid_requestline(POTENTIAL_HTTP_PORTS, ht socket.connect(('foo.com', 80)) # When I try to send data - socket.sendall("GET /foobar HTTP/1.1\r\nContent-Type: application/json\r\n\r\n") + socket.sendall(b"GET /foobar HTTP/1.1\r\nContent-Type: application/json\r\n\r\n") @patch('httpretty.core.old_socket') @@ -482,7 +482,7 @@ def test_fakesock_socket_sendall_with_valid_requestline(POTENTIAL_HTTP_PORTS, ht socket.connect(('foo.com', 80)) # When I try to send data - socket.sendall("GET /foobar HTTP/1.1\r\nContent-Type: application/json\r\n\r\n") + socket.sendall(b"GET /foobar HTTP/1.1\r\nContent-Type: application/json\r\n\r\n") @patch('httpretty.core.old_socket') @@ -493,7 +493,7 @@ def test_fakesock_socket_sendall_with_body_data_no_entry(POTENTIAL_HTTP_PORTS, o # Using a subclass of socket that mocks out real_sendall class MySocket(fakesock.socket): def real_sendall(self, data): - data.should.equal('BLABLABLABLA') + data.should.equal(b'BLABLABLABLA') return 'cool' # Given an instance of that socket @@ -504,7 +504,7 @@ def test_fakesock_socket_sendall_with_body_data_no_entry(POTENTIAL_HTTP_PORTS, o socket.connect(('foo.com', 80)) # When I try to send data - result = socket.sendall("BLABLABLABLA") + result = socket.sendall(b"BLABLABLABLA") # Then the result should be the return value from real_sendall result.should.equal('cool') @@ -533,7 +533,7 @@ def test_fakesock_socket_sendall_with_body_data_with_entry(POTENTIAL_HTTP_PORTS, socket.connect(('foo.com', 80)) # When I try to send data - socket.sendall("BLABLABLABLA") + socket.sendall(b"BLABLABLABLA") # Then the entry should have that body entry.request.body.should.equal('BLABLABLABLA') @@ -563,7 +563,7 @@ def test_fakesock_socket_sendall_with_body_data_with_chunked_entry(POTENTIAL_HTT socket.connect(('foo.com', 80)) # When I try to send data - socket.sendall("BLABLABLABLA") + socket.sendall(b"BLABLABLABLA") # Then the entry should have that body httpretty.last_request.body.should.equal('BLABLABLABLA') From a132fda76668e0ba305303eab67639c80291e765 Mon Sep 17 00:00:00 2001 From: Cyril Roelandt Date: Fri, 10 Jan 2014 21:23:28 +0100 Subject: [PATCH 04/11] Python 3: cast return values of map() to list(). This ensures Python 3 compatibility. --- httpretty/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/httpretty/core.py b/httpretty/core.py index 9a0aea3..029a5c5 100644 --- a/httpretty/core.py +++ b/httpretty/core.py @@ -187,7 +187,7 @@ class HTTPrettyRequest(BaseHTTPRequestHandler, BaseClass): parsed = parse_qs(expanded) result = {} for k in parsed: - result[k] = map(decode_utf8, parsed[k]) + result[k] = list(map(decode_utf8, parsed[k])) return result @@ -373,7 +373,7 @@ class fakesock(object): # path might come with s = urlsplit(path) POTENTIAL_HTTP_PORTS.add(int(s.port or 80)) - headers, body = map(utf8, data.split('\r\n\r\n', 1)) + headers, body = list(map(utf8, data.split(b'\r\n\r\n', 1))) request = httpretty.historify_request(headers, body) @@ -394,7 +394,7 @@ class fakesock(object): def debug(self, func, *a, **kw): if self.is_http: frame = inspect.stack()[0][0] - lines = map(utf8, traceback.format_stack(frame)) + lines = list(map(utf8, traceback.format_stack(frame))) message = [ "HTTPretty intercepted and unexpected socket method call.", From a0ed19aa774ede5f14e3b35059d2154f7911a7da Mon Sep 17 00:00:00 2001 From: Cyril Roelandt Date: Sat, 11 Jan 2014 00:50:27 +0100 Subject: [PATCH 05/11] Python 3: fix parse_requestline(). The parse_requestline() function needs a text string as an input. In Python 3, it had bytes. --- httpretty/core.py | 2 +- httpretty/http.py | 23 ++++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/httpretty/core.py b/httpretty/core.py index 029a5c5..e0d9fc1 100644 --- a/httpretty/core.py +++ b/httpretty/core.py @@ -345,7 +345,7 @@ class fakesock(object): try: requestline, _ = data.split(b'\r\n', 1) - method, path, version = parse_requestline(requestline) + method, path, version = parse_requestline(decode_utf8(requestline)) is_parsing_headers = True except ValueError: is_parsing_headers = False diff --git a/httpretty/http.py b/httpretty/http.py index 388a73a..7e9a568 100644 --- a/httpretty/http.py +++ b/httpretty/http.py @@ -27,6 +27,7 @@ from __future__ import unicode_literals import re from .compat import BaseClass +from .utils import decode_utf8 STATUSES = { @@ -109,14 +110,14 @@ STATUSES = { class HttpBaseClass(BaseClass): - GET = b'GET' - PUT = b'PUT' - POST = b'POST' - DELETE = b'DELETE' - HEAD = b'HEAD' - PATCH = b'PATCH' - OPTIONS = b'OPTIONS' - CONNECT = b'CONNECT' + GET = 'GET' + PUT = 'PUT' + POST = 'POST' + DELETE = 'DELETE' + HEAD = 'HEAD' + PATCH = 'PATCH' + OPTIONS = 'OPTIONS' + CONNECT = 'CONNECT' METHODS = (GET, PUT, POST, DELETE, HEAD, PATCH, OPTIONS, CONNECT) @@ -133,8 +134,8 @@ def parse_requestline(s): ... ValueError: Not a Request-Line """ - methods = b'|'.join(HttpBaseClass.METHODS) - m = re.match(br'(' + methods + b')\s+(.*)\s+HTTP/(1.[0|1])', s, re.I) + methods = '|'.join(HttpBaseClass.METHODS) + m = re.match(r'(' + methods + ')\s+(.*)\s+HTTP/(1.[0|1])', s, re.I) if m: return m.group(1).upper(), m.group(2), m.group(3) else: @@ -147,7 +148,7 @@ def last_requestline(sent_data): """ for line in reversed(sent_data): try: - parse_requestline(line) + parse_requestline(decode_utf8(line)) except ValueError: pass else: From 6921c58cef317f6f02d4c7ea1da40d89523d1226 Mon Sep 17 00:00:00 2001 From: Cyril Roelandt Date: Sat, 11 Jan 2014 03:04:22 +0100 Subject: [PATCH 06/11] Python 3: do not hardcode class name in test_request_stubs_internals(). StringIO might be either 'StringIO.StringIO' or 'io.BytesIO' depending on the version of Python used. Do not hardcode the type in the test. --- tests/unit/test_core.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_core.py b/tests/unit/test_core.py index 5296406..9a2938e 100644 --- a/tests/unit/test_core.py +++ b/tests/unit/test_core.py @@ -8,6 +8,7 @@ from datetime import datetime from mock import Mock, patch, call from sure import expect +from httpretty.compat import StringIO from httpretty.core import HTTPrettyRequest, FakeSSLSocket, fakesock, httpretty @@ -43,10 +44,12 @@ def test_request_stubs_internals(): }) # And the `rfile` should be a StringIO - request.should.have.property('rfile').being.a('StringIO.StringIO') + type_as_str = StringIO.__module__ + '.' + StringIO.__name__ + + request.should.have.property('rfile').being.a(type_as_str) # And the `wfile` should be a StringIO - request.should.have.property('wfile').being.a('StringIO.StringIO') + request.should.have.property('wfile').being.a(type_as_str) # And the `method` should be available request.should.have.property('method').being.equal('POST') From 1086c3d26d0e227891d66b9713f735f0810fa0f3 Mon Sep 17 00:00:00 2001 From: Cyril Roelandt Date: Sat, 11 Jan 2014 03:27:29 +0100 Subject: [PATCH 07/11] Python3: Fix test_recording_calls() * in record_request, do not try to call json.dumps() on a dict() containing bytes; * in test_recording_calls(), do not hardcode the expected version of Tornado. There was a json.dumps(some_dict) and soe_dict had bytes values --- httpretty/core.py | 4 ++-- tests/functional/test_requests.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/httpretty/core.py b/httpretty/core.py index e0d9fc1..4ed913c 100644 --- a/httpretty/core.py +++ b/httpretty/core.py @@ -808,12 +808,12 @@ class httpretty(HttpBaseClass): 'uri': uri, 'method': request.method, 'headers': dict(request.headers), - 'body': request.body, + 'body': decode_utf8(request.body), 'querystring': request.querystring }, 'response': { 'status': response.status, - 'body': response.data, + 'body': decode_utf8(response.data), 'headers': dict(response.headers) } }) diff --git a/tests/functional/test_requests.py b/tests/functional/test_requests.py index 09a1a2f..50027c9 100644 --- a/tests/functional/test_requests.py +++ b/tests/functional/test_requests.py @@ -32,7 +32,9 @@ import re import json import requests from sure import within, microseconds, expect +from tornado import version as tornado_version from httpretty import HTTPretty, httprettified +from httpretty.compat import text_type from httpretty.core import decode_utf8 from .base import FIXTURE_FILE, use_tornado_server @@ -697,7 +699,7 @@ def test_recording_calls(): ] }) response['response'].should.have.key("status").being.equal(200) - response['response'].should.have.key("body").being.an(unicode) + response['response'].should.have.key("body").being.an(text_type) response['response'].should.have.key("headers").being.a(dict) response['response']["headers"].should.have.key("server").being.equal("TornadoServer/" + tornado_version) From 07559d76dc58c74f4d051c181d7ce8f14475eb24 Mon Sep 17 00:00:00 2001 From: Cyril Roelandt Date: Mon, 13 Jan 2014 00:38:21 +0100 Subject: [PATCH 08/11] Python 3: fix test_request_stubs_internals In Python 2, dict(request.headers) will have all keys in lower case, while In Python 3, the capital letters will still be there. This is because of internal changes in the requests library. Let's use lower-case headers, so that this is not an issue. --- tests/unit/test_core.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_core.py b/tests/unit/test_core.py index 9a2938e..ce02df8 100644 --- a/tests/unit/test_core.py +++ b/tests/unit/test_core.py @@ -24,11 +24,11 @@ def test_request_stubs_internals(): # Given a valid HTTP request header string headers = "\r\n".join([ 'POST /somewhere/?name=foo&age=bar HTTP/1.1', - 'Accept-Encoding: identity', - 'Host: github.com', - 'Content-Type: application/json', - 'Connection: close', - 'User-Agent: Python-urllib/2.7', + 'accept-encoding: identity', + 'host: github.com', + 'content-type: application/json', + 'connection: close', + 'user-agent: Python-urllib/2.7', ]) # When I create a HTTPrettyRequest with an empty body From d272b60051d3258625d6b654ae5c001d97d9e85f Mon Sep 17 00:00:00 2001 From: Cyril Roelandt Date: Mon, 13 Jan 2014 12:46:32 +0100 Subject: [PATCH 09/11] Python 3: fix encoding issue In Python 3, the parse_request() method uses iso-8859-1 to decode the path, but not in Python 2. This causes test failures in Python 3. Fix that by encoding the path back to iso-8859-1. --- httpretty/core.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/httpretty/core.py b/httpretty/core.py index 4ed913c..b2c5778 100644 --- a/httpretty/core.py +++ b/httpretty/core.py @@ -166,6 +166,10 @@ class HTTPrettyRequest(BaseHTTPRequestHandler, BaseClass): # Now 2 convenient attributes for the HTTPretty API: # `querystring` holds a dictionary with the parsed query string + try: + self.path = self.path.encode('iso-8859-1') + except UnicodeDecodeError: + pass self.path = decode_utf8(self.path) qstring = self.path.split("?", 1)[-1] From 4b5db343a9c33a7b7600363231fcdf50e82f903d Mon Sep 17 00:00:00 2001 From: Cyril Roelandt Date: Mon, 13 Jan 2014 16:21:13 +0100 Subject: [PATCH 10/11] Python 3: fix test_httpretty_should_allow_registering_regexes_with_streaming_responses() This was cause by the use of text strings instead of bytes. --- httpretty/core.py | 4 ++-- tests/functional/test_requests.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/httpretty/core.py b/httpretty/core.py index b2c5778..df6b15b 100644 --- a/httpretty/core.py +++ b/httpretty/core.py @@ -363,10 +363,10 @@ class fakesock(object): if not is_parsing_headers: if len(self._sent_data) > 1: headers = utf8(last_requestline(self._sent_data)) - meta = dict(self._entry.request.headers) + meta = self._entry.request.headers body = utf8(self._sent_data[-1]) if meta.get('transfer-encoding', '') == 'chunked': - if not body.isdigit() and body != '\r\n' and body != '0\r\n\r\n': + if not body.isdigit() and body != b'\r\n' and body != b'0\r\n\r\n': self._entry.request.body += body else: self._entry.request.body += body diff --git a/tests/functional/test_requests.py b/tests/functional/test_requests.py index 50027c9..daada92 100644 --- a/tests/functional/test_requests.py +++ b/tests/functional/test_requests.py @@ -575,7 +575,7 @@ def test_httpretty_should_allow_registering_regexes_with_streaming_responses(): os.environ['DEBUG'] = 'true' def my_callback(request, url, headers): - request.body.should.equal('hithere') + request.body.should.equal(b'hithere') return 200, headers, "Received" HTTPretty.register_uri( @@ -592,7 +592,7 @@ def test_httpretty_should_allow_registering_regexes_with_streaming_responses(): 'https://api.yipit.com/v1/deal;brand=gap?first_name=chuck&last_name=norris', data=gen(), ) - expect(response.content).to.equal("Received") + expect(response.content).to.equal(b"Received") expect(HTTPretty.last_request.method).to.equal('POST') expect(HTTPretty.last_request.path).to.equal('/v1/deal;brand=gap?first_name=chuck&last_name=norris') From f2004b0261f8d9c816b476935224ba05636d7e4b Mon Sep 17 00:00:00 2001 From: Cyril Roelandt Date: Tue, 14 Jan 2014 14:34:41 +0100 Subject: [PATCH 11/11] Python3: use bytes where needed. Bytes were needed in some place, instead of text strings. --- tests/unit/test_core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_core.py b/tests/unit/test_core.py index ce02df8..faed4cd 100644 --- a/tests/unit/test_core.py +++ b/tests/unit/test_core.py @@ -525,7 +525,7 @@ def test_fakesock_socket_sendall_with_body_data_with_entry(POTENTIAL_HTTP_PORTS, # Using a mocked entry entry = Mock() entry.request.headers = {} - entry.request.body = '' + entry.request.body = b'' # Given an instance of that socket socket = MySocket() @@ -539,7 +539,7 @@ def test_fakesock_socket_sendall_with_body_data_with_entry(POTENTIAL_HTTP_PORTS, socket.sendall(b"BLABLABLABLA") # Then the entry should have that body - entry.request.body.should.equal('BLABLABLABLA') + entry.request.body.should.equal(b'BLABLABLABLA') @patch('httpretty.core.old_socket') @@ -556,7 +556,7 @@ def test_fakesock_socket_sendall_with_body_data_with_chunked_entry(POTENTIAL_HTT entry.request.headers = { 'transfer-encoding': 'chunked', } - entry.request.body = '' + entry.request.body = b'' # Given an instance of that socket socket = MySocket() @@ -569,4 +569,4 @@ def test_fakesock_socket_sendall_with_body_data_with_chunked_entry(POTENTIAL_HTT socket.sendall(b"BLABLABLABLA") # Then the entry should have that body - httpretty.last_request.body.should.equal('BLABLABLABLA') + httpretty.last_request.body.should.equal(b'BLABLABLABLA')