openstack/deb-python-httpretty
Code Issues Proposed changes

Fix bug that occurs when using custom schema/port/regex

Browse Source 
Reopens https://github.com/gabrielfalcao/HTTPretty/pull/145. Reopened as a new PR because Travis didn't seem to be running the correct tests. Removed the irrelevant changes, such as PEP8 and the `self.truesock.settimeout(0)` change.

HTTPretty does not behave correctly when using regex matching, HTTPS and custom ports. When this scenario is triggered, a timeout/max retries exceeded error occurs.

To duplicate run:

```python
@httpretty.activate
def exceed_max_retries_with_custom_port_and_https():
    HTTPretty.register_uri(
        HTTPretty.GET,
        re.compile('https://api.yipit.com:1234/v1/deal;brand=(?P<brand_name>\w+)'),
        body='meow'
    )
    uri = 'https://api.yipit.com:1234/v1/deal;brand=gap?first_name=chuck&last_name=norris'
    response = requests.get(uri)
    return response.content
```

Cause
--

The combination of a regex URI, custom port, and HTTPS causes HTTPretty to get stuck at 2e814635ff/httpretty/core.py (L323) and eventually raise this error:

```
ConnectionError: HTTPSConnectionPool(host='api.yipit.com', port=1234): Max retries exceeded with url: /v1/deal;brand=gap?first_name=chuck&last_name=norris (Caused by <class 'socket.error'>: [Errno 36] Operation now in progress).
```

This error happens because URI schema's are reconstructed incorrectly during the URI matching.

This should fail (http != https), but it does not!

```python
@httpretty.activate
def broken_reconstruction_of_uri_schema():
    uri = 'api.yipit.com:1234/v1/deal'
    HTTPretty.register_uri(HTTPretty.GET,
        'https://' + uri,
        body=lambda method, uri, headers: [200, headers, uri]
    )
    response = requests.get(uri)
    expect(response.text).to.equal('http://' + uri)  # incorrect!
```

Solution
--

To correct the internal confusion between HTTP and HTTPS ports, we need to separate the two in our DEFAULT/POTENTIAL PORTS lists. When URIMatcher encounters a non-regex URI it uses URIInfo.from_uri to add
the URIs port to the known ports. This behavior is now added for regex URIs.

We now use the DEFAULT_PORTS lists in HTTPretty.reset() to reset the POTENTIAL_PORTS lists. Also, to avoid using the global keyword, we do an in-place reset with intersection_update.

Added the following tests:

- test_httpretty_should_work_with_non_standard_ports
- test_httpretty_reset_by_switching_protocols_for_same_port
- test_httpretty_should_allow_registering_regexes_with_port_and_give_a_proper_match_to_the_callback
This commit is contained in:
Luqmaan
2014-04-03 23:25:09 -05:00
parent 4c2b10925c
commit 9eecb1c94c
3 changed files with 177 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
httpretty/core.py
View File

@@ -100,8 +100,10 @@ except ImportError: # pragma: no cover
ssl = None
POTENTIAL_HTTP_PORTS = set([80, 443])
DEFAULT_HTTP_PORTS = tuple(POTENTIAL_HTTP_PORTS)
DEFAULT_HTTP_PORTS = frozenset([80])
POTENTIAL_HTTP_PORTS = set(DEFAULT_HTTP_PORTS)
DEFAULT_HTTPS_PORTS = frozenset([443])
POTENTIAL_HTTPS_PORTS = set(DEFAULT_HTTPS_PORTS)
class HTTPrettyRequest(BaseHTTPRequestHandler, BaseClass):
@@ -288,7 +290,7 @@ class fakesock(object):
def connect(self, address):
self._address = (self._host, self._port) = address
self._closed = False
self.is_http = self._port in POTENTIAL_HTTP_PORTS
self.is_http = self._port in POTENTIAL_HTTP_PORTS | POTENTIAL_HTTPS_PORTS
if not self.is_http:
self.truesock.connect(self._address)
@@ -635,7 +637,12 @@ class URIInfo(BaseClass):
self.port = port or 80
self.path = path or ''
self.query = query or ''
self.scheme = scheme or (self.port == 443 and "https" or "http")
if scheme:
self.scheme = scheme
elif self.port in POTENTIAL_HTTPS_PORTS:
self.scheme = 'https'
else:
self.scheme = 'http'
self.fragment = fragment or ''
self.last_request = last_request
@@ -687,7 +694,8 @@ class URIInfo(BaseClass):
def get_full_domain(self):
hostname = decode_utf8(self.hostname)
if self.port not in DEFAULT_HTTP_PORTS:
# Port 80/443 should not be appended to the url
if self.port not in DEFAULT_HTTP_PORTS | DEFAULT_HTTPS_PORTS:
return ":".join([hostname, str(self.port)])
return hostname
@@ -695,7 +703,10 @@ class URIInfo(BaseClass):
@classmethod
def from_uri(cls, uri, entry):
result = urlsplit(uri)
POTENTIAL_HTTP_PORTS.add(int(result.port or 80))
if result.scheme == 'https':
POTENTIAL_HTTPS_PORTS.add(int(result.port or 443))
else:
POTENTIAL_HTTP_PORTS.add(int(result.port or 80))
return cls(result.username,
result.password,
result.hostname,
@@ -715,6 +726,11 @@ class URIMatcher(object):
self._match_querystring = match_querystring
if type(uri).__name__ == 'SRE_Pattern':
self.regex = uri
result = urlsplit(uri.pattern)
if result.scheme == 'https':
POTENTIAL_HTTPS_PORTS.add(int(result.port or 443))
else:
POTENTIAL_HTTP_PORTS.add(int(result.port or 80))
else:
self.info = URIInfo.from_uri(uri, entries)
@@ -848,8 +864,8 @@ class httpretty(HttpBaseClass):
@classmethod
def reset(cls):
global POTENTIAL_HTTP_PORTS
POTENTIAL_HTTP_PORTS = set([80, 443])
POTENTIAL_HTTP_PORTS.intersection_update(DEFAULT_HTTP_PORTS)
POTENTIAL_HTTPS_PORTS.intersection_update(DEFAULT_HTTPS_PORTS)
cls._entries.clear()
cls.latest_requests = []
cls.last_request = HTTPrettyRequestEmpty()

151
tests/functional/test_requests.py
View File

@@ -768,6 +768,157 @@ def test_py26_callback_response():
expect(request_callback.call_count).equal(1)
@httprettified
def test_httpretty_should_work_with_non_standard_ports():
"HTTPretty should work with a non-standard port number"
HTTPretty.register_uri(
HTTPretty.GET,
re.compile("https://api.yipit.com:1234/v1/deal;brand=(?P<brand_name>\w+)"),
body=lambda method, uri, headers: [200, headers, uri]
)
HTTPretty.register_uri(
HTTPretty.POST,
"https://asdf.com:666/meow",
body=lambda method, uri, headers: [200, headers, uri]
)
response = requests.get('https://api.yipit.com:1234/v1/deal;brand=gap?first_name=chuck&last_name=norris')
expect(response.text).to.equal('https://api.yipit.com:1234/v1/deal;brand=gap?first_name=chuck&last_name=norris')
expect(HTTPretty.last_request.method).to.equal('GET')
expect(HTTPretty.last_request.path).to.equal('/v1/deal;brand=gap?first_name=chuck&last_name=norris')
response = requests.post('https://asdf.com:666/meow')
expect(response.text).to.equal('https://asdf.com:666/meow')
expect(HTTPretty.last_request.method).to.equal('POST')
expect(HTTPretty.last_request.path).to.equal('/meow')
@httprettified
def test_httpretty_reset_by_switching_protocols_for_same_port():
"HTTPretty should reset protocol/port associations"
HTTPretty.register_uri(
HTTPretty.GET,
"http://api.yipit.com:1234/v1/deal",
body=lambda method, uri, headers: [200, headers, uri]
)
response = requests.get('http://api.yipit.com:1234/v1/deal')
expect(response.text).to.equal('http://api.yipit.com:1234/v1/deal')
expect(HTTPretty.last_request.method).to.equal('GET')
expect(HTTPretty.last_request.path).to.equal('/v1/deal')
HTTPretty.reset()
HTTPretty.register_uri(
HTTPretty.GET,
"https://api.yipit.com:1234/v1/deal",
body=lambda method, uri, headers: [200, headers, uri]
)
response = requests.get('https://api.yipit.com:1234/v1/deal')
expect(response.text).to.equal('https://api.yipit.com:1234/v1/deal')
expect(HTTPretty.last_request.method).to.equal('GET')
expect(HTTPretty.last_request.path).to.equal('/v1/deal')
@httprettified
def test_httpretty_should_allow_registering_regexes_with_port_and_give_a_proper_match_to_the_callback():
"HTTPretty should allow registering regexes with requests and giva a proper match to the callback"
HTTPretty.register_uri(
HTTPretty.GET,
re.compile("https://api.yipit.com:1234/v1/deal;brand=(?P<brand_name>\w+)"),
body=lambda method, uri, headers: [200, headers, uri]
)
response = requests.get('https://api.yipit.com:1234/v1/deal;brand=gap?first_name=chuck&last_name=norris')
expect(response.text).to.equal('https://api.yipit.com:1234/v1/deal;brand=gap?first_name=chuck&last_name=norris')
expect(HTTPretty.last_request.method).to.equal('GET')
expect(HTTPretty.last_request.path).to.equal('/v1/deal;brand=gap?first_name=chuck&last_name=norris')
@httprettified
def test_httpretty_should_work_with_non_standard_ports():
"HTTPretty should work with a non-standard port number"
HTTPretty.register_uri(
HTTPretty.GET,
re.compile("https://api.yipit.com:1234/v1/deal;brand=(?P<brand_name>\w+)"),
body=lambda method, uri, headers: [200, headers, uri]
)
HTTPretty.register_uri(
HTTPretty.POST,
"https://asdf.com:666/meow",
body=lambda method, uri, headers: [200, headers, uri]
)
response = requests.get('https://api.yipit.com:1234/v1/deal;brand=gap?first_name=chuck&last_name=norris')
expect(response.text).to.equal('https://api.yipit.com:1234/v1/deal;brand=gap?first_name=chuck&last_name=norris')
expect(HTTPretty.last_request.method).to.equal('GET')
expect(HTTPretty.last_request.path).to.equal('/v1/deal;brand=gap?first_name=chuck&last_name=norris')
response = requests.post('https://asdf.com:666/meow')
expect(response.text).to.equal('https://asdf.com:666/meow')
expect(HTTPretty.last_request.method).to.equal('POST')
expect(HTTPretty.last_request.path).to.equal('/meow')
@httprettified
def test_httpretty_reset_by_switching_protocols_for_same_port():
"HTTPretty should reset protocol/port associations"
HTTPretty.register_uri(
HTTPretty.GET,
"http://api.yipit.com:1234/v1/deal",
body=lambda method, uri, headers: [200, headers, uri]
)
response = requests.get('http://api.yipit.com:1234/v1/deal')
expect(response.text).to.equal('http://api.yipit.com:1234/v1/deal')
expect(HTTPretty.last_request.method).to.equal('GET')
expect(HTTPretty.last_request.path).to.equal('/v1/deal')
HTTPretty.reset()
HTTPretty.register_uri(
HTTPretty.GET,
"https://api.yipit.com:1234/v1/deal",
body=lambda method, uri, headers: [200, headers, uri]
)
response = requests.get('https://api.yipit.com:1234/v1/deal')
expect(response.text).to.equal('https://api.yipit.com:1234/v1/deal')
expect(HTTPretty.last_request.method).to.equal('GET')
expect(HTTPretty.last_request.path).to.equal('/v1/deal')
@httprettified
def test_httpretty_should_allow_registering_regexes_with_port_and_give_a_proper_match_to_the_callback():
"HTTPretty should allow registering regexes with requests and giva a proper match to the callback"
HTTPretty.register_uri(
HTTPretty.GET,
re.compile("https://api.yipit.com:1234/v1/deal;brand=(?P<brand_name>\w+)"),
body=lambda method, uri, headers: [200, headers, uri]
)
response = requests.get('https://api.yipit.com:1234/v1/deal;brand=gap?first_name=chuck&last_name=norris')
expect(response.text).to.equal('https://api.yipit.com:1234/v1/deal;brand=gap?first_name=chuck&last_name=norris')
expect(HTTPretty.last_request.method).to.equal('GET')
expect(HTTPretty.last_request.path).to.equal('/v1/deal;brand=gap?first_name=chuck&last_name=norris')
import json

3
tests/unit/test_httpretty.py
View File

@@ -69,7 +69,8 @@ def test_httpretty_should_raise_on_socket_send_when_uri_registered():
HTTPretty.register_uri(HTTPretty.GET,
'http://127.0.0.1:5000')
expect(core.POTENTIAL_HTTP_PORTS).to.be.equal(set([80, 443, 5000]))
expect(core.POTENTIAL_HTTP_PORTS).to.be.equal(set([80, 5000]))
expect(core.POTENTIAL_HTTPS_PORTS).to.be.equal(set([443]))
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(('127.0.0.1', 5000))