From f2107512ee4be7bfbc1aaefdef12e1cba1147777 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 15 Jan 2015 16:09:23 -0600 Subject: [PATCH] Close streamed requests explicitly If we don't explicitly close a response after streaming its download, then we can run into HTTPConnectionPool full warnings. It also will hurt performance if we have to continuously create new sockets for new responses. Calling close will return the connection to the pool so it can be reused. Note this is only necessary when streaming a response. If we don't stream it, then requests will return the connection to the pool for us. Change-Id: I803bd4dd0e769c233501d5e5ff07a19705fbe233 Closes-bug: 1341777 --- glanceclient/common/http.py | 13 ++++++++++++- tests/utils.py | 3 +++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/glanceclient/common/http.py b/glanceclient/common/http.py index a2ade225..6363dc83 100644 --- a/glanceclient/common/http.py +++ b/glanceclient/common/http.py @@ -236,7 +236,7 @@ class HTTPClient(object): if content_type == 'application/octet-stream': # Do not read all response in memory when # downloading an image. - body_iter = resp.iter_content(chunk_size=CHUNKSIZE) + body_iter = _close_after_stream(resp, CHUNKSIZE) self.log_http_response(resp) else: content = resp.content @@ -271,3 +271,14 @@ class HTTPClient(object): def delete(self, url, **kwargs): return self._request('DELETE', url, **kwargs) + + +def _close_after_stream(response, chunk_size): + """Iterate over the content and ensure the response is closed after.""" + # Yield each chunk in the response body + for chunk in response.iter_content(chunk_size=chunk_size): + yield chunk + # Once we're done streaming the body, ensure everything is closed. + # This will return the connection to the HTTPConnectionPool in urllib3 + # and ideally reduce the number of HTTPConnectionPool full warnings. + response.close() diff --git a/tests/utils.py b/tests/utils.py index b2849e5f..3b4b51a0 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -126,6 +126,9 @@ class FakeResponse(object): def read(self, amt): return self.body.read(amt) + def close(self): + pass + @property def content(self): if hasattr(self.body, "read"):