Close streamed requests explicitly

If we don't explicitly close a response after streaming its download,
then we can run into HTTPConnectionPool full warnings. It also will hurt
performance if we have to continuously create new sockets for new
responses. Calling close will return the connection to the pool so it
can be reused. Note this is only necessary when streaming a response. If
we don't stream it, then requests will return the connection to the pool
for us.

Change-Id: I803bd4dd0e769c233501d5e5ff07a19705fbe233
Closes-bug: 1341777

glanceclient/common/http.py

@@ -236,7 +236,7 @@ class HTTPClient(object):
         if content_type == 'application/octet-stream':
             # Do not read all response in memory when
             # downloading an image.
-            body_iter = resp.iter_content(chunk_size=CHUNKSIZE)
+            body_iter = _close_after_stream(resp, CHUNKSIZE)
             self.log_http_response(resp)
         else:
             content = resp.content
@@ -271,3 +271,14 @@ class HTTPClient(object):
 
     def delete(self, url, **kwargs):
         return self._request('DELETE', url, **kwargs)
+
+
+def _close_after_stream(response, chunk_size):
+    """Iterate over the content and ensure the response is closed after."""
+    # Yield each chunk in the response body
+    for chunk in response.iter_content(chunk_size=chunk_size):
+        yield chunk
+    # Once we're done streaming the body, ensure everything is closed.
+    # This will return the connection to the HTTPConnectionPool in urllib3
+    # and ideally reduce the number of HTTPConnectionPool full warnings.
+    response.close()

tests/utils.py

@@ -126,6 +126,9 @@ class FakeResponse(object):
     def read(self, amt):
         return self.body.read(amt)
 
+    def close(self):
+        pass
+
     @property
     def content(self):
         if hasattr(self.body, "read"):