Replace httplib2 with httplib as http driver

* This allows us to send truly chunked responses to users
* Handle bad connection url schemes with a new InvalidEndpoint exception
* Fixes bug 1023240

Change-Id: I34500987f51d4e0c6e1f89ecf93853de3fcbb1c3

glanceclient/common/http.py

@@ -3,12 +3,11 @@ OpenStack Client interface. Handles the REST calls and responses.
 """
 
 import copy
+import httplib
 import logging
 import os
-import StringIO
 import urlparse
 
-import httplib2
 
 try:
     import json
@@ -29,18 +28,26 @@ USER_AGENT = 'python-glanceclient'
 CHUNKSIZE = 1024 * 64  # 64kB
 
 
-class HTTPClient(httplib2.Http):
+class HTTPClient(object):
 
     def __init__(self, endpoint, token=None, timeout=600, insecure=False):
-        super(HTTPClient, self).__init__(timeout=timeout)
+        parts = urlparse.urlparse(endpoint)
-        self.endpoint = endpoint
+        self.connection_class = self.get_connection_class(parts.scheme)
+        self.endpoint = (parts.hostname, parts.port)
         self.auth_token = token
 
-        # httplib2 overrides
+    @staticmethod
-        self.force_exception_to_status_code = True
+    def get_connection_class(scheme):
-        self.disable_ssl_certificate_validation = insecure
+        try:
+            return getattr(httplib, '%sConnection' % scheme.upper())
+        except AttributeError:
+            msg = 'Unsupported scheme: %s' % scheme
+            raise exc.InvalidEndpoint(msg)
 
-    def http_log(self, args, kwargs, resp, body):
+    def get_connection(self):
+        return self.connection_class(*self.endpoint)
+
+    def http_log(self, args, kwargs, resp):
         if os.environ.get('GLANCECLIENT_DEBUG', False):
             ch = logging.StreamHandler()
             logger.setLevel(logging.DEBUG)
@@ -64,37 +71,34 @@ class HTTPClient(httplib2.Http):
             logger.debug("REQ BODY (RAW): %s\n" % (kwargs['raw_body']))
         if 'body' in kwargs:
             logger.debug("REQ BODY: %s\n" % (kwargs['body']))
-        logger.debug("RESP: %s\nRESP BODY: %s\n", resp, body)
+        logger.debug("RESP: %s", resp)
 
     def _http_request(self, url, method, **kwargs):
         """ Send an http request with the specified characteristics.
 
-        Wrapper around httplib2.Http.request to handle tasks such as
+        Wrapper around httplib.HTTP(S)Connection.request to handle tasks such
-        setting headers, JSON encoding/decoding, and error handling.
+        as setting headers and error handling.
         """
-        url = self.endpoint + url
-
         # Copy the kwargs so we can reuse the original in case of redirects
         kwargs['headers'] = copy.deepcopy(kwargs.get('headers', {}))
         kwargs['headers'].setdefault('User-Agent', USER_AGENT)
         if self.auth_token:
             kwargs['headers'].setdefault('X-Auth-Token', self.auth_token)
 
-        resp, body = super(HTTPClient, self).request(url, method, **kwargs)
+        conn = self.get_connection()
-        self.http_log((url, method,), kwargs, resp, body)
+        conn.request(method, url, **kwargs)
+        resp = conn.getresponse()
+
+        self.http_log((url, method,), kwargs, resp)
 
         if 400 <= resp.status < 600:
             logger.exception("Request returned failure status.")
-            raise exc.from_response(resp, body)
+            raise exc.from_response(resp)
         elif resp.status in (301, 302, 305):
             # Redirected. Reissue the request to the new location.
             return self._http_request(resp['location'], method, **kwargs)
 
-        #NOTE(bcwaldon): body has been loaded to a string at this point,
+        body_iter = ResponseBodyIterator(resp)
-        # but we want to move to a world where it can be read from a
-        # socket-level cache. This is here until we can do that.
-        body_iter = ResponseBodyIterator(StringIO.StringIO(body))
-
         return resp, body_iter
 
     def json_request(self, method, url, **kwargs):

glanceclient/exc.py

@@ -23,6 +23,11 @@ class SchemaNotFound(KeyError):
     pass
 
 
+class InvalidEndpoint(ValueError):
+    """The provided endpoint could not be used"""
+    pass
+
+
 class ClientException(Exception):
     """
     The base exception class for all exceptions this library raises.
@@ -104,28 +109,7 @@ _code_map = dict((c.http_status, c) for c in [BadRequest, Unauthorized,
                    Forbidden, NotFound, OverLimit, HTTPNotImplemented])
 
 
-def from_response(response, body):
+def from_response(response):
-    """
+    """Return an instance of an ClientException based on httplib response."""
-    Return an instance of an ClientException or subclass
-    based on an httplib2 response.
-
-    Usage::
-
-        resp, body = http.request(...)
-        if resp.status != 200:
-            raise exception_from_response(resp, body)
-    """
     cls = _code_map.get(response.status, ClientException)
-    if body:
+    return cls(code=response.status)
-        if hasattr(body, 'keys'):
-            error = body[body.keys()[0]]
-            message = error.get('message', None)
-            details = error.get('details', None)
-        else:
-            # If we didn't get back a properly formed error message we
-            # probably couldn't communicate with Keystone at all.
-            message = "Unable to communicate with image service: %s." % body
-            details = None
-        return cls(code=response.status, message=message, details=details)
-    else:
-        return cls(code=response.status)

glanceclient/shell.py

@@ -18,7 +18,6 @@ Command-line interface to the OpenStack Images API.
 """
 
 import argparse
-import httplib2
 import re
 import sys
 
@@ -234,10 +233,6 @@ class OpenStackImagesShell(object):
         # Parse args again and call whatever callback was selected
         args = subcommand_parser.parse_args(argv)
 
-        # Deal with global arguments
-        if args.debug:
-            httplib2.debuglevel = 1
-
         # Short-circuit and deal with help command right away.
         if args.func == self.do_help:
             self.do_help(args)
@@ -315,8 +310,5 @@ def main():
         OpenStackImagesShell().main(sys.argv[1:])
 
     except Exception, e:
-        if httplib2.debuglevel == 1:
+        print >> sys.stderr, e
-            raise
-        else:
-            print >> sys.stderr, e
         sys.exit(1)

glanceclient/v1/images.py

@@ -74,7 +74,7 @@ class ImageManager(base.Manager):
         :rtype: :class:`Image`
         """
         resp, body = self.api.raw_request('HEAD', '/v1/images/%s' % image_id)
-        meta = self._image_meta_from_headers(resp)
+        meta = self._image_meta_from_headers(dict(resp.getheaders()))
         return Image(self, meta)
 
     def data(self, image):
@@ -84,7 +84,7 @@ class ImageManager(base.Manager):
         :rtype: iterable containing image data
         """
         image_id = base.getid(image)
-        resp, body = self.api.raw_request('GET', '/v1/images/%s' % image_id)
+        _, body = self.api.raw_request('GET', '/v1/images/%s' % image_id)
         return body
 
     def list(self, **kwargs):

tests/utils.py

@@ -13,6 +13,11 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import copy
+import StringIO
+
+from glanceclient.common import http
+
 
 class FakeAPI(object):
     def __init__(self, fixtures):
@@ -25,7 +30,18 @@ class FakeAPI(object):
         return self.fixtures[url][method]
 
     def raw_request(self, *args, **kwargs):
-        return self._request(*args, **kwargs)
+        fixture = self._request(*args, **kwargs)
+        body_iter = http.ResponseBodyIterator(StringIO.StringIO(fixture[1]))
+        return FakeResponse(fixture[0]), body_iter
 
     def json_request(self, *args, **kwargs):
-        return self._request(*args, **kwargs)
+        fixture = self._request(*args, **kwargs)
+        return FakeResponse(fixture[0]), fixture[1]
+
+
+class FakeResponse(object):
+    def __init__(self, headers):
+        self.headers = headers
+
+    def getheaders(self):
+        return copy.deepcopy(self.headers).items()

tools/pip-requires

@@ -1,5 +1,4 @@
 argparse
-httplib2
 prettytable>=0.6,<0.7
 python-keystoneclient>=0.1,<0.2
 warlock==0.1.0