Finalize Python3 support

Set the environment variable PYTHONHASHSEED to 0 to have
predictive tests under Python 3.

Change-Id: Ia15a9383e0f20bd0e4572e9f9b9772f1704dff86

glanceclient/common/http.py

@@ -19,6 +19,7 @@ import hashlib
 import logging
 import posixpath
 import socket
+import ssl
 import struct
 
 import six
@@ -63,6 +64,13 @@ USER_AGENT = 'python-glanceclient'
 CHUNKSIZE = 1024 * 64  # 64kB
 
 
+def to_bytes(s):
+    if isinstance(s, six.string_types):
+        return six.b(s)
+    else:
+        return s
+
+
 class HTTPClient(object):
 
     def __init__(self, endpoint, **kwargs):
@@ -149,8 +157,9 @@ class HTTPClient(object):
         dump.extend(['%s: %s' % (k, v) for k, v in resp.getheaders()])
         dump.append('')
         if body:
+            body = strutils.safe_decode(body)
             dump.extend([body, ''])
-        LOG.debug(strutils.safe_encode('\n'.join(dump)))
+        LOG.debug('\n'.join([strutils.safe_encode(x) for x in dump]))
 
     @staticmethod
     def encode_headers(headers):
@@ -239,9 +248,9 @@ class HTTPClient(object):
 
         # Read body into string if it isn't obviously image data
         if resp.getheader('content-type', None) != 'application/octet-stream':
-            body_str = ''.join([chunk for chunk in body_iter])
+            body_str = b''.join([to_bytes(chunk) for chunk in body_iter])
             self.log_http_response(resp, body_str)
-            body_iter = six.StringIO(body_str)
+            body_iter = six.BytesIO(body_str)
         else:
             self.log_http_response(resp)
 
@@ -349,16 +358,28 @@ class VerifiedHTTPSConnection(HTTPSConnection):
     def __init__(self, host, port=None, key_file=None, cert_file=None,
                  cacert=None, timeout=None, insecure=False,
                  ssl_compression=True):
-        HTTPSConnection.__init__(self, host, port,
-                                 key_file=key_file,
-                                 cert_file=cert_file)
-        self.key_file = key_file
-        self.cert_file = cert_file
-        self.timeout = timeout
-        self.insecure = insecure
-        self.ssl_compression = ssl_compression
-        self.cacert = cacert
-        self.setcontext()
+        # List of exceptions reported by Python3 instead of
+        # SSLConfigurationError
+        if six.PY3:
+            excp_lst = (TypeError, FileNotFoundError, ssl.SSLError)
+        else:
+            excp_lst = ()
+        try:
+            HTTPSConnection.__init__(self, host, port,
+                                     key_file=key_file,
+                                     cert_file=cert_file)
+            self.key_file = key_file
+            self.cert_file = cert_file
+            self.timeout = timeout
+            self.insecure = insecure
+            self.ssl_compression = ssl_compression
+            self.cacert = cacert
+            self.setcontext()
+            # ssl exceptions are reported in various form in Python 3
+            # so to be compatible, we report the same kind as under
+            # Python2
+        except excp_lst as e:
+            raise exc.SSLConfigurationError(str(e))
 
     @staticmethod
     def host_matches_cert(host, x509):
@@ -388,7 +409,7 @@ class VerifiedHTTPSConnection(HTTPSConnection):
         san_list = None
         for i in range(x509.get_extension_count()):
             ext = x509.get_extension(i)
-            if ext.get_short_name() == 'subjectAltName':
+            if ext.get_short_name() == b'subjectAltName':
                 san_list = str(ext)
                 for san in ''.join(san_list.split()).split(','):
                     if san.startswith('DNS:'):
@@ -458,7 +479,7 @@ class VerifiedHTTPSConnection(HTTPSConnection):
 
         if self.cacert:
             try:
-                self.context.load_verify_locations(self.cacert)
+                self.context.load_verify_locations(to_bytes(self.cacert))
             except Exception as e:
                 msg = ('Unable to load CA from "%(cacert)s" %(exc)s' %
                        dict(cacert=self.cacert, exc=e))
@@ -537,6 +558,8 @@ class ResponseBodyIterator(object):
                 raise
             else:
                 yield chunk
+                if isinstance(chunk, six.string_types):
+                    chunk = six.b(chunk)
                 md5sum.update(chunk)
 
     def next(self):

glanceclient/common/utils.py

@@ -267,7 +267,8 @@ def get_file_size(file_obj):
     :param file_obj: file-like object.
     :retval The file's size or None if it cannot be determined.
     """
-    if hasattr(file_obj, 'seek') and hasattr(file_obj, 'tell'):
+    if (hasattr(file_obj, 'seek') and hasattr(file_obj, 'tell') and
+            (six.PY2 or six.PY3 and file_obj.seekable())):
         try:
             curr = file_obj.tell()
             file_obj.seek(0, os.SEEK_END)

glanceclient/shell.py

@@ -20,6 +20,7 @@ Command-line interface to the OpenStack Images API.
 from __future__ import print_function
 
 import argparse
+import copy
 import json
 import logging
 import os
@@ -440,8 +441,12 @@ class OpenStackImagesShell(object):
 
     def main(self, argv):
         # Parse args once to find version
+
+        #NOTE(flepied) Under Python3, parsed arguments are removed
+        # from the list so make a copy for the first parsing
+        base_argv = copy.deepcopy(argv)
         parser = self.get_base_parser()
-        (options, args) = parser.parse_known_args(argv)
+        (options, args) = parser.parse_known_args(base_argv)
 
         # build available subcommands based on version
         api_version = options.os_image_api_version

setup.cfg

@@ -16,6 +16,8 @@ classifier =
     Programming Language :: Python :: 2
     Programming Language :: Python :: 2.7
     Programming Language :: Python :: 2.6
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.3
 
 [files]
 packages =

tests/test_http.py

@@ -105,7 +105,7 @@ class TestClient(testtools.TestCase):
 
     def test_request_redirected(self):
         resp = utils.FakeResponse({'location': 'http://www.example.com'},
-                                  status=302, body=six.StringIO())
+                                  status=302, body=six.BytesIO())
         http_client.HTTPConnection.request(
             mox.IgnoreArg(),
             mox.IgnoreArg(),
@@ -114,8 +114,8 @@ class TestClient(testtools.TestCase):
         http_client.HTTPConnection.getresponse().AndReturn(resp)
 
         # The second request should be to the redirected location
-        expected_response = 'Ok'
-        resp2 = utils.FakeResponse({}, six.StringIO(expected_response))
+        expected_response = b'Ok'
+        resp2 = utils.FakeResponse({}, six.BytesIO(expected_response))
         http_client.HTTPConnection.request(
             'GET',
             'http://www.example.com',
@@ -135,8 +135,8 @@ class TestClient(testtools.TestCase):
 
         # Lets fake the response
         # returned by httplib
-        expected_response = 'Ok'
-        fake = utils.FakeResponse({}, six.StringIO(expected_response))
+        expected_response = b'Ok'
+        fake = utils.FakeResponse({}, six.BytesIO(expected_response))
         http_client.HTTPConnection.getresponse().AndReturn(fake)
         self.mock.ReplayAll()
 
@@ -146,9 +146,13 @@ class TestClient(testtools.TestCase):
         self.assertEqual(fake, resp)
 
     def test_headers_encoding(self):
-        headers = {"test": u'ni\xf1o'}
+        value = u'ni\xf1o'
+        headers = {"test": value}
         encoded = self.client.encode_headers(headers)
-        self.assertEqual("ni\xc3\xb1o", encoded["test"])
+        if six.PY2:
+            self.assertEqual("ni\xc3\xb1o", encoded["test"])
+        else:
+            self.assertEqual(value, encoded["test"])
 
     def test_raw_request(self):
         " Verify the path being used for HTTP requests reflects accurately. "
@@ -164,7 +168,7 @@ class TestClient(testtools.TestCase):
             headers=mox.IgnoreArg()).WithSideEffects(check_request)
 
         # fake the response returned by httplib
-        fake = utils.FakeResponse({}, six.StringIO('Ok'))
+        fake = utils.FakeResponse({}, six.BytesIO(b'Ok'))
         http_client.HTTPConnection.getresponse().AndReturn(fake)
         self.mock.ReplayAll()
 
@@ -192,7 +196,7 @@ class TestClient(testtools.TestCase):
             headers=mox.IgnoreArg()).WithSideEffects(check_request)
 
         # fake the response returned by httplib
-        fake = utils.FakeResponse({}, six.StringIO('Ok'))
+        fake = utils.FakeResponse({}, six.BytesIO(b'Ok'))
         http_client.HTTPConnection.getresponse().AndReturn(fake)
         self.mock.ReplayAll()
 
@@ -396,19 +400,19 @@ class TestVerifiedHTTPSConnection(testtools.TestCase):
 class TestResponseBodyIterator(testtools.TestCase):
 
     def test_iter_default_chunk_size_64k(self):
-        resp = utils.FakeResponse({}, six.StringIO('X' * 98304))
+        resp = utils.FakeResponse({}, six.BytesIO(b'X' * 98304))
         iterator = http.ResponseBodyIterator(resp)
         chunks = list(iterator)
-        self.assertEqual(['X' * 65536, 'X' * 32768], chunks)
+        self.assertEqual([b'X' * 65536, b'X' * 32768], chunks)
 
     def test_integrity_check_with_correct_checksum(self):
-        resp = utils.FakeResponse({}, six.StringIO('CCC'))
+        resp = utils.FakeResponse({}, six.BytesIO(b'CCC'))
         body = http.ResponseBodyIterator(resp)
         body.set_checksum('defb99e69a9f1f6e06f15006b1f166ae')
         list(body)
 
     def test_integrity_check_with_wrong_checksum(self):
-        resp = utils.FakeResponse({}, six.StringIO('BB'))
+        resp = utils.FakeResponse({}, six.BytesIO(b'BB'))
         body = http.ResponseBodyIterator(resp)
         body.set_checksum('wrong')
         try:
@@ -418,7 +422,7 @@ class TestResponseBodyIterator(testtools.TestCase):
             self.assertEqual(errno.EPIPE, e.errno)
 
     def test_set_checksum_in_consumed_iterator(self):
-        resp = utils.FakeResponse({}, six.StringIO('CCC'))
+        resp = utils.FakeResponse({}, six.BytesIO(b'CCC'))
         body = http.ResponseBodyIterator(resp)
         list(body)
         # Setting checksum for an already consumed iterator should raise an
@@ -430,6 +434,6 @@ class TestResponseBodyIterator(testtools.TestCase):
     def test_body_size(self):
         size = 1000000007
         resp = utils.FakeResponse(
-            {'content-length': str(size)}, six.StringIO('BB'))
+            {'content-length': str(size)}, six.BytesIO(b'BB'))
         body = http.ResponseBodyIterator(resp)
         self.assertEqual(size, len(body))

tests/test_ssl.py

@@ -66,6 +66,8 @@ class TestVerifiedHTTPSConnection(testtools.TestCase):
             conn = http.VerifiedHTTPSConnection('127.0.0.1', 0,
                                                 key_file=key_file,
                                                 cacert=cacert)
+        except exc.SSLConfigurationError:
+            pass
         except Exception:
             self.fail('Failed to init VerifiedHTTPSConnection.')
 

tests/test_utils.py

@@ -109,7 +109,10 @@ class TestUtils(testtools.TestCase):
         self.assertEqual('error message', ret)
 
         ret = utils.exception_to_str(Exception('\xa5 error message'))
-        self.assertEqual(' error message', ret)
+        if six.PY2:
+            self.assertEqual(' error message', ret)
+        else:
+            self.assertEqual('\xa5 error message', ret)
 
         ret = utils.exception_to_str(FakeException('\xa5 error message'))
         self.assertEqual("Caught '%(exception)s' exception." %

tests/v1/test_images.py

@@ -347,7 +347,7 @@ fixtures = {
         'HEAD': (
             {
                 'x-image-meta-id': '3',
-                'x-image-meta-name': "ni\xc3\xb1o"
+                'x-image-meta-name': u"ni\xf1o"
             },
             None,
         ),
@@ -622,9 +622,13 @@ class ImageManagerTest(testtools.TestCase):
         self.assertEqual(expect, self.api.calls)
 
     def test_image_meta_from_headers_encoding(self):
-        fields = {"x-image-meta-name": "ni\xc3\xb1o"}
+        value = u"ni\xf1o"
+        if six.PY2:
+            fields = {"x-image-meta-name": "ni\xc3\xb1o"}
+        else:
+            fields = {"x-image-meta-name": value}
         headers = self.mgr._image_meta_from_headers(fields)
-        self.assertEqual(u"ni\xf1o", headers["name"])
+        self.assertEqual(value, headers["name"])
 
     def test_image_list_with_owner(self):
         images = self.mgr.list(owner='A', page_size=20)

tests/v2/test_images.py

@@ -16,6 +16,7 @@
 import errno
 import testtools
 
+import six
 import warlock
 
 from glanceclient.v2 import images
@@ -403,8 +404,10 @@ class TestController(testtools.TestCase):
             #   /v2/images?owner=ni%C3%B1o&limit=20
             # We just want to make sure filters are correctly encoded.
             pass
-
-        self.assertEqual("ni\xc3\xb1o", filters["owner"])
+        if six.PY2:
+            self.assertEqual("ni\xc3\xb1o", filters["owner"])
+        else:
+            self.assertEqual("ni\xf1o", filters["owner"])
 
     def test_list_images_for_tag_single_image(self):
         img_id = '3a4560a1-e585-443e-9b39-553b46ec92d1'

tox.ini

@@ -9,6 +9,7 @@ install_command = pip install -U {opts} {packages}
 setenv = VIRTUAL_ENV={envdir}
          OS_STDOUT_NOCAPTURE=False
          OS_STDERR_NOCAPTURE=False
+	 PYTHONHASHSEED=0
 
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt