Merge "Fix swiftclient output regression"

swiftclient/client.py

@@ -212,6 +212,15 @@ def encode_meta_headers(headers):
     return ret
 
 
+class LowerKeyCaseInsensitiveDict(CaseInsensitiveDict):
+    """
+    CaseInsensitiveDict returning lower case keys for items()
+    """
+
+    def __iter__(self):
+        return iter(self._store.keys())
+
+
 class _ObjectBody:
     """
     Readable and iterable object body response wrapper.
@@ -738,7 +747,7 @@ def get_auth(auth_url, user, key, **kwargs):
 
 
 def resp_header_dict(resp):
-    resp_headers = CaseInsensitiveDict()
+    resp_headers = LowerKeyCaseInsensitiveDict()
     for header, value in resp.getheaders():
         header = parse_header_string(header)
         resp_headers[header] = parse_header_string(value)

test/unit/test_command_helpers.py

@@ -18,6 +18,7 @@ import unittest
 from unittest import mock
 
 from swiftclient import command_helpers as h
+from swiftclient.client import LowerKeyCaseInsensitiveDict
 from swiftclient.multithreading import OutputManager
 
 
@@ -245,5 +246,32 @@ Content-Encoding: gzip
             ETag: 68b329da9893e34099c7d8ad5cb9c940
       Meta Color: blue
 Content-Encoding: gzip
+"""
+        self.assertOut(expected)
+
+    def test_stat_object_case_insensitive_headers(self):
+        self.options['verbose'] += 1
+        # stub head object request
+        stub_headers = LowerKeyCaseInsensitiveDict({
+            'content-length': 2 ** 20,
+            'x-object-meta-color': 'blue',
+            'ETag': '68b329da9893e34099c7d8ad5cb9c940',
+            'content-encoding': 'gzip',
+        })
+        self.conn.head_object.return_value = stub_headers
+        args = ('c', 'o')
+        with self.output_manager as output_manager:
+            items, headers = h.stat_object(self.conn, self.options, *args)
+            h.print_object_stats(items, headers, output_manager)
+        expected = """
+             URL: http://storage/v1/a/c/o
+      Auth Token: tk12345
+         Account: a
+       Container: c
+          Object: o
+  Content Length: 1048576
+            ETag: 68b329da9893e34099c7d8ad5cb9c940
+      Meta Color: blue
+Content-Encoding: gzip
 """
         self.assertOut(expected)

test/unit/test_service.py

@@ -27,12 +27,13 @@ from unittest import mock
 from concurrent.futures import Future
 from hashlib import md5
 from queue import Queue, Empty as QueueEmptyError
-from requests.structures import CaseInsensitiveDict
 from time import sleep
 
 import swiftclient
 import swiftclient.utils as utils
-from swiftclient.client import Connection, ClientException
+from swiftclient.client import (
+    Connection, ClientException, LowerKeyCaseInsensitiveDict
+)
 from swiftclient.service import (
     SwiftService, SwiftError, SwiftUploadObject, SwiftDeleteObject
 )
@@ -242,6 +243,26 @@ class TestSwiftReader(unittest.TestCase):
         self.assertEqual(sr._actual_md5.hexdigest(),
                          md5('abc'.encode() * 3).hexdigest())
 
+    def test_swift_reader_knows_slo_etag_is_not_md5(self):
+        segment_bodies = [b'abc', b'def', b'ghi']
+        # slo etag is md5 of the sum of md5 of segments
+        slo_etag = md5(b''.join(
+            md5(b).hexdigest().encode()
+            for b in segment_bodies
+        )).hexdigest()
+        headers = LowerKeyCaseInsensitiveDict({
+            'Content-Length': len(b''.join(segment_bodies)),
+            'X-Static-Large-Object': 'true',
+            'ETag': '"%s"' % slo_etag
+        })
+        sr = self.sr('path', segment_bodies, headers)
+        # x-static-large-object; so no exception is raised!
+        actual_md5 = md5(b''.join(sr)).hexdigest()
+        self.assertEqual(sr._actual_read, 9)
+        self.assertIsNone(sr._actual_md5)
+        self.assertEqual(actual_md5,
+                         md5(b''.join(segment_bodies)).hexdigest())
+
 
 class _TestServiceBase(unittest.TestCase):
     def _get_mock_connection(self, attempts=2):
@@ -674,7 +695,7 @@ class TestSwiftError(unittest.TestCase):
     def test_swifterror_clientexception_creation(self):
         test_exc = ClientException(
             Exception('test exc'),
-            http_response_headers=CaseInsensitiveDict({
+            http_response_headers=LowerKeyCaseInsensitiveDict({
                 'x-trans-id': 'someTransId'})
         )
         se = SwiftError(5, 'con', 'obj', 'seg', test_exc)

test/unit/test_shell.py

@@ -22,7 +22,6 @@ import hashlib
 import json
 import logging
 import os
-from requests.structures import CaseInsensitiveDict
 import tempfile
 import unittest
 from unittest import mock
@@ -32,6 +31,7 @@ from requests.exceptions import RequestException
 from urllib3.exceptions import HTTPError
 
 import swiftclient
+from swiftclient.client import LowerKeyCaseInsensitiveDict
 from swiftclient.service import SwiftError
 import swiftclient.shell
 import swiftclient.utils
@@ -245,7 +245,7 @@ class TestShell(unittest.TestCase):
             swiftclient.ClientException(
                 'test',
                 http_status=404,
-                http_response_headers=CaseInsensitiveDict({
+                http_response_headers=LowerKeyCaseInsensitiveDict({
                     'x-trans-id': 'someTransId'})
             )
         argv = ["", "stat", "container"]
@@ -344,7 +344,7 @@ class TestShell(unittest.TestCase):
         connection.return_value.head_object.side_effect = \
             swiftclient.ClientException(
                 'test', http_status=404,
-                http_response_headers=CaseInsensitiveDict({
+                http_response_headers=LowerKeyCaseInsensitiveDict({
                     'x-trans-id': 'someTransId'})
             )
         argv = ["", "stat", "container", "object"]
@@ -791,7 +791,7 @@ class TestShell(unittest.TestCase):
 
             body = mock.MagicMock()
             body.resp.read.side_effect = RequestException('test_exc')
-            return (CaseInsensitiveDict({
+            return (LowerKeyCaseInsensitiveDict({
                 'content-type': 'text/plain',
                 'etag': '2cbbfe139a744d6abbe695e17f3c1991',
                 'x-trans-id': 'someTransId'}),
@@ -841,7 +841,7 @@ class TestShell(unittest.TestCase):
 
             body = mock.MagicMock()
             body.__iter__.side_effect = RequestException('test_exc')
-            return (CaseInsensitiveDict({
+            return (LowerKeyCaseInsensitiveDict({
                 'content-type': 'text/plain',
                 'etag': '2cbbfe139a744d6abbe695e17f3c1991',
                 'x-trans-id': 'someTransId'}),
@@ -871,7 +871,7 @@ class TestShell(unittest.TestCase):
     def test_download_bad_content_length(self, connection):
         objcontent = io.BytesIO(b'objcontent')
         connection.return_value.get_object.side_effect = [
-            (CaseInsensitiveDict({
+            (LowerKeyCaseInsensitiveDict({
                 'content-type': 'text/plain',
                 'content-length': 'BAD',
                 'etag': '2cbbfe139a744d6abbe695e17f3c1991',

test/unit/test_swiftclient.py

@@ -1117,6 +1117,17 @@ class TestGetObject(MockHttpTest):
         self.assertEqual('t\xe9st', headers.get('x-utf-8-header', ''))
         self.assertEqual('%ff', headers.get('x-non-utf-8-header', ''))
         self.assertEqual('%FF', headers.get('x-binary-header', ''))
+        for k, v in headers.items():
+            # N.B. k is always lower case!
+            self.assertTrue(k.islower())
+        for k in headers.keys():
+            # N.B. k is always lower case!
+            self.assertTrue(k.islower())
+        self.assertTrue(set([
+            'x-utf-8-header',
+            'x-non-utf-8-header',
+            'x-binary-header',
+        ]).intersection(headers))
 
         self.assertEqual('t\xe9st', headers.get('X-Utf-8-Header', ''))
         self.assertEqual('%ff', headers.get('X-Non-Utf-8-Header', ''))