Fix swiftclient output regression
Fix swiftclient output regression introduced by the related change: - output for SLO object download: fix incorrect error from SwiftReader about SLO object ETag header not matching MD5 checksum - output for object stat: fix duplicated ETag - output for account/container stat: fix duplicated byte/object counts Co-Authored-By: Yan Xiao <yanxiao@nvidia.com> Related-Change: Ice9cc9fe68684563f18ee527996e5a4292230a96 Change-Id: I5b2d79f89d1b6016de69d6b58879e5c2ef31e107
This commit is contained in:
parent
ce4fb27b53
commit
ed6fd60915
@ -212,6 +212,15 @@ def encode_meta_headers(headers):
|
||||
return ret
|
||||
|
||||
|
||||
class LowerKeyCaseInsensitiveDict(CaseInsensitiveDict):
|
||||
"""
|
||||
CaseInsensitiveDict returning lower case keys for items()
|
||||
"""
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self._store.keys())
|
||||
|
||||
|
||||
class _ObjectBody:
|
||||
"""
|
||||
Readable and iterable object body response wrapper.
|
||||
@ -738,7 +747,7 @@ def get_auth(auth_url, user, key, **kwargs):
|
||||
|
||||
|
||||
def resp_header_dict(resp):
|
||||
resp_headers = CaseInsensitiveDict()
|
||||
resp_headers = LowerKeyCaseInsensitiveDict()
|
||||
for header, value in resp.getheaders():
|
||||
header = parse_header_string(header)
|
||||
resp_headers[header] = parse_header_string(value)
|
||||
|
@ -18,6 +18,7 @@ import unittest
|
||||
from unittest import mock
|
||||
|
||||
from swiftclient import command_helpers as h
|
||||
from swiftclient.client import LowerKeyCaseInsensitiveDict
|
||||
from swiftclient.multithreading import OutputManager
|
||||
|
||||
|
||||
@ -245,5 +246,32 @@ Content-Encoding: gzip
|
||||
ETag: 68b329da9893e34099c7d8ad5cb9c940
|
||||
Meta Color: blue
|
||||
Content-Encoding: gzip
|
||||
"""
|
||||
self.assertOut(expected)
|
||||
|
||||
def test_stat_object_case_insensitive_headers(self):
|
||||
self.options['verbose'] += 1
|
||||
# stub head object request
|
||||
stub_headers = LowerKeyCaseInsensitiveDict({
|
||||
'content-length': 2 ** 20,
|
||||
'x-object-meta-color': 'blue',
|
||||
'ETag': '68b329da9893e34099c7d8ad5cb9c940',
|
||||
'content-encoding': 'gzip',
|
||||
})
|
||||
self.conn.head_object.return_value = stub_headers
|
||||
args = ('c', 'o')
|
||||
with self.output_manager as output_manager:
|
||||
items, headers = h.stat_object(self.conn, self.options, *args)
|
||||
h.print_object_stats(items, headers, output_manager)
|
||||
expected = """
|
||||
URL: http://storage/v1/a/c/o
|
||||
Auth Token: tk12345
|
||||
Account: a
|
||||
Container: c
|
||||
Object: o
|
||||
Content Length: 1048576
|
||||
ETag: 68b329da9893e34099c7d8ad5cb9c940
|
||||
Meta Color: blue
|
||||
Content-Encoding: gzip
|
||||
"""
|
||||
self.assertOut(expected)
|
||||
|
@ -27,12 +27,13 @@ from unittest import mock
|
||||
from concurrent.futures import Future
|
||||
from hashlib import md5
|
||||
from queue import Queue, Empty as QueueEmptyError
|
||||
from requests.structures import CaseInsensitiveDict
|
||||
from time import sleep
|
||||
|
||||
import swiftclient
|
||||
import swiftclient.utils as utils
|
||||
from swiftclient.client import Connection, ClientException
|
||||
from swiftclient.client import (
|
||||
Connection, ClientException, LowerKeyCaseInsensitiveDict
|
||||
)
|
||||
from swiftclient.service import (
|
||||
SwiftService, SwiftError, SwiftUploadObject, SwiftDeleteObject
|
||||
)
|
||||
@ -242,6 +243,26 @@ class TestSwiftReader(unittest.TestCase):
|
||||
self.assertEqual(sr._actual_md5.hexdigest(),
|
||||
md5('abc'.encode() * 3).hexdigest())
|
||||
|
||||
def test_swift_reader_knows_slo_etag_is_not_md5(self):
|
||||
segment_bodies = [b'abc', b'def', b'ghi']
|
||||
# slo etag is md5 of the sum of md5 of segments
|
||||
slo_etag = md5(b''.join(
|
||||
md5(b).hexdigest().encode()
|
||||
for b in segment_bodies
|
||||
)).hexdigest()
|
||||
headers = LowerKeyCaseInsensitiveDict({
|
||||
'Content-Length': len(b''.join(segment_bodies)),
|
||||
'X-Static-Large-Object': 'true',
|
||||
'ETag': '"%s"' % slo_etag
|
||||
})
|
||||
sr = self.sr('path', segment_bodies, headers)
|
||||
# x-static-large-object; so no exception is raised!
|
||||
actual_md5 = md5(b''.join(sr)).hexdigest()
|
||||
self.assertEqual(sr._actual_read, 9)
|
||||
self.assertIsNone(sr._actual_md5)
|
||||
self.assertEqual(actual_md5,
|
||||
md5(b''.join(segment_bodies)).hexdigest())
|
||||
|
||||
|
||||
class _TestServiceBase(unittest.TestCase):
|
||||
def _get_mock_connection(self, attempts=2):
|
||||
@ -674,7 +695,7 @@ class TestSwiftError(unittest.TestCase):
|
||||
def test_swifterror_clientexception_creation(self):
|
||||
test_exc = ClientException(
|
||||
Exception('test exc'),
|
||||
http_response_headers=CaseInsensitiveDict({
|
||||
http_response_headers=LowerKeyCaseInsensitiveDict({
|
||||
'x-trans-id': 'someTransId'})
|
||||
)
|
||||
se = SwiftError(5, 'con', 'obj', 'seg', test_exc)
|
||||
|
@ -22,7 +22,6 @@ import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from requests.structures import CaseInsensitiveDict
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
@ -32,6 +31,7 @@ from requests.exceptions import RequestException
|
||||
from urllib3.exceptions import HTTPError
|
||||
|
||||
import swiftclient
|
||||
from swiftclient.client import LowerKeyCaseInsensitiveDict
|
||||
from swiftclient.service import SwiftError
|
||||
import swiftclient.shell
|
||||
import swiftclient.utils
|
||||
@ -245,7 +245,7 @@ class TestShell(unittest.TestCase):
|
||||
swiftclient.ClientException(
|
||||
'test',
|
||||
http_status=404,
|
||||
http_response_headers=CaseInsensitiveDict({
|
||||
http_response_headers=LowerKeyCaseInsensitiveDict({
|
||||
'x-trans-id': 'someTransId'})
|
||||
)
|
||||
argv = ["", "stat", "container"]
|
||||
@ -344,7 +344,7 @@ class TestShell(unittest.TestCase):
|
||||
connection.return_value.head_object.side_effect = \
|
||||
swiftclient.ClientException(
|
||||
'test', http_status=404,
|
||||
http_response_headers=CaseInsensitiveDict({
|
||||
http_response_headers=LowerKeyCaseInsensitiveDict({
|
||||
'x-trans-id': 'someTransId'})
|
||||
)
|
||||
argv = ["", "stat", "container", "object"]
|
||||
@ -791,7 +791,7 @@ class TestShell(unittest.TestCase):
|
||||
|
||||
body = mock.MagicMock()
|
||||
body.resp.read.side_effect = RequestException('test_exc')
|
||||
return (CaseInsensitiveDict({
|
||||
return (LowerKeyCaseInsensitiveDict({
|
||||
'content-type': 'text/plain',
|
||||
'etag': '2cbbfe139a744d6abbe695e17f3c1991',
|
||||
'x-trans-id': 'someTransId'}),
|
||||
@ -841,7 +841,7 @@ class TestShell(unittest.TestCase):
|
||||
|
||||
body = mock.MagicMock()
|
||||
body.__iter__.side_effect = RequestException('test_exc')
|
||||
return (CaseInsensitiveDict({
|
||||
return (LowerKeyCaseInsensitiveDict({
|
||||
'content-type': 'text/plain',
|
||||
'etag': '2cbbfe139a744d6abbe695e17f3c1991',
|
||||
'x-trans-id': 'someTransId'}),
|
||||
@ -871,7 +871,7 @@ class TestShell(unittest.TestCase):
|
||||
def test_download_bad_content_length(self, connection):
|
||||
objcontent = io.BytesIO(b'objcontent')
|
||||
connection.return_value.get_object.side_effect = [
|
||||
(CaseInsensitiveDict({
|
||||
(LowerKeyCaseInsensitiveDict({
|
||||
'content-type': 'text/plain',
|
||||
'content-length': 'BAD',
|
||||
'etag': '2cbbfe139a744d6abbe695e17f3c1991',
|
||||
|
@ -1117,6 +1117,17 @@ class TestGetObject(MockHttpTest):
|
||||
self.assertEqual('t\xe9st', headers.get('x-utf-8-header', ''))
|
||||
self.assertEqual('%ff', headers.get('x-non-utf-8-header', ''))
|
||||
self.assertEqual('%FF', headers.get('x-binary-header', ''))
|
||||
for k, v in headers.items():
|
||||
# N.B. k is always lower case!
|
||||
self.assertTrue(k.islower())
|
||||
for k in headers.keys():
|
||||
# N.B. k is always lower case!
|
||||
self.assertTrue(k.islower())
|
||||
self.assertTrue(set([
|
||||
'x-utf-8-header',
|
||||
'x-non-utf-8-header',
|
||||
'x-binary-header',
|
||||
]).intersection(headers))
|
||||
|
||||
self.assertEqual('t\xe9st', headers.get('X-Utf-8-Header', ''))
|
||||
self.assertEqual('%ff', headers.get('X-Non-Utf-8-Header', ''))
|
||||
|
Loading…
x
Reference in New Issue
Block a user