Fix swiftclient output regression

Fix swiftclient output regression introduced by the related change:
  - output for SLO object download: fix incorrect error from
    SwiftReader about SLO object ETag header not matching MD5 checksum
  - output for object stat: fix duplicated ETag
  - output for account/container stat: fix duplicated byte/object counts

Co-Authored-By: Yan Xiao <yanxiao@nvidia.com>
Related-Change: Ice9cc9fe68684563f18ee527996e5a4292230a96
Change-Id: I5b2d79f89d1b6016de69d6b58879e5c2ef31e107
This commit is contained in:
Clay Gerrard 2024-04-17 13:59:56 -05:00 committed by Tim Burke
parent ce4fb27b53
commit ed6fd60915
5 changed files with 79 additions and 10 deletions

@ -212,6 +212,15 @@ def encode_meta_headers(headers):
return ret
class LowerKeyCaseInsensitiveDict(CaseInsensitiveDict):
"""
CaseInsensitiveDict returning lower case keys for items()
"""
def __iter__(self):
return iter(self._store.keys())
class _ObjectBody:
"""
Readable and iterable object body response wrapper.
@ -738,7 +747,7 @@ def get_auth(auth_url, user, key, **kwargs):
def resp_header_dict(resp):
resp_headers = CaseInsensitiveDict()
resp_headers = LowerKeyCaseInsensitiveDict()
for header, value in resp.getheaders():
header = parse_header_string(header)
resp_headers[header] = parse_header_string(value)

@ -18,6 +18,7 @@ import unittest
from unittest import mock
from swiftclient import command_helpers as h
from swiftclient.client import LowerKeyCaseInsensitiveDict
from swiftclient.multithreading import OutputManager
@ -245,5 +246,32 @@ Content-Encoding: gzip
ETag: 68b329da9893e34099c7d8ad5cb9c940
Meta Color: blue
Content-Encoding: gzip
"""
self.assertOut(expected)
def test_stat_object_case_insensitive_headers(self):
self.options['verbose'] += 1
# stub head object request
stub_headers = LowerKeyCaseInsensitiveDict({
'content-length': 2 ** 20,
'x-object-meta-color': 'blue',
'ETag': '68b329da9893e34099c7d8ad5cb9c940',
'content-encoding': 'gzip',
})
self.conn.head_object.return_value = stub_headers
args = ('c', 'o')
with self.output_manager as output_manager:
items, headers = h.stat_object(self.conn, self.options, *args)
h.print_object_stats(items, headers, output_manager)
expected = """
URL: http://storage/v1/a/c/o
Auth Token: tk12345
Account: a
Container: c
Object: o
Content Length: 1048576
ETag: 68b329da9893e34099c7d8ad5cb9c940
Meta Color: blue
Content-Encoding: gzip
"""
self.assertOut(expected)

@ -27,12 +27,13 @@ from unittest import mock
from concurrent.futures import Future
from hashlib import md5
from queue import Queue, Empty as QueueEmptyError
from requests.structures import CaseInsensitiveDict
from time import sleep
import swiftclient
import swiftclient.utils as utils
from swiftclient.client import Connection, ClientException
from swiftclient.client import (
Connection, ClientException, LowerKeyCaseInsensitiveDict
)
from swiftclient.service import (
SwiftService, SwiftError, SwiftUploadObject, SwiftDeleteObject
)
@ -242,6 +243,26 @@ class TestSwiftReader(unittest.TestCase):
self.assertEqual(sr._actual_md5.hexdigest(),
md5('abc'.encode() * 3).hexdigest())
def test_swift_reader_knows_slo_etag_is_not_md5(self):
segment_bodies = [b'abc', b'def', b'ghi']
# slo etag is md5 of the sum of md5 of segments
slo_etag = md5(b''.join(
md5(b).hexdigest().encode()
for b in segment_bodies
)).hexdigest()
headers = LowerKeyCaseInsensitiveDict({
'Content-Length': len(b''.join(segment_bodies)),
'X-Static-Large-Object': 'true',
'ETag': '"%s"' % slo_etag
})
sr = self.sr('path', segment_bodies, headers)
# x-static-large-object; so no exception is raised!
actual_md5 = md5(b''.join(sr)).hexdigest()
self.assertEqual(sr._actual_read, 9)
self.assertIsNone(sr._actual_md5)
self.assertEqual(actual_md5,
md5(b''.join(segment_bodies)).hexdigest())
class _TestServiceBase(unittest.TestCase):
def _get_mock_connection(self, attempts=2):
@ -674,7 +695,7 @@ class TestSwiftError(unittest.TestCase):
def test_swifterror_clientexception_creation(self):
test_exc = ClientException(
Exception('test exc'),
http_response_headers=CaseInsensitiveDict({
http_response_headers=LowerKeyCaseInsensitiveDict({
'x-trans-id': 'someTransId'})
)
se = SwiftError(5, 'con', 'obj', 'seg', test_exc)

@ -22,7 +22,6 @@ import hashlib
import json
import logging
import os
from requests.structures import CaseInsensitiveDict
import tempfile
import unittest
from unittest import mock
@ -32,6 +31,7 @@ from requests.exceptions import RequestException
from urllib3.exceptions import HTTPError
import swiftclient
from swiftclient.client import LowerKeyCaseInsensitiveDict
from swiftclient.service import SwiftError
import swiftclient.shell
import swiftclient.utils
@ -245,7 +245,7 @@ class TestShell(unittest.TestCase):
swiftclient.ClientException(
'test',
http_status=404,
http_response_headers=CaseInsensitiveDict({
http_response_headers=LowerKeyCaseInsensitiveDict({
'x-trans-id': 'someTransId'})
)
argv = ["", "stat", "container"]
@ -344,7 +344,7 @@ class TestShell(unittest.TestCase):
connection.return_value.head_object.side_effect = \
swiftclient.ClientException(
'test', http_status=404,
http_response_headers=CaseInsensitiveDict({
http_response_headers=LowerKeyCaseInsensitiveDict({
'x-trans-id': 'someTransId'})
)
argv = ["", "stat", "container", "object"]
@ -791,7 +791,7 @@ class TestShell(unittest.TestCase):
body = mock.MagicMock()
body.resp.read.side_effect = RequestException('test_exc')
return (CaseInsensitiveDict({
return (LowerKeyCaseInsensitiveDict({
'content-type': 'text/plain',
'etag': '2cbbfe139a744d6abbe695e17f3c1991',
'x-trans-id': 'someTransId'}),
@ -841,7 +841,7 @@ class TestShell(unittest.TestCase):
body = mock.MagicMock()
body.__iter__.side_effect = RequestException('test_exc')
return (CaseInsensitiveDict({
return (LowerKeyCaseInsensitiveDict({
'content-type': 'text/plain',
'etag': '2cbbfe139a744d6abbe695e17f3c1991',
'x-trans-id': 'someTransId'}),
@ -871,7 +871,7 @@ class TestShell(unittest.TestCase):
def test_download_bad_content_length(self, connection):
objcontent = io.BytesIO(b'objcontent')
connection.return_value.get_object.side_effect = [
(CaseInsensitiveDict({
(LowerKeyCaseInsensitiveDict({
'content-type': 'text/plain',
'content-length': 'BAD',
'etag': '2cbbfe139a744d6abbe695e17f3c1991',

@ -1117,6 +1117,17 @@ class TestGetObject(MockHttpTest):
self.assertEqual('t\xe9st', headers.get('x-utf-8-header', ''))
self.assertEqual('%ff', headers.get('x-non-utf-8-header', ''))
self.assertEqual('%FF', headers.get('x-binary-header', ''))
for k, v in headers.items():
# N.B. k is always lower case!
self.assertTrue(k.islower())
for k in headers.keys():
# N.B. k is always lower case!
self.assertTrue(k.islower())
self.assertTrue(set([
'x-utf-8-header',
'x-non-utf-8-header',
'x-binary-header',
]).intersection(headers))
self.assertEqual('t\xe9st', headers.get('X-Utf-8-Header', ''))
self.assertEqual('%ff', headers.get('X-Non-Utf-8-Header', ''))