From 4350152828dc8cade3ae9ec1116e7ceb8d8d98a5 Mon Sep 17 00:00:00 2001 From: gholt Date: Tue, 20 May 2014 20:19:47 +0000 Subject: [PATCH] Container sync no longer sending swift_bytes value Container sync had a bug where it'd send out the trailing "; swift_bytes=xxx" part of the content-type header. That trailing part is just for internal cluster usage by SLO. Since that needed to be stripped in two places now, I separated it out to a function that both spots call. Change-Id: Ibd6035d7a6b78205344bcc9d98bc1b7a9d463427 --- swift/common/utils.py | 8 ++++++++ swift/container/sync.py | 10 +++++++--- swift/proxy/controllers/obj.py | 14 ++++++-------- test/unit/common/test_utils.py | 16 ++++++++++++++++ test/unit/container/test_sync.py | 12 ++++++++---- 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/swift/common/utils.py b/swift/common/utils.py index 5f8bbc80eb..3538d8fb42 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -2658,6 +2658,14 @@ def override_bytes_from_content_type(listing_dict, logger=None): listing_dict['content_type'] = content_type +def clean_content_type(value): + if ';' in value: + left, right = value.rsplit(';', 1) + if right.lstrip().startswith('swift_bytes='): + return left + return value + + def quote(value, safe='/'): """ Patched version of urllib.quote that encodes utf-8 strings before quoting diff --git a/swift/container/sync.py b/swift/container/sync.py index bdde696722..65d81a9b6f 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -29,9 +29,10 @@ from swift.common.direct_client import direct_get_object from swift.common.internal_client import delete_object, put_object from swift.common.exceptions import ClientException from swift.common.ring import Ring -from swift.common.utils import audit_location_generator, get_logger, \ - hash_path, config_true_value, validate_sync_to, whataremyips, \ - FileLikeIter, urlparse, quote +from swift.common.utils import ( + audit_location_generator, clean_content_type, config_true_value, + FileLikeIter, get_logger, hash_path, quote, urlparse, validate_sync_to, + whataremyips) from swift.common.daemon import Daemon from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND @@ -403,6 +404,9 @@ class ContainerSync(Daemon): del headers[key] if 'etag' in headers: headers['etag'] = headers['etag'].strip('"') + if 'content-type' in headers: + headers['content-type'] = clean_content_type( + headers['content-type']) headers['x-timestamp'] = row['created_at'] if realm and realm_key: nonce = uuid.uuid4().hex diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 20841f30f0..af0efb6d34 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -35,9 +35,10 @@ from eventlet import GreenPile from eventlet.queue import Queue from eventlet.timeout import Timeout -from swift.common.utils import ContextPool, normalize_timestamp, \ - config_true_value, public, json, csv_append, GreenthreadSafeIterator, \ - quorum_size, GreenAsyncPile, normalize_delete_at_timestamp +from swift.common.utils import ( + clean_content_type, config_true_value, ContextPool, csv_append, + GreenAsyncPile, GreenthreadSafeIterator, json, + normalize_delete_at_timestamp, normalize_timestamp, public, quorum_size) from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_metadata, check_object_creation, \ check_copy_from_header @@ -206,11 +207,8 @@ class ObjectController(Controller): req.swift_entity_path) if ';' in resp.headers.get('content-type', ''): - # strip off swift_bytes from content-type - content_type, check_extra_meta = \ - resp.headers['content-type'].rsplit(';', 1) - if check_extra_meta.lstrip().startswith('swift_bytes='): - resp.content_type = content_type + resp.content_type = clean_content_type( + resp.headers['content-type']) return resp @public diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index b6e7b7a1b9..29045513c9 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -1879,6 +1879,22 @@ cluster_dfw1 = http://dfw1.host/v1/ self.assertEquals(listing_dict['content_type'], 'text/plain;hello="world"') + def test_clean_content_type(self): + subtests = { + '': '', 'text/plain': 'text/plain', + 'text/plain; someother=thing': 'text/plain; someother=thing', + 'text/plain; swift_bytes=123': 'text/plain', + 'text/plain; someother=thing; swift_bytes=123': + 'text/plain; someother=thing', + # Since Swift always tacks on the swift_bytes, clean_content_type() + # only strips swift_bytes if it's last. The next item simply shows + # that if for some other odd reason it's not last, + # clean_content_type() will not remove it from the header. + 'text/plain; swift_bytes=123; someother=thing': + 'text/plain; swift_bytes=123; someother=thing'} + for before, after in subtests.items(): + self.assertEqual(utils.clean_content_type(before), after) + def test_quote(self): res = utils.quote('/v1/a/c3/subdirx/') assert res == '/v1/a/c3/subdirx/' diff --git a/test/unit/container/test_sync.py b/test/unit/container/test_sync.py index 0f45aa2acb..f70494c841 100644 --- a/test/unit/container/test_sync.py +++ b/test/unit/container/test_sync.py @@ -751,13 +751,15 @@ class TestContainerSync(unittest.TestCase): 'US abcdef ef62c64bb88a33fa00722daa23d5d43253164962', 'x-timestamp': '1.2', 'etag': 'etagvalue', - 'other-header': 'other header value'}) + 'other-header': 'other header value', + 'content-type': 'text/plain'}) else: self.assertEquals(headers, { 'x-container-sync-key': 'key', 'x-timestamp': '1.2', 'other-header': 'other header value', - 'etag': 'etagvalue'}) + 'etag': 'etagvalue', + 'content-type': 'text/plain'}) self.assertEquals(contents.read(), 'contents') self.assertEquals(proxy, 'http://proxy') @@ -770,7 +772,8 @@ class TestContainerSync(unittest.TestCase): def fake_direct_get_object(node, part, account, container, obj, resp_chunk_size=1): return ({'other-header': 'other header value', - 'etag': '"etagvalue"', 'x-timestamp': '1.2'}, + 'etag': '"etagvalue"', 'x-timestamp': '1.2', + 'content-type': 'text/plain; swift_bytes=123'}, iter('contents')) sync.direct_get_object = fake_direct_get_object @@ -790,7 +793,8 @@ class TestContainerSync(unittest.TestCase): 'last-modified': 'last modified value', 'x-timestamp': '1.2', 'other-header': 'other header value', - 'etag': '"etagvalue"'}, + 'etag': '"etagvalue"', + 'content-type': 'text/plain; swift_bytes=123'}, iter('contents')) sync.direct_get_object = fake_direct_get_object