Merge "Properly read py2 object metadata on py3"

This commit is contained in:
Zuul 2023-05-15 08:23:37 +00:00 committed by Gerrit Code Review
commit e6203384c2
4 changed files with 256 additions and 33 deletions

View File

@ -30,6 +30,7 @@ from swift.container.backend import ContainerBroker, DATADIR as CBDATADIR
from swift.obj.diskfile import get_data_dir, read_metadata, DATADIR_BASE, \
extract_policy
from swift.common.storage_policy import POLICIES
from swift.common.swob import wsgi_to_str
from swift.common.middleware.crypto.crypto_utils import load_crypto_meta
from swift.common.utils import md5
@ -537,6 +538,8 @@ def print_obj(datafile, check_etag=True, swift_dir='/etc/swift',
except EOFError:
print("Invalid metadata")
raise InfoSystemExit()
metadata = {wsgi_to_str(k): v if k == 'name' else wsgi_to_str(v)
for k, v in metadata.items()}
etag = metadata.pop('ETag', '')
length = metadata.pop('Content-Length', '')

View File

@ -167,24 +167,36 @@ def _encode_metadata(metadata):
return dict(((encode_str(k), encode_str(v)) for k, v in metadata.items()))
def _decode_metadata(metadata):
def _decode_metadata(metadata, metadata_written_by_py3):
"""
Given a metadata dict from disk, convert keys and values to native strings.
:param metadata: a dict
:param metadata_written_by_py3:
"""
if six.PY2:
def to_str(item):
def to_str(item, is_name=False):
# For years, py2 and py3 handled non-ascii metadata differently;
# see https://bugs.launchpad.net/swift/+bug/2012531
if metadata_written_by_py3 and not is_name:
# do our best to read new-style data replicated from a py3 node
item = item.decode('utf8').encode('latin1')
if isinstance(item, six.text_type):
return item.encode('utf8')
return item
else:
def to_str(item):
def to_str(item, is_name=False):
# For years, py2 and py3 handled non-ascii metadata differently;
# see https://bugs.launchpad.net/swift/+bug/2012531
if not metadata_written_by_py3 and isinstance(item, bytes) \
and not is_name:
# do our best to read old py2 data
item = item.decode('latin1')
if isinstance(item, six.binary_type):
return item.decode('utf8', 'surrogateescape')
return item
return dict(((to_str(k), to_str(v)) for k, v in metadata.items()))
return {to_str(k): to_str(v, k == b'name') for k, v in metadata.items()}
def read_metadata(fd, add_missing_checksum=False):
@ -238,6 +250,7 @@ def read_metadata(fd, add_missing_checksum=False):
"stored checksum='%s', computed='%s'" % (
fd, metadata_checksum, computed_checksum))
metadata_written_by_py3 = (b'_codecs\nencode' in metadata[:32])
# strings are utf-8 encoded when written, but have not always been
# (see https://bugs.launchpad.net/swift/+bug/1678018) so encode them again
# when read
@ -245,7 +258,7 @@ def read_metadata(fd, add_missing_checksum=False):
metadata = pickle.loads(metadata)
else:
metadata = pickle.loads(metadata, encoding='bytes')
return _decode_metadata(metadata)
return _decode_metadata(metadata, metadata_written_by_py3)
def write_metadata(fd, metadata, xattr_size=65536):

View File

@ -1219,7 +1219,7 @@ class TestPrintObjFullMeta(TestCliInfoBase):
md = {'name': '/AUTH_admin/c/obj',
'Content-Type': 'application/octet-stream',
'ETag': 'd41d8cd98f00b204e9800998ecf8427e',
'Content-Length': 0}
'Content-Length': '0'}
write_metadata(fp, md)
def test_print_obj(self):
@ -1256,7 +1256,7 @@ class TestPrintObjFullMeta(TestCliInfoBase):
md = {'name': '/AUTH_admin/c/obj',
'Content-Type': 'application/octet-stream',
'ETag': 'd41d8cd98f00b204e9800998ecf8427e',
'Content-Length': 0}
'Content-Length': '0'}
write_metadata(fp, md)
object_ring = ring.Ring(self.testdir, ring_name='object-2')
@ -1297,7 +1297,7 @@ class TestPrintObjFullMeta(TestCliInfoBase):
md = {'name': '/AUTH_admin/c/obj',
'Content-Type': 'application/octet-stream',
'ETag': 'd41d8cd98f00b204e9800998ecf8427e',
'Content-Length': 0}
'Content-Length': '0'}
write_metadata(fp, md)
object_ring = ring.Ring(self.testdir, ring_name='object-3')
@ -1368,7 +1368,7 @@ class TestPrintObjFullMeta(TestCliInfoBase):
md = {'name': '/AUTH_admin/c/obj',
'Content-Type': 'application/octet-stream',
'ETag': 'badetag',
'Content-Length': 0}
'Content-Length': '0'}
write_metadata(fp, md)
out = StringIO()
@ -1703,5 +1703,5 @@ class TestPrintObjWeirdPath(TestPrintObjFullMeta):
md = {'name': '/AUTH_admin/c/obj',
'Content-Type': 'application/octet-stream',
'ETag': 'd41d8cd98f00b204e9800998ecf8427e',
'Content-Length': 0}
'Content-Length': '0'}
write_metadata(fp, md)

View File

@ -15,6 +15,8 @@
"""Tests for swift.obj.diskfile"""
from __future__ import print_function
import binascii
import six.moves.cPickle as pickle
import os
import errno
@ -28,6 +30,7 @@ import uuid
import xattr
import re
import six
import sys
from collections import defaultdict
from random import shuffle, randint
from shutil import rmtree
@ -554,6 +557,233 @@ class TestDiskFileModuleMethods(unittest.TestCase):
exp_dir = '/srv/node/sdb5/objects-1/123'
self.assertEqual(part_dir, exp_dir)
def test_can_read_old_meta(self):
# outputs taken from `xattr -l <diskfile>`
cases = {
'python_2.7.18_swift_2.13_replicated': '''
0000 80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D ..}q.(U.Content-
0010 4C 65 6E 67 74 68 71 02 55 02 31 33 55 04 6E 61 Lengthq.U.13U.na
0020 6D 65 71 03 55 12 2F 41 55 54 48 5F 74 65 73 74 meq.U./AUTH_test
0030 2F E2 98 83 2F E2 98 83 71 04 55 13 58 2D 4F 62 /.../...q.U.X-Ob
0040 6A 65 63 74 2D 4D 65 74 61 2D 4D 74 69 6D 65 55 ject-Meta-MtimeU
0050 11 31 36 38 32 39 35 39 38 37 34 2E 37 35 36 32 .1682959874.7562
0060 30 35 71 05 55 04 45 54 61 67 71 06 55 20 36 62 05q.U.ETagq.U 6b
0070 37 64 39 61 31 63 35 64 31 36 37 63 63 35 30 30 7d9a1c5d167cc500
0080 33 37 66 32 39 66 32 39 30 62 62 33 37 35 71 07 37f29f290bb375q.
0090 55 0B 58 2D 54 69 6D 65 73 74 61 6D 70 71 08 55 U.X-Timestampq.U
00A0 10 31 36 38 32 39 36 32 36 35 31 2E 39 37 34 39 .1682962651.9749
00B0 34 55 11 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 61 4U.X-Object-Meta
00C0 2D E2 98 83 55 03 E2 98 83 71 09 55 0C 43 6F 6E -...U....q.U.Con
00D0 74 65 6E 74 2D 54 79 70 65 71 0A 55 18 61 70 70 tent-Typeq.U.app
00E0 6C 69 63 61 74 69 6F 6E 2F 6F 63 74 65 74 2D 73 lication/octet-s
00F0 74 72 65 61 6D 71 0B 75 2E treamq.u.
''',
'python_2.7.18_swift_2.13_ec': '''
0000 80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D ..}q.(U.Content-
0010 4C 65 6E 67 74 68 71 02 55 02 38 34 55 04 6E 61 Lengthq.U.84U.na
0020 6D 65 71 03 55 12 2F 41 55 54 48 5F 74 65 73 74 meq.U./AUTH_test
0030 2F E2 98 83 2F E2 98 83 71 04 58 1E 00 00 00 58 /.../...q.X....X
0040 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61 2D -Object-Sysmeta-
0050 45 63 2D 46 72 61 67 2D 49 6E 64 65 78 71 05 55 Ec-Frag-Indexq.U
0060 01 35 55 13 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 .5U.X-Object-Met
0070 61 2D 4D 74 69 6D 65 55 11 31 36 38 32 39 35 39 a-MtimeU.1682959
0080 38 37 34 2E 37 35 36 32 30 35 71 06 58 22 00 00 874.756205q.X"..
0090 00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 .X-Object-Sysmet
00A0 61 2D 45 63 2D 43 6F 6E 74 65 6E 74 2D 4C 65 6E a-Ec-Content-Len
00B0 67 74 68 71 07 55 02 31 33 71 08 58 18 00 00 00 gthq.U.13q.X....
00C0 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61 X-Object-Sysmeta
00D0 2D 45 63 2D 45 74 61 67 71 09 55 20 36 62 37 64 -Ec-Etagq.U 6b7d
00E0 39 61 31 63 35 64 31 36 37 63 63 35 30 30 33 37 9a1c5d167cc50037
00F0 66 32 39 66 32 39 30 62 62 33 37 35 71 0A 55 04 f29f290bb375q.U.
0100 45 54 61 67 71 0B 55 20 65 32 66 64 34 33 30 65 ETagq.U e2fd430e
0110 61 66 37 32 32 33 63 32 35 30 33 63 34 65 38 33 af7223c2503c4e83
0120 30 31 63 66 66 33 37 63 71 0C 55 0B 58 2D 54 69 01cff37cq.U.X-Ti
0130 6D 65 73 74 61 6D 70 71 0D 55 10 31 36 38 32 39 mestampq.U.16829
0140 36 32 32 36 32 2E 31 36 31 39 39 55 11 58 2D 4F 62262.16199U.X-O
0150 62 6A 65 63 74 2D 4D 65 74 61 2D E2 98 83 55 03 bject-Meta-...U.
0160 E2 98 83 71 0E 58 1A 00 00 00 58 2D 4F 62 6A 65 ...q.X....X-Obje
0170 63 74 2D 53 79 73 6D 65 74 61 2D 45 63 2D 53 63 ct-Sysmeta-Ec-Sc
0180 68 65 6D 65 71 0F 55 1A 6C 69 62 65 72 61 73 75 hemeq.U.liberasu
0190 72 65 63 6F 64 65 5F 72 73 5F 76 61 6E 64 20 34 recode_rs_vand 4
01A0 2B 32 71 10 55 0C 43 6F 6E 74 65 6E 74 2D 54 79 +2q.U.Content-Ty
01B0 70 65 71 11 55 18 61 70 70 6C 69 63 61 74 69 6F peq.U.applicatio
01C0 6E 2F 6F 63 74 65 74 2D 73 74 72 65 61 6D 71 12 n/octet-streamq.
01D0 58 20 00 00 00 58 2D 4F 62 6A 65 63 74 2D 53 79 X ...X-Object-Sy
01E0 73 6D 65 74 61 2D 45 63 2D 53 65 67 6D 65 6E 74 smeta-Ec-Segment
01F0 2D 53 69 7A 65 71 13 55 07 31 30 34 38 35 37 36 -Sizeq.U.1048576
0200 71 14 75 2E q.u.
''',
'python_2.7.18_swift_2.23_replicated': '''
0000 80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D ..}q.(U.Content-
0010 4C 65 6E 67 74 68 71 02 55 02 31 33 71 03 55 04 Lengthq.U.13q.U.
0020 6E 61 6D 65 71 04 55 12 2F 41 55 54 48 5F 74 65 nameq.U./AUTH_te
0030 73 74 2F E2 98 83 2F E2 98 83 71 05 55 0C 43 6F st/.../...q.U.Co
0040 6E 74 65 6E 74 2D 54 79 70 65 71 06 55 18 61 70 ntent-Typeq.U.ap
0050 70 6C 69 63 61 74 69 6F 6E 2F 6F 63 74 65 74 2D plication/octet-
0060 73 74 72 65 61 6D 71 07 55 04 45 54 61 67 71 08 streamq.U.ETagq.
0070 55 20 36 62 37 64 39 61 31 63 35 64 31 36 37 63 U 6b7d9a1c5d167c
0080 63 35 30 30 33 37 66 32 39 66 32 39 30 62 62 33 c50037f29f290bb3
0090 37 35 71 09 55 0B 58 2D 54 69 6D 65 73 74 61 6D 75q.U.X-Timestam
00A0 70 71 0A 55 10 31 36 38 32 39 36 33 32 30 39 2E pq.U.1682963209.
00B0 38 32 32 37 32 71 0B 55 11 58 2D 4F 62 6A 65 63 82272q.U.X-Objec
00C0 74 2D 4D 65 74 61 2D E2 98 83 71 0C 55 03 E2 98 t-Meta-...q.U...
00D0 83 71 0D 55 13 58 2D 4F 62 6A 65 63 74 2D 4D 65 .q.U.X-Object-Me
00E0 74 61 2D 4D 74 69 6D 65 71 0E 55 11 31 36 38 32 ta-Mtimeq.U.1682
00F0 39 35 39 38 37 34 2E 37 35 36 32 30 35 71 0F 75 959874.756205q.u
0100 2E .
''',
'python_3.10.6_swift_2.23_replicated': '''
0000 80 02 7D 71 00 28 63 5F 63 6F 64 65 63 73 0A 65 ..}q.(c_codecs.e
0010 6E 63 6F 64 65 0A 71 01 58 0B 00 00 00 58 2D 54 ncode.q.X....X-T
0020 69 6D 65 73 74 61 6D 70 71 02 58 06 00 00 00 6C imestampq.X....l
0030 61 74 69 6E 31 71 03 86 71 04 52 71 05 68 01 58 atin1q..q.Rq.h.X
0040 10 00 00 00 31 36 38 32 39 36 33 30 31 37 2E 31 ....1682963017.1
0050 30 34 37 32 71 06 68 03 86 71 07 52 71 08 68 01 0472q.h..q.Rq.h.
0060 58 0C 00 00 00 43 6F 6E 74 65 6E 74 2D 54 79 70 X....Content-Typ
0070 65 71 09 68 03 86 71 0A 52 71 0B 68 01 58 18 00 eq.h..q.Rq.h.X..
0080 00 00 61 70 70 6C 69 63 61 74 69 6F 6E 2F 6F 63 ..application/oc
0090 74 65 74 2D 73 74 72 65 61 6D 71 0C 68 03 86 71 tet-streamq.h..q
00A0 0D 52 71 0E 68 01 58 0E 00 00 00 43 6F 6E 74 65 .Rq.h.X....Conte
00B0 6E 74 2D 4C 65 6E 67 74 68 71 0F 68 03 86 71 10 nt-Lengthq.h..q.
00C0 52 71 11 68 01 58 02 00 00 00 31 33 71 12 68 03 Rq.h.X....13q.h.
00D0 86 71 13 52 71 14 68 01 58 04 00 00 00 45 54 61 .q.Rq.h.X....ETa
00E0 67 71 15 68 03 86 71 16 52 71 17 68 01 58 20 00 gq.h..q.Rq.h.X .
00F0 00 00 36 62 37 64 39 61 31 63 35 64 31 36 37 63 ..6b7d9a1c5d167c
0100 63 35 30 30 33 37 66 32 39 66 32 39 30 62 62 33 c50037f29f290bb3
0110 37 35 71 18 68 03 86 71 19 52 71 1A 68 01 58 13 75q.h..q.Rq.h.X.
0120 00 00 00 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 61 ...X-Object-Meta
0130 2D 4D 74 69 6D 65 71 1B 68 03 86 71 1C 52 71 1D -Mtimeq.h..q.Rq.
0140 68 01 58 11 00 00 00 31 36 38 32 39 35 39 38 37 h.X....168295987
0150 34 2E 37 35 36 32 30 35 71 1E 68 03 86 71 1F 52 4.756205q.h..q.R
0160 71 20 68 01 58 1A 00 00 00 58 2D 4F 62 6A 65 63 q h.X....X-Objec
0170 74 2D 4D 65 74 61 2D C3 83 C2 A2 C3 82 C2 98 C3 t-Meta-.........
0180 82 C2 83 71 21 68 03 86 71 22 52 71 23 68 01 58 ...q!h..q"Rq#h.X
0190 0C 00 00 00 C3 83 C2 A2 C3 82 C2 98 C3 82 C2 83 ................
01A0 71 24 68 03 86 71 25 52 71 26 68 01 58 04 00 00 q$h..q%Rq&h.X...
01B0 00 6E 61 6D 65 71 27 68 03 86 71 28 52 71 29 68 .nameq'h..q(Rq)h
01C0 01 58 18 00 00 00 2F 41 55 54 48 5F 74 65 73 74 .X..../AUTH_test
01D0 2F C3 A2 C2 98 C2 83 2F C3 A2 C2 98 C2 83 71 2A /....../......q*
01E0 68 03 86 71 2B 52 71 2C 75 2E h..q+Rq,u.
''',
'python_2.7.18_swift_2.23_ec': '''
0000 80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D ..}q.(U.Content-
0010 4C 65 6E 67 74 68 71 02 55 02 38 34 71 03 55 04 Lengthq.U.84q.U.
0020 6E 61 6D 65 71 04 55 12 2F 41 55 54 48 5F 74 65 nameq.U./AUTH_te
0030 73 74 2F E2 98 83 2F E2 98 83 71 05 55 1E 58 2D st/.../...q.U.X-
0040 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61 2D 45 Object-Sysmeta-E
0050 63 2D 46 72 61 67 2D 49 6E 64 65 78 55 01 35 55 c-Frag-IndexU.5U
0060 0C 43 6F 6E 74 65 6E 74 2D 54 79 70 65 71 06 55 .Content-Typeq.U
0070 18 61 70 70 6C 69 63 61 74 69 6F 6E 2F 6F 63 74 .application/oct
0080 65 74 2D 73 74 72 65 61 6D 71 07 55 22 58 2D 4F et-streamq.U"X-O
0090 62 6A 65 63 74 2D 53 79 73 6D 65 74 61 2D 45 63 bject-Sysmeta-Ec
00A0 2D 43 6F 6E 74 65 6E 74 2D 4C 65 6E 67 74 68 55 -Content-LengthU
00B0 02 31 33 71 08 55 18 58 2D 4F 62 6A 65 63 74 2D .13q.U.X-Object-
00C0 53 79 73 6D 65 74 61 2D 45 63 2D 45 74 61 67 55 Sysmeta-Ec-EtagU
00D0 20 36 62 37 64 39 61 31 63 35 64 31 36 37 63 63 6b7d9a1c5d167cc
00E0 35 30 30 33 37 66 32 39 66 32 39 30 62 62 33 37 50037f29f290bb37
00F0 35 71 09 55 04 45 54 61 67 71 0A 55 20 65 32 66 5q.U.ETagq.U e2f
0100 64 34 33 30 65 61 66 37 32 32 33 63 32 35 30 33 d430eaf7223c2503
0110 63 34 65 38 33 30 31 63 66 66 33 37 63 71 0B 55 c4e8301cff37cq.U
0120 0B 58 2D 54 69 6D 65 73 74 61 6D 70 71 0C 55 10 .X-Timestampq.U.
0130 31 36 38 32 39 36 33 31 33 30 2E 33 35 39 38 36 1682963130.35986
0140 71 0D 55 11 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 q.U.X-Object-Met
0150 61 2D E2 98 83 71 0E 55 03 E2 98 83 71 0F 55 1A a-...q.U....q.U.
0160 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61 X-Object-Sysmeta
0170 2D 45 63 2D 53 63 68 65 6D 65 55 1A 6C 69 62 65 -Ec-SchemeU.libe
0180 72 61 73 75 72 65 63 6F 64 65 5F 72 73 5F 76 61 rasurecode_rs_va
0190 6E 64 20 34 2B 32 71 10 55 13 58 2D 4F 62 6A 65 nd 4+2q.U.X-Obje
01A0 63 74 2D 4D 65 74 61 2D 4D 74 69 6D 65 71 11 55 ct-Meta-Mtimeq.U
01B0 11 31 36 38 32 39 35 39 38 37 34 2E 37 35 36 32 .1682959874.7562
01C0 30 35 71 12 55 20 58 2D 4F 62 6A 65 63 74 2D 53 05q.U X-Object-S
01D0 79 73 6D 65 74 61 2D 45 63 2D 53 65 67 6D 65 6E ysmeta-Ec-Segmen
01E0 74 2D 53 69 7A 65 55 07 31 30 34 38 35 37 36 71 t-SizeU.1048576q
01F0 13 75 2E .u.
''',
'python_3.10.6_swift_2.23_ec': '''
0000 80 02 7D 71 00 28 63 5F 63 6F 64 65 63 73 0A 65 ..}q.(c_codecs.e
0010 6E 63 6F 64 65 0A 71 01 58 0B 00 00 00 58 2D 54 ncode.q.X....X-T
0020 69 6D 65 73 74 61 6D 70 71 02 58 06 00 00 00 6C imestampq.X....l
0030 61 74 69 6E 31 71 03 86 71 04 52 71 05 68 01 58 atin1q..q.Rq.h.X
0040 10 00 00 00 31 36 38 32 39 36 32 39 35 35 2E 33 ....1682962955.3
0050 37 35 34 36 71 06 68 03 86 71 07 52 71 08 68 01 7546q.h..q.Rq.h.
0060 58 0C 00 00 00 43 6F 6E 74 65 6E 74 2D 54 79 70 X....Content-Typ
0070 65 71 09 68 03 86 71 0A 52 71 0B 68 01 58 18 00 eq.h..q.Rq.h.X..
0080 00 00 61 70 70 6C 69 63 61 74 69 6F 6E 2F 6F 63 ..application/oc
0090 74 65 74 2D 73 74 72 65 61 6D 71 0C 68 03 86 71 tet-streamq.h..q
00A0 0D 52 71 0E 68 01 58 0E 00 00 00 43 6F 6E 74 65 .Rq.h.X....Conte
00B0 6E 74 2D 4C 65 6E 67 74 68 71 0F 68 03 86 71 10 nt-Lengthq.h..q.
00C0 52 71 11 68 01 58 02 00 00 00 38 34 71 12 68 03 Rq.h.X....84q.h.
00D0 86 71 13 52 71 14 68 01 58 04 00 00 00 45 54 61 .q.Rq.h.X....ETa
00E0 67 71 15 68 03 86 71 16 52 71 17 68 01 58 20 00 gq.h..q.Rq.h.X .
00F0 00 00 65 32 66 64 34 33 30 65 61 66 37 32 32 33 ..e2fd430eaf7223
0100 63 32 35 30 33 63 34 65 38 33 30 31 63 66 66 33 c2503c4e8301cff3
0110 37 63 71 18 68 03 86 71 19 52 71 1A 68 01 58 13 7cq.h..q.Rq.h.X.
0120 00 00 00 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 61 ...X-Object-Meta
0130 2D 4D 74 69 6D 65 71 1B 68 03 86 71 1C 52 71 1D -Mtimeq.h..q.Rq.
0140 68 01 58 11 00 00 00 31 36 38 32 39 35 39 38 37 h.X....168295987
0150 34 2E 37 35 36 32 30 35 71 1E 68 03 86 71 1F 52 4.756205q.h..q.R
0160 71 20 68 01 58 1A 00 00 00 58 2D 4F 62 6A 65 63 q h.X....X-Objec
0170 74 2D 4D 65 74 61 2D C3 83 C2 A2 C3 82 C2 98 C3 t-Meta-.........
0180 82 C2 83 71 21 68 03 86 71 22 52 71 23 68 01 58 ...q!h..q"Rq#h.X
0190 0C 00 00 00 C3 83 C2 A2 C3 82 C2 98 C3 82 C2 83 ................
01A0 71 24 68 03 86 71 25 52 71 26 68 01 58 18 00 00 q$h..q%Rq&h.X...
01B0 00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 .X-Object-Sysmet
01C0 61 2D 45 63 2D 45 74 61 67 71 27 68 03 86 71 28 a-Ec-Etagq'h..q(
01D0 52 71 29 68 01 58 20 00 00 00 36 62 37 64 39 61 Rq)h.X ...6b7d9a
01E0 31 63 35 64 31 36 37 63 63 35 30 30 33 37 66 32 1c5d167cc50037f2
01F0 39 66 32 39 30 62 62 33 37 35 71 2A 68 03 86 71 9f290bb375q*h..q
0200 2B 52 71 2C 68 01 58 22 00 00 00 58 2D 4F 62 6A +Rq,h.X"...X-Obj
0210 65 63 74 2D 53 79 73 6D 65 74 61 2D 45 63 2D 43 ect-Sysmeta-Ec-C
0220 6F 6E 74 65 6E 74 2D 4C 65 6E 67 74 68 71 2D 68 ontent-Lengthq-h
0230 03 86 71 2E 52 71 2F 68 01 58 02 00 00 00 31 33 ..q.Rq/h.X....13
0240 71 30 68 03 86 71 31 52 71 32 68 01 58 1E 00 00 q0h..q1Rq2h.X...
0250 00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 .X-Object-Sysmet
0260 61 2D 45 63 2D 46 72 61 67 2D 49 6E 64 65 78 71 a-Ec-Frag-Indexq
0270 33 68 03 86 71 34 52 71 35 68 01 58 01 00 00 00 3h..q4Rq5h.X....
0280 35 71 36 68 03 86 71 37 52 71 38 68 01 58 1A 00 5q6h..q7Rq8h.X..
0290 00 00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 ..X-Object-Sysme
02A0 74 61 2D 45 63 2D 53 63 68 65 6D 65 71 39 68 03 ta-Ec-Schemeq9h.
02B0 86 71 3A 52 71 3B 68 01 58 1A 00 00 00 6C 69 62 .q:Rq;h.X....lib
02C0 65 72 61 73 75 72 65 63 6F 64 65 5F 72 73 5F 76 erasurecode_rs_v
02D0 61 6E 64 20 34 2B 32 71 3C 68 03 86 71 3D 52 71 and 4+2q<h..q=Rq
02E0 3E 68 01 58 20 00 00 00 58 2D 4F 62 6A 65 63 74 >h.X ...X-Object
02F0 2D 53 79 73 6D 65 74 61 2D 45 63 2D 53 65 67 6D -Sysmeta-Ec-Segm
0300 65 6E 74 2D 53 69 7A 65 71 3F 68 03 86 71 40 52 ent-Sizeq?h..q@R
0310 71 41 68 01 58 07 00 00 00 31 30 34 38 35 37 36 qAh.X....1048576
0320 71 42 68 03 86 71 43 52 71 44 68 01 58 04 00 00 qBh..qCRqDh.X...
0330 00 6E 61 6D 65 71 45 68 03 86 71 46 52 71 47 68 .nameqEh..qFRqGh
0340 01 58 18 00 00 00 2F 41 55 54 48 5F 74 65 73 74 .X..../AUTH_test
0350 2F C3 A2 C2 98 C2 83 2F C3 A2 C2 98 C2 83 71 48 /....../......qH
0360 68 03 86 71 49 52 71 4A 75 2E h..qIRqJu.
''',
}
def raw_xattr(output):
return binascii.unhexlify(''.join(
line[7:55] for line in output.split('\n')
).replace(' ', ''))
path = os.path.join(self.testdir, str(uuid.uuid4()))
for case, xattr_output in cases.items():
try:
to_write = raw_xattr(xattr_output)
with open(path, 'wb') as fp:
xattr.setxattr(
fp.fileno(), 'user.swift.metadata', to_write)
with open(path, 'rb') as fd:
actual = diskfile.read_metadata(fd)
# name should come out as native strings
expected_name = b'/AUTH_test/\xe2\x98\x83/\xe2\x98\x83'
if not six.PY2:
expected_name = expected_name.decode('utf8')
self.assertEqual(actual['name'], expected_name)
# other meta will be WSGI strings, though
self.assertEqual(
actual['X-Object-Meta-\xe2\x98\x83'], '\xe2\x98\x83')
except Exception:
print('Failure in %s' % case, file=sys.stderr)
raise
def test_write_read_metadata(self):
path = os.path.join(self.testdir, str(uuid.uuid4()))
metadata = {'name': '/a/c/o',
@ -597,29 +827,6 @@ class TestDiskFileModuleMethods(unittest.TestCase):
diskfile.write_metadata(fd, metadata)
check_metadata(as_native, str)
# mock the read path to check the write path encoded persisted metadata
with mock.patch.object(diskfile, '_decode_metadata', lambda x: x):
check_metadata(as_bytes, bytes)
# simulate a legacy diskfile that might have persisted
# (some) unicode metadata
with mock.patch.object(diskfile, '_encode_metadata', lambda x: x):
with open(path, 'wb') as fd:
diskfile.write_metadata(fd, metadata)
# sanity check: mock read path again to see that we did persist unicode
with mock.patch.object(diskfile, '_decode_metadata', lambda x: x):
with open(path, 'rb') as fd:
actual = diskfile.read_metadata(fd)
for k, v in actual.items():
if isinstance(k, six.text_type) and \
k == u'X-Object-Meta-Strange':
self.assertIsInstance(v, six.text_type)
break
else:
self.fail('Did not find X-Object-Meta-Strange')
# check that read_metadata converts binary_type
check_metadata(as_native, str)
@patch_policies
class TestObjectAuditLocationGenerator(unittest.TestCase):