openstack/swift
Code Issues Proposed changes

func tests: work with etag-quoter on by default

Browse Source 
Also, run the in-process encryption func tests like that.

Change-Id: I984ab8d1304d23b89589973950b10dda4aea0db3
This commit is contained in:
Tim Burke 2019-11-19 21:25:45 -08:00 committed by Pete Zaitcev
parent 6c1bc3949d
commit 99947150dd
9 changed files with 152 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
test/functional/__init__.py
View File

@ -322,12 +322,16 @@ def _load_encryption(proxy_conf_file, swift_conf_file, **kwargs):
pipeline = pipeline.replace(
"proxy-logging proxy-server",
"keymaster encryption proxy-logging proxy-server")
pipeline = pipeline.replace(
"cache listing_formats",
"cache etag-quoter listing_formats")
conf.set(section, 'pipeline', pipeline)
root_secret = base64.b64encode(os.urandom(32))
if not six.PY2:
root_secret = root_secret.decode('ascii')
conf.set('filter:keymaster', 'encryption_root_secret', root_secret)
conf.set('filter:versioned_writes', 'allow_object_versioning', 'true')
conf.set('filter:etag-quoter', 'enable_by_default', 'true')
except NoSectionError as err:
msg = 'Error problem with proxy conf file %s: %s' % \
(proxy_conf_file, err)

9
test/functional/test_object.py
View File

@ -1726,7 +1726,7 @@ class TestObject(unittest.TestCase):
if 'etag_quoter' not in tf.cluster_info:
raise SkipTest("etag-quoter middleware is not enabled")
def do_head(expect_quoted=False):
def do_head(expect_quoted=None):
def head(url, token, parsed, conn):
conn.request('HEAD', '%s/%s/%s' % (
parsed.path, self.container, self.obj), '',
@ -1736,6 +1736,11 @@ class TestObject(unittest.TestCase):
resp = retry(head)
resp.read()
self.assertEqual(resp.status, 200)
if expect_quoted is None:
expect_quoted = tf.cluster_info.get('etag_quoter', {}).get(
'enable_by_default', False)
expected_etag = hashlib.md5(b'test').hexdigest()
if expect_quoted:
expected_etag = '"%s"' % expected_etag
@ -1771,7 +1776,7 @@ class TestObject(unittest.TestCase):
post_container('')
do_head(expect_quoted=True)
post_container('f')
do_head()
do_head(expect_quoted=False)
finally:
# Don't leave a dirty account
post_account('')

19
test/functional/test_object_versioning.py
View File

@ -26,6 +26,7 @@ from six.moves.urllib.parse import quote, unquote
import test.functional as tf
from swift.common.swob import normalize_etag
from swift.common.utils import MD5_OF_EMPTY_STRING, config_true_value
from swift.common.middleware.versioned_writes.object_versioning import \
DELETE_MARKER_CONTENT_TYPE
@ -337,7 +338,7 @@ class TestObjectVersioning(TestObjectVersioningBase):
obj = self.env.unversioned_container.file(oname)
resp = obj.write(body, return_resp=True)
etag = resp.getheader('etag')
self.assertEqual(md5(body).hexdigest(), etag)
self.assertEqual(md5(body).hexdigest(), normalize_etag(etag))
# un-versioned object is cool with with if-match
self.assertEqual(body, obj.read(hdrs={'if-match': etag}))
@ -350,7 +351,11 @@ class TestObjectVersioning(TestObjectVersioningBase):
self.assertEqual(resp.getheader('etag'), etag)
# versioned object is too with with if-match
self.assertEqual(body, v_obj.read(hdrs={'if-match': etag}))
self.assertEqual(body, v_obj.read(hdrs={
'if-match': normalize_etag(etag)}))
# works quoted, too
self.assertEqual(body, v_obj.read(hdrs={
'if-match': '"%s"' % normalize_etag(etag)}))
with self.assertRaises(ResponseError) as cm:
v_obj.read(hdrs={'if-match': 'not-the-etag'})
self.assertEqual(412, cm.exception.status)
@ -989,13 +994,13 @@ class TestObjectVersioning(TestObjectVersioningBase):
'Content-Type': 'text/jibberish32'
}, return_resp=True)
v1_version_id = resp.getheader('x-object-version-id')
v1_etag = resp.getheader('etag')
v1_etag = normalize_etag(resp.getheader('etag'))
resp = obj.write(b'version2', hdrs={
'Content-Type': 'text/jibberish33'
}, return_resp=True)
v2_version_id = resp.getheader('x-object-version-id')
v2_etag = resp.getheader('etag')
v2_etag = normalize_etag(resp.getheader('etag'))
# sanity
self.assertEqual(b'version2', obj.read())
@ -1062,7 +1067,7 @@ class TestObjectVersioning(TestObjectVersioningBase):
self.assertEqual(b'version1', obj.read())
obj_info = obj.info()
self.assertEqual('text/jibberish32', obj_info['content_type'])
self.assertEqual(v1_etag, obj_info['etag'])
self.assertEqual(v1_etag, normalize_etag(obj_info['etag']))
def test_delete_with_version_api_old_object(self):
versioned_obj_name = Utils.create_name()
@ -2378,7 +2383,7 @@ class TestSloWithVersioning(TestObjectVersioningBase):
expected = {
'bytes': file_info['content_length'],
'content_type': 'application/octet-stream',
'hash': manifest_info['etag'],
'hash': normalize_etag(manifest_info['etag']),
'name': 'my-slo-manifest',
'slo_etag': file_info['etag'],
'version_symlink': True,
@ -2410,7 +2415,7 @@ class TestSloWithVersioning(TestObjectVersioningBase):
expected = {
'bytes': file_info['content_length'],
'content_type': 'application/octet-stream',
'hash': manifest_info['etag'],
'hash': normalize_etag(manifest_info['etag']),
'name': 'my-slo-manifest',
'slo_etag': file_info['etag'],
'version_symlink': True,

31
test/functional/test_slo.py
View File

@ -23,6 +23,8 @@ from copy import deepcopy
import six
from swift.common.swob import normalize_etag
import test.functional as tf
from test.functional import cluster_info, SkipTest
from test.functional.tests import Utils, Base, Base2, BaseEnv
@ -299,8 +301,14 @@ class TestSlo(Base):
# a POST.
file_item.initialize(parms={'multipart-manifest': 'get'})
manifest_etag = file_item.etag
self.assertFalse(manifest_etag.startswith('"'))
self.assertFalse(manifest_etag.endswith('"'))
if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'):
self.assertTrue(manifest_etag.startswith('"'))
self.assertTrue(manifest_etag.endswith('"'))
# ...but in the listing, it'll be stripped
manifest_etag = manifest_etag[1:-1]
else:
self.assertFalse(manifest_etag.startswith('"'))
self.assertFalse(manifest_etag.endswith('"'))
file_item.initialize()
slo_etag = file_item.etag
@ -715,6 +723,8 @@ class TestSlo(Base):
source_contents = source.read(parms={'multipart-manifest': 'get'})
source_json = json.loads(source_contents)
manifest_etag = hashlib.md5(source_contents).hexdigest()
if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'):
manifest_etag = '"%s"' % manifest_etag
self.assertEqual(manifest_etag, source.etag)
source.initialize()
@ -752,14 +762,14 @@ class TestSlo(Base):
actual = names['manifest-abcde']
self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes'])
self.assertEqual('application/octet-stream', actual['content_type'])
self.assertEqual(manifest_etag, actual['hash'])
self.assertEqual(normalize_etag(manifest_etag), actual['hash'])
self.assertEqual(slo_etag, actual['slo_etag'])
self.assertIn('copied-abcde-manifest-only', names)
actual = names['copied-abcde-manifest-only']
self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes'])
self.assertEqual('application/octet-stream', actual['content_type'])
self.assertEqual(manifest_etag, actual['hash'])
self.assertEqual(normalize_etag(manifest_etag), actual['hash'])
self.assertEqual(slo_etag, actual['slo_etag'])
# Test copy manifest including data segments
@ -789,6 +799,8 @@ class TestSlo(Base):
source_contents = source.read(parms={'multipart-manifest': 'get'})
source_json = json.loads(source_contents)
manifest_etag = hashlib.md5(source_contents).hexdigest()
if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'):
manifest_etag = '"%s"' % manifest_etag
self.assertEqual(manifest_etag, source.etag)
source.initialize()
@ -831,14 +843,14 @@ class TestSlo(Base):
self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes'])
self.assertEqual('application/octet-stream', actual['content_type'])
# the container listing should have the etag of the manifest contents
self.assertEqual(manifest_etag, actual['hash'])
self.assertEqual(normalize_etag(manifest_etag), actual['hash'])
self.assertEqual(slo_etag, actual['slo_etag'])
self.assertIn('copied-abcde-manifest-only', names)
actual = names['copied-abcde-manifest-only']
self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes'])
self.assertEqual('image/jpeg', actual['content_type'])
self.assertEqual(manifest_etag, actual['hash'])
self.assertEqual(normalize_etag(manifest_etag), actual['hash'])
self.assertEqual(slo_etag, actual['slo_etag'])
def test_slo_copy_the_manifest_account(self):
@ -1098,12 +1110,7 @@ class TestSlo(Base):
manifest = self.env.container.file("manifest-db")
got_body = manifest.read(parms={'multipart-manifest': 'get',
'format': 'raw'})
body_md5 = hashlib.md5(got_body).hexdigest()
headers = dict(
(h.lower(), v)
for h, v in manifest.conn.response.getheaders())
self.assertIn('etag', headers)
self.assertEqual(headers['etag'], body_md5)
self.assert_etag(hashlib.md5(got_body).hexdigest())
# raw format should have the actual manifest object content-type
self.assertEqual('application/octet-stream', manifest.content_type)

79
test/functional/test_symlink.py
View File

@ -25,6 +25,7 @@ from six.moves import urllib
from uuid import uuid4
from swift.common.http import is_success
from swift.common.swob import normalize_etag
from swift.common.utils import json, MD5_OF_EMPTY_STRING
from swift.common.middleware.slo import SloGetContext
from test.functional import check_response, retry, requires_acls, \
@ -1135,7 +1136,7 @@ class TestSymlink(Base):
etag=self.env.tgt_etag)
# overwrite tgt object
old_tgt_etag = self.env.tgt_etag
old_tgt_etag = normalize_etag(self.env.tgt_etag)
self.env._create_tgt_object(body='updated target body')
# sanity
@ -1380,7 +1381,7 @@ class TestSymlink(Base):
object_list[0]['symlink_path'])
obj_info = object_list[0]
self.assertIn('symlink_etag', obj_info)
self.assertEqual(self.env.tgt_etag,
self.assertEqual(normalize_etag(self.env.tgt_etag),
obj_info['symlink_etag'])
self.assertEqual(int(self.env.tgt_length),
obj_info['symlink_bytes'])
@ -1550,7 +1551,7 @@ class TestSymlinkSlo(Base):
'symlink_path': '/v1/%s/%s/manifest-abcde' % (
self.account_name, self.env.container2.name),
'symlink_bytes': 4 * 2 ** 20 + 1,
'symlink_etag': manifest_etag,
'symlink_etag': normalize_etag(manifest_etag),
})
def test_static_link_target_slo_manifest_wrong_etag(self):
@ -1740,7 +1741,11 @@ class TestSymlinkToSloSegments(Base):
self.assertEqual(1024 * 1024, f_dict['bytes'])
self.assertEqual('application/octet-stream',
f_dict['content_type'])
self.assertEqual(manifest_etag, f_dict['hash'])
if tf.cluster_info.get('etag_quoter', {}).get(
'enable_by_default'):
self.assertEqual(manifest_etag, '"%s"' % f_dict['hash'])
else:
self.assertEqual(manifest_etag, f_dict['hash'])
self.assertEqual(slo_etag, f_dict['slo_etag'])
break
else:
@ -1759,7 +1764,11 @@ class TestSymlinkToSloSegments(Base):
self.assertEqual(1024 * 1024, f_dict['bytes'])
self.assertEqual(file_item.content_type,
f_dict['content_type'])
self.assertEqual(manifest_etag, f_dict['hash'])
if tf.cluster_info.get('etag_quoter', {}).get(
'enable_by_default'):
self.assertEqual(manifest_etag, '"%s"' % f_dict['hash'])
else:
self.assertEqual(manifest_etag, f_dict['hash'])
self.assertEqual(slo_etag, f_dict['slo_etag'])
break
else:
@ -1778,7 +1787,11 @@ class TestSymlinkToSloSegments(Base):
self.assertEqual(1024 * 1024, f_dict['bytes'])
self.assertEqual(file_item.content_type,
f_dict['content_type'])
self.assertEqual(manifest_etag, f_dict['hash'])
if tf.cluster_info.get('etag_quoter', {}).get(
'enable_by_default'):
self.assertEqual(manifest_etag, '"%s"' % f_dict['hash'])
else:
self.assertEqual(manifest_etag, f_dict['hash'])
self.assertEqual(slo_etag, f_dict['slo_etag'])
break
else:
@ -1811,6 +1824,8 @@ class TestSymlinkToSloSegments(Base):
source_contents = source.read(parms={'multipart-manifest': 'get'})
source_json = json.loads(source_contents)
manifest_etag = hashlib.md5(source_contents).hexdigest()
if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'):
manifest_etag = '"%s"' % manifest_etag
source.initialize()
slo_etag = source.etag
@ -1857,14 +1872,20 @@ class TestSymlinkToSloSegments(Base):
actual = names['manifest-linkto-ab']
self.assertEqual(2 * 1024 * 1024, actual['bytes'])
self.assertEqual('application/octet-stream', actual['content_type'])
self.assertEqual(manifest_etag, actual['hash'])
if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'):
self.assertEqual(manifest_etag, '"%s"' % actual['hash'])
else:
self.assertEqual(manifest_etag, actual['hash'])
self.assertEqual(slo_etag, actual['slo_etag'])
self.assertIn('copied-ab-manifest-only', names)
actual = names['copied-ab-manifest-only']
self.assertEqual(2 * 1024 * 1024, actual['bytes'])
self.assertEqual('application/octet-stream', actual['content_type'])
self.assertEqual(manifest_etag, actual['hash'])
if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'):
self.assertEqual(manifest_etag, '"%s"' % actual['hash'])
else:
self.assertEqual(manifest_etag, actual['hash'])
self.assertEqual(slo_etag, actual['slo_etag'])
@ -2000,13 +2021,13 @@ class TestSymlinkTargetObjectComparison(Base):
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_header('etag', md5)
self.assert_etag(md5)
hdrs = {'If-Match': 'bogus'}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(412)
self.assert_header('etag', md5)
self.assert_etag(md5)
def testIfMatchMultipleEtags(self):
for file_item in self.env.files:
@ -2022,13 +2043,13 @@ class TestSymlinkTargetObjectComparison(Base):
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_header('etag', md5)
self.assert_etag(md5)
hdrs = {'If-Match': '"bogus1", "bogus2", "bogus3"'}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(412)
self.assert_header('etag', md5)
self.assert_etag(md5)
def testIfNoneMatch(self):
for file_item in self.env.files:
@ -2044,13 +2065,13 @@ class TestSymlinkTargetObjectComparison(Base):
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_header('etag', md5)
self.assert_etag(md5)
hdrs = {'If-None-Match': md5}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(304)
self.assert_header('etag', md5)
self.assert_etag(md5)
self.assert_header('accept-ranges', 'bytes')
def testIfNoneMatchMultipleEtags(self):
@ -2067,14 +2088,14 @@ class TestSymlinkTargetObjectComparison(Base):
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_header('etag', md5)
self.assert_etag(md5)
hdrs = {'If-None-Match':
'"bogus1", "bogus2", "%s"' % md5}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(304)
self.assert_header('etag', md5)
self.assert_etag(md5)
self.assert_header('accept-ranges', 'bytes')
def testIfModifiedSince(self):
@ -2091,19 +2112,19 @@ class TestSymlinkTargetObjectComparison(Base):
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_header('etag', md5)
self.assert_etag(md5)
self.assertTrue(file_symlink.info(hdrs=hdrs, parms=self.env.parms))
hdrs = {'If-Modified-Since': self.env.time_new}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(304)
self.assert_header('etag', md5)
self.assert_etag(md5)
self.assert_header('accept-ranges', 'bytes')
self.assertRaises(ResponseError, file_symlink.info, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(304)
self.assert_header('etag', md5)
self.assert_etag(md5)
self.assert_header('accept-ranges', 'bytes')
def testIfUnmodifiedSince(self):
@ -2120,18 +2141,18 @@ class TestSymlinkTargetObjectComparison(Base):
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_header('etag', md5)
self.assert_etag(md5)
self.assertTrue(file_symlink.info(hdrs=hdrs, parms=self.env.parms))
hdrs = {'If-Unmodified-Since': self.env.time_old_f2}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(412)
self.assert_header('etag', md5)
self.assert_etag(md5)
self.assertRaises(ResponseError, file_symlink.info, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(412)
self.assert_header('etag', md5)
self.assert_etag(md5)
def testIfMatchAndUnmodified(self):
for file_item in self.env.files:
@ -2148,21 +2169,21 @@ class TestSymlinkTargetObjectComparison(Base):
else:
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_header('etag', md5)
self.assert_etag(md5)
hdrs = {'If-Match': 'bogus',
'If-Unmodified-Since': self.env.time_new}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(412)
self.assert_header('etag', md5)
self.assert_etag(md5)
hdrs = {'If-Match': md5,
'If-Unmodified-Since': self.env.time_old_f3}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(412)
self.assert_header('etag', md5)
self.assert_etag(md5)
def testLastModified(self):
file_item = self.env.container.file(Utils.create_name())
@ -2186,7 +2207,7 @@ class TestSymlinkTargetObjectComparison(Base):
hdrs = {'If-Modified-Since': last_modified}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs)
self.assert_status(304)
self.assert_header('etag', md5)
self.assert_etag(md5)
self.assert_header('accept-ranges', 'bytes')
hdrs = {'If-Unmodified-Since': last_modified}
@ -2227,20 +2248,20 @@ class TestSymlinkComparison(TestSymlinkTargetObjectComparison):
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_header('etag', md5)
self.assert_etag(md5)
hdrs = {'If-Modified-Since': last_modified}
self.assertRaises(ResponseError, file_symlink.read, hdrs=hdrs,
parms=self.env.parms)
self.assert_status(304)
self.assert_header('etag', md5)
self.assert_etag(md5)
self.assert_header('accept-ranges', 'bytes')
hdrs = {'If-Unmodified-Since': last_modified}
body = file_symlink.read(hdrs=hdrs, parms=self.env.parms)
self.assertEqual(b'', body)
self.assert_status(200)
self.assert_header('etag', md5)
self.assert_etag(md5)
class TestSymlinkAccountTempurl(Base):

11
test/functional/test_versioned_writes.py
View File

@ -684,7 +684,11 @@ class TestObjectVersioning(Base):
prev_version = versions_container.file(versioned_obj_name)
prev_version_info = prev_version.info(parms={'symlink': 'get'})
self.assertEqual(b"aaaaa", prev_version.read())
self.assertEqual(MD5_OF_EMPTY_STRING, prev_version_info['etag'])
symlink_etag = prev_version_info['etag']
if symlink_etag.startswith('"') and symlink_etag.endswith('"') and \
symlink_etag[1:-1]:
symlink_etag = symlink_etag[1:-1]
self.assertEqual(MD5_OF_EMPTY_STRING, symlink_etag)
self.assertEqual(sym_tgt_header,
prev_version_info['x_symlink_target'])
return symlink, tgt_a
@ -698,7 +702,10 @@ class TestObjectVersioning(Base):
symlink.delete()
sym_info = symlink.info(parms={'symlink': 'get'})
self.assertEqual(b"aaaaa", symlink.read())
self.assertEqual(MD5_OF_EMPTY_STRING, sym_info['etag'])
if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'):
self.assertEqual('"%s"' % MD5_OF_EMPTY_STRING, sym_info['etag'])
else:
self.assertEqual(MD5_OF_EMPTY_STRING, sym_info['etag'])
self.assertEqual(
quote(unquote('%s/%s' % (self.env.container.name, target.name))),
sym_info['x_symlink_target'])

75
test/functional/tests.py
View File

@ -27,6 +27,7 @@ import uuid
from copy import deepcopy
import eventlet
from swift.common.http import is_success, is_client_error
from swift.common.swob import normalize_etag
from email.utils import parsedate
if six.PY2:
@ -131,6 +132,13 @@ class Base(unittest.TestCase):
'Expected header name %r not found in response.' % header_name)
self.assertEqual(expected_value, actual_value)
def assert_etag(self, unquoted_value):
if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'):
expected = '"%s"' % unquoted_value
else:
expected = unquoted_value
self.assert_header('etag', expected)
class Base2(object):
@classmethod
@ -874,7 +882,11 @@ class TestContainer(Base):
for actual in file_list:
name = actual['name']
self.assertIn(name, expected)
self.assertEqual(expected[name]['etag'], actual['hash'])
if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'):
self.assertEqual(expected[name]['etag'],
'"%s"' % actual['hash'])
else:
self.assertEqual(expected[name]['etag'], actual['hash'])
self.assertEqual(
expected[name]['content_type'], actual['content_type'])
self.assertEqual(
@ -1365,6 +1377,8 @@ class TestFile(Base):
'x-delete-at': mock.ANY,
'x-trans-id': mock.ANY,
'x-openstack-request-id': mock.ANY}
if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'):
expected_headers['etag'] = '"%s"' % expected_headers['etag']
unexpected_headers = ['connection', 'x-delete-after']
do_test(put_headers, {}, expected_headers, unexpected_headers)
@ -1420,7 +1434,7 @@ class TestFile(Base):
self.fail('Failed to find %s in listing' % dest_filename)
self.assertEqual(file_item.size, obj['bytes'])
self.assertEqual(file_item.etag, obj['hash'])
self.assertEqual(normalize_etag(file_item.etag), obj['hash'])
self.assertEqual(file_item.content_type, obj['content_type'])
file_copy = cont.file(dest_filename)
@ -1470,7 +1484,7 @@ class TestFile(Base):
self.fail('Failed to find %s in listing' % dest_filename)
self.assertEqual(file_item.size, obj['bytes'])
self.assertEqual(file_item.etag, obj['hash'])
self.assertEqual(normalize_etag(file_item.etag), obj['hash'])
self.assertEqual(
'application/test-changed', obj['content_type'])
@ -1505,7 +1519,7 @@ class TestFile(Base):
self.fail('Failed to find %s in listing' % dest_filename)
self.assertEqual(file_item.size, obj['bytes'])
self.assertEqual(file_item.etag, obj['hash'])
self.assertEqual(normalize_etag(file_item.etag), obj['hash'])
self.assertEqual(
'application/test-updated', obj['content_type'])
@ -2088,7 +2102,7 @@ class TestFile(Base):
self.assertEqual(file_item.read(hdrs=hdrs), data[-i:])
self.assert_header('content-range', 'bytes %d-%d/%d' % (
file_length - i, file_length - 1, file_length))
self.assert_header('etag', file_item.md5)
self.assert_etag(file_item.md5)
self.assert_header('accept-ranges', 'bytes')
range_string = 'bytes=%d-' % (i)
@ -2102,7 +2116,7 @@ class TestFile(Base):
self.assertRaises(ResponseError, file_item.read, hdrs=hdrs)
self.assert_status(416)
self.assert_header('content-range', 'bytes */%d' % file_length)
self.assert_header('etag', file_item.md5)
self.assert_etag(file_item.md5)
self.assert_header('accept-ranges', 'bytes')
range_string = 'bytes=%d-%d' % (file_length - 1000, file_length + 2000)
@ -2416,14 +2430,16 @@ class TestFile(Base):
file_item.content_type = content_type
file_item.write_random(self.env.file_size)
md5 = file_item.md5
expected_etag = file_item.md5
if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'):
expected_etag = '"%s"' % expected_etag
file_item = self.env.container.file(file_name)
info = file_item.info()
self.assert_status(200)
self.assertEqual(info['content_length'], self.env.file_size)
self.assertEqual(info['etag'], md5)
self.assertEqual(info['etag'], expected_etag)
self.assertEqual(info['content_type'], content_type)
self.assertIn('last_modified', info)
@ -2612,14 +2628,7 @@ class TestFile(Base):
file_item = self.env.container.file(Utils.create_name())
data = io.BytesIO(file_item.write_random(512))
etag = File.compute_md5sum(data)
headers = dict((h.lower(), v)
for h, v in self.env.conn.response.getheaders())
self.assertIn('etag', headers.keys())
header_etag = headers['etag'].strip('"')
self.assertEqual(etag, header_etag)
self.assert_etag(File.compute_md5sum(data))
def testChunkedPut(self):
if (tf.web_front_end == 'apache2'):
@ -2645,7 +2654,7 @@ class TestFile(Base):
self.assertEqual(data, file_item.read())
info = file_item.info()
self.assertEqual(etag, info['etag'])
self.assertEqual(normalize_etag(info['etag']), etag)
def test_POST(self):
# verify consistency between object and container listing metadata
@ -2670,7 +2679,10 @@ class TestFile(Base):
self.fail('Failed to find file %r in listing' % file_name)
self.assertEqual(1024, f_dict['bytes'])
self.assertEqual('text/foobar', f_dict['content_type'])
self.assertEqual(etag, f_dict['hash'])
if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'):
self.assertEqual(etag, '"%s"' % f_dict['hash'])
else:
self.assertEqual(etag, f_dict['hash'])
put_last_modified = f_dict['last_modified']
# now POST updated content-type to each file
@ -2697,7 +2709,10 @@ class TestFile(Base):
self.assertEqual(1024, f_dict['bytes'])
self.assertEqual('image/foobarbaz', f_dict['content_type'])
self.assertLess(put_last_modified, f_dict['last_modified'])
self.assertEqual(etag, f_dict['hash'])
if tf.cluster_info.get('etag_quoter', {}).get('enable_by_default'):
self.assertEqual(etag, '"%s"' % f_dict['hash'])
else:
self.assertEqual(etag, f_dict['hash'])
class TestFileUTF8(Base2, TestFile):
@ -2742,7 +2757,7 @@ class TestFileComparison(Base):
hdrs = {'If-Match': 'bogus'}
self.assertRaises(ResponseError, file_item.read, hdrs=hdrs)
self.assert_status(412)
self.assert_header('etag', file_item.md5)
self.assert_etag(file_item.md5)
def testIfMatchMultipleEtags(self):
for file_item in self.env.files:
@ -2752,7 +2767,7 @@ class TestFileComparison(Base):
hdrs = {'If-Match': '"bogus1", "bogus2", "bogus3"'}
self.assertRaises(ResponseError, file_item.read, hdrs=hdrs)
self.assert_status(412)
self.assert_header('etag', file_item.md5)
self.assert_etag(file_item.md5)
def testIfNoneMatch(self):
for file_item in self.env.files:
@ -2762,7 +2777,7 @@ class TestFileComparison(Base):
hdrs = {'If-None-Match': file_item.md5}
self.assertRaises(ResponseError, file_item.read, hdrs=hdrs)
self.assert_status(304)
self.assert_header('etag', file_item.md5)
self.assert_etag(file_item.md5)
self.assert_header('accept-ranges', 'bytes')
def testIfNoneMatchMultipleEtags(self):
@ -2774,7 +2789,7 @@ class TestFileComparison(Base):
'"bogus1", "bogus2", "%s"' % file_item.md5}
self.assertRaises(ResponseError, file_item.read, hdrs=hdrs)
self.assert_status(304)
self.assert_header('etag', file_item.md5)
self.assert_etag(file_item.md5)
self.assert_header('accept-ranges', 'bytes')
def testIfModifiedSince(self):
@ -2786,11 +2801,11 @@ class TestFileComparison(Base):
hdrs = {'If-Modified-Since': self.env.time_new}
self.assertRaises(ResponseError, file_item.read, hdrs=hdrs)
self.assert_status(304)
self.assert_header('etag', file_item.md5)
self.assert_etag(file_item.md5)
self.assert_header('accept-ranges', 'bytes')
self.assertRaises(ResponseError, file_item.info, hdrs=hdrs)
self.assert_status(304)
self.assert_header('etag', file_item.md5)
self.assert_etag(file_item.md5)
self.assert_header('accept-ranges', 'bytes')
def testIfUnmodifiedSince(self):
@ -2802,10 +2817,10 @@ class TestFileComparison(Base):
hdrs = {'If-Unmodified-Since': self.env.time_old_f2}
self.assertRaises(ResponseError, file_item.read, hdrs=hdrs)
self.assert_status(412)
self.assert_header('etag', file_item.md5)
self.assert_etag(file_item.md5)
self.assertRaises(ResponseError, file_item.info, hdrs=hdrs)
self.assert_status(412)
self.assert_header('etag', file_item.md5)
self.assert_etag(file_item.md5)
def testIfMatchAndUnmodified(self):
for file_item in self.env.files:
@ -2817,13 +2832,13 @@ class TestFileComparison(Base):
'If-Unmodified-Since': self.env.time_new}
self.assertRaises(ResponseError, file_item.read, hdrs=hdrs)
self.assert_status(412)
self.assert_header('etag', file_item.md5)
self.assert_etag(file_item.md5)
hdrs = {'If-Match': file_item.md5,
'If-Unmodified-Since': self.env.time_old_f3}
self.assertRaises(ResponseError, file_item.read, hdrs=hdrs)
self.assert_status(412)
self.assert_header('etag', file_item.md5)
self.assert_etag(file_item.md5)
def testLastModified(self):
file_name = Utils.create_name()
@ -2844,7 +2859,7 @@ class TestFileComparison(Base):
hdrs = {'If-Modified-Since': last_modified}
self.assertRaises(ResponseError, file_item.read, hdrs=hdrs)
self.assert_status(304)
self.assert_header('etag', etag)
self.assert_etag(etag)
self.assert_header('accept-ranges', 'bytes')
hdrs = {'If-Unmodified-Since': last_modified}

7
test/probe/test_object_async_update.py
View File

@ -23,6 +23,7 @@ from swiftclient.exceptions import ClientException
from swift.common import direct_client
from swift.common.manager import Manager
from swift.common.swob import normalize_etag
from test.probe.common import kill_nonprimary_server, \
kill_server, ReplProbeTest, start_server, ECProbeTest
@ -210,7 +211,7 @@ class TestUpdateOverridesEC(ECProbeTest):
self.assertEqual(1, len(listing))
self.assertEqual('o1', listing[0]['name'])
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual(meta['etag'], listing[0]['hash'])
self.assertEqual(normalize_etag(meta['etag']), listing[0]['hash'])
self.assertEqual('test/ctype', listing[0]['content_type'])
def test_update_during_POST_only(self):
@ -261,7 +262,7 @@ class TestUpdateOverridesEC(ECProbeTest):
self.assertEqual(1, len(listing))
self.assertEqual('o1', listing[0]['name'])
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual(meta['etag'], listing[0]['hash'])
self.assertEqual(normalize_etag(meta['etag']), listing[0]['hash'])
self.assertEqual('test/ctype', listing[0]['content_type'])
# Run the object-updaters to send the async pending from the PUT
@ -328,7 +329,7 @@ class TestUpdateOverridesEC(ECProbeTest):
self.assertEqual(1, len(listing))
self.assertEqual('o1', listing[0]['name'])
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual(meta['etag'], listing[0]['hash'])
self.assertEqual(normalize_etag(meta['etag']), listing[0]['hash'])
self.assertEqual('test/ctype', listing[0]['content_type'])

3
test/probe/test_object_metadata_replication.py
View File

@ -22,6 +22,7 @@ import uuid
from swift.common.direct_client import direct_get_suffix_hashes
from swift.common.exceptions import DiskFileDeleted
from swift.common.internal_client import UnexpectedResponse
from swift.common.swob import normalize_etag
from swift.container.backend import ContainerBroker
from swift.common import utils
from swiftclient import client
@ -129,7 +130,7 @@ class Test(ReplProbeTest):
def _assert_object_metadata_matches_listing(self, listing, metadata):
self.assertEqual(listing['bytes'], int(metadata['content-length']))
self.assertEqual(listing['hash'], metadata['etag'])
self.assertEqual(listing['hash'], normalize_etag(metadata['etag']))
self.assertEqual(listing['content_type'], metadata['content-type'])
modified = Timestamp(metadata['x-timestamp']).isoformat
self.assertEqual(listing['last_modified'], modified)