openstack/swift
Code Issues Proposed changes

Provide a method for retrieving on-disk metadata

Browse Source 
We hide the internal dictionary for the metadata providing a method to
retrieve it to abstract away the implementation details of how
DiskFile object provides and maintains that metadata.

This is in anticipation of the DiskFile API refactoring.

Change-Id: I1c0dc01a4680bd435512405e2d31fba24421720a
Signed-off-by: Peter Portante <peter.portante@redhat.com>
This commit is contained in:
Peter Portante
2013-09-03 11:10:22 -04:00
parent 8a255a3415
commit 698023f477
3 changed files with 82 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
swift/obj/diskfile.py
View File

@@ -380,7 +380,7 @@ class DiskFile(object):
self.device_path = join(path, device)
self.tmpdir = join(path, device, 'tmp')
self.logger = logger
self.metadata = {}
self._metadata = {}
self.data_file = None
self.fp = None
self.iter_etag = None
@@ -464,8 +464,8 @@ class DiskFile(object):
pull the metadata from the tombstone file which has the timestamp.
"""
with open(ts_file) as fp:
self.metadata = read_metadata(fp)
self.metadata['deleted'] = True
self._metadata = read_metadata(fp)
self._metadata['deleted'] = True
def _verify_name(self):
"""
@@ -473,7 +473,7 @@ class DiskFile(object):
named.
"""
try:
mname = self.metadata['name']
mname = self._metadata['name']
except KeyError:
pass
else:
@@ -495,13 +495,13 @@ class DiskFile(object):
datafile_metadata = read_metadata(fp)
if meta_file:
with open(meta_file) as mfp:
self.metadata = read_metadata(mfp)
self._metadata = read_metadata(mfp)
sys_metadata = dict(
[(key, val) for key, val in datafile_metadata.iteritems()
if key.lower() in DATAFILE_SYSTEM_META])
self.metadata.update(sys_metadata)
self._metadata.update(sys_metadata)
else:
self.metadata = datafile_metadata
self._metadata = datafile_metadata
self._verify_name()
self.data_file = data_file
return fp
@@ -586,8 +586,8 @@ class DiskFile(object):
return
if self.iter_etag and self.started_at_0 and self.read_to_eof and \
'ETag' in self.metadata and \
self.iter_etag.hexdigest() != self.metadata.get('ETag'):
'ETag' in self._metadata and \
self.iter_etag.hexdigest() != self._metadata.get('ETag'):
self.quarantine()
def close(self, verify_file=True):
@@ -611,6 +611,14 @@ class DiskFile(object):
self.fp.close()
self.fp = None
def get_metadata(self):
"""
Provide the metadata for an object as a dictionary.
:returns: object's metadata dictionary
"""
return self._metadata
def is_deleted(self):
"""
Check if the file is deleted.
@@ -618,7 +626,7 @@ class DiskFile(object):
:returns: True if the file doesn't exist or has been flagged as
deleted.
"""
return not self.data_file or 'deleted' in self.metadata
return not self.data_file or 'deleted' in self._metadata
def is_expired(self):
"""
@@ -626,8 +634,8 @@ class DiskFile(object):
:returns: True if the file has an X-Delete-At in the past
"""
return ('X-Delete-At' in self.metadata and
int(self.metadata['X-Delete-At']) <= time.time())
return ('X-Delete-At' in self._metadata and
int(self._metadata['X-Delete-At']) <= time.time())
@contextmanager
def create(self, size=None):
@@ -713,8 +721,8 @@ class DiskFile(object):
if self.data_file:
file_size = self.threadpool.run_in_thread(
getsize, self.data_file)
if 'Content-Length' in self.metadata:
metadata_size = int(self.metadata['Content-Length'])
if 'Content-Length' in self._metadata:
metadata_size = int(self._metadata['Content-Length'])
if file_size != metadata_size:
raise DiskFileError(
'Content-Length of %s does not match file size '

65
swift/obj/server.py
View File

@@ -305,7 +305,8 @@ class ObjectController(object):
except (DiskFileError, DiskFileNotExist):
disk_file.quarantine()
return HTTPNotFound(request=request)
orig_timestamp = disk_file.metadata.get('X-Timestamp', '0')
orig_metadata = disk_file.get_metadata()
orig_timestamp = orig_metadata.get('X-Timestamp', '0')
if orig_timestamp >= request.headers['x-timestamp']:
return HTTPConflict(request=request)
metadata = {'X-Timestamp': request.headers['x-timestamp']}
@@ -315,7 +316,7 @@ class ObjectController(object):
if header_key in request.headers:
header_caps = header_key.title()
metadata[header_caps] = request.headers[header_key]
old_delete_at = int(disk_file.metadata.get('X-Delete-At') or 0)
old_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
if old_delete_at != new_delete_at:
if new_delete_at:
self.delete_at_update('PUT', new_delete_at, account, container,
@@ -354,8 +355,9 @@ class ObjectController(object):
obj)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
old_delete_at = int(disk_file.metadata.get('X-Delete-At') or 0)
orig_timestamp = disk_file.metadata.get('X-Timestamp')
orig_metadata = disk_file.get_metadata()
old_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
orig_timestamp = orig_metadata.get('X-Timestamp')
if orig_timestamp and orig_timestamp >= request.headers['x-timestamp']:
return HTTPConflict(request=request)
upload_expiration = time.time() + self.max_upload_time
@@ -414,10 +416,10 @@ class ObjectController(object):
self.container_update(
'PUT', account, container, obj, request,
HeaderKeyDict({
'x-size': disk_file.metadata['Content-Length'],
'x-content-type': disk_file.metadata['Content-Type'],
'x-timestamp': disk_file.metadata['X-Timestamp'],
'x-etag': disk_file.metadata['ETag']}),
'x-size': metadata['Content-Length'],
'x-content-type': metadata['Content-Type'],
'x-timestamp': metadata['X-Timestamp'],
'x-etag': metadata['ETag']}),
device)
resp = HTTPCreated(request=request, etag=etag)
return resp
@@ -443,14 +445,15 @@ class ObjectController(object):
except (DiskFileError, DiskFileNotExist):
disk_file.quarantine()
return HTTPNotFound(request=request)
metadata = disk_file.get_metadata()
if request.headers.get('if-match') not in (None, '*') and \
disk_file.metadata['ETag'] not in request.if_match:
metadata['ETag'] not in request.if_match:
disk_file.close()
return HTTPPreconditionFailed(request=request)
if request.headers.get('if-none-match') is not None:
if disk_file.metadata['ETag'] in request.if_none_match:
if metadata['ETag'] in request.if_none_match:
resp = HTTPNotModified(request=request)
resp.etag = disk_file.metadata['ETag']
resp.etag = metadata['ETag']
disk_file.close()
return resp
try:
@@ -460,7 +463,7 @@ class ObjectController(object):
return HTTPPreconditionFailed(request=request)
if if_unmodified_since and \
datetime.fromtimestamp(
float(disk_file.metadata['X-Timestamp']), UTC) > \
float(metadata['X-Timestamp']), UTC) > \
if_unmodified_since:
disk_file.close()
return HTTPPreconditionFailed(request=request)
@@ -471,29 +474,29 @@ class ObjectController(object):
return HTTPPreconditionFailed(request=request)
if if_modified_since and \
datetime.fromtimestamp(
float(disk_file.metadata['X-Timestamp']), UTC) < \
float(metadata['X-Timestamp']), UTC) < \
if_modified_since:
disk_file.close()
return HTTPNotModified(request=request)
response = Response(app_iter=disk_file,
request=request, conditional_response=True)
response.headers['Content-Type'] = disk_file.metadata.get(
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in disk_file.metadata.iteritems():
for key, value in metadata.iteritems():
if key.lower().startswith('x-object-meta-') or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = disk_file.metadata['ETag']
response.last_modified = float(disk_file.metadata['X-Timestamp'])
response.etag = metadata['ETag']
response.last_modified = float(metadata['X-Timestamp'])
response.content_length = file_size
if response.content_length < self.keep_cache_size and \
(self.keep_cache_private or
('X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers)):
disk_file.keep_cache = True
if 'Content-Encoding' in disk_file.metadata:
response.content_encoding = disk_file.metadata['Content-Encoding']
response.headers['X-Timestamp'] = disk_file.metadata['X-Timestamp']
if 'Content-Encoding' in metadata:
response.content_encoding = metadata['Content-Encoding']
response.headers['X-Timestamp'] = metadata['X-Timestamp']
return request.get_response(response)
@public
@@ -515,19 +518,20 @@ class ObjectController(object):
disk_file.quarantine()
return HTTPNotFound(request=request)
response = Response(request=request, conditional_response=True)
response.headers['Content-Type'] = disk_file.metadata.get(
metadata = disk_file.get_metadata()
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in disk_file.metadata.iteritems():
for key, value in metadata.iteritems():
if key.lower().startswith('x-object-meta-') or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = disk_file.metadata['ETag']
response.last_modified = float(disk_file.metadata['X-Timestamp'])
response.etag = metadata['ETag']
response.last_modified = float(metadata['X-Timestamp'])
# Needed for container sync feature
response.headers['X-Timestamp'] = disk_file.metadata['X-Timestamp']
response.headers['X-Timestamp'] = metadata['X-Timestamp']
response.content_length = file_size
if 'Content-Encoding' in disk_file.metadata:
response.content_encoding = disk_file.metadata['Content-Encoding']
if 'Content-Encoding' in metadata:
response.content_encoding = metadata['Content-Encoding']
return response
@public
@@ -545,17 +549,18 @@ class ObjectController(object):
obj)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
orig_metadata = disk_file.get_metadata()
if 'x-if-delete-at' in request.headers and \
int(request.headers['x-if-delete-at']) != \
int(disk_file.metadata.get('X-Delete-At') or 0):
int(orig_metadata.get('X-Delete-At') or 0):
return HTTPPreconditionFailed(
request=request,
body='X-If-Delete-At and X-Delete-At do not match')
old_delete_at = int(disk_file.metadata.get('X-Delete-At') or 0)
old_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
if old_delete_at:
self.delete_at_update('DELETE', old_delete_at, account,
container, obj, request, device)
orig_timestamp = disk_file.metadata.get('X-Timestamp', 0)
orig_timestamp = orig_metadata.get('X-Timestamp', 0)
req_timestamp = request.headers['X-Timestamp']
if disk_file.is_deleted() or disk_file.is_expired():
response_class = HTTPNotFound

45
test/unit/obj/test_diskfile.py
View File

@@ -387,12 +387,17 @@ class TestDiskFile(unittest.TestCase):
FakeLogger(), keep_data_fp=keep_data_fp)
return df
def test_get_metadata(self):
df = self._create_test_file('1234567890', timestamp=42)
md = df.get_metadata()
self.assertEquals(md['X-Timestamp'], normalize_timestamp(42))
def test_disk_file_default_disallowed_metadata(self):
# build an object with some meta (ts 41)
orig_metadata = {'X-Object-Meta-Key1': 'Value1',
'Content-Type': 'text/garbage'}
df = self._get_disk_file(ts=41, extra_metadata=orig_metadata)
self.assertEquals('1024', df.metadata['Content-Length'])
self.assertEquals('1024', df._metadata['Content-Length'])
# write some new metadata (fast POST, don't send orig meta, ts 42)
df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
FakeLogger())
@@ -400,11 +405,11 @@ class TestDiskFile(unittest.TestCase):
df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
FakeLogger())
# non-fast-post updateable keys are preserved
self.assertEquals('text/garbage', df.metadata['Content-Type'])
self.assertEquals('text/garbage', df._metadata['Content-Type'])
# original fast-post updateable keys are removed
self.assert_('X-Object-Meta-Key1' not in df.metadata)
self.assert_('X-Object-Meta-Key1' not in df._metadata)
# new fast-post updateable keys are added
self.assertEquals('Value2', df.metadata['X-Object-Meta-Key2'])
self.assertEquals('Value2', df._metadata['X-Object-Meta-Key2'])
def test_disk_file_app_iter_corners(self):
df = self._create_test_file('1234567890')
@@ -721,10 +726,10 @@ class TestDiskFile(unittest.TestCase):
self._create_ondisk_file(df, 'A', ext='.data', timestamp=5)
df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
FakeLogger())
self.assertTrue('X-Timestamp' in df.metadata)
self.assertEquals(df.metadata['X-Timestamp'], normalize_timestamp(10))
self.assertTrue('deleted' in df.metadata)
self.assertTrue(df.metadata['deleted'])
self.assertTrue('X-Timestamp' in df._metadata)
self.assertEquals(df._metadata['X-Timestamp'], normalize_timestamp(10))
self.assertTrue('deleted' in df._metadata)
self.assertTrue(df._metadata['deleted'])
def test_ondisk_search_loop_meta_ts_data(self):
df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
@@ -737,9 +742,9 @@ class TestDiskFile(unittest.TestCase):
self._create_ondisk_file(df, 'A', ext='.data', timestamp=5)
df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
FakeLogger())
self.assertTrue('X-Timestamp' in df.metadata)
self.assertEquals(df.metadata['X-Timestamp'], normalize_timestamp(8))
self.assertTrue('deleted' in df.metadata)
self.assertTrue('X-Timestamp' in df._metadata)
self.assertEquals(df._metadata['X-Timestamp'], normalize_timestamp(8))
self.assertTrue('deleted' in df._metadata)
def test_ondisk_search_loop_meta_data_ts(self):
df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
@@ -752,9 +757,9 @@ class TestDiskFile(unittest.TestCase):
self._create_ondisk_file(df, '', ext='.ts', timestamp=5)
df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
FakeLogger())
self.assertTrue('X-Timestamp' in df.metadata)
self.assertEquals(df.metadata['X-Timestamp'], normalize_timestamp(10))
self.assertTrue('deleted' not in df.metadata)
self.assertTrue('X-Timestamp' in df._metadata)
self.assertEquals(df._metadata['X-Timestamp'], normalize_timestamp(10))
self.assertTrue('deleted' not in df._metadata)
def test_ondisk_search_loop_data_meta_ts(self):
df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
@@ -767,9 +772,9 @@ class TestDiskFile(unittest.TestCase):
self._create_ondisk_file(df, '', ext='.meta', timestamp=5)
df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
FakeLogger())
self.assertTrue('X-Timestamp' in df.metadata)
self.assertEquals(df.metadata['X-Timestamp'], normalize_timestamp(10))
self.assertTrue('deleted' not in df.metadata)
self.assertTrue('X-Timestamp' in df._metadata)
self.assertEquals(df._metadata['X-Timestamp'], normalize_timestamp(10))
self.assertTrue('deleted' not in df._metadata)
def test_ondisk_search_loop_wayward_files_ignored(self):
df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
@@ -783,9 +788,9 @@ class TestDiskFile(unittest.TestCase):
self._create_ondisk_file(df, '', ext='.meta', timestamp=5)
df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
FakeLogger())
self.assertTrue('X-Timestamp' in df.metadata)
self.assertEquals(df.metadata['X-Timestamp'], normalize_timestamp(10))
self.assertTrue('deleted' not in df.metadata)
self.assertTrue('X-Timestamp' in df._metadata)
self.assertEquals(df._metadata['X-Timestamp'], normalize_timestamp(10))
self.assertTrue('deleted' not in df._metadata)
def test_ondisk_search_loop_listdir_error(self):
df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',