openstack/swift
Code Issues Proposed changes

remove hash_cleanup_listdir legacy translation method

Browse Source 
Change-Id: I0b96dfde32b4c666eebda6e88228516dd693ef92
closes-bug:#1550569
This commit is contained in:
Mahati Chamarthy
2016-03-08 19:01:53 +05:30
parent 643dbce134
commit 82c8023b89
2 changed files with 33 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
swift/obj/diskfile.py
View File

@@ -751,20 +751,6 @@ class BaseDiskFileManager(object):
results['files'] = files
return results
def hash_cleanup_listdir(self, hsh_path, reclaim_age=ONE_WEEK):
"""
List contents of a hash directory and clean up any old files.
For EC policy, delete files older than a .durable or .ts file.
:param hsh_path: object hash path
:param reclaim_age: age in seconds at which to remove tombstones
:returns: list of files remaining in the directory, reverse sorted
"""
# maintain compatibility with 'legacy' hash_cleanup_listdir
# return value
return self.cleanup_ondisk_files(
hsh_path, reclaim_age=reclaim_age)['files']
def _update_suffix_hashes(self, hashes, ondisk_info):
"""
Applies policy specific updates to the given dict of md5 hashes for
@@ -1065,8 +1051,8 @@ class BaseDiskFileManager(object):
dev_path, get_data_dir(policy), str(partition), object_hash[-3:],
object_hash)
try:
filenames = self.hash_cleanup_listdir(object_path,
self.reclaim_age)
filenames = self.cleanup_ondisk_files(object_path,
self.reclaim_age)['files']
except OSError as err:
if err.errno == errno.ENOTDIR:
quar_path = self.quarantine_renamer(dev_path, object_path)
@@ -1322,7 +1308,7 @@ class BaseDiskFileWriter(object):
self._put_succeeded = True
if cleanup:
try:
self.manager.hash_cleanup_listdir(self._datadir)
self.manager.cleanup_ondisk_files(self._datadir)['files']
except OSError:
logging.exception(_('Problem cleaning up %s'), self._datadir)
@@ -2411,7 +2397,7 @@ class ECDiskFileWriter(BaseDiskFileWriter):
exc = DiskFileNoSpace(str(err))
else:
try:
self.manager.hash_cleanup_listdir(self._datadir)
self.manager.cleanup_ondisk_files(self._datadir)['files']
except OSError as os_err:
self.manager.logger.exception(
_('Problem cleaning up %s (%s)') %

58
test/unit/obj/test_diskfile.py
View File

@@ -569,13 +569,13 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
expected_after_cleanup = set([f[0] for f in test
if (f[2] if len(f) > 2 else f[1])])
if reclaim_age:
class_under_test.hash_cleanup_listdir(
hashdir, reclaim_age=reclaim_age)
class_under_test.cleanup_ondisk_files(
hashdir, reclaim_age=reclaim_age)['files']
else:
with mock.patch('swift.obj.diskfile.time') as mock_time:
# don't reclaim anything
mock_time.time.return_value = 0.0
class_under_test.hash_cleanup_listdir(hashdir)
class_under_test.cleanup_ondisk_files(hashdir)['files']
after_cleanup = set(os.listdir(hashdir))
errmsg = "expected %r, got %r for test %r" % (
sorted(expected_after_cleanup), sorted(after_cleanup), test
@@ -746,9 +746,9 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value=None)
with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock(
'hash_cleanup_listdir')) as hclistdir, \
'cleanup_ondisk_files')) as hclistdir, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
hclistdir.return_value = ['1381679759.90941.data']
hclistdir.return_value = {'files': ['1381679759.90941.data']}
readmeta.return_value = {'name': '/a/c/o'}
self.assertRaises(
DiskFileDeviceUnavailable,
@@ -759,7 +759,7 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock(
'hash_cleanup_listdir')) as hclistdir, \
'cleanup_ondisk_files')) as hclistdir, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta, \
mock.patch(self._manager_mock(
'quarantine_renamer')) as quarantine_renamer:
@@ -779,7 +779,7 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock(
'hash_cleanup_listdir')) as hclistdir, \
'cleanup_ondisk_files')) as hclistdir, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
osexc = OSError()
osexc.errno = errno.ENOENT
@@ -794,7 +794,7 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock(
'hash_cleanup_listdir')) as hclistdir, \
'cleanup_ondisk_files')) as hclistdir, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
osexc = OSError()
hclistdir.side_effect = osexc
@@ -808,9 +808,9 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock(
'hash_cleanup_listdir')) as hclistdir, \
'cleanup_ondisk_files')) as hclistdir, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
hclistdir.return_value = []
hclistdir.return_value = {'files': []}
readmeta.return_value = {'name': '/a/c/o'}
self.assertRaises(
DiskFileNotExist,
@@ -821,9 +821,9 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock(
'hash_cleanup_listdir')) as hclistdir, \
'cleanup_ondisk_files')) as hclistdir, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
hclistdir.return_value = ['1381679759.90941.data']
hclistdir.return_value = {'files': ['1381679759.90941.data']}
readmeta.side_effect = EOFError()
self.assertRaises(
DiskFileNotExist,
@@ -834,9 +834,9 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock(
'hash_cleanup_listdir')) as hclistdir, \
'cleanup_ondisk_files')) as hclistdir, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
hclistdir.return_value = ['1381679759.90941.data']
hclistdir.return_value = {'files': ['1381679759.90941.data']}
readmeta.return_value = {}
try:
self.df_mgr.get_diskfile_from_hash(
@@ -850,9 +850,9 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock(
'hash_cleanup_listdir')) as hclistdir, \
'cleanup_ondisk_files')) as hclistdir, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
hclistdir.return_value = ['1381679759.90941.data']
hclistdir.return_value = {'files': ['1381679759.90941.data']}
readmeta.return_value = {'name': 'bad'}
try:
self.df_mgr.get_diskfile_from_hash(
@@ -866,9 +866,9 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')) as dfclass, \
mock.patch(self._manager_mock(
'hash_cleanup_listdir')) as hclistdir, \
'cleanup_ondisk_files')) as hclistdir, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
hclistdir.return_value = ['1381679759.90941.data']
hclistdir.return_value = {'files': ['1381679759.90941.data']}
readmeta.return_value = {'name': '/a/c/o'}
self.df_mgr.get_diskfile_from_hash(
'dev', '9', '9a7175077c01a23ade5956b8a2bba900', POLICIES[0])
@@ -3091,7 +3091,7 @@ class DiskFileMixin(BaseDiskFileTestMixin):
}
writer.put(metadata)
with mock.patch(self._manager_mock(
'hash_cleanup_listdir', df), mock_hcl):
'cleanup_ondisk_files', df), mock_hcl):
writer.commit(timestamp)
expected = {
EC_POLICY: 1,
@@ -3125,11 +3125,11 @@ class DiskFileMixin(BaseDiskFileTestMixin):
'Content-Length': '0',
}
with mock.patch(self._manager_mock(
'hash_cleanup_listdir', df)) as mock_hcl:
'cleanup_ondisk_files', df)) as mock_hcl:
writer.put(metadata)
self.assertEqual(expected[0], mock_hcl.call_count)
with mock.patch(self._manager_mock(
'hash_cleanup_listdir', df)) as mock_hcl:
'cleanup_ondisk_files', df)) as mock_hcl:
writer.commit(timestamp)
self.assertEqual(expected[1], mock_hcl.call_count)
@@ -3151,15 +3151,15 @@ class DiskFileMixin(BaseDiskFileTestMixin):
'Content-Length': '0',
}
with mock.patch(self._manager_mock(
'hash_cleanup_listdir', df)) as mock_hcl:
'cleanup_ondisk_files', df)) as mock_hcl:
writer.put(metadata)
self.assertEqual(expected[0], mock_hcl.call_count)
with mock.patch(self._manager_mock(
'hash_cleanup_listdir', df)) as mock_hcl:
'cleanup_ondisk_files', df)) as mock_hcl:
writer.commit(timestamp)
self.assertEqual(expected[1], mock_hcl.call_count)
with mock.patch(self._manager_mock(
'hash_cleanup_listdir', df)) as mock_hcl:
'cleanup_ondisk_files', df)) as mock_hcl:
timestamp = Timestamp(time())
df.delete(timestamp)
self.assertEqual(expected[2], mock_hcl.call_count)
@@ -3594,7 +3594,7 @@ class DiskFileMixin(BaseDiskFileTestMixin):
df = self._get_open_disk_file()
file_count = len(os.listdir(df._datadir))
ts = time()
with mock.patch(self._manager_mock('hash_cleanup_listdir'), mock_hcl):
with mock.patch(self._manager_mock('cleanup_ondisk_files'), mock_hcl):
try:
df.delete(ts)
except OSError:
@@ -4506,9 +4506,9 @@ class TestSuffixHashes(unittest.TestCase):
if isinstance(output_files, Exception):
path = os.path.join(self.testdir, 'does-not-matter')
self.assertRaises(output_files.__class__,
df_mgr.hash_cleanup_listdir, path)
df_mgr.cleanup_ondisk_files, path)
return
files = df_mgr.hash_cleanup_listdir('/whatever')
files = df_mgr.cleanup_ondisk_files('/whatever')['files']
self.assertEqual(files, output_files)
# hash_cleanup_listdir tests - behaviors
@@ -4721,7 +4721,7 @@ class TestSuffixHashes(unittest.TestCase):
df_mgr = self.df_router[policy]
# common.utils.listdir *completely* mutes ENOENT
path = os.path.join(self.testdir, 'does-not-exist')
self.assertEqual(df_mgr.hash_cleanup_listdir(path), [])
self.assertEqual(df_mgr.cleanup_ondisk_files(path)['files'], [])
def test_hash_cleanup_listdir_hsh_path_other_oserror(self):
for policy in self.iter_policies():
@@ -4730,7 +4730,7 @@ class TestSuffixHashes(unittest.TestCase):
mock_listdir.side_effect = OSError('kaboom!')
# but it will raise other OSErrors
path = os.path.join(self.testdir, 'does-not-matter')
self.assertRaises(OSError, df_mgr.hash_cleanup_listdir,
self.assertRaises(OSError, df_mgr.cleanup_ondisk_files,
path)
def test_hash_cleanup_listdir_reclaim_tombstone_remove_file_error(self):