openstack/swift
Code Issues Proposed changes

remove hash_cleanup_listdir legacy translation method

Browse Source 
Change-Id: I0b96dfde32b4c666eebda6e88228516dd693ef92
closes-bug:#1550569
This commit is contained in:
Mahati Chamarthy
2016-03-08 19:01:53 +05:30
parent 643dbce134
commit 82c8023b89
2 changed files with 33 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
swift/obj/diskfile.py
View File

@@ -751,20 +751,6 @@ class BaseDiskFileManager(object):
results['files'] = files results['files'] = files
return results return results
def hash_cleanup_listdir(self, hsh_path, reclaim_age=ONE_WEEK):
"""
List contents of a hash directory and clean up any old files.
For EC policy, delete files older than a .durable or .ts file.
:param hsh_path: object hash path
:param reclaim_age: age in seconds at which to remove tombstones
:returns: list of files remaining in the directory, reverse sorted
"""
# maintain compatibility with 'legacy' hash_cleanup_listdir
# return value
return self.cleanup_ondisk_files(
hsh_path, reclaim_age=reclaim_age)['files']
def _update_suffix_hashes(self, hashes, ondisk_info): def _update_suffix_hashes(self, hashes, ondisk_info):
""" """
Applies policy specific updates to the given dict of md5 hashes for Applies policy specific updates to the given dict of md5 hashes for
@@ -1065,8 +1051,8 @@ class BaseDiskFileManager(object):
dev_path, get_data_dir(policy), str(partition), object_hash[-3:], dev_path, get_data_dir(policy), str(partition), object_hash[-3:],
object_hash) object_hash)
try: try:
filenames = self.hash_cleanup_listdir(object_path, filenames = self.cleanup_ondisk_files(object_path,
self.reclaim_age) self.reclaim_age)['files']
except OSError as err: except OSError as err:
if err.errno == errno.ENOTDIR: if err.errno == errno.ENOTDIR:
quar_path = self.quarantine_renamer(dev_path, object_path) quar_path = self.quarantine_renamer(dev_path, object_path)
@@ -1322,7 +1308,7 @@ class BaseDiskFileWriter(object):
self._put_succeeded = True self._put_succeeded = True
if cleanup: if cleanup:
try: try:
self.manager.hash_cleanup_listdir(self._datadir) self.manager.cleanup_ondisk_files(self._datadir)['files']
except OSError: except OSError:
logging.exception(_('Problem cleaning up %s'), self._datadir) logging.exception(_('Problem cleaning up %s'), self._datadir)
@@ -2411,7 +2397,7 @@ class ECDiskFileWriter(BaseDiskFileWriter):
exc = DiskFileNoSpace(str(err)) exc = DiskFileNoSpace(str(err))
else: else:
try: try:
self.manager.hash_cleanup_listdir(self._datadir) self.manager.cleanup_ondisk_files(self._datadir)['files']
except OSError as os_err: except OSError as os_err:
self.manager.logger.exception( self.manager.logger.exception(
_('Problem cleaning up %s (%s)') % _('Problem cleaning up %s (%s)') %

58
test/unit/obj/test_diskfile.py
View File

@@ -569,13 +569,13 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
expected_after_cleanup = set([f[0] for f in test expected_after_cleanup = set([f[0] for f in test
if (f[2] if len(f) > 2 else f[1])]) if (f[2] if len(f) > 2 else f[1])])
if reclaim_age: if reclaim_age:
class_under_test.hash_cleanup_listdir( class_under_test.cleanup_ondisk_files(
hashdir, reclaim_age=reclaim_age) hashdir, reclaim_age=reclaim_age)['files']
else: else:
with mock.patch('swift.obj.diskfile.time') as mock_time: with mock.patch('swift.obj.diskfile.time') as mock_time:
# don't reclaim anything # don't reclaim anything
mock_time.time.return_value = 0.0 mock_time.time.return_value = 0.0
class_under_test.hash_cleanup_listdir(hashdir) class_under_test.cleanup_ondisk_files(hashdir)['files']
after_cleanup = set(os.listdir(hashdir)) after_cleanup = set(os.listdir(hashdir))
errmsg = "expected %r, got %r for test %r" % ( errmsg = "expected %r, got %r for test %r" % (
sorted(expected_after_cleanup), sorted(after_cleanup), test sorted(expected_after_cleanup), sorted(after_cleanup), test
@@ -746,9 +746,9 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value=None) self.df_mgr.get_dev_path = mock.MagicMock(return_value=None)
with mock.patch(self._manager_mock('diskfile_cls')), \ with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'hash_cleanup_listdir')) as hclistdir, \ 'cleanup_ondisk_files')) as hclistdir, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta: mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
hclistdir.return_value = ['1381679759.90941.data'] hclistdir.return_value = {'files': ['1381679759.90941.data']}
readmeta.return_value = {'name': '/a/c/o'} readmeta.return_value = {'name': '/a/c/o'}
self.assertRaises( self.assertRaises(
DiskFileDeviceUnavailable, DiskFileDeviceUnavailable,
@@ -759,7 +759,7 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \ with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'hash_cleanup_listdir')) as hclistdir, \ 'cleanup_ondisk_files')) as hclistdir, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta, \ mock.patch('swift.obj.diskfile.read_metadata') as readmeta, \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'quarantine_renamer')) as quarantine_renamer: 'quarantine_renamer')) as quarantine_renamer:
@@ -779,7 +779,7 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \ with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'hash_cleanup_listdir')) as hclistdir, \ 'cleanup_ondisk_files')) as hclistdir, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta: mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
osexc = OSError() osexc = OSError()
osexc.errno = errno.ENOENT osexc.errno = errno.ENOENT
@@ -794,7 +794,7 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \ with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'hash_cleanup_listdir')) as hclistdir, \ 'cleanup_ondisk_files')) as hclistdir, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta: mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
osexc = OSError() osexc = OSError()
hclistdir.side_effect = osexc hclistdir.side_effect = osexc
@@ -808,9 +808,9 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \ with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'hash_cleanup_listdir')) as hclistdir, \ 'cleanup_ondisk_files')) as hclistdir, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta: mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
hclistdir.return_value = [] hclistdir.return_value = {'files': []}
readmeta.return_value = {'name': '/a/c/o'} readmeta.return_value = {'name': '/a/c/o'}
self.assertRaises( self.assertRaises(
DiskFileNotExist, DiskFileNotExist,
@@ -821,9 +821,9 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \ with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'hash_cleanup_listdir')) as hclistdir, \ 'cleanup_ondisk_files')) as hclistdir, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta: mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
hclistdir.return_value = ['1381679759.90941.data'] hclistdir.return_value = {'files': ['1381679759.90941.data']}
readmeta.side_effect = EOFError() readmeta.side_effect = EOFError()
self.assertRaises( self.assertRaises(
DiskFileNotExist, DiskFileNotExist,
@@ -834,9 +834,9 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \ with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'hash_cleanup_listdir')) as hclistdir, \ 'cleanup_ondisk_files')) as hclistdir, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta: mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
hclistdir.return_value = ['1381679759.90941.data'] hclistdir.return_value = {'files': ['1381679759.90941.data']}
readmeta.return_value = {} readmeta.return_value = {}
try: try:
self.df_mgr.get_diskfile_from_hash( self.df_mgr.get_diskfile_from_hash(
@@ -850,9 +850,9 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')), \ with mock.patch(self._manager_mock('diskfile_cls')), \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'hash_cleanup_listdir')) as hclistdir, \ 'cleanup_ondisk_files')) as hclistdir, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta: mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
hclistdir.return_value = ['1381679759.90941.data'] hclistdir.return_value = {'files': ['1381679759.90941.data']}
readmeta.return_value = {'name': 'bad'} readmeta.return_value = {'name': 'bad'}
try: try:
self.df_mgr.get_diskfile_from_hash( self.df_mgr.get_diskfile_from_hash(
@@ -866,9 +866,9 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/')
with mock.patch(self._manager_mock('diskfile_cls')) as dfclass, \ with mock.patch(self._manager_mock('diskfile_cls')) as dfclass, \
mock.patch(self._manager_mock( mock.patch(self._manager_mock(
'hash_cleanup_listdir')) as hclistdir, \ 'cleanup_ondisk_files')) as hclistdir, \
mock.patch('swift.obj.diskfile.read_metadata') as readmeta: mock.patch('swift.obj.diskfile.read_metadata') as readmeta:
hclistdir.return_value = ['1381679759.90941.data'] hclistdir.return_value = {'files': ['1381679759.90941.data']}
readmeta.return_value = {'name': '/a/c/o'} readmeta.return_value = {'name': '/a/c/o'}
self.df_mgr.get_diskfile_from_hash( self.df_mgr.get_diskfile_from_hash(
'dev', '9', '9a7175077c01a23ade5956b8a2bba900', POLICIES[0]) 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', POLICIES[0])
@@ -3091,7 +3091,7 @@ class DiskFileMixin(BaseDiskFileTestMixin):
} }
writer.put(metadata) writer.put(metadata)
with mock.patch(self._manager_mock( with mock.patch(self._manager_mock(
'hash_cleanup_listdir', df), mock_hcl): 'cleanup_ondisk_files', df), mock_hcl):
writer.commit(timestamp) writer.commit(timestamp)
expected = { expected = {
EC_POLICY: 1, EC_POLICY: 1,
@@ -3125,11 +3125,11 @@ class DiskFileMixin(BaseDiskFileTestMixin):
'Content-Length': '0', 'Content-Length': '0',
} }
with mock.patch(self._manager_mock( with mock.patch(self._manager_mock(
'hash_cleanup_listdir', df)) as mock_hcl: 'cleanup_ondisk_files', df)) as mock_hcl:
writer.put(metadata) writer.put(metadata)
self.assertEqual(expected[0], mock_hcl.call_count) self.assertEqual(expected[0], mock_hcl.call_count)
with mock.patch(self._manager_mock( with mock.patch(self._manager_mock(
'hash_cleanup_listdir', df)) as mock_hcl: 'cleanup_ondisk_files', df)) as mock_hcl:
writer.commit(timestamp) writer.commit(timestamp)
self.assertEqual(expected[1], mock_hcl.call_count) self.assertEqual(expected[1], mock_hcl.call_count)
@@ -3151,15 +3151,15 @@ class DiskFileMixin(BaseDiskFileTestMixin):
'Content-Length': '0', 'Content-Length': '0',
} }
with mock.patch(self._manager_mock( with mock.patch(self._manager_mock(
'hash_cleanup_listdir', df)) as mock_hcl: 'cleanup_ondisk_files', df)) as mock_hcl:
writer.put(metadata) writer.put(metadata)
self.assertEqual(expected[0], mock_hcl.call_count) self.assertEqual(expected[0], mock_hcl.call_count)
with mock.patch(self._manager_mock( with mock.patch(self._manager_mock(
'hash_cleanup_listdir', df)) as mock_hcl: 'cleanup_ondisk_files', df)) as mock_hcl:
writer.commit(timestamp) writer.commit(timestamp)
self.assertEqual(expected[1], mock_hcl.call_count) self.assertEqual(expected[1], mock_hcl.call_count)
with mock.patch(self._manager_mock( with mock.patch(self._manager_mock(
'hash_cleanup_listdir', df)) as mock_hcl: 'cleanup_ondisk_files', df)) as mock_hcl:
timestamp = Timestamp(time()) timestamp = Timestamp(time())
df.delete(timestamp) df.delete(timestamp)
self.assertEqual(expected[2], mock_hcl.call_count) self.assertEqual(expected[2], mock_hcl.call_count)
@@ -3594,7 +3594,7 @@ class DiskFileMixin(BaseDiskFileTestMixin):
df = self._get_open_disk_file() df = self._get_open_disk_file()
file_count = len(os.listdir(df._datadir)) file_count = len(os.listdir(df._datadir))
ts = time() ts = time()
with mock.patch(self._manager_mock('hash_cleanup_listdir'), mock_hcl): with mock.patch(self._manager_mock('cleanup_ondisk_files'), mock_hcl):
try: try:
df.delete(ts) df.delete(ts)
except OSError: except OSError:
@@ -4506,9 +4506,9 @@ class TestSuffixHashes(unittest.TestCase):
if isinstance(output_files, Exception): if isinstance(output_files, Exception):
path = os.path.join(self.testdir, 'does-not-matter') path = os.path.join(self.testdir, 'does-not-matter')
self.assertRaises(output_files.__class__, self.assertRaises(output_files.__class__,
df_mgr.hash_cleanup_listdir, path) df_mgr.cleanup_ondisk_files, path)
return return
files = df_mgr.hash_cleanup_listdir('/whatever') files = df_mgr.cleanup_ondisk_files('/whatever')['files']
self.assertEqual(files, output_files) self.assertEqual(files, output_files)
# hash_cleanup_listdir tests - behaviors # hash_cleanup_listdir tests - behaviors
@@ -4721,7 +4721,7 @@ class TestSuffixHashes(unittest.TestCase):
df_mgr = self.df_router[policy] df_mgr = self.df_router[policy]
# common.utils.listdir *completely* mutes ENOENT # common.utils.listdir *completely* mutes ENOENT
path = os.path.join(self.testdir, 'does-not-exist') path = os.path.join(self.testdir, 'does-not-exist')
self.assertEqual(df_mgr.hash_cleanup_listdir(path), []) self.assertEqual(df_mgr.cleanup_ondisk_files(path)['files'], [])
def test_hash_cleanup_listdir_hsh_path_other_oserror(self): def test_hash_cleanup_listdir_hsh_path_other_oserror(self):
for policy in self.iter_policies(): for policy in self.iter_policies():
@@ -4730,7 +4730,7 @@ class TestSuffixHashes(unittest.TestCase):
mock_listdir.side_effect = OSError('kaboom!') mock_listdir.side_effect = OSError('kaboom!')
# but it will raise other OSErrors # but it will raise other OSErrors
path = os.path.join(self.testdir, 'does-not-matter') path = os.path.join(self.testdir, 'does-not-matter')
self.assertRaises(OSError, df_mgr.hash_cleanup_listdir, self.assertRaises(OSError, df_mgr.cleanup_ondisk_files,
path) path)
def test_hash_cleanup_listdir_reclaim_tombstone_remove_file_error(self): def test_hash_cleanup_listdir_reclaim_tombstone_remove_file_error(self):