diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py index 0cadc22c22..ff84da4a69 100644 --- a/test/unit/obj/test_diskfile.py +++ b/test/unit/obj/test_diskfile.py @@ -2053,13 +2053,15 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin): # Make a couple of (soon-to-be-)expired tombstones df1 = self.df_mgr.get_diskfile( self.existing_device, 0, 'a', 'c', 'o1', POLICIES[0]) - df1.delete(Timestamp(the_time[0])) + ts_delete1 = Timestamp(the_time[0]) + df1.delete(ts_delete1) df1_hash = utils.hash_path('a', 'c', 'o1') df1_suffix = df1_hash[-3:] df2 = self.df_mgr.get_diskfile( self.existing_device, 0, 'a', 'c', 'o2', POLICIES[0]) - df2.delete(Timestamp(the_time[0] + 1)) + ts_delete2 = Timestamp(the_time[0] + 1) + df2.delete(ts_delete2) df2_hash = utils.hash_path('a', 'c', 'o2') df2_suffix = df2_hash[-3:] @@ -2067,11 +2069,11 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin): self.assertTrue(os.path.exists(os.path.join( self.testdir, self.existing_device, 'objects', '0', df1_suffix, df1_hash, - "1525354555.65758.ts"))) + "%s.ts" % ts_delete1.internal))) self.assertTrue(os.path.exists(os.path.join( self.testdir, self.existing_device, 'objects', '0', df2_suffix, df2_hash, - "1525354556.65758.ts"))) + "%s.ts" % ts_delete2.internal))) # Cache the hashes and expire the tombstones self.df_mgr.get_hashes(self.existing_device, '0', [], POLICIES[0]) @@ -2085,11 +2087,11 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin): self.assertFalse(os.path.exists(os.path.join( self.testdir, self.existing_device, 'objects', '0', df1_suffix, df1_hash, - "1525354555.65758.ts"))) + "%s.ts" % ts_delete1.internal))) self.assertFalse(os.path.exists(os.path.join( self.testdir, self.existing_device, 'objects', '0', df2_suffix, df2_hash, - "1525354556.65758.ts"))) + "%s.ts" % ts_delete2.internal))) # The empty hash dirs are gone self.assertFalse(os.path.exists(os.path.join( @@ -2145,7 +2147,8 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin): with mock.patch('time.time', mock_time): df = self.df_mgr.get_diskfile( self.existing_device, 0, 'a', 'c', 'o', POLICIES[0]) - df.delete(Timestamp(the_time[0])) + ts_delete = Timestamp(the_time[0]) + df.delete(ts_delete) df_hash = utils.hash_path('a', 'c', 'o') df_suffix = df_hash[-3:] @@ -2153,7 +2156,7 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin): self.assertTrue(os.path.exists(os.path.join( self.testdir, self.existing_device, 'objects', '0', df_suffix, df_hash, - "1525354555.65758.ts"))) + "%s.ts" % ts_delete.internal))) # Expire the tombstone the_time[0] += 2 * self.df_mgr.reclaim_age @@ -5444,9 +5447,9 @@ class DiskFileMixin(BaseDiskFileTestMixin): def test_open_deleted(self): df = self._get_open_disk_file() - ts = time() + ts = Timestamp.now() df.delete(ts) - exp_name = '%s.ts' % str(Timestamp(ts).internal) + exp_name = '%s.ts' % ts.internal dl = os.listdir(df._datadir) self.assertEqual(len(dl), 1) self.assertIn(exp_name, set(dl)) @@ -5455,9 +5458,9 @@ class DiskFileMixin(BaseDiskFileTestMixin): def test_open_deleted_with_corrupt_tombstone(self): df = self._get_open_disk_file() - ts = time() + ts = Timestamp.now() df.delete(ts) - exp_name = '%s.ts' % str(Timestamp(ts).internal) + exp_name = '%s.ts' % ts.internal dl = os.listdir(df._datadir) self.assertEqual(len(dl), 1) self.assertIn(exp_name, set(dl)) @@ -5590,45 +5593,49 @@ class DiskFileMixin(BaseDiskFileTestMixin): def test_ondisk_search_loop_ts_meta_data(self): df = self._simple_get_diskfile() - self._create_ondisk_file(df, b'', ext='.ts', timestamp=10) - self._create_ondisk_file(df, b'', ext='.ts', timestamp=9) - self._create_ondisk_file(df, b'', ext='.meta', timestamp=8) - self._create_ondisk_file(df, b'', ext='.meta', timestamp=7) - self._create_ondisk_file(df, b'B', ext='.data', timestamp=6) - self._create_ondisk_file(df, b'A', ext='.data', timestamp=5) + ts = [Timestamp(i) for i in range(11)] + self._create_ondisk_file(df, b'', ext='.ts', timestamp=ts[10]) + self._create_ondisk_file(df, b'', ext='.ts', timestamp=ts[9]) + self._create_ondisk_file(df, b'', ext='.meta', timestamp=ts[8]) + self._create_ondisk_file(df, b'', ext='.meta', timestamp=ts[7]) + self._create_ondisk_file(df, b'B', ext='.data', timestamp=ts[6]) + self._create_ondisk_file(df, b'A', ext='.data', timestamp=ts[5]) df = self._simple_get_diskfile() with self.assertRaises(DiskFileDeleted) as raised: df.open() - self.assertEqual(raised.exception.timestamp, Timestamp(10).internal) + self.assertEqual(raised.exception.timestamp, ts[10].internal) def test_ondisk_search_loop_meta_ts_data(self): df = self._simple_get_diskfile() - self._create_ondisk_file(df, b'', ext='.meta', timestamp=10) - self._create_ondisk_file(df, b'', ext='.meta', timestamp=9) - self._create_ondisk_file(df, b'', ext='.ts', timestamp=8) - self._create_ondisk_file(df, b'', ext='.ts', timestamp=7) - self._create_ondisk_file(df, b'B', ext='.data', timestamp=6) - self._create_ondisk_file(df, b'A', ext='.data', timestamp=5) + ts = [Timestamp(i) for i in range(11)] + self._create_ondisk_file(df, b'', ext='.meta', timestamp=ts[10]) + self._create_ondisk_file(df, b'', ext='.meta', timestamp=ts[9]) + self._create_ondisk_file(df, b'', ext='.ts', timestamp=ts[8]) + self._create_ondisk_file(df, b'', ext='.ts', timestamp=ts[7]) + self._create_ondisk_file(df, b'B', ext='.data', timestamp=ts[6]) + self._create_ondisk_file(df, b'A', ext='.data', timestamp=ts[5]) df = self._simple_get_diskfile() with self.assertRaises(DiskFileDeleted) as raised: df.open() - self.assertEqual(raised.exception.timestamp, Timestamp(8).internal) + self.assertEqual(raised.exception.timestamp, ts[8].internal) def _test_ondisk_search_loop_meta_data_ts(self, legacy_durable=False): df = self._simple_get_diskfile() - self._create_ondisk_file(df, b'', ext='.meta', timestamp=10) - self._create_ondisk_file(df, b'', ext='.meta', timestamp=9) + ts = [Timestamp(i) for i in range(11)] + self._create_ondisk_file(df, b'', ext='.meta', timestamp=ts[10]) + self._create_ondisk_file(df, b'', ext='.meta', timestamp=ts[9]) self._create_ondisk_file( - df, b'B', ext='.data', legacy_durable=legacy_durable, timestamp=8) + df, b'B', ext='.data', legacy_durable=legacy_durable, + timestamp=ts[8]) self._create_ondisk_file( - df, b'A', ext='.data', legacy_durable=legacy_durable, timestamp=7) - self._create_ondisk_file(df, b'', ext='.ts', timestamp=6) - self._create_ondisk_file(df, b'', ext='.ts', timestamp=5) + df, b'A', ext='.data', legacy_durable=legacy_durable, + timestamp=ts[7]) + self._create_ondisk_file(df, b'', ext='.ts', timestamp=ts[6]) + self._create_ondisk_file(df, b'', ext='.ts', timestamp=ts[5]) df = self._simple_get_diskfile() with df.open(): self.assertIn('X-Timestamp', df._metadata) - self.assertEqual(df._metadata['X-Timestamp'], - Timestamp(10).internal) + self.assertEqual(df._metadata['X-Timestamp'], ts[10].internal) self.assertNotIn('deleted', df._metadata) def test_ondisk_search_loop_meta_data_ts(self): @@ -5640,23 +5647,25 @@ class DiskFileMixin(BaseDiskFileTestMixin): def _test_ondisk_search_loop_multiple_meta_data(self, legacy_durable=False): df = self._simple_get_diskfile() - self._create_ondisk_file(df, b'', ext='.meta', timestamp=10, + ts = [Timestamp(i) for i in range(11)] + self._create_ondisk_file(df, b'', ext='.meta', timestamp=ts[10], metadata={'X-Object-Meta-User': 'user-meta'}) - self._create_ondisk_file(df, b'', ext='.meta', timestamp=9, - ctype_timestamp=9, + self._create_ondisk_file(df, b'', ext='.meta', timestamp=ts[9], + ctype_timestamp=ts[9], metadata={'Content-Type': 'newest', 'X-Object-Meta-User': 'blah'}) self._create_ondisk_file( - df, b'B', ext='.data', legacy_durable=legacy_durable, timestamp=8, + df, b'B', ext='.data', legacy_durable=legacy_durable, + timestamp=ts[8], metadata={'Content-Type': 'newer'}) self._create_ondisk_file( - df, b'A', ext='.data', legacy_durable=legacy_durable, timestamp=7, + df, b'A', ext='.data', legacy_durable=legacy_durable, + timestamp=ts[7], metadata={'Content-Type': 'oldest'}) df = self._simple_get_diskfile() with df.open(): self.assertTrue('X-Timestamp' in df._metadata) - self.assertEqual(df._metadata['X-Timestamp'], - Timestamp(10).internal) + self.assertEqual(df._metadata['X-Timestamp'], ts[10].internal) self.assertTrue('Content-Type' in df._metadata) self.assertEqual(df._metadata['Content-Type'], 'newest') self.assertTrue('X-Object-Meta-User' in df._metadata) @@ -5670,20 +5679,20 @@ class DiskFileMixin(BaseDiskFileTestMixin): def _test_ondisk_search_loop_stale_meta_data(self, legacy_durable=False): df = self._simple_get_diskfile() - self._create_ondisk_file(df, b'', ext='.meta', timestamp=10, + ts = [Timestamp(i) for i in range(11)] + self._create_ondisk_file(df, b'', ext='.meta', timestamp=ts[10], metadata={'X-Object-Meta-User': 'user-meta'}) - self._create_ondisk_file(df, b'', ext='.meta', timestamp=9, - ctype_timestamp=7, + self._create_ondisk_file(df, b'', ext='.meta', timestamp=ts[9], + ctype_timestamp=ts[7], metadata={'Content-Type': 'older', 'X-Object-Meta-User': 'blah'}) self._create_ondisk_file( - df, b'B', ext='.data', legacy_durable=legacy_durable, timestamp=8, - metadata={'Content-Type': 'newer'}) + df, b'B', ext='.data', legacy_durable=legacy_durable, + timestamp=ts[8], metadata={'Content-Type': 'newer'}) df = self._simple_get_diskfile() with df.open(): self.assertTrue('X-Timestamp' in df._metadata) - self.assertEqual(df._metadata['X-Timestamp'], - Timestamp(10).internal) + self.assertEqual(df._metadata['X-Timestamp'], ts[10].internal) self.assertTrue('Content-Type' in df._metadata) self.assertEqual(df._metadata['Content-Type'], 'newer') self.assertTrue('X-Object-Meta-User' in df._metadata) @@ -5697,19 +5706,21 @@ class DiskFileMixin(BaseDiskFileTestMixin): def _test_ondisk_search_loop_data_ts_meta(self, legacy_durable=False): df = self._simple_get_diskfile() + ts = [Timestamp(i) for i in range(11)] self._create_ondisk_file( - df, b'B', ext='.data', legacy_durable=legacy_durable, timestamp=10) + df, b'B', ext='.data', legacy_durable=legacy_durable, + timestamp=ts[10]) self._create_ondisk_file( - df, b'A', ext='.data', legacy_durable=legacy_durable, timestamp=9) - self._create_ondisk_file(df, b'', ext='.ts', timestamp=8) - self._create_ondisk_file(df, b'', ext='.ts', timestamp=7) - self._create_ondisk_file(df, b'', ext='.meta', timestamp=6) - self._create_ondisk_file(df, b'', ext='.meta', timestamp=5) + df, b'A', ext='.data', legacy_durable=legacy_durable, + timestamp=ts[9]) + self._create_ondisk_file(df, b'', ext='.ts', timestamp=ts[8]) + self._create_ondisk_file(df, b'', ext='.ts', timestamp=ts[7]) + self._create_ondisk_file(df, b'', ext='.meta', timestamp=ts[6]) + self._create_ondisk_file(df, b'', ext='.meta', timestamp=ts[5]) df = self._simple_get_diskfile() with df.open(): self.assertIn('X-Timestamp', df._metadata) - self.assertEqual(df._metadata['X-Timestamp'], - Timestamp(10).internal) + self.assertEqual(df._metadata['X-Timestamp'], ts[10].internal) self.assertNotIn('deleted', df._metadata) def test_ondisk_search_loop_data_ts_meta(self): @@ -5721,20 +5732,22 @@ class DiskFileMixin(BaseDiskFileTestMixin): def _test_ondisk_search_loop_wayward_files_ignored(self, legacy_durable=False): df = self._simple_get_diskfile() - self._create_ondisk_file(df, b'X', ext='.bar', timestamp=11) + ts = [Timestamp(i) for i in range(12)] + self._create_ondisk_file(df, b'X', ext='.bar', timestamp=ts[11]) self._create_ondisk_file( - df, b'B', ext='.data', legacy_durable=legacy_durable, timestamp=10) + df, b'B', ext='.data', legacy_durable=legacy_durable, + timestamp=ts[10]) self._create_ondisk_file( - df, b'A', ext='.data', legacy_durable=legacy_durable, timestamp=9) - self._create_ondisk_file(df, b'', ext='.ts', timestamp=8) - self._create_ondisk_file(df, b'', ext='.ts', timestamp=7) - self._create_ondisk_file(df, b'', ext='.meta', timestamp=6) - self._create_ondisk_file(df, b'', ext='.meta', timestamp=5) + df, b'A', ext='.data', legacy_durable=legacy_durable, + timestamp=ts[9]) + self._create_ondisk_file(df, b'', ext='.ts', timestamp=ts[8]) + self._create_ondisk_file(df, b'', ext='.ts', timestamp=ts[7]) + self._create_ondisk_file(df, b'', ext='.meta', timestamp=ts[6]) + self._create_ondisk_file(df, b'', ext='.meta', timestamp=ts[5]) df = self._simple_get_diskfile() with df.open(): self.assertIn('X-Timestamp', df._metadata) - self.assertEqual(df._metadata['X-Timestamp'], - Timestamp(10).internal) + self.assertEqual(df._metadata['X-Timestamp'], ts[10].internal) self.assertNotIn('deleted', df._metadata) def test_ondisk_search_loop_wayward_files_ignored(self): @@ -5800,9 +5813,9 @@ class DiskFileMixin(BaseDiskFileTestMixin): def test_diskfile_content_length_deleted(self): df = self._get_open_disk_file() - ts = time() + ts = Timestamp.now() df.delete(ts) - exp_name = '%s.ts' % str(Timestamp(ts).internal) + exp_name = '%s.ts' % ts.internal dl = os.listdir(df._datadir) self.assertEqual(len(dl), 1) self.assertIn(exp_name, set(dl)) @@ -5837,9 +5850,9 @@ class DiskFileMixin(BaseDiskFileTestMixin): def test_diskfile_timestamp_deleted(self): df = self._get_open_disk_file() - ts = time() + ts = Timestamp.now() df.delete(ts) - exp_name = '%s.ts' % str(Timestamp(ts).internal) + exp_name = '%s.ts' % ts.internal dl = os.listdir(df._datadir) self.assertEqual(len(dl), 1) self.assertIn(exp_name, set(dl)) @@ -5951,12 +5964,12 @@ class DiskFileMixin(BaseDiskFileTestMixin): df = self._get_open_disk_file() file_count = len(os.listdir(df._datadir)) - ts = time() + ts = Timestamp.now() with mock.patch( self._manager_mock('cleanup_ondisk_files'), mock_cleanup): # Expect to swallow the OSError df.delete(ts) - exp_name = '%s.ts' % str(Timestamp(ts).internal) + exp_name = '%s.ts' % Timestamp(ts).internal dl = os.listdir(df._datadir) self.assertEqual(len(dl), file_count + 1) self.assertIn(exp_name, set(dl)) @@ -7293,7 +7306,8 @@ class TestECDiskFile(DiskFileMixin, unittest.TestCase): def test_ondisk_data_info_has_durable_key(self): # non-durable; use frag_prefs=[] to allow it to be opened df = self._simple_get_diskfile(obj='o1', frag_prefs=[]) - self._create_ondisk_file(df, b'', ext='.data', timestamp=10, + ts_data = Timestamp(10) + self._create_ondisk_file(df, b'', ext='.data', timestamp=ts_data, metadata={'name': '/a/c/o1'}, commit=False) with df.open(): self.assertIn('durable', df._ondisk_info['data_info']) @@ -7301,7 +7315,7 @@ class TestECDiskFile(DiskFileMixin, unittest.TestCase): # durable df = self._simple_get_diskfile(obj='o2') - self._create_ondisk_file(df, b'', ext='.data', timestamp=10, + self._create_ondisk_file(df, b'', ext='.data', timestamp=ts_data, metadata={'name': '/a/c/o2'}) with df.open(): self.assertIn('durable', df._ondisk_info['data_info']) @@ -7309,7 +7323,8 @@ class TestECDiskFile(DiskFileMixin, unittest.TestCase): # legacy durable df = self._simple_get_diskfile(obj='o3') - self._create_ondisk_file(df, b'', ext='.data', timestamp=10, + ts_legacy = Timestamp('10') + self._create_ondisk_file(df, b'', ext='.data', timestamp=ts_legacy, metadata={'name': '/a/c/o3'}, legacy_durable=True) with df.open(): @@ -7368,8 +7383,7 @@ class TestSuffixHashes(unittest.TestCase): 'mount_check': False, } self.df_router = diskfile.DiskFileRouter(self.conf, self.logger) - self._ts_iter = (Timestamp(t) for t in - itertools.count(int(time()))) + self._ts_iter = make_timestamp_iter() self.policy = None def ts(self): @@ -8568,9 +8582,10 @@ class TestSuffixHashes(unittest.TestCase): mkdirs(df._datadir) now = time() # go behind the scenes and setup a bunch of weird file names - for tdiff in [500, 100, 10, 1]: + timestamps = [Timestamp(now - tdiff) + for tdiff in [500, 100, 10, 1]] + for timestamp in timestamps: for suff in ['.meta', '.data', '.ts']: - timestamp = Timestamp(now - tdiff) filename = timestamp.internal if policy.policy_type == EC_POLICY and suff == '.data': filename += '#%s' % df._frag_index @@ -8603,14 +8618,15 @@ class TestSuffixHashes(unittest.TestCase): now = time() timestamp = None # go behind the scenes and setup a bunch of weird file names - for tdiff in [500, 100, 10, 1]: + timestamps = [Timestamp(now - tdiff) + for tdiff in [500, 100, 10, 1]] + for i, timestamp in enumerate(timestamps): suffs = ['.meta', '.data'] - if tdiff > 50: + if i < 2: suffs.append('.ts') if policy.policy_type == EC_POLICY and legacy_durable: suffs.append('.durable') for suff in suffs: - timestamp = Timestamp(now - tdiff) if suff == '.data': filename = _make_datafilename( timestamp, policy, frag_index,