diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 7b0cbed597..1cbd7dadc1 100644 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -995,8 +995,14 @@ class TestObjectController(BaseTestCase): resp, head_resp = self._do_test_diskfile_metadata_unavailable(req) self.assertEqual(resp.status_int, 201) self.assertEqual(head_resp.status_int, 200) - self.assertEqual(req.headers['X-Timestamp'], + # response x-timestamp uses the *normal* format + self.assertEqual(Timestamp(req.headers['X-Timestamp']).normal, head_resp.headers['X-Timestamp']) + # response x-<*>-timestamp uses the *internal* format + self.assertEqual(Timestamp(req.headers['X-Timestamp']).internal, + head_resp.headers['X-Backend-Timestamp']) + self.assertEqual(Timestamp(req.headers['X-Timestamp']).internal, + head_resp.headers['X-Backend-Data-Timestamp']) self.assertNotIn('X-Object-Meta-Test', head_resp.headers) def test_POST_metafile_unavailable(self): @@ -2015,7 +2021,7 @@ class TestObjectController(BaseTestCase): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, - headers={'X-Timestamp': old_timestamp.normal, + headers={'X-Timestamp': old_timestamp.internal, 'Content-Length': '0', 'Content-Type': 'application/octet-stream'}) resp = req.get_response(self.object_controller) @@ -2023,7 +2029,7 @@ class TestObjectController(BaseTestCase): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': new_timestamp.normal, + headers={'X-Timestamp': new_timestamp.internal, 'Content-Type': 'text/plain', 'Content-Encoding': 'gzip'}) req.body = 'VERIFY TWO' @@ -2052,14 +2058,14 @@ class TestObjectController(BaseTestCase): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, - headers={'X-Timestamp': new_timestamp.normal, + headers={'X-Timestamp': new_timestamp.internal, 'Content-Length': '0', 'Content-Type': 'application/octet-stream'}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 404) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': old_timestamp.normal, + headers={'X-Timestamp': old_timestamp.internal, 'Content-Type': 'text/plain', 'Content-Encoding': 'gzip'}) req.body = 'VERIFY TWO' @@ -2125,11 +2131,11 @@ class TestObjectController(BaseTestCase): 'Content-Encoding': 'gzip'}) def test_PUT_old_timestamp(self): - ts = time() - orig_timestamp = utils.Timestamp(ts).internal + older_timestamp = next(self.ts) + orig_timestamp = next(self.ts) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': orig_timestamp, + headers={'X-Timestamp': orig_timestamp.internal, 'Content-Length': '6', 'Content-Type': 'application/octet-stream'}) req.body = 'VERIFY' @@ -2137,23 +2143,25 @@ class TestObjectController(BaseTestCase): self.assertEqual(resp.status_int, 201) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(ts), + headers={'X-Timestamp': orig_timestamp.internal, 'Content-Type': 'text/plain', 'Content-Encoding': 'gzip'}) req.body = 'VERIFY TWO' resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 409) - self.assertEqual(resp.headers['X-Backend-Timestamp'], orig_timestamp) + self.assertEqual(resp.headers['X-Backend-Timestamp'], + orig_timestamp.internal) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ - 'X-Timestamp': normalize_timestamp(ts - 1), + 'X-Timestamp': older_timestamp.internal, 'Content-Type': 'text/plain', 'Content-Encoding': 'gzip'}) req.body = 'VERIFY THREE' resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 409) - self.assertEqual(resp.headers['X-Backend-Timestamp'], orig_timestamp) + self.assertEqual(resp.headers['X-Backend-Timestamp'], + orig_timestamp.internal) def test_PUT_new_object_really_old_timestamp(self): req = Request.blank( @@ -3603,10 +3611,10 @@ class TestObjectController(BaseTestCase): self.assertEqual(resp.status_int, 404) sleep(.00001) - timestamp = normalize_timestamp(time()) + timestamp = Timestamp.now() req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ - 'X-Timestamp': timestamp, + 'X-Timestamp': timestamp.internal, 'Content-Type': 'application/octet-stream', 'Content-length': '6'}) req.body = b'VERIFY' @@ -3614,10 +3622,10 @@ class TestObjectController(BaseTestCase): self.assertEqual(resp.status_int, 201) sleep(.00001) - timestamp = normalize_timestamp(time()) + timestamp = Timestamp.now() req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, - headers={'X-Timestamp': timestamp}) + headers={'X-Timestamp': timestamp.internal}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 204) @@ -3626,7 +3634,7 @@ class TestObjectController(BaseTestCase): resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 404) self.assertEqual(resp.headers['X-Backend-Timestamp'], - utils.Timestamp(timestamp).internal) + timestamp.internal) def test_HEAD_quarantine_zbyte(self): # Test swift.obj.server.ObjectController.GET @@ -5644,9 +5652,9 @@ class TestObjectController(BaseTestCase): def capture_updates(ip, port, method, path, headers, *args, **kwargs): container_updates.append((ip, port, method, path, headers)) # create a new object - create_timestamp = next(self.ts).internal + create_timestamp = next(self.ts) req = Request.blank('/sda1/p/a/c/o', method='PUT', body=b'test1', - headers={'X-Timestamp': create_timestamp, + headers={'X-Timestamp': create_timestamp.internal, 'X-Container-Host': '10.0.0.1:8080', 'X-Container-Device': 'sda1', 'X-Container-Partition': 'p', @@ -5668,7 +5676,7 @@ class TestObjectController(BaseTestCase): 'X-Size': len(b'test1'), 'X-Etag': md5(b'test1', usedforsecurity=False).hexdigest(), 'X-Content-Type': 'text/plain', - 'X-Timestamp': create_timestamp, + 'X-Timestamp': create_timestamp.internal, } for key, value in expected.items(): self.assertEqual(headers[key], str(value)) @@ -5677,16 +5685,16 @@ class TestObjectController(BaseTestCase): req = Request.blank('/sda1/p/a/c/o', method='GET') resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.headers['X-Timestamp'], - utils.Timestamp(create_timestamp).normal) + self.assertEqual(resp.headers['X-Timestamp'], create_timestamp.normal) self.assertEqual(resp.headers['X-Backend-Timestamp'], - create_timestamp) + create_timestamp.internal) + self.assertEqual(resp.headers['X-Backend-Data-Timestamp'], + create_timestamp.internal) self.assertEqual(resp.body, b'test1') # send an update with an offset - offset_timestamp = utils.Timestamp( - create_timestamp, offset=1).internal + offset_timestamp = utils.Timestamp(create_timestamp, offset=1) req = Request.blank('/sda1/p/a/c/o', method='PUT', body=b'test2', - headers={'X-Timestamp': offset_timestamp, + headers={'X-Timestamp': offset_timestamp.internal, 'X-Container-Host': '10.0.0.1:8080', 'X-Container-Device': 'sda1', 'X-Container-Partition': 'p', @@ -5708,7 +5716,7 @@ class TestObjectController(BaseTestCase): 'X-Size': len(b'test2'), 'X-Etag': md5(b'test2', usedforsecurity=False).hexdigest(), 'X-Content-Type': 'text/html', - 'X-Timestamp': offset_timestamp, + 'X-Timestamp': offset_timestamp.internal, } for key, value in expected.items(): self.assertEqual(headers[key], str(value)) @@ -5717,19 +5725,19 @@ class TestObjectController(BaseTestCase): req = Request.blank('/sda1/p/a/c/o', method='GET') resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.headers['X-Timestamp'], - utils.Timestamp(offset_timestamp).normal) + self.assertEqual(resp.headers['X-Timestamp'], offset_timestamp.normal) self.assertEqual(resp.headers['X-Backend-Timestamp'], - offset_timestamp) + offset_timestamp.internal) self.assertEqual(resp.body, b'test2') # now overwrite with a newer time - overwrite_timestamp = next(self.ts).internal - req = Request.blank('/sda1/p/a/c/o', method='PUT', body=b'test3', - headers={'X-Timestamp': overwrite_timestamp, - 'X-Container-Host': '10.0.0.1:8080', - 'X-Container-Device': 'sda1', - 'X-Container-Partition': 'p', - 'Content-Type': 'text/enriched'}) + overwrite_timestamp = next(self.ts) + req = Request.blank( + '/sda1/p/a/c/o', method='PUT', body=b'test3', + headers={'X-Timestamp': overwrite_timestamp.internal, + 'X-Container-Host': '10.0.0.1:8080', + 'X-Container-Device': 'sda1', + 'X-Container-Partition': 'p', + 'Content-Type': 'text/enriched'}) with mocked_http_conn(200, give_connect=capture_updates) as fake_conn: with fake_spawn(): resp = req.get_response(self.object_controller) @@ -5747,7 +5755,7 @@ class TestObjectController(BaseTestCase): 'X-Size': len(b'test3'), 'X-Etag': md5(b'test3', usedforsecurity=False).hexdigest(), 'X-Content-Type': 'text/enriched', - 'X-Timestamp': overwrite_timestamp, + 'X-Timestamp': overwrite_timestamp.internal, } for key, value in expected.items(): self.assertEqual(headers[key], str(value)) @@ -5757,18 +5765,20 @@ class TestObjectController(BaseTestCase): resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 200) self.assertEqual(resp.headers['X-Timestamp'], - utils.Timestamp(overwrite_timestamp).normal) + overwrite_timestamp.normal) self.assertEqual(resp.headers['X-Backend-Timestamp'], - overwrite_timestamp) + overwrite_timestamp.internal) + self.assertEqual(resp.headers['X-Backend-Data-Timestamp'], + overwrite_timestamp.internal) self.assertEqual(resp.body, b'test3') # delete with an offset - offset_delete = utils.Timestamp(overwrite_timestamp, - offset=1).internal - req = Request.blank('/sda1/p/a/c/o', method='DELETE', - headers={'X-Timestamp': offset_delete, - 'X-Container-Host': '10.0.0.1:8080', - 'X-Container-Device': 'sda1', - 'X-Container-Partition': 'p'}) + offset_delete = utils.Timestamp(overwrite_timestamp, offset=1) + req = Request.blank( + '/sda1/p/a/c/o', method='DELETE', + headers={'X-Timestamp': offset_delete.internal, + 'X-Container-Host': '10.0.0.1:8080', + 'X-Container-Device': 'sda1', + 'X-Container-Partition': 'p'}) with mocked_http_conn(200, give_connect=capture_updates) as fake_conn: with fake_spawn(): resp = req.get_response(self.object_controller) @@ -5783,7 +5793,7 @@ class TestObjectController(BaseTestCase): self.assertEqual(method, 'DELETE') self.assertEqual(path, '/sda1/p/a/c/o') expected = { - 'X-Timestamp': offset_delete, + 'X-Timestamp': offset_delete.internal, } for key, value in expected.items(): self.assertEqual(headers[key], str(value)) @@ -5793,11 +5803,12 @@ class TestObjectController(BaseTestCase): resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 404) self.assertIsNone(resp.headers['X-Timestamp']) - self.assertEqual(resp.headers['X-Backend-Timestamp'], offset_delete) + self.assertEqual(resp.headers['X-Backend-Timestamp'], + offset_delete.internal) # and one more delete with a newer timestamp - delete_timestamp = next(self.ts).internal + delete_timestamp = next(self.ts) req = Request.blank('/sda1/p/a/c/o', method='DELETE', - headers={'X-Timestamp': delete_timestamp, + headers={'X-Timestamp': delete_timestamp.internal, 'X-Container-Host': '10.0.0.1:8080', 'X-Container-Device': 'sda1', 'X-Container-Partition': 'p'}) @@ -5815,7 +5826,7 @@ class TestObjectController(BaseTestCase): self.assertEqual(method, 'DELETE') self.assertEqual(path, '/sda1/p/a/c/o') expected = { - 'X-Timestamp': delete_timestamp, + 'X-Timestamp': delete_timestamp.internal, } for key, value in expected.items(): self.assertEqual(headers[key], str(value)) @@ -5825,7 +5836,8 @@ class TestObjectController(BaseTestCase): resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 404) self.assertIsNone(resp.headers['X-Timestamp']) - self.assertEqual(resp.headers['X-Backend-Timestamp'], delete_timestamp) + self.assertEqual(resp.headers['X-Backend-Timestamp'], + delete_timestamp.internal) def test_call_bad_request(self): # Test swift.obj.server.ObjectController.__call__ @@ -6533,14 +6545,15 @@ class TestObjectController(BaseTestCase): def fake_http_connect(*args): raise Exception('test') + timestamp = next(self.ts) orig_http_connect = object_server.http_connect try: object_server.http_connect = fake_http_connect self.object_controller.async_update( 'PUT', 'a', 'c', 'o', '127.0.0.1:1234', 1, 'sdc1', - {'x-timestamp': '1', 'x-out': 'set', - 'X-Backend-Storage-Policy-Index': int(policy)}, 'sda1', - policy, db_state='unsharded') + {'x-timestamp': timestamp.internal, 'x-out': 'set', + 'X-Backend-Storage-Policy-Index': int(policy)}, + 'sda1', policy, db_state='unsharded') finally: object_server.http_connect = orig_http_connect utils.HASH_PATH_PREFIX = _prefix @@ -6548,9 +6561,9 @@ class TestObjectController(BaseTestCase): self.assertEqual( pickle.load(open(os.path.join( self.testdir, 'sda1', async_dir, 'a83', - '06fbf0b514e5199dfc4e00f42eb5ea83-%s' % - utils.Timestamp(1).internal), 'rb')), - {'headers': {'x-timestamp': '1', 'x-out': 'set', + '06fbf0b514e5199dfc4e00f42eb5ea83-%s' % timestamp.internal), + 'rb')), + {'headers': {'x-timestamp': timestamp.internal, 'x-out': 'set', 'user-agent': 'object-server %s' % os.getpid(), 'X-Backend-Storage-Policy-Index': int(policy)}, 'account': 'a', 'container': 'c', 'obj': 'o', 'op': 'PUT', @@ -6580,19 +6593,21 @@ class TestObjectController(BaseTestCase): orig_http_connect = object_server.http_connect try: for status in (199, 300, 503): + timestamp = next(self.ts) object_server.http_connect = fake_http_connect(status) self.object_controller.async_update( 'PUT', 'a', 'c', 'o', '127.0.0.1:1234', 1, 'sdc1', - {'x-timestamp': '1', 'x-out': str(status), - 'X-Backend-Storage-Policy-Index': int(policy)}, 'sda1', - policy, db_state='unsharded') + {'x-timestamp': timestamp.internal, 'x-out': str(status), + 'X-Backend-Storage-Policy-Index': int(policy)}, + 'sda1', policy, db_state='unsharded') async_dir = diskfile.get_async_dir(policy) self.assertEqual( pickle.load(open(os.path.join( self.testdir, 'sda1', async_dir, 'a83', '06fbf0b514e5199dfc4e00f42eb5ea83-%s' % - utils.Timestamp(1).internal), 'rb')), - {'headers': {'x-timestamp': '1', 'x-out': str(status), + timestamp.internal), 'rb')), + {'headers': {'x-timestamp': timestamp.internal, + 'x-out': str(status), 'user-agent': 'object-server %s' % os.getpid(), 'X-Backend-Storage-Policy-Index': @@ -6625,14 +6640,18 @@ class TestObjectController(BaseTestCase): orig_http_connect = object_server.http_connect try: for status in (200, 299): + timestamp = next(self.ts) object_server.http_connect = fake_http_connect(status) self.object_controller.async_update( 'PUT', 'a', 'c', 'o', '127.0.0.1:1234', 1, 'sdc1', - {'x-timestamp': '1', 'x-out': str(status)}, 'sda1', 0) + {'x-timestamp': timestamp.internal, + 'x-out': str(status)}, + 'sda1', 0) self.assertFalse( os.path.exists(os.path.join( self.testdir, 'sda1', 'async_pending', 'a83', - '06fbf0b514e5199dfc4e00f42eb5ea83-0000000001.00000'))) + '06fbf0b514e5199dfc4e00f42eb5ea83-%s' + % timestamp.internal))) finally: object_server.http_connect = orig_http_connect utils.HASH_PATH_PREFIX = _prefix @@ -6655,18 +6674,20 @@ class TestObjectController(BaseTestCase): orig_http_connect = object_server.http_connect try: for status in (200, 299): + timestamp = next(self.ts) object_server.http_connect = fake_http_connect() self.object_controller.node_timeout = 0.001 self.object_controller.async_update( 'PUT', 'a', 'c', 'o', '127.0.0.1:1234', 1, 'sdc1', - {'x-timestamp': '1', 'x-out': str(status)}, 'sda1', - policy) + {'x-timestamp': timestamp.internal, + 'x-out': str(status)}, + 'sda1', policy) async_dir = diskfile.get_async_dir(policy) self.assertTrue( os.path.exists(os.path.join( self.testdir, 'sda1', async_dir, 'a83', '06fbf0b514e5199dfc4e00f42eb5ea83-%s' % - utils.Timestamp(1).internal))) + timestamp.internal))) finally: object_server.http_connect = orig_http_connect utils.HASH_PATH_PREFIX = _prefix @@ -6698,10 +6719,11 @@ class TestObjectController(BaseTestCase): def capture_updates(ip, port, method, path, headers, *args, **kwargs): container_updates.append((ip, port, method, path, headers)) + timestamp = next(self.ts) req = Request.blank( '/sda1/0/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': 1, + headers={'X-Timestamp': timestamp.internal, 'X-Trans-Id': '123', 'X-Container-Host': 'chost:cport', 'X-Container-Partition': 'cpartition', @@ -6724,7 +6746,7 @@ class TestObjectController(BaseTestCase): 'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-content-type': 'text/plain', - 'x-timestamp': utils.Timestamp(1).internal, + 'x-timestamp': timestamp.internal, 'X-Backend-Storage-Policy-Index': '0', # default when not given 'x-trans-id': '123', 'referer': 'PUT http://localhost/sda1/0/a/c/o'})) @@ -6991,10 +7013,11 @@ class TestObjectController(BaseTestCase): def test_container_update_async(self): policy = random.choice(list(POLICIES)) + ts = next(self.ts) req = Request.blank( '/sda1/0/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': 1, + headers={'X-Timestamp': ts.internal, 'X-Trans-Id': '123', 'X-Container-Host': 'chost:cport', 'X-Container-Partition': 'cpartition', @@ -7023,14 +7046,14 @@ class TestObjectController(BaseTestCase): self.assertEqual(account, 'a') self.assertEqual(container, 'c') self.assertEqual(obj, 'o') - self.assertEqual(timestamp, utils.Timestamp(1).internal) + self.assertEqual(timestamp, ts.internal) self.assertEqual(policy, policy) self.assertEqual(data, { 'headers': HeaderKeyDict({ 'X-Size': '0', 'User-Agent': 'object-server %s' % os.getpid(), 'X-Content-Type': 'text/plain', - 'X-Timestamp': utils.Timestamp(1).internal, + 'X-Timestamp': ts.internal, 'X-Trans-Id': '123', 'Referer': 'PUT http://localhost/sda1/0/a/c/o', 'X-Backend-Storage-Policy-Index': int(policy), @@ -7054,10 +7077,11 @@ class TestObjectController(BaseTestCase): # just capture the args to see that we would have called called_async_update_args.append([a, kw]) + ts = next(self.ts) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': '12345', + headers={'X-Timestamp': ts.internal, 'Content-Type': 'application/burrito', 'Content-Length': '0', 'X-Backend-Storage-Policy-Index': 0, @@ -7083,7 +7107,7 @@ class TestObjectController(BaseTestCase): # check that the calls to async_update have happened headers_out = {'X-Size': '0', 'X-Content-Type': 'application/burrito', - 'X-Timestamp': '0000012345.00000', + 'X-Timestamp': ts.internal, 'X-Trans-Id': '-', 'Referer': 'PUT http://localhost/sda1/p/a/c/o', 'X-Backend-Storage-Policy-Index': '0', @@ -7113,10 +7137,11 @@ class TestObjectController(BaseTestCase): # just capture the args to see that we would have called called_async_update_args.append([a, kw]) + ts = next(self.ts) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': '12345', + headers={'X-Timestamp': ts.internal, 'Content-Type': 'application/burrito', 'Content-Length': '0', 'X-Backend-Storage-Policy-Index': 0, @@ -7145,10 +7170,11 @@ class TestObjectController(BaseTestCase): def fake_async_update(*args): given_args.extend(args) + ts = next(self.ts) req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': 1, + headers={'X-Timestamp': ts.internal, 'X-Trans-Id': '123', 'X-Container-Host': 'chost,badhost', 'X-Container-Partition': 'cpartition', @@ -7175,6 +7201,7 @@ class TestObjectController(BaseTestCase): # Test how delete_at_update works with a request to overwrite an object # with delete-at metadata policy = random.choice(list(POLICIES)) + ts = next(self.ts) def do_test(method, headers, expected_args): given_args = [] @@ -7182,7 +7209,7 @@ class TestObjectController(BaseTestCase): def fake_async_update(*args): given_args.extend(args) - headers.update({'X-Timestamp': 1, + headers.update({'X-Timestamp': ts.internal, 'X-Trans-Id': '123', 'X-Backend-Storage-Policy-Index': int(policy)}) req = Request.blank( @@ -7201,7 +7228,7 @@ class TestObjectController(BaseTestCase): '0000000002-a/c/o', None, None, None, HeaderKeyDict({ 'X-Backend-Storage-Policy-Index': 0, - 'x-timestamp': utils.Timestamp('1').internal, + 'x-timestamp': ts.internal, 'x-trans-id': '123', 'referer': '%s http://localhost/v1/a/c/o' % method}), 'sda1', policy] @@ -7226,11 +7253,12 @@ class TestObjectController(BaseTestCase): def fake_async_update(*args): given_args.extend(args) + ts = next(self.ts) self.object_controller.async_update = fake_async_update req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': 1, + headers={'X-Timestamp': ts.internal, 'X-Trans-Id': '1234', 'X-Backend-Storage-Policy-Index': int(policy)}) self.object_controller.delete_at_update( @@ -7241,7 +7269,7 @@ class TestObjectController(BaseTestCase): HeaderKeyDict({ # the expiring objects account is always 0 'X-Backend-Storage-Policy-Index': 0, - 'x-timestamp': utils.Timestamp('1').internal, + 'x-timestamp': ts.internal, 'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1', policy]) @@ -7695,13 +7723,13 @@ class TestObjectController(BaseTestCase): def test_GET_but_expired(self): # Start off with an existing object that will expire - now = time() - delete_at_timestamp = int(now + 100) + ts_now = Timestamp.now() + delete_at_seconds = int(ts_now) + 100 req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=self._update_delete_at_headers({ - 'X-Timestamp': normalize_timestamp(now), - 'X-Delete-At': str(delete_at_timestamp), + 'X-Timestamp': ts_now.internal, + 'X-Delete-At': delete_at_seconds, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'})) req.body = 'TEST' @@ -7711,7 +7739,7 @@ class TestObjectController(BaseTestCase): # It expires in the future, so it's accessible via GET req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Timestamp': normalize_timestamp(now)}) + headers={'X-Timestamp': ts_now.internal}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 200) @@ -7719,18 +7747,15 @@ class TestObjectController(BaseTestCase): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Timestamp': normalize_timestamp( - delete_at_timestamp + 1)}) + headers={'X-Timestamp': Timestamp(delete_at_seconds + 1).internal}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 404) - self.assertEqual(resp.headers['X-Backend-Timestamp'], - utils.Timestamp(now)) + self.assertEqual(resp.headers['X-Backend-Timestamp'], ts_now.internal) # ...unless X-Backend-Replication is sent req = Request.blank( '/sda1/p/a/c/o', method='GET', - headers={'X-Timestamp': - normalize_timestamp(delete_at_timestamp + 1), + headers={'X-Timestamp': Timestamp(delete_at_seconds + 1).internal, 'X-Backend-Replication': 'True'}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 200) @@ -7739,8 +7764,7 @@ class TestObjectController(BaseTestCase): # ...or x-backend-open-expired is sent req = Request.blank( '/sda1/p/a/c/o', method='GET', - headers={'X-Timestamp': - normalize_timestamp(delete_at_timestamp + 1), + headers={'X-Timestamp': Timestamp(delete_at_seconds + 1).internal, 'x-backend-open-expired': 'True'}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 200) @@ -7748,13 +7772,13 @@ class TestObjectController(BaseTestCase): def test_HEAD_but_expired(self): # We have an object that expires in the future - now = time() - delete_at_timestamp = int(now + 100) + ts_now = Timestamp.now() + delete_at_seconds = int(ts_now) + 100 req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=self._update_delete_at_headers({ - 'X-Timestamp': normalize_timestamp(now), - 'X-Delete-At': str(delete_at_timestamp), + 'X-Timestamp': ts_now.internal, + 'X-Delete-At': delete_at_seconds, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'})) req.body = b'TEST' @@ -7765,7 +7789,7 @@ class TestObjectController(BaseTestCase): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}, - headers={'X-Timestamp': normalize_timestamp(now)}) + headers={'X-Timestamp': ts_now.internal}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 200) @@ -7773,19 +7797,18 @@ class TestObjectController(BaseTestCase): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}, - headers={'X-Timestamp': normalize_timestamp( - delete_at_timestamp + 1)}) + headers={'X-Timestamp': Timestamp(delete_at_seconds + 1).internal}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 404) self.assertEqual(resp.headers['X-Backend-Timestamp'], - utils.Timestamp(now)) + utils.Timestamp(ts_now)) # It should be accessible with x-backend-open-expired req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}, - headers={'X-Timestamp': normalize_timestamp( - delete_at_timestamp + 2), 'x-backend-open-expired': 'true'}) + headers={'X-Timestamp': Timestamp(delete_at_seconds + 2).internal, + 'x-backend-open-expired': 'true'}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 200) @@ -7793,8 +7816,8 @@ class TestObjectController(BaseTestCase): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}, - headers={'X-Timestamp': normalize_timestamp( - delete_at_timestamp + 2), 'x-backend-replication': 'true'}) + headers={'X-Timestamp': Timestamp(delete_at_seconds + 2).internal, + 'x-backend-replication': 'true'}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 200) self.assertEqual(b'', resp.body) @@ -9623,15 +9646,15 @@ class TestObjectController(BaseTestCase): self.assertFalse(os.path.isdir(object_dir)) def test_race_doesnt_quarantine(self): - existing_timestamp = normalize_timestamp(time()) - delete_timestamp = normalize_timestamp(time() + 1) - put_timestamp = normalize_timestamp(time() + 2) - head_timestamp = normalize_timestamp(time() + 3) + existing_timestamp = next(self.ts) + delete_timestamp = next(self.ts) + put_timestamp = next(self.ts) + head_timestamp = next(self.ts) # make a .ts req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, - headers={'X-Timestamp': existing_timestamp}) + headers={'X-Timestamp': existing_timestamp.internal}) req.get_response(self.object_controller) # force a PUT between the listdir and read_metadata of a DELETE @@ -9644,7 +9667,7 @@ class TestObjectController(BaseTestCase): put_once[0] = True req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': put_timestamp, + headers={'X-Timestamp': put_timestamp.internal, 'Content-Length': '9', 'Content-Type': 'application/octet-stream'}) req.body = 'some data' @@ -9655,7 +9678,7 @@ class TestObjectController(BaseTestCase): with mock.patch('os.listdir', mock_listdir): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, - headers={'X-Timestamp': delete_timestamp}) + headers={'X-Timestamp': delete_timestamp.internal}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 404) @@ -9664,21 +9687,21 @@ class TestObjectController(BaseTestCase): req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}, - headers={'X-Timestamp': head_timestamp}) + headers={'X-Timestamp': head_timestamp.internal}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.headers['X-Timestamp'], put_timestamp) + self.assertEqual(resp.headers['X-Timestamp'], put_timestamp.normal) def test_race_with_PUT_POST_PUT(self): - existing_timestamp = normalize_timestamp(time()) - post_timestamp = normalize_timestamp(time() + 1) - put_timestamp = normalize_timestamp(time() + 2) - head_timestamp = normalize_timestamp(time() + 3) + existing_timestamp = next(self.ts) + post_timestamp = next(self.ts) + put_timestamp = next(self.ts) + head_timestamp = next(self.ts) # make a .data req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': existing_timestamp, + headers={'X-Timestamp': existing_timestamp.internal, 'Content-Type': 'application/octet-stream'}, body=b'orig data') resp = req.get_response(self.object_controller) @@ -9694,7 +9717,7 @@ class TestObjectController(BaseTestCase): put_once[0] = True req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': put_timestamp, + headers={'X-Timestamp': put_timestamp.internal, 'Content-Type': 'application/octet-stream'}, body=b'some data') resp = req.get_response(self.object_controller) @@ -9704,7 +9727,7 @@ class TestObjectController(BaseTestCase): with mock.patch('os.listdir', mock_listdir): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': post_timestamp}) + headers={'X-Timestamp': post_timestamp.internal}) resp = req.get_response(self.object_controller) self.assertNotIn('X-Backend-Timestamp', resp.headers) self.assertEqual(resp.status_int, 503) @@ -9714,10 +9737,10 @@ class TestObjectController(BaseTestCase): req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}, - headers={'X-Timestamp': head_timestamp}) + headers={'X-Timestamp': head_timestamp.internal}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 200) - self.assertEqual(resp.headers['X-Timestamp'], put_timestamp) + self.assertEqual(resp.headers['X-Timestamp'], put_timestamp.normal) def test_multiphase_put_draining(self): # We want to ensure that we read the whole response body even if @@ -10224,7 +10247,7 @@ class TestObjectServer(unittest.TestCase): expected_meta = {'Content-Length': '82', 'name': '/a/c/o', 'X-Object-Sysmeta-Ec-Frag-Index': '2', - 'X-Timestamp': put_timestamp.normal, + 'X-Timestamp': put_timestamp.internal, 'Content-Type': 'text/plain'} for k, v in actual_meta.items(): # See diskfile.py:_decode_metadata diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index 606f6dbf49..12ec48104d 100644 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -198,8 +198,7 @@ class BaseObjectControllerMixin(object): # default policy and ring references self.policy = POLICIES.default self.obj_ring = self.policy.object_ring - self._ts_iter = (utils.Timestamp(t) for t in - itertools.count(int(time.time()))) + self._ts_iter = make_timestamp_iter() def _make_app(self): self.app = PatchedObjControllerApp( @@ -1081,9 +1080,8 @@ class CommonObjectControllerMixin(BaseObjectControllerMixin): def test_HEAD_x_newest_different_timestamps(self): req = swob.Request.blank('/v1/a/c/o', method='HEAD', headers={'X-Newest': 'true'}) - ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) num_expected_requests = 2 * self.replicas() - timestamps = [next(ts) for i in range(num_expected_requests)] + timestamps = [self.ts() for i in range(num_expected_requests)] newest_timestamp = timestamps[-1] random.shuffle(timestamps) backend_response_headers = [{ @@ -1099,11 +1097,13 @@ class CommonObjectControllerMixin(BaseObjectControllerMixin): def test_HEAD_x_newest_with_two_vector_timestamps(self): req = swob.Request.blank('/v1/a/c/o', method='HEAD', headers={'X-Newest': 'true'}) - ts = (utils.Timestamp.now(offset=offset) - for offset in itertools.count()) + # constant float part, varying offset... + now = Timestamp.now() + ts = (Timestamp(now, offset) for offset in itertools.count()) num_expected_requests = 2 * self.replicas() timestamps = [next(ts) for i in range(num_expected_requests)] newest_timestamp = timestamps[-1] + self.assertGreater(newest_timestamp, timestamps[0]) random.shuffle(timestamps) backend_response_headers = [{ 'X-Backend-Timestamp': t.internal, @@ -1114,15 +1114,14 @@ class CommonObjectControllerMixin(BaseObjectControllerMixin): resp = req.get_response(self.app) self.assertEqual(resp.status_int, 200) self.assertEqual(resp.headers['x-backend-timestamp'], - newest_timestamp.internal) + newest_timestamp.internal, timestamps) def test_HEAD_x_newest_with_some_missing(self): req = swob.Request.blank('/v1/a/c/o', method='HEAD', headers={'X-Newest': 'true'}) - ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) request_count = self.app.request_node_count(self.obj_ring.replicas) backend_response_headers = [{ - 'x-timestamp': next(ts).normal, + 'x-timestamp': self.ts().normal, } for i in range(request_count)] responses = [404] * (request_count - 1) responses.append(200) @@ -1167,14 +1166,13 @@ class CommonObjectControllerMixin(BaseObjectControllerMixin): self.assertEqual(resp.headers['X-Backend-Timestamp'], '2') def test_container_sync_delete(self): - ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) test_indexes = [None] + [int(p) for p in POLICIES] for policy_index in test_indexes: req = swob.Request.blank( '/v1/a/c/o', method='DELETE', headers={ - 'X-Timestamp': next(ts).internal}) + 'X-Timestamp': self.ts().internal}) codes = [409] * self.obj_ring.replicas - ts_iter = itertools.repeat(next(ts).internal) + ts_iter = itertools.repeat(self.ts().internal) with set_http_connect(*codes, timestamps=ts_iter): resp = req.get_response(self.app) self.assertEqual(resp.status_int, 409) @@ -2778,29 +2776,27 @@ class TestReplicatedObjController(CommonObjectControllerMixin, self.assertEqual(resp.status_int, 202) def test_container_sync_put_x_timestamp_older(self): - ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) test_indexes = [None] + [int(p) for p in POLICIES] for policy_index in test_indexes: self.app.container_info['storage_policy'] = policy_index req = swob.Request.blank( '/v1/a/c/o', method='PUT', headers={ 'Content-Length': 0, - 'X-Timestamp': next(ts).internal}) - ts_iter = itertools.repeat(next(ts).internal) + 'X-Timestamp': self.ts().internal}) + ts_iter = itertools.repeat(self.ts().internal) codes = [409] * self.obj_ring.replicas with set_http_connect(*codes, timestamps=ts_iter): resp = req.get_response(self.app) self.assertEqual(resp.status_int, 202) def test_container_sync_put_x_timestamp_newer(self): - ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) test_indexes = [None] + [int(p) for p in POLICIES] for policy_index in test_indexes: - orig_timestamp = next(ts).internal + orig_timestamp = self.ts().internal req = swob.Request.blank( '/v1/a/c/o', method='PUT', headers={ 'Content-Length': 0, - 'X-Timestamp': next(ts).internal}) + 'X-Timestamp': self.ts().internal}) ts_iter = itertools.repeat(orig_timestamp) codes = [201] * self.obj_ring.replicas with set_http_connect(*codes, timestamps=ts_iter): @@ -2808,23 +2804,21 @@ class TestReplicatedObjController(CommonObjectControllerMixin, self.assertEqual(resp.status_int, 201) def test_put_x_timestamp_conflict(self): - ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) req = swob.Request.blank( '/v1/a/c/o', method='PUT', headers={ 'Content-Length': 0, - 'X-Timestamp': next(ts).internal}) - ts_iter = iter([next(ts).internal, None, None]) + 'X-Timestamp': self.ts().internal}) + ts_iter = iter([self.ts().internal, None, None]) codes = [409] + [201] * (self.obj_ring.replicas - 1) with set_http_connect(*codes, timestamps=ts_iter): resp = req.get_response(self.app) self.assertEqual(resp.status_int, 202) def test_put_x_timestamp_conflict_with_missing_backend_timestamp(self): - ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) req = swob.Request.blank( '/v1/a/c/o', method='PUT', headers={ 'Content-Length': 0, - 'X-Timestamp': next(ts).internal}) + 'X-Timestamp': self.ts().internal}) ts_iter = iter([None, None, None]) codes = [409] * self.obj_ring.replicas with set_http_connect(*codes, timestamps=ts_iter): @@ -2832,35 +2826,32 @@ class TestReplicatedObjController(CommonObjectControllerMixin, self.assertEqual(resp.status_int, 202) def test_put_x_timestamp_conflict_with_other_weird_success_response(self): - ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) req = swob.Request.blank( '/v1/a/c/o', method='PUT', headers={ 'Content-Length': 0, - 'X-Timestamp': next(ts).internal}) - ts_iter = iter([next(ts).internal, None, None]) + 'X-Timestamp': self.ts().internal}) + ts_iter = iter([self.ts().internal, None, None]) codes = [409] + [(201, 'notused')] * (self.obj_ring.replicas - 1) with set_http_connect(*codes, timestamps=ts_iter): resp = req.get_response(self.app) self.assertEqual(resp.status_int, 202) def test_put_x_timestamp_conflict_with_if_none_match(self): - ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) req = swob.Request.blank( '/v1/a/c/o', method='PUT', headers={ 'Content-Length': 0, 'If-None-Match': '*', - 'X-Timestamp': next(ts).internal}) - ts_iter = iter([next(ts).internal, None, None]) + 'X-Timestamp': self.ts().internal}) + ts_iter = iter([self.ts().internal, None, None]) codes = [409] + [(412, 'notused')] * (self.obj_ring.replicas - 1) with set_http_connect(*codes, timestamps=ts_iter): resp = req.get_response(self.app) self.assertEqual(resp.status_int, 412) def test_container_sync_put_x_timestamp_race(self): - ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) test_indexes = [None] + [int(p) for p in POLICIES] for policy_index in test_indexes: - put_timestamp = next(ts).internal + put_timestamp = self.ts().internal req = swob.Request.blank( '/v1/a/c/o', method='PUT', headers={ 'Content-Length': 0, @@ -2877,10 +2868,9 @@ class TestReplicatedObjController(CommonObjectControllerMixin, self.assertEqual(resp.status_int, 202) def test_container_sync_put_x_timestamp_unsynced_race(self): - ts = (utils.Timestamp(t) for t in itertools.count(int(time.time()))) test_indexes = [None] + [int(p) for p in POLICIES] for policy_index in test_indexes: - put_timestamp = next(ts).internal + put_timestamp = self.ts().internal req = swob.Request.blank( '/v1/a/c/o', method='PUT', headers={ 'Content-Length': 0,