diff --git a/swift/common/container_sync_realms.py b/swift/common/container_sync_realms.py index 51adcddf34..a9832ba437 100644 --- a/swift/common/container_sync_realms.py +++ b/swift/common/container_sync_realms.py @@ -146,7 +146,7 @@ class ContainerSyncRealms(object): :param request_method: HTTP method of the request. :param path: The path to the resource (url-encoded). - :param x_timestamp: The X-Timestamp header value for the request. + :param x_timestamp: (str) The X-Timestamp header value for the request. :param nonce: A unique value for the request. :param realm_key: Shared secret at the cluster operator level. :param user_key: Shared secret at the user's container level. diff --git a/swift/common/db.py b/swift/common/db.py index f9fee01864..447228f130 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -413,7 +413,7 @@ class DatabaseBroker(object): END; """) if not put_timestamp: - put_timestamp = Timestamp(0).internal + put_timestamp = Timestamp.zero().internal self._initialize(conn, put_timestamp, storage_policy_index=storage_policy_index) conn.commit() diff --git a/swift/common/exceptions.py b/swift/common/exceptions.py index a3e1f2b68a..74d1540c0d 100644 --- a/swift/common/exceptions.py +++ b/swift/common/exceptions.py @@ -14,7 +14,7 @@ # limitations under the License. from eventlet import Timeout -import swift.common.utils +from swift.common.utils.timestamp import Timestamp class MessageTimeout(Timeout): @@ -93,8 +93,8 @@ class DiskFileDeleted(DiskFileNotExist): def __init__(self, metadata=None): self.metadata = metadata or {} - self.timestamp = swift.common.utils.Timestamp( - self.metadata.get('X-Timestamp', 0)) + self.timestamp = Timestamp( + self.metadata.get('X-Timestamp', Timestamp.zero())) class DiskFileExpired(DiskFileDeleted): diff --git a/swift/common/middleware/s3api/s3request.py b/swift/common/middleware/s3api/s3request.py index dafcd90ff2..9f3641f067 100644 --- a/swift/common/middleware/s3api/s3request.py +++ b/swift/common/middleware/s3api/s3request.py @@ -1322,7 +1322,7 @@ class S3Request(swob.Request): reason='invalid_date') # Anyways, request timestamp should be validated - epoch = S3Timestamp(0) + epoch = S3Timestamp.zero() if self.timestamp < epoch: raise AccessDenied(reason='invalid_date') diff --git a/swift/common/utils/timestamp.py b/swift/common/utils/timestamp.py index 60c942ed08..7067f4fe0d 100644 --- a/swift/common/utils/timestamp.py +++ b/swift/common/utils/timestamp.py @@ -124,8 +124,18 @@ class Timestamp(object): @classmethod def now(cls, offset=0, delta=0): + """ + Returns an instance of a Timestamp at the current time. + """ return cls(time.time(), offset=offset, delta=delta) + @classmethod + def zero(cls): + """ + Returns an instance of the smallest possible Timestamp. + """ + return cls(0) + def __repr__(self): return INTERNAL_FORMAT % (self.timestamp, self.offset) diff --git a/swift/container/backend.py b/swift/container/backend.py index 2971ba873a..94070ab46b 100644 --- a/swift/container/backend.py +++ b/swift/container/backend.py @@ -588,7 +588,7 @@ class ContainerBroker(DatabaseBroker): :param storage_policy_index: storage policy index """ if put_timestamp is None: - put_timestamp = Timestamp(0).internal + put_timestamp = Timestamp.zero().internal # The container_stat view is for compatibility; old versions of Swift # expected a container_stat table with columns "object_count" and # "bytes_used", but when that stuff became per-storage-policy and diff --git a/swift/container/reconciler.py b/swift/container/reconciler.py index 75a25d9e74..d61206a30c 100644 --- a/swift/container/reconciler.py +++ b/swift/container/reconciler.py @@ -77,7 +77,7 @@ def cmp_policy_info(info, remote_info): def has_been_recreated(info): return (info['put_timestamp'] > info['delete_timestamp'] > - Timestamp(0)) + Timestamp.zero()) remote_recreated = has_been_recreated(remote_info) recreated = has_been_recreated(info) @@ -112,7 +112,7 @@ def incorrect_policy_index(info, remote_info): def translate_container_headers_to_info(headers): - default_timestamp = Timestamp(0).internal + default_timestamp = Timestamp.zero().internal return { 'storage_policy_index': int(headers['X-Backend-Storage-Policy-Index']), 'put_timestamp': headers.get('x-backend-put-timestamp', @@ -538,7 +538,8 @@ class ContainerReconciler(Daemon): 'determine the destination timestamp, if any', path, q_ts) return False - dest_ts = Timestamp(dest_obj.get('x-backend-timestamp', 0)) + dest_ts = Timestamp( + dest_obj.get('x-backend-timestamp', Timestamp.zero())) if dest_ts >= q_ts: self.stats_log('found_object', '%r (%f) in policy_index %s ' 'is newer than queue (%f)', path, dest_ts, @@ -566,7 +567,8 @@ class ContainerReconciler(Daemon): source_obj_info = {} source_obj_iter = None - source_ts = Timestamp(source_obj_info.get('x-backend-timestamp', 0)) + source_ts = Timestamp( + source_obj_info.get('x-backend-timestamp', Timestamp.zero())) if source_obj_status == 404 and q_op == 'DELETE': return self.ensure_tombstone_in_right_location( q_policy_index, account, container, obj, q_ts, path, diff --git a/swift/container/server.py b/swift/container/server.py index 7ca28d8ad0..cd99b1ebf3 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -60,15 +60,17 @@ def gen_resp_headers(info, is_deleted=False): """ Convert container info dict to headers. """ + ts_zero = Timestamp.zero() # backend headers are always included headers = { - 'X-Backend-Timestamp': Timestamp(info.get('created_at', 0)).internal, - 'X-Backend-PUT-Timestamp': Timestamp(info.get( - 'put_timestamp', 0)).internal, + 'X-Backend-Timestamp': Timestamp( + info.get('created_at', ts_zero)).internal, + 'X-Backend-PUT-Timestamp': Timestamp( + info.get('put_timestamp', ts_zero)).internal, 'X-Backend-DELETE-Timestamp': Timestamp( - info.get('delete_timestamp', 0)).internal, + info.get('delete_timestamp', ts_zero)).internal, 'X-Backend-Status-Changed-At': Timestamp( - info.get('status_changed_at', 0)).internal, + info.get('status_changed_at', ts_zero)).internal, 'X-Backend-Storage-Policy-Index': info.get('storage_policy_index', 0), } if not is_deleted: @@ -76,9 +78,9 @@ def gen_resp_headers(info, is_deleted=False): headers.update({ 'X-Container-Object-Count': info.get('object_count', 0), 'X-Container-Bytes-Used': info.get('bytes_used', 0), - 'X-Timestamp': Timestamp(info.get('created_at', 0)).normal, + 'X-Timestamp': Timestamp(info.get('created_at', ts_zero)).normal, 'X-PUT-Timestamp': Timestamp( - info.get('put_timestamp', 0)).normal, + info.get('put_timestamp', ts_zero)).normal, 'X-Backend-Sharding-State': info.get('db_state', UNSHARDED), }) return headers diff --git a/swift/container/sync.py b/swift/container/sync.py index 28bda61240..ee9e62a130 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -493,7 +493,8 @@ class ContainerSync(Daemon): proxy=self.select_http_proxy(), logger=self.logger, retries=0) - remote_ts = Timestamp(metadata.get('x-timestamp', 0)) + remote_ts = Timestamp( + metadata.get('x-timestamp', Timestamp.zero())) self.logger.debug("remote obj timestamp %s local obj %s" % (timestamp.internal, remote_ts.internal)) if timestamp <= remote_ts: @@ -590,7 +591,8 @@ class ContainerSync(Daemon): row['name'])) return True - timestamp = Timestamp(headers.get('x-timestamp', 0)) + timestamp = Timestamp( + headers.get('x-timestamp', Timestamp.zero())) if timestamp < ts_meta: if exc: raise exc diff --git a/swift/obj/server.py b/swift/obj/server.py index 9aa027b936..b3d7b44071 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -689,7 +689,8 @@ class ObjectController(BaseStorageServer): return HTTPNotFound(request=request) except DiskFileStateChanged: return HTTPServiceUnavailable(request=request) - orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) + orig_timestamp = Timestamp( + orig_metadata.get('X-Timestamp', Timestamp.zero())) orig_ctype_timestamp = disk_file.content_type_timestamp req_ctype_time = '0' req_ctype = request.headers.get('Content-Type') @@ -847,7 +848,7 @@ class ObjectController(BaseStorageServer): except (DiskFileNotExist, DiskFileQuarantined, DiskFileStateChanged): orig_metadata = {} - orig_timestamp = Timestamp(0) + orig_timestamp = Timestamp.zero() # Checks for If-None-Match if request.if_none_match is not None and orig_metadata: if '*' in request.if_none_match: @@ -1309,7 +1310,8 @@ class ObjectController(BaseStorageServer): else: response_class = HTTPConflict response_timestamp = max(orig_timestamp, req_timestamp) - orig_delete_at = Timestamp(orig_metadata.get('X-Delete-At') or 0) + orig_delete_at = Timestamp( + orig_metadata.get('X-Delete-At') or Timestamp.zero()) try: req_if_delete_at_val = request.headers['x-if-delete-at'] req_if_delete_at = Timestamp(req_if_delete_at_val) diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 34f4688231..3279f01949 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -1176,7 +1176,8 @@ class GetterSource(object): return Timestamp(self.resp.getheader('x-backend-data-timestamp') or self.resp.getheader('x-backend-timestamp') or self.resp.getheader('x-put-timestamp') or - self.resp.getheader('x-timestamp') or 0) + self.resp.getheader('x-timestamp') or + Timestamp.zero()) @property def parts_iter(self): @@ -1412,7 +1413,7 @@ class GetOrHeadHandler(GetterBase): self.used_nodes = [] self.used_source_etag = None self.concurrency = concurrency - self.latest_404_timestamp = Timestamp(0) + self.latest_404_timestamp = Timestamp.zero() policy_options = self.app.get_policy_options(self.policy) self.rebalance_missing_suppression_count = min( policy_options.rebalance_missing_suppression_count, @@ -1601,7 +1602,8 @@ class GetOrHeadHandler(GetterBase): src_headers.get('x-backend-data-timestamp') or src_headers.get('x-backend-timestamp') or src_headers.get('x-put-timestamp') or - src_headers.get('x-timestamp') or 0) + src_headers.get('x-timestamp') or + Timestamp.zero()) if ps_timestamp >= self.latest_404_timestamp: self.statuses.append(possible_source.status) self.reasons.append(possible_source.reason) @@ -1615,14 +1617,16 @@ class GetOrHeadHandler(GetterBase): if 'handoff_index' in node and \ (is_server_error(possible_source.status) or possible_source.status == HTTP_NOT_FOUND) and \ - not Timestamp(src_headers.get('x-backend-timestamp', 0)): + not Timestamp(src_headers.get('x-backend-timestamp', + Timestamp.zero())): # throw out 5XX and 404s from handoff nodes unless the data is # really on disk and had been DELETEd return False if self.rebalance_missing_suppression_count > 0 and \ possible_source.status == HTTP_NOT_FOUND and \ - not Timestamp(src_headers.get('x-backend-timestamp', 0)): + not Timestamp(src_headers.get('x-backend-timestamp', + Timestamp.zero())): self.rebalance_missing_suppression_count -= 1 return False @@ -1638,7 +1642,8 @@ class GetOrHeadHandler(GetterBase): if self.server_type == 'Object' and \ possible_source.status == HTTP_NOT_FOUND: hdrs = HeaderKeyDict(possible_source.getheaders()) - ts = Timestamp(hdrs.get('X-Backend-Timestamp', 0)) + ts = Timestamp(hdrs.get('X-Backend-Timestamp', + Timestamp.zero())) if ts > self.latest_404_timestamp: self.latest_404_timestamp = ts self.app.check_response(node, self.server_type, possible_source, diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index f740064542..b34c17509c 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -2705,7 +2705,8 @@ class ECFragGetter(GetterBase): if 'handoff_index' in node and \ (is_server_error(possible_source.status) or possible_source.status == HTTP_NOT_FOUND) and \ - not Timestamp(src_headers.get('x-backend-timestamp', 0)): + not Timestamp( + src_headers.get('x-backend-timestamp', Timestamp.zero())): # throw out 5XX and 404s from handoff nodes unless the data is # really on disk and had been DELETEd self.logger.debug('Ignoring %s from handoff' % @@ -3010,7 +3011,9 @@ class ECObjectController(BaseObjectController): t_obj = bad_resp_headers.get( 'X-Backend-Timestamp', bad_resp_headers.get('X-Timestamp')) - bad_ts = Timestamp(t_data_file or t_obj or '0') + bad_ts = Timestamp(t_data_file or + t_obj or + Timestamp.zero()) if bad_ts <= best_bucket.timestamp: # We have reason to believe there's still good data # out there, it's just currently unavailable @@ -3018,7 +3021,8 @@ class ECObjectController(BaseObjectController): if getter.status: timestamp = Timestamp(getter.last_headers.get( 'X-Backend-Timestamp', - getter.last_headers.get('X-Timestamp', 0))) + getter.last_headers.get( + 'X-Timestamp', Timestamp.zero()))) if (rebalance_missing_suppression_count > 0 and getter.status == HTTP_NOT_FOUND and not timestamp): diff --git a/test/unit/common/middleware/test_container_sync.py b/test/unit/common/middleware/test_container_sync.py index 0418fcd724..eb01a7c11a 100644 --- a/test/unit/common/middleware/test_container_sync.py +++ b/test/unit/common/middleware/test_container_sync.py @@ -23,6 +23,7 @@ from unittest import mock from swift.common import swob from swift.common.middleware import container_sync +from swift.common.utils.timestamp import Timestamp from swift.proxy.controllers.base import get_cache_key from swift.proxy.controllers.info import InfoController @@ -255,6 +256,24 @@ cluster_dfw1 = http://dfw1.host/v1/ self.assertIn('swift.slo_override', req.environ) self.assertIn('swift.symlink_override', req.environ) + def test_valid_sig3(self): + ts = Timestamp.zero() + sig = self.sync.realms_conf.get_sig( + 'GET', '/v1/a/c', ts.internal, 'nonce', + self.sync.realms_conf.key2('US'), 'abc') + req = swob.Request.blank( + '/v1/a/c', headers={'x-container-sync-auth': 'US nonce ' + sig, + 'x-timestamp': ts.internal}) + infocache = req.environ.setdefault('swift.infocache', {}) + infocache[get_cache_key('a', 'c')] = {'sync_key': 'abc'} + resp = req.get_response(self.sync) + self.assertEqual(resp.status, '200 OK') + self.assertEqual(resp.body, b'Response to Authorized Request') + self.assertIn('cs:valid', req.environ.get('swift.log_info')) + self.assertIn('swift.authorize_override', req.environ) + self.assertIn('swift.slo_override', req.environ) + self.assertIn('swift.symlink_override', req.environ) + def test_info(self): req = swob.Request.blank('/info') resp = req.get_response(self.sync) diff --git a/test/unit/common/test_db.py b/test/unit/common/test_db.py index b88a25bf81..e591e733ce 100644 --- a/test/unit/common/test_db.py +++ b/test/unit/common/test_db.py @@ -353,6 +353,31 @@ class TestExampleBroker(TestDbBase): def setUp(self): super(TestExampleBroker, self).setUp() + def test_initialize(self): + broker = self.broker_class(self.db_path, account='a', container='c') + put_timestamp = next(self.ts) + created_at = next(self.ts) + with patch('swift.common.db.Timestamp.now', return_value=created_at): + broker.initialize(put_timestamp.internal) + info = broker.get_info() + self.assertEqual(info['created_at'], created_at.internal) + self.assertEqual(info['put_timestamp'], put_timestamp.internal) + self.assertEqual(info['delete_timestamp'], '0') + self.assertEqual(info['status_changed_at'], put_timestamp.internal) + self.assertFalse(broker.is_deleted()) + + def test_initialize_default_put_timestamp(self): + broker = self.broker_class(self.db_path, account='a', container='c') + created_at = next(self.ts) + with patch('swift.common.db.Timestamp.now', return_value=created_at): + broker.initialize() + info = broker.get_info() + self.assertEqual(info['created_at'], created_at.internal) + self.assertEqual(info['put_timestamp'], Timestamp.zero().internal) + self.assertEqual(info['delete_timestamp'], '0') + self.assertEqual(info['status_changed_at'], Timestamp.zero().internal) + self.assertFalse(broker.is_deleted()) + def test_delete_db(self): broker = self.broker_class(self.db_path, account='a', container='c') broker.initialize(next(self.ts).internal) diff --git a/test/unit/common/utils/test_timestamp.py b/test/unit/common/utils/test_timestamp.py index 91a7d3e998..1cbdb7673e 100644 --- a/test/unit/common/utils/test_timestamp.py +++ b/test/unit/common/utils/test_timestamp.py @@ -25,12 +25,25 @@ from swift.common.utils import timestamp class TestTimestamp(unittest.TestCase): """Tests for swift.common.utils.timestamp.Timestamp""" + def test_zero(self): + ts_zero = timestamp.Timestamp.zero() + self.assertEqual(0.0, float(ts_zero)) + self.assertEqual(0, ts_zero.offset) + self.assertEqual(timestamp.Timestamp.zero(), + timestamp.Timestamp(timestamp.Timestamp.zero())) + self.assertEqual(ts_zero.internal, '0000000000.00000') + # for now this is true... + self.assertEqual(timestamp.Timestamp(0), ts_zero) def test_invalid_input(self): with self.assertRaises(ValueError): timestamp.Timestamp(time.time(), offset=-1) with self.assertRaises(ValueError): timestamp.Timestamp('123.456_78_90') + with self.assertRaises(ValueError): + timestamp.Timestamp('') + with self.assertRaises(TypeError): + timestamp.Timestamp(None) def test_invalid_string_conversion(self): t = timestamp.Timestamp.now()