From c1dc2fa624d2abaea1c872a4e98bf7880c07c236 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Tue, 10 Jun 2014 22:17:47 -0700 Subject: [PATCH] Add two vector timestamps The normalized form of the X-Timestamp header looks like a float with a fixed width to ensure stable string sorting - normalized timestamps look like "1402464677.04188" To support overwrites of existing data without modifying the original timestamp but still maintain consistency a second internal offset vector is append to the normalized timestamp form which compares and sorts greater than the fixed width float format but less than a newer timestamp. The internalized format of timestamps looks like "1402464677.04188_0000000000000000" - the portion after the underscore is the offset and is a formatted hexadecimal integer. The internalized form is not exposed to clients in responses from Swift. Normal client operations will not create a timestamp with an offset. The Timestamp class in common.utils supports internalized and normalized formatting of timestamps and also comparison of timestamp values. When the offset value of a Timestamp is 0 - it's considered insignificant and need not be represented in the string format; to support backwards compatibility during a Swift upgrade the internalized and normalized form of a Timestamp with an insignificant offset are identical. When a timestamp includes an offset it will always be represented in the internalized form, but is still excluded from the normalized form. Timestamps with an equivalent timestamp portion (the float part) will compare and order by their offset. Timestamps with a greater timestamp portion will always compare and order greater than a Timestamp with a lesser timestamp regardless of it's offset. String comparison and ordering is guaranteed for the internalized string format, and is backwards compatible for normalized timestamps which do not include an offset. The reconciler currently uses a offset bump to ensure that objects can move to the wrong storage policy and be moved back. This use-case is valid because the content represented by the user-facing timestamp is not modified in way. Future consumers of the offset vector of timestamps should be mindful of HTTP semantics of If-Modified and take care to avoid deviation in the response from the object server without an accompanying change to the user facing timestamp. DocImpact Implements: blueprint storage-policies Change-Id: Id85c960b126ec919a481dc62469bf172b7fb8549 --- swift/account/backend.py | 6 +- swift/account/reaper.py | 12 +- swift/account/server.py | 35 +- swift/account/utils.py | 8 +- swift/cli/info.py | 23 +- swift/common/constraints.py | 18 +- swift/common/db.py | 19 +- swift/common/db_replicator.py | 14 +- swift/common/direct_client.py | 4 +- swift/common/exceptions.py | 8 +- swift/common/swob.py | 20 +- swift/common/utils.py | 130 +++++- swift/container/backend.py | 10 +- swift/container/reconciler.py | 53 +-- swift/container/replicator.py | 8 +- swift/container/server.py | 69 ++- swift/container/sync.py | 7 +- swift/container/updater.py | 4 +- swift/obj/diskfile.py | 12 +- swift/obj/mem_diskfile.py | 5 +- swift/obj/server.py | 57 ++- swift/proxy/controllers/base.py | 6 +- swift/proxy/controllers/container.py | 4 +- swift/proxy/controllers/obj.py | 26 +- test/probe/test_object_expirer.py | 11 +- test/unit/account/test_backend.py | 194 ++++----- test/unit/account/test_utils.py | 128 ++++++ test/unit/cli/test_info.py | 35 +- test/unit/common/test_constraints.py | 15 + test/unit/common/test_db.py | 51 ++- test/unit/common/test_direct_client.py | 14 +- test/unit/common/test_swob.py | 20 + test/unit/common/test_utils.py | 556 +++++++++++++++++++++++++ test/unit/container/test_backend.py | 366 ++++++++-------- test/unit/container/test_reconciler.py | 113 ++--- test/unit/container/test_replicator.py | 63 ++- test/unit/container/test_server.py | 385 +++++++++++------ test/unit/obj/test_diskfile.py | 120 +++--- test/unit/obj/test_server.py | 302 +++++++++++--- 39 files changed, 2035 insertions(+), 896 deletions(-) create mode 100644 test/unit/account/test_utils.py diff --git a/swift/account/backend.py b/swift/account/backend.py index 10cf18c134..595c4969ec 100644 --- a/swift/account/backend.py +++ b/swift/account/backend.py @@ -24,7 +24,7 @@ import errno import sqlite3 -from swift.common.utils import normalize_timestamp, lock_parent_directory +from swift.common.utils import Timestamp, lock_parent_directory from swift.common.db import DatabaseBroker, DatabaseConnectionError, \ PENDING_CAP, PICKLE_PROTOCOL, utf8encode @@ -155,7 +155,7 @@ class AccountBroker(DatabaseBroker): conn.execute(''' UPDATE account_stat SET account = ?, created_at = ?, id = ?, put_timestamp = ?, status_changed_at = ? - ''', (self.account, normalize_timestamp(time.time()), str(uuid4()), + ''', (self.account, Timestamp(time.time()).internal, str(uuid4()), put_timestamp, put_timestamp)) def create_policy_stat_table(self, conn): @@ -293,7 +293,7 @@ class AccountBroker(DatabaseBroker): """ return status == 'DELETED' or ( container_count in (None, '', 0, '0') and - float(delete_timestamp) > float(put_timestamp)) + Timestamp(delete_timestamp) > Timestamp(put_timestamp)) def _is_deleted(self, conn): """ diff --git a/swift/account/reaper.py b/swift/account/reaper.py index c83041182d..2f0bb0f8d7 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -18,7 +18,7 @@ import random from swift import gettext_ as _ from logging import DEBUG from math import sqrt -from time import time, ctime +from time import time from eventlet import GreenPool, sleep, Timeout @@ -29,7 +29,7 @@ from swift.common.direct_client import direct_delete_container, \ from swift.common.exceptions import ClientException from swift.common.ring import Ring from swift.common.utils import get_logger, whataremyips, ismount, \ - config_true_value + config_true_value, Timestamp from swift.common.daemon import Daemon from swift.common.storage_policy import POLICIES, POLICY_INDEX @@ -229,7 +229,8 @@ class AccountReaper(Daemon): """ begin = time() info = broker.get_info() - if time() - float(info['delete_timestamp']) <= self.delay_reaping: + if time() - float(Timestamp(info['delete_timestamp'])) <= \ + self.delay_reaping: return False account = info['account'] self.logger.info(_('Beginning pass on account %s'), account) @@ -281,10 +282,11 @@ class AccountReaper(Daemon): log += _(', elapsed: %.02fs') % (time() - begin) self.logger.info(log) self.logger.timing_since('timing', self.start_time) + delete_timestamp = Timestamp(info['delete_timestamp']) if self.stats_containers_remaining and \ - begin - float(info['delete_timestamp']) >= self.reap_not_done_after: + begin - float(delete_timestamp) >= self.reap_not_done_after: self.logger.warn(_('Account %s has not been reaped since %s') % - (account, ctime(float(info['delete_timestamp'])))) + (account, delete_timestamp.isoformat)) return True def reap_container(self, account, account_partition, account_nodes, diff --git a/swift/account/server.py b/swift/account/server.py index 6427b6b7dc..58dfd607ad 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -27,9 +27,9 @@ from swift.common.db import DatabaseConnectionError, DatabaseAlreadyExists from swift.common.request_helpers import get_param, get_listing_content_type, \ split_and_validate_path from swift.common.utils import get_logger, hash_path, public, \ - normalize_timestamp, storage_directory, config_true_value, \ + Timestamp, storage_directory, config_true_value, \ json, timing_stats, replication, get_log_line -from swift.common.constraints import check_mount, check_float, check_utf8 +from swift.common.constraints import check_mount, valid_timestamp, check_utf8 from swift.common import constraints from swift.common.db_replicator import ReplicatorRpc from swift.common.swob import HTTPAccepted, HTTPBadRequest, \ @@ -90,14 +90,11 @@ class AccountController(object): drive, part, account = split_and_validate_path(req, 3) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) - if 'x-timestamp' not in req.headers or \ - not check_float(req.headers['x-timestamp']): - return HTTPBadRequest(body='Missing timestamp', request=req, - content_type='text/plain') + req_timestamp = valid_timestamp(req) broker = self._get_account_broker(drive, part, account) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) - broker.delete_db(req.headers['x-timestamp']) + broker.delete_db(req_timestamp.internal) return self._deleted_response(broker, req, HTTPNoContent) @public @@ -108,6 +105,10 @@ class AccountController(object): if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) if container: # put account container + if 'x-timestamp' not in req.headers: + timestamp = Timestamp(time.time()) + else: + timestamp = valid_timestamp(req) pending_timeout = None container_policy_index = req.headers.get(POLICY_INDEX, 0) if 'x-trans-id' in req.headers: @@ -117,8 +118,7 @@ class AccountController(object): if account.startswith(self.auto_create_account_prefix) and \ not os.path.exists(broker.db_file): try: - broker.initialize(normalize_timestamp( - req.headers.get('x-timestamp') or time.time())) + broker.initialize(timestamp.internal) except DatabaseAlreadyExists: pass if req.headers.get('x-account-override-deleted', 'no').lower() != \ @@ -135,11 +135,11 @@ class AccountController(object): else: return HTTPCreated(request=req) else: # put account + timestamp = valid_timestamp(req) broker = self._get_account_broker(drive, part, account) - timestamp = normalize_timestamp(req.headers['x-timestamp']) if not os.path.exists(broker.db_file): try: - broker.initialize(timestamp) + broker.initialize(timestamp.internal) created = True except DatabaseAlreadyExists: created = False @@ -148,11 +148,11 @@ class AccountController(object): body='Recently deleted') else: created = broker.is_deleted() - broker.update_put_timestamp(timestamp) + broker.update_put_timestamp(timestamp.internal) if broker.is_deleted(): return HTTPConflict(request=req) metadata = {} - metadata.update((key, (value, timestamp)) + metadata.update((key, (value, timestamp.internal)) for key, value in req.headers.iteritems() if is_sys_or_user_meta('account', key)) if metadata: @@ -238,19 +238,14 @@ class AccountController(object): def POST(self, req): """Handle HTTP POST request.""" drive, part, account = split_and_validate_path(req, 3) - if 'x-timestamp' not in req.headers or \ - not check_float(req.headers['x-timestamp']): - return HTTPBadRequest(body='Missing or bad timestamp', - request=req, - content_type='text/plain') + req_timestamp = valid_timestamp(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) - timestamp = normalize_timestamp(req.headers['x-timestamp']) metadata = {} - metadata.update((key, (value, timestamp)) + metadata.update((key, (value, req_timestamp.internal)) for key, value in req.headers.iteritems() if is_sys_or_user_meta('account', key)) if metadata: diff --git a/swift/account/utils.py b/swift/account/utils.py index 13dec505cc..6cc8700961 100644 --- a/swift/account/utils.py +++ b/swift/account/utils.py @@ -17,7 +17,7 @@ import time from xml.sax import saxutils from swift.common.swob import HTTPOk, HTTPNoContent -from swift.common.utils import json, normalize_timestamp +from swift.common.utils import json, Timestamp from swift.common.storage_policy import POLICIES @@ -27,7 +27,7 @@ class FakeAccountBroker(object): like an account broker would for a real, empty account with no metadata. """ def get_info(self): - now = normalize_timestamp(time.time()) + now = Timestamp(time.time()).internal return {'container_count': 0, 'object_count': 0, 'bytes_used': 0, @@ -51,8 +51,8 @@ def get_response_headers(broker): 'X-Account-Container-Count': info['container_count'], 'X-Account-Object-Count': info['object_count'], 'X-Account-Bytes-Used': info['bytes_used'], - 'X-Timestamp': info['created_at'], - 'X-PUT-Timestamp': info['put_timestamp']} + 'X-Timestamp': Timestamp(info['created_at']).normal, + 'X-PUT-Timestamp': Timestamp(info['put_timestamp']).normal} policy_stats = broker.get_policy_stats() for policy_idx, stats in policy_stats.items(): policy = POLICIES.get_by_index(policy_idx) diff --git a/swift/cli/info.py b/swift/cli/info.py index fd0fa53206..7d1557c08e 100644 --- a/swift/cli/info.py +++ b/swift/cli/info.py @@ -14,10 +14,10 @@ import itertools import os import sqlite3 import urllib -from datetime import datetime from hashlib import md5 -from swift.common.utils import hash_path, storage_directory +from swift.common.utils import hash_path, storage_directory, \ + Timestamp from swift.common.ring import Ring from swift.common.request_helpers import is_sys_meta, is_user_meta, \ strip_sys_meta_prefix, strip_user_meta_prefix @@ -174,16 +174,16 @@ def print_db_info_metadata(db_type, info, metadata): print 'Metadata:' print (' Created at: %s (%s)' % - (datetime.utcfromtimestamp(float(info['created_at'])), + (Timestamp(info['created_at']).isoformat, info['created_at'])) print (' Put Timestamp: %s (%s)' % - (datetime.utcfromtimestamp(float(info['put_timestamp'])), + (Timestamp(info['put_timestamp']).isoformat, info['put_timestamp'])) print (' Delete Timestamp: %s (%s)' % - (datetime.utcfromtimestamp(float(info['delete_timestamp'])), + (Timestamp(info['delete_timestamp']).isoformat, info['delete_timestamp'])) print (' Status Timestamp: %s (%s)' % - (datetime.utcfromtimestamp(float(info['status_changed_at'])), + (Timestamp(info['status_changed_at']).isoformat, info['status_changed_at'])) if db_type == 'account': print ' Container Count: %s' % info['container_count'] @@ -197,12 +197,10 @@ def print_db_info_metadata(db_type, info, metadata): print (' Storage Policy: %s (%s)' % ( policy_name, info['storage_policy_index'])) print (' Reported Put Timestamp: %s (%s)' % - (datetime.utcfromtimestamp( - float(info['reported_put_timestamp'])), + (Timestamp(info['reported_put_timestamp']).isoformat, info['reported_put_timestamp'])) print (' Reported Delete Timestamp: %s (%s)' % - (datetime.utcfromtimestamp - (float(info['reported_delete_timestamp'])), + (Timestamp(info['reported_delete_timestamp']).isoformat, info['reported_delete_timestamp'])) print ' Reported Object Count: %s' % info['reported_object_count'] print ' Reported Bytes Used: %s' % info['reported_bytes_used'] @@ -255,7 +253,7 @@ def print_obj_metadata(metadata): raise ValueError('Metadata is None') path = metadata.pop('name', '') content_type = metadata.pop('Content-Type', '') - ts = metadata.pop('X-Timestamp', 0) + ts = Timestamp(metadata.pop('X-Timestamp', 0)) account = container = obj = obj_hash = None if path: try: @@ -276,8 +274,7 @@ def print_obj_metadata(metadata): else: print 'Content-Type: Not found in metadata' if ts: - print ('Timestamp: %s (%s)' % - (datetime.utcfromtimestamp(float(ts)), ts)) + print ('Timestamp: %s (%s)' % (ts.isoformat, ts.internal)) else: print 'Timestamp: Not found in metadata' diff --git a/swift/common/constraints.py b/swift/common/constraints.py index d3cbea0863..24dcbf034c 100644 --- a/swift/common/constraints.py +++ b/swift/common/constraints.py @@ -18,7 +18,7 @@ import urllib from urllib import unquote from ConfigParser import ConfigParser, NoSectionError, NoOptionError -from swift.common import utils +from swift.common import utils, exceptions from swift.common.swob import HTTPBadRequest, HTTPLengthRequired, \ HTTPRequestEntityTooLarge, HTTPPreconditionFailed @@ -209,6 +209,22 @@ def check_float(string): return False +def valid_timestamp(request): + """ + Helper function to extract a timestamp from requests that require one. + + :param request: the swob request object + + :returns: a valid Timestamp instance + :raises: HTTPBadRequest on missing or invalid X-Timestamp + """ + try: + return request.timestamp + except exceptions.InvalidTimestamp as e: + raise HTTPBadRequest(body=str(e), request=request, + content_type='text/plain') + + def check_utf8(string): """ Validate if a string is valid UTF-8 str or unicode and that it diff --git a/swift/common/db.py b/swift/common/db.py index 10fcc1be9f..9e654e5058 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -29,7 +29,7 @@ from tempfile import mkstemp from eventlet import sleep, Timeout import sqlite3 -from swift.common.utils import json, normalize_timestamp, renamer, \ +from swift.common.utils import json, Timestamp, renamer, \ mkdirs, lock_parent_directory, fallocate from swift.common.exceptions import LockTimeout @@ -144,7 +144,7 @@ def chexor(old, name, timestamp): :param old: hex representation of the current DB hash :param name: name of the object or container being inserted - :param timestamp: timestamp of the new record + :param timestamp: internalized timestamp of the new record :returns: a hex representation of the new hash value """ if name is None: @@ -222,7 +222,7 @@ class DatabaseBroker(object): The storage_policy_index is passed through to the subclass's ``_initialize`` method. It is ignored by ``AccountBroker``. - :param put_timestamp: timestamp of initial PUT request + :param put_timestamp: internalized timestamp of initial PUT request :param storage_policy_index: only required for containers """ if self.db_file == ':memory:': @@ -280,7 +280,7 @@ class DatabaseBroker(object): END; """) if not put_timestamp: - put_timestamp = normalize_timestamp(0) + put_timestamp = Timestamp(0).internal self._initialize(conn, put_timestamp, storage_policy_index=storage_policy_index) conn.commit() @@ -302,9 +302,8 @@ class DatabaseBroker(object): """ Mark the DB as deleted - :param timestamp: delete timestamp + :param timestamp: internalized delete timestamp """ - timestamp = normalize_timestamp(timestamp) # first, clear the metadata cleared_meta = {} for k in self.metadata: @@ -463,8 +462,8 @@ class DatabaseBroker(object): delete_timestamp=MAX(?, delete_timestamp) ''' % self.db_type, (created_at, put_timestamp, delete_timestamp)) if old_status != self._is_deleted(conn): - timestamp = normalize_timestamp(time.time()) - self._update_status_changed_at(conn, timestamp) + timestamp = Timestamp(time.time()) + self._update_status_changed_at(conn, timestamp.internal) conn.commit() @@ -790,7 +789,7 @@ class DatabaseBroker(object): Update the put_timestamp. Only modifies it if it is greater than the current timestamp. - :param timestamp: put timestamp + :param timestamp: internalized put timestamp """ with self.get() as conn: conn.execute( @@ -804,6 +803,8 @@ class DatabaseBroker(object): Update the status_changed_at field in the stat table. Only modifies status_changed_at if the timestamp is greater than the current status_changed_at timestamp. + + :param timestamp: internalized timestamp """ with self.get() as conn: self._update_status_changed_at(conn, timestamp) diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index 952130ae8e..c03a13790a 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -31,7 +31,7 @@ import swift.common.db from swift.common.direct_client import quote from swift.common.utils import get_logger, whataremyips, storage_directory, \ renamer, mkdirs, lock_parent_directory, config_true_value, \ - unlink_older_than, dump_recon_cache, rsync_ip, ismount, json + unlink_older_than, dump_recon_cache, rsync_ip, ismount, json, Timestamp from swift.common import ring from swift.common.http import HTTP_NOT_FOUND, HTTP_INSUFFICIENT_STORAGE from swift.common.bufferedhttp import BufferedHTTPConnection @@ -458,16 +458,8 @@ class Replicator(Daemon): return # The db is considered deleted if the delete_timestamp value is greater # than the put_timestamp, and there are no objects. - delete_timestamp = 0 - try: - delete_timestamp = float(info['delete_timestamp']) - except ValueError: - pass - put_timestamp = 0 - try: - put_timestamp = float(info['put_timestamp']) - except ValueError: - pass + delete_timestamp = Timestamp(info.get('delete_timestamp') or 0) + put_timestamp = Timestamp(info.get('put_timestamp') or 0) if delete_timestamp < (now - self.reclaim_age) and \ delete_timestamp > put_timestamp and \ info['count'] in (None, '', 0, '0'): diff --git a/swift/common/direct_client.py b/swift/common/direct_client.py index 6a6efdea59..3fe2c236ae 100644 --- a/swift/common/direct_client.py +++ b/swift/common/direct_client.py @@ -27,7 +27,7 @@ from eventlet import sleep, Timeout from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ClientException -from swift.common.utils import normalize_timestamp, FileLikeIter +from swift.common.utils import Timestamp, FileLikeIter from swift.common.http import HTTP_NO_CONTENT, HTTP_INSUFFICIENT_STORAGE, \ is_success, is_server_error from swift.common.swob import HeaderKeyDict @@ -91,7 +91,7 @@ def _get_direct_account_container(path, stype, node, part, def gen_headers(hdrs_in=None, add_ts=False): hdrs_out = HeaderKeyDict(hdrs_in) if hdrs_in else HeaderKeyDict() if add_ts: - hdrs_out['X-Timestamp'] = normalize_timestamp(time()) + hdrs_out['X-Timestamp'] = Timestamp(time()).internal hdrs_out['User-Agent'] = 'direct-client %s' % os.getpid() return hdrs_out diff --git a/swift/common/exceptions.py b/swift/common/exceptions.py index a3e6cba0f7..e7999bab97 100644 --- a/swift/common/exceptions.py +++ b/swift/common/exceptions.py @@ -14,6 +14,7 @@ # limitations under the License. from eventlet import Timeout +import swift.common.utils class MessageTimeout(Timeout): @@ -30,6 +31,10 @@ class SwiftException(Exception): pass +class InvalidTimestamp(SwiftException): + pass + + class DiskFileError(SwiftException): pass @@ -54,7 +59,8 @@ class DiskFileDeleted(DiskFileNotExist): def __init__(self, metadata=None): self.metadata = metadata or {} - self.timestamp = self.metadata.get('X-Timestamp', 0) + self.timestamp = swift.common.utils.Timestamp( + self.metadata.get('X-Timestamp', 0)) class DiskFileExpired(DiskFileDeleted): diff --git a/swift/common/swob.py b/swift/common/swob.py index 69f4f2f694..c83856f953 100644 --- a/swift/common/swob.py +++ b/swift/common/swob.py @@ -49,7 +49,8 @@ import random import functools import inspect -from swift.common.utils import reiterate, split_path +from swift.common.utils import reiterate, split_path, Timestamp +from swift.common.exceptions import InvalidTimestamp RESPONSE_REASONS = { @@ -762,6 +763,7 @@ class Request(object): body = _req_body_property() charset = None _params_cache = None + _timestamp = None acl = _req_environ_property('swob.ACL') def __init__(self, environ): @@ -843,6 +845,22 @@ class Request(object): return self._params_cache str_params = params + @property + def timestamp(self): + """ + Provides HTTP_X_TIMESTAMP as a :class:`~swift.common.utils.Timestamp` + """ + if self._timestamp is None: + try: + raw_timestamp = self.environ['HTTP_X_TIMESTAMP'] + except KeyError: + raise InvalidTimestamp('Missing X-Timestamp header') + try: + self._timestamp = Timestamp(raw_timestamp) + except ValueError: + raise InvalidTimestamp('Invalid X-Timestamp header') + return self._timestamp + @property def path_qs(self): """The path of the request, without host but with query string.""" diff --git a/swift/common/utils.py b/swift/common/utils.py index da68cb1303..a5badaab9b 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -63,7 +63,7 @@ utf8_decoder = codecs.getdecoder('utf-8') utf8_encoder = codecs.getencoder('utf-8') from swift import gettext_ as _ -from swift.common.exceptions import LockTimeout, MessageTimeout +import swift.common.exceptions from swift.common.http import is_success, is_redirection, HTTP_NOT_FOUND # logging doesn't import patched as cleanly as one would like @@ -562,6 +562,120 @@ def drop_buffer_cache(fd, offset, length): 'length': length, 'ret': ret}) +NORMAL_FORMAT = "%016.05f" +INTERNAL_FORMAT = NORMAL_FORMAT + '_%016x' +# Setting this to True will cause the internal format to always display +# extended digits - even when the value is equivalent to the normalized form. +# This isn't ideal during an upgrade when some servers might not understand +# the new time format - but flipping it to True works great for testing. +FORCE_INTERNAL = False # or True + + +class Timestamp(object): + """ + Internal Representation of Swift Time. + + The normalized form of the X-Timestamp header looks like a float + with a fixed width to ensure stable string sorting - normalized + timestamps look like "1402464677.04188" + + To support overwrites of existing data without modifying the original + timestamp but still maintain consistency a second internal offset vector + is append to the normalized timestamp form which compares and sorts + greater than the fixed width float format but less than a newer timestamp. + The internalized format of timestamps looks like + "1402464677.04188_0000000000000000" - the portion after the underscore is + the offset and is a formatted hexadecimal integer. + + The internalized form is not exposed to clients in responses from + Swift. Normal client operations will not create a timestamp with an + offset. + + The Timestamp class in common.utils supports internalized and + normalized formatting of timestamps and also comparison of timestamp + values. When the offset value of a Timestamp is 0 - it's considered + insignificant and need not be represented in the string format; to + support backwards compatibility during a Swift upgrade the + internalized and normalized form of a Timestamp with an + insignificant offset are identical. When a timestamp includes an + offset it will always be represented in the internalized form, but + is still excluded from the normalized form. Timestamps with an + equivalent timestamp portion (the float part) will compare and order + by their offset. Timestamps with a greater timestamp portion will + always compare and order greater than a Timestamp with a lesser + timestamp regardless of it's offset. String comparison and ordering + is guaranteed for the internalized string format, and is backwards + compatible for normalized timestamps which do not include an offset. + """ + + def __init__(self, timestamp, offset=0): + if isinstance(timestamp, basestring): + parts = timestamp.split('_', 1) + self.timestamp = float(parts.pop(0)) + if parts: + self.offset = int(parts[0], 16) + else: + self.offset = 0 + else: + self.timestamp = float(timestamp) + self.offset = getattr(timestamp, 'offset', 0) + # increment offset + if offset >= 0: + self.offset += offset + else: + raise ValueError('offset must be non-negative') + + def __repr__(self): + return INTERNAL_FORMAT % (self.timestamp, self.offset) + + def __str__(self): + raise TypeError('You must specificy which string format is required') + + def __float__(self): + return self.timestamp + + def __int__(self): + return int(self.timestamp) + + def __nonzero__(self): + return bool(self.timestamp or self.offset) + + @property + def normal(self): + return NORMAL_FORMAT % self.timestamp + + @property + def internal(self): + if self.offset or FORCE_INTERNAL: + return INTERNAL_FORMAT % (self.timestamp, self.offset) + else: + return self.normal + + @property + def isoformat(self): + isoformat = datetime.datetime.utcfromtimestamp( + float(self.normal)).isoformat() + # python isoformat() doesn't include msecs when zero + if len(isoformat) < len("1970-01-01T00:00:00.000000"): + isoformat += ".000000" + return isoformat + + def __eq__(self, other): + if not isinstance(other, Timestamp): + other = Timestamp(other) + return self.internal == other.internal + + def __ne__(self, other): + if not isinstance(other, Timestamp): + other = Timestamp(other) + return self.internal != other.internal + + def __cmp__(self, other): + if not isinstance(other, Timestamp): + other = Timestamp(other) + return cmp(self.internal, other.internal) + + def normalize_timestamp(timestamp): """ Format a timestamp (string or numeric) into a standardized @@ -574,15 +688,15 @@ def normalize_timestamp(timestamp): :param timestamp: unix timestamp :returns: normalized timestamp as a string """ - return "%016.05f" % (float(timestamp)) + return Timestamp(timestamp).normal def last_modified_date_to_timestamp(last_modified_date_str): """ - Convert a last modified date (liked you'd get from a container listing, + Convert a last modified date (like you'd get from a container listing, e.g. 2014-02-28T23:22:36.698390) to a float. """ - return float( + return Timestamp( datetime.datetime.strptime( last_modified_date_str, '%Y-%m-%dT%H:%M:%S.%f' ).strftime('%s.%f') @@ -1007,7 +1121,7 @@ class LogAdapter(logging.LoggerAdapter, object): emsg = exc.__class__.__name__ if hasattr(exc, 'seconds'): emsg += ' (%ss)' % exc.seconds - if isinstance(exc, MessageTimeout): + if isinstance(exc, swift.common.exceptions.MessageTimeout): if exc.msg: emsg += ' %s' % exc.msg else: @@ -1441,7 +1555,7 @@ def hash_path(account, container=None, object=None, raw_digest=False): @contextmanager -def lock_path(directory, timeout=10, timeout_class=LockTimeout): +def lock_path(directory, timeout=10, timeout_class=None): """ Context manager that acquires a lock on a directory. This will block until the lock can be acquired, or the timeout time has expired (whichever occurs @@ -1458,6 +1572,8 @@ def lock_path(directory, timeout=10, timeout_class=LockTimeout): constructed as timeout_class(timeout, lockpath). Default: LockTimeout """ + if timeout_class is None: + timeout_class = swift.common.exceptions.LockTimeout mkdirs(directory) lockpath = '%s/.lock' % directory fd = os.open(lockpath, os.O_WRONLY | os.O_CREAT) @@ -1497,7 +1613,7 @@ def lock_file(filename, timeout=10, append=False, unlink=True): fd = os.open(filename, flags) file_obj = os.fdopen(fd, mode) try: - with LockTimeout(timeout, filename): + with swift.common.exceptions.LockTimeout(timeout, filename): while True: try: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) diff --git a/swift/container/backend.py b/swift/container/backend.py index 6a2f92bebd..ff5129f142 100644 --- a/swift/container/backend.py +++ b/swift/container/backend.py @@ -24,7 +24,7 @@ import errno import sqlite3 -from swift.common.utils import normalize_timestamp, lock_parent_directory +from swift.common.utils import Timestamp, lock_parent_directory from swift.common.db import DatabaseBroker, DatabaseConnectionError, \ PENDING_CAP, PICKLE_PROTOCOL, utf8encode @@ -202,7 +202,7 @@ class ContainerBroker(DatabaseBroker): :param storage_policy_index: storage policy index """ if put_timestamp is None: - put_timestamp = normalize_timestamp(0) + put_timestamp = Timestamp(0).internal # The container_stat view is for compatibility; old versions of Swift # expected a container_stat table with columns "object_count" and # "bytes_used", but when that stuff became per-storage-policy and @@ -224,7 +224,7 @@ class ContainerBroker(DatabaseBroker): INSERT INTO container_info (account, container, created_at, id, put_timestamp, status_changed_at, storage_policy_index) VALUES (?, ?, ?, ?, ?, ?, ?); - """, (self.account, self.container, normalize_timestamp(time.time()), + """, (self.account, self.container, Timestamp(time.time()).internal, str(uuid4()), put_timestamp, put_timestamp, storage_policy_index)) @@ -372,7 +372,7 @@ class ContainerBroker(DatabaseBroker): # value is greater than the put_timestamp, and there are no # objects in the container. return (object_count in (None, '', 0, '0')) and ( - float(delete_timestamp) > float(put_timestamp)) + Timestamp(delete_timestamp) > Timestamp(put_timestamp)) def _is_deleted(self, conn): """ @@ -524,7 +524,7 @@ class ContainerBroker(DatabaseBroker): Update the container_stat policy_index and status_changed_at. """ if timestamp is None: - timestamp = normalize_timestamp(time.time()) + timestamp = Timestamp(time.time()).internal def _setit(conn): conn.execute(''' diff --git a/swift/container/reconciler.py b/swift/container/reconciler.py index 6231a83d35..7bde84147b 100644 --- a/swift/container/reconciler.py +++ b/swift/container/reconciler.py @@ -27,7 +27,7 @@ from swift.common.direct_client import ( from swift.common.internal_client import InternalClient, UnexpectedResponse from swift.common.storage_policy import POLICY_INDEX from swift.common.utils import get_logger, split_path, quorum_size, \ - FileLikeIter, normalize_timestamp, last_modified_date_to_timestamp, \ + FileLikeIter, Timestamp, last_modified_date_to_timestamp, \ LRUCache @@ -68,7 +68,7 @@ def cmp_policy_info(info, remote_info): def has_been_recreated(info): return (info['put_timestamp'] > info['delete_timestamp'] > - normalize_timestamp(0)) + Timestamp(0)) remote_recreated = has_been_recreated(remote_info) recreated = has_been_recreated(info) @@ -98,7 +98,7 @@ def incorrect_policy_index(info, remote_info): def translate_container_headers_to_info(headers): - default_timestamp = normalize_timestamp(0) + default_timestamp = Timestamp(0).internal return { 'storage_policy_index': int(headers[POLICY_INDEX]), 'put_timestamp': headers.get('x-backend-put-timestamp', @@ -117,7 +117,7 @@ def best_policy_index(headers): def get_reconciler_container_name(obj_timestamp): - return str(int(float(obj_timestamp)) // + return str(int(Timestamp(obj_timestamp)) // MISPLACED_OBJECTS_CONTAINER_DIVISOR * MISPLACED_OBJECTS_CONTAINER_DIVISOR) @@ -195,7 +195,7 @@ def add_to_reconciler_queue(container_ring, account, container, obj, # already been popped from the queue to be reprocessed, but # could potentially prevent out of order updates from making it # into the queue - x_timestamp = normalize_timestamp(time.time()) + x_timestamp = Timestamp(time.time()).internal else: x_timestamp = obj_timestamp q_op_type = get_reconciler_content_type(op) @@ -230,10 +230,7 @@ def add_to_reconciler_queue(container_ring, account, container, obj, def slightly_later_timestamp(ts, offset=1): - # I'm guessing to avoid rounding errors Swift uses a 10-microsecond - # resolution instead of Python's 1-microsecond resolution. - offset *= 0.00001 - return normalize_timestamp(float(ts) + offset) + return Timestamp(ts, offset=offset).internal def parse_raw_obj(obj_info): @@ -266,7 +263,7 @@ def parse_raw_obj(obj_info): 'container': container, 'obj': obj, 'q_op': q_op, - 'q_ts': float(obj_info['hash']), + 'q_ts': Timestamp(obj_info['hash']), 'q_record': last_modified_date_to_timestamp( obj_info['last_modified']), 'path': '/%s/%s/%s' % (account, container, obj) @@ -407,7 +404,7 @@ class ContainerReconciler(Daemon): success = False try: self.swift.delete_object(account, container, obj, - acceptable_statuses=(2, 4), + acceptable_statuses=(2, 404), headers=headers) except UnexpectedResponse as err: self.stats_log('cleanup_failed', '%r (%f) was not cleaned up ' @@ -430,7 +427,7 @@ class ContainerReconciler(Daemon): :param obj: the object name :param q_policy_index: the policy index of the source indicated by the queue entry. - :param q_ts: a float, the timestamp of the misplaced object + :param q_ts: the timestamp of the misplaced object :param q_op: the operation of the misplaced request :param path: the full path of the misplaced object for logging @@ -459,12 +456,7 @@ class ContainerReconciler(Daemon): dest_obj = self.swift.get_object_metadata(account, container, obj, headers=headers, acceptable_statuses=(2, 4)) - dest_ts = float( - dest_obj.get('x-timestamp', - dest_obj.get('x-backend-timestamp', - '0.0') - ) - ) + dest_ts = Timestamp(dest_obj.get('x-backend-timestamp', 0)) if dest_ts >= q_ts: self.stats_log('found_object', '%r (%f) in policy_index %s ' 'is newer than queue (%f)', path, dest_ts, @@ -492,7 +484,7 @@ class ContainerReconciler(Daemon): source_obj_info = {} source_obj_iter = None - source_ts = float(source_obj_info.get("X-Timestamp", 0)) + source_ts = Timestamp(source_obj_info.get('x-backend-timestamp', 0)) if source_obj_status == 404 and q_op == 'DELETE': return self.ensure_tombstone_in_right_location( q_policy_index, account, container, obj, q_ts, path, @@ -516,10 +508,10 @@ class ContainerReconciler(Daemon): :param account: the account name of the misplaced object :param container: the container name of the misplaced object :param obj: the name of the misplaced object - :param q_ts: a float, the timestamp of the misplaced object + :param q_ts: the timestamp of the misplaced object :param path: the full path of the misplaced object for logging :param container_policy_index: the policy_index of the destination - :param source_ts: a float, the timestamp of the source object + :param source_ts: the timestamp of the source object :param source_obj_status: the HTTP status source object request :param source_obj_info: the HTTP headers of the source object request :param source_obj_iter: the body iter of the source object request @@ -527,21 +519,22 @@ class ContainerReconciler(Daemon): if source_obj_status // 100 != 2 or source_ts < q_ts: if q_ts < time.time() - self.reclaim_age: # it's old and there are no tombstones or anything; give up - self.stats_log('lost_source', '%r (%f) was not available in ' - 'policy_index %s and has expired', path, q_ts, - q_policy_index, level=logging.CRITICAL) + self.stats_log('lost_source', '%r (%s) was not available in ' + 'policy_index %s and has expired', path, + q_ts.internal, q_policy_index, + level=logging.CRITICAL) return True # the source object is unavailable or older than the queue # entry; a version that will satisfy the queue entry hopefully # exists somewhere in the cluster, so wait and try again - self.stats_log('unavailable_source', '%r (%f) in ' - 'policy_index %s responded %s (%f)', path, - q_ts, q_policy_index, source_obj_status, - source_ts, level=logging.WARNING) + self.stats_log('unavailable_source', '%r (%s) in ' + 'policy_index %s responded %s (%s)', path, + q_ts.internal, q_policy_index, source_obj_status, + source_ts.internal, level=logging.WARNING) return False # optimistically move any source with a timestamp >= q_ts - ts = max(float(source_ts), q_ts) + ts = max(Timestamp(source_ts), q_ts) # move the object put_timestamp = slightly_later_timestamp(ts, offset=2) self.stats_log('copy_attempt', '%r (%f) in policy_index %s will be ' @@ -638,7 +631,7 @@ class ContainerReconciler(Daemon): success = self._reconcile_object(**info) except: # noqa self.logger.exception('Unhandled Exception trying to ' - 'reconcile %r (%s) in policy_index %s', + 'reconcile %r (%f) in policy_index %s', info['path'], info['q_ts'], info['q_policy_index']) if success: diff --git a/swift/container/replicator.py b/swift/container/replicator.py index d60b53c305..aed6042637 100644 --- a/swift/container/replicator.py +++ b/swift/container/replicator.py @@ -28,7 +28,7 @@ from swift.common.storage_policy import POLICIES from swift.common.exceptions import DeviceUnavailable from swift.common.http import is_success from swift.common.db import DatabaseAlreadyExists -from swift.common.utils import (json, normalize_timestamp, hash_path, +from swift.common.utils import (json, Timestamp, hash_path, storage_directory, quorum_size) @@ -59,10 +59,10 @@ class ContainerReplicator(db_replicator.Replicator): if is_success(response.status): remote_info = json.loads(response.data) if incorrect_policy_index(info, remote_info): - status_changed_at = normalize_timestamp(time.time()) + status_changed_at = Timestamp(time.time()) broker.set_storage_policy_index( remote_info['storage_policy_index'], - timestamp=status_changed_at) + timestamp=status_changed_at.internal) broker.merge_timestamps(*(remote_info[key] for key in ( 'created_at', 'put_timestamp', 'delete_timestamp'))) rv = parent._handle_sync_response( @@ -256,7 +256,7 @@ class ContainerReplicatorRpc(db_replicator.ReplicatorRpc): """ info = broker.get_replication_info() if incorrect_policy_index(info, remote_info): - status_changed_at = normalize_timestamp(time.time()) + status_changed_at = Timestamp(time.time()).internal broker.set_storage_policy_index( remote_info['storage_policy_index'], timestamp=status_changed_at) diff --git a/swift/container/server.py b/swift/container/server.py index e028ddf9ef..74c0bb2821 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -16,7 +16,6 @@ import os import time import traceback -from datetime import datetime from swift import gettext_ as _ from xml.etree.cElementTree import Element, SubElement, tostring @@ -30,10 +29,10 @@ from swift.common.container_sync_realms import ContainerSyncRealms from swift.common.request_helpers import get_param, get_listing_content_type, \ split_and_validate_path, is_sys_or_user_meta from swift.common.utils import get_logger, hash_path, public, \ - normalize_timestamp, storage_directory, validate_sync_to, \ + Timestamp, storage_directory, validate_sync_to, \ config_true_value, json, timing_stats, replication, \ override_bytes_from_content_type, get_log_line -from swift.common.constraints import check_mount, check_float, check_utf8 +from swift.common.constraints import check_mount, valid_timestamp, check_utf8 from swift.common import constraints from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ConnectionTimeout @@ -51,13 +50,13 @@ def gen_resp_headers(info, is_deleted=False): """ # backend headers are always included headers = { - 'X-Backend-Timestamp': normalize_timestamp(info.get('created_at', 0)), - 'X-Backend-PUT-Timestamp': normalize_timestamp( - info.get('put_timestamp', 0)), - 'X-Backend-DELETE-Timestamp': normalize_timestamp( - info.get('delete_timestamp', 0)), - 'X-Backend-Status-Changed-At': normalize_timestamp( - info.get('status_changed_at', 0)), + 'X-Backend-Timestamp': Timestamp(info.get('created_at', 0)).internal, + 'X-Backend-PUT-Timestamp': Timestamp(info.get( + 'put_timestamp', 0)).internal, + 'X-Backend-DELETE-Timestamp': Timestamp( + info.get('delete_timestamp', 0)).internal, + 'X-Backend-Status-Changed-At': Timestamp( + info.get('status_changed_at', 0)).internal, POLICY_INDEX: info.get('storage_policy_index', 0), } if not is_deleted: @@ -65,9 +64,9 @@ def gen_resp_headers(info, is_deleted=False): headers.update({ 'X-Container-Object-Count': info.get('object_count', 0), 'X-Container-Bytes-Used': info.get('bytes_used', 0), - 'X-Timestamp': normalize_timestamp(info.get('created_at', 0)), - 'X-PUT-Timestamp': normalize_timestamp( - info.get('put_timestamp', 0)), + 'X-Timestamp': Timestamp(info.get('created_at', 0)).normal, + 'X-PUT-Timestamp': Timestamp( + info.get('put_timestamp', 0)).normal, }) return headers @@ -245,10 +244,7 @@ class ContainerController(object): """Handle HTTP DELETE request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) - if 'x-timestamp' not in req.headers or \ - not check_float(req.headers['x-timestamp']): - return HTTPBadRequest(body='Missing timestamp', request=req, - content_type='text/plain') + req_timestamp = valid_timestamp(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) # policy index is only relevant for delete_obj (and transitively for @@ -258,10 +254,7 @@ class ContainerController(object): if account.startswith(self.auto_create_account_prefix) and obj and \ not os.path.exists(broker.db_file): try: - broker.initialize( - normalize_timestamp( - req.headers.get('x-timestamp') or time.time()), - obj_policy_index) + broker.initialize(req_timestamp.internal, obj_policy_index) except DatabaseAlreadyExists: pass if not os.path.exists(broker.db_file): @@ -274,9 +267,9 @@ class ContainerController(object): # delete container if not broker.empty(): return HTTPConflict(request=req) - existed = float(broker.get_info()['put_timestamp']) and \ + existed = Timestamp(broker.get_info()['put_timestamp']) and \ not broker.is_deleted() - broker.delete_db(req.headers['X-Timestamp']) + broker.delete_db(req_timestamp.internal) if not broker.is_deleted(): return HTTPConflict(request=req) resp = self.account_update(req, account, container, broker) @@ -298,6 +291,7 @@ class ContainerController(object): when creating the container :param requested_policy_index: the storage policy index sent in the request, may be None + :returns: created, a bool, if database did not previously exist """ if not os.path.exists(broker.db_file): @@ -329,10 +323,7 @@ class ContainerController(object): """Handle HTTP PUT request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) - if 'x-timestamp' not in req.headers or \ - not check_float(req.headers['x-timestamp']): - return HTTPBadRequest(body='Missing timestamp', request=req, - content_type='text/plain') + req_timestamp = valid_timestamp(req) if 'x-container-sync-to' in req.headers: err, sync_to, realm, realm_key = validate_sync_to( req.headers['x-container-sync-to'], self.allowed_sync_hosts, @@ -342,7 +333,6 @@ class ContainerController(object): if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) requested_policy_index = self.get_and_validate_policy_index(req) - timestamp = normalize_timestamp(req.headers['x-timestamp']) broker = self._get_container_broker(drive, part, account, container) if obj: # put container object # obj put expects the policy_index header, default is for @@ -351,12 +341,12 @@ class ContainerController(object): if account.startswith(self.auto_create_account_prefix) and \ not os.path.exists(broker.db_file): try: - broker.initialize(timestamp, obj_policy_index) + broker.initialize(req_timestamp.internal, obj_policy_index) except DatabaseAlreadyExists: pass if not os.path.exists(broker.db_file): return HTTPNotFound() - broker.put_object(obj, timestamp, + broker.put_object(obj, req_timestamp.internal, int(req.headers['x-size']), req.headers['x-content-type'], req.headers['x-etag'], 0, @@ -370,12 +360,12 @@ class ContainerController(object): else: new_container_policy = requested_policy_index created = self._update_or_create(req, broker, - timestamp, + req_timestamp.internal, new_container_policy, requested_policy_index) metadata = {} metadata.update( - (key, (value, timestamp)) + (key, (value, req_timestamp.internal)) for key, value in req.headers.iteritems() if key.lower() in self.save_headers or is_sys_or_user_meta('container', key)) @@ -433,11 +423,7 @@ class ContainerController(object): return {'subdir': name} response = {'bytes': size, 'hash': etag, 'name': name, 'content_type': content_type} - last_modified = datetime.utcfromtimestamp(float(created)).isoformat() - # python isoformat() doesn't include msecs when zero - if len(last_modified) < len("1970-01-01T00:00:00.000000"): - last_modified += ".000000" - response['last_modified'] = last_modified + response['last_modified'] = Timestamp(created).isoformat override_bytes_from_content_type(response, logger=self.logger) return response @@ -541,10 +527,7 @@ class ContainerController(object): def POST(self, req): """Handle HTTP POST request.""" drive, part, account, container = split_and_validate_path(req, 4) - if 'x-timestamp' not in req.headers or \ - not check_float(req.headers['x-timestamp']): - return HTTPBadRequest(body='Missing or bad timestamp', - request=req, content_type='text/plain') + req_timestamp = valid_timestamp(req) if 'x-container-sync-to' in req.headers: err, sync_to, realm, realm_key = validate_sync_to( req.headers['x-container-sync-to'], self.allowed_sync_hosts, @@ -556,10 +539,10 @@ class ContainerController(object): broker = self._get_container_broker(drive, part, account, container) if broker.is_deleted(): return HTTPNotFound(request=req) - timestamp = normalize_timestamp(req.headers['x-timestamp']) metadata = {} metadata.update( - (key, (value, timestamp)) for key, value in req.headers.iteritems() + (key, (value, req_timestamp.internal)) + for key, value in req.headers.iteritems() if key.lower() in self.save_headers or is_sys_or_user_meta('container', key)) if metadata: diff --git a/swift/container/sync.py b/swift/container/sync.py index aa1ecc6929..b5fb5e203a 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -32,7 +32,7 @@ from swift.common.ring import Ring from swift.common.utils import ( audit_location_generator, clean_content_type, config_true_value, FileLikeIter, get_logger, hash_path, quote, urlparse, validate_sync_to, - whataremyips) + whataremyips, Timestamp) from swift.common.daemon import Daemon from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND from swift.common.storage_policy import POLICIES, POLICY_INDEX @@ -373,7 +373,7 @@ class ContainerSync(Daemon): row['name']) shuffle(nodes) exc = None - looking_for_timestamp = float(row['created_at']) + looking_for_timestamp = Timestamp(row['created_at']) timestamp = -1 headers = body = None headers_out = {POLICY_INDEX: str(info['storage_policy_index'])} @@ -383,7 +383,8 @@ class ContainerSync(Daemon): node, part, info['account'], info['container'], row['name'], headers=headers_out, resp_chunk_size=65536) - this_timestamp = float(these_headers['x-timestamp']) + this_timestamp = Timestamp( + these_headers['x-timestamp']) if this_timestamp > timestamp: timestamp = this_timestamp headers = these_headers diff --git a/swift/container/updater.py b/swift/container/updater.py index 2a13164479..4097f136b6 100644 --- a/swift/container/updater.py +++ b/swift/container/updater.py @@ -30,7 +30,7 @@ from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ConnectionTimeout from swift.common.ring import Ring from swift.common.utils import get_logger, config_true_value, ismount, \ - dump_recon_cache, quorum_size + dump_recon_cache, quorum_size, Timestamp from swift.common.daemon import Daemon from swift.common.http import is_success, HTTP_INTERNAL_SERVER_ERROR from swift.common.storage_policy import POLICY_INDEX @@ -210,7 +210,7 @@ class ContainerUpdater(Daemon): info = broker.get_info() # Don't send updates if the container was auto-created since it # definitely doesn't have up to date statistics. - if float(info['put_timestamp']) <= 0: + if Timestamp(info['put_timestamp']) <= 0: return if self.account_suppressions.get(info['account'], 0) > time.time(): return diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py index 91de3d4fa0..be3cde7416 100644 --- a/swift/obj/diskfile.py +++ b/swift/obj/diskfile.py @@ -49,7 +49,7 @@ from eventlet import Timeout from swift import gettext_ as _ from swift.common.constraints import check_mount -from swift.common.utils import mkdirs, normalize_timestamp, \ +from swift.common.utils import mkdirs, Timestamp, \ storage_directory, hash_path, renamer, fallocate, fsync, \ fdatasync, drop_buffer_cache, ThreadPool, lock_path, write_pickle, \ config_true_value, listdir, split_path, ismount, remove_file @@ -228,7 +228,7 @@ def hash_cleanup_listdir(hsh_path, reclaim_age=ONE_WEEK): if files[0].endswith('.ts'): # remove tombstones older than reclaim_age ts = files[0].rsplit('.', 1)[0] - if (time.time() - float(ts)) > reclaim_age: + if (time.time() - float(Timestamp(ts))) > reclaim_age: remove_file(join(hsh_path, files[0])) files.remove(files[0]) elif files: @@ -552,7 +552,7 @@ class DiskFileManager(object): write_pickle, data, os.path.join(async_dir, ohash[-3:], ohash + '-' + - normalize_timestamp(timestamp)), + Timestamp(timestamp).internal), os.path.join(device_path, get_tmp_dir(policy_idx))) self.logger.increment('async_pendings') @@ -794,7 +794,7 @@ class DiskFileWriter(object): :param metadata: dictionary of metadata to be associated with the object """ - timestamp = normalize_timestamp(metadata['X-Timestamp']) + timestamp = Timestamp(metadata['X-Timestamp']).internal metadata['name'] = self._name target_path = join(self._datadir, timestamp + self._extension) @@ -1060,7 +1060,7 @@ class DiskFile(object): def timestamp(self): if self._metadata is None: raise DiskFileNotOpen() - return self._metadata.get('X-Timestamp') + return Timestamp(self._metadata.get('X-Timestamp')) @classmethod def from_hash_dir(cls, mgr, hash_dir_path, device_path, partition): @@ -1449,7 +1449,7 @@ class DiskFile(object): :raises DiskFileError: this implementation will raise the same errors as the `create()` method. """ - timestamp = normalize_timestamp(timestamp) + timestamp = Timestamp(timestamp).internal with self.create() as deleter: deleter._extension = '.ts' diff --git a/swift/obj/mem_diskfile.py b/swift/obj/mem_diskfile.py index 939e783703..d02cdbe43e 100644 --- a/swift/obj/mem_diskfile.py +++ b/swift/obj/mem_diskfile.py @@ -22,7 +22,7 @@ from contextlib import contextmanager from eventlet import Timeout -from swift.common.utils import normalize_timestamp +from swift.common.utils import Timestamp from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist, \ DiskFileCollision, DiskFileDeleted, DiskFileNotOpen from swift.common.swob import multi_range_iterator @@ -394,7 +394,6 @@ class DiskFile(object): :param timestamp: timestamp to compare with each file """ - timestamp = normalize_timestamp(timestamp) fp, md = self._filesystem.get_object(self._name) - if md['X-Timestamp'] < timestamp: + if md['X-Timestamp'] < Timestamp(timestamp): self._filesystem.del_object(self._name) diff --git a/swift/obj/server.py b/swift/obj/server.py index 4d771ae7ff..d7b4ef292d 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -29,10 +29,10 @@ from eventlet import sleep, Timeout from swift.common.utils import public, get_logger, \ config_true_value, timing_stats, replication, \ - normalize_delete_at_timestamp, get_log_line + normalize_delete_at_timestamp, get_log_line, Timestamp from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, \ - check_float, check_utf8 + valid_timestamp, check_utf8 from swift.common.exceptions import ConnectionTimeout, DiskFileQuarantined, \ DiskFileNotExist, DiskFileCollision, DiskFileNoSpace, DiskFileDeleted, \ DiskFileDeviceUnavailable, DiskFileExpired, ChunkReadTimeout @@ -271,7 +271,7 @@ class ObjectController(object): headers_in = request.headers headers_out = HeaderKeyDict({ POLICY_INDEX: 0, # system accounts are always Policy-0 - 'x-timestamp': headers_in['x-timestamp'], + 'x-timestamp': request.timestamp.internal, 'x-trans-id': headers_in.get('x-trans-id', '-'), 'referer': request.as_referer()}) if op != 'DELETE': @@ -325,10 +325,7 @@ class ObjectController(object): """Handle HTTP POST requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) - if 'x-timestamp' not in request.headers or \ - not check_float(request.headers['x-timestamp']): - return HTTPBadRequest(body='Missing timestamp', request=request, - content_type='text/plain') + req_timestamp = valid_timestamp(request) new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, @@ -343,10 +340,10 @@ class ObjectController(object): orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) - orig_timestamp = orig_metadata.get('X-Timestamp', '0') - if orig_timestamp >= request.headers['x-timestamp']: + orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) + if orig_timestamp >= req_timestamp: return HTTPConflict(request=request) - metadata = {'X-Timestamp': request.headers['x-timestamp']} + metadata = {'X-Timestamp': req_timestamp.internal} metadata.update(val for val in request.headers.iteritems() if is_user_meta('object', val[0])) for header_key in self.allowed_headers: @@ -371,10 +368,7 @@ class ObjectController(object): """Handle HTTP PUT requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) - if 'x-timestamp' not in request.headers or \ - not check_float(request.headers['x-timestamp']): - return HTTPBadRequest(body='Missing timestamp', request=request, - content_type='text/plain') + req_timestamp = valid_timestamp(request) error_response = check_object_creation(request, obj) if error_response: return error_response @@ -407,8 +401,8 @@ class ObjectController(object): # The current ETag matches, so return 412 return HTTPPreconditionFailed(request=request) - orig_timestamp = orig_metadata.get('X-Timestamp') - if orig_timestamp and orig_timestamp >= request.headers['x-timestamp']: + orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) + if orig_timestamp and orig_timestamp >= req_timestamp: return HTTPConflict(request=request) orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) upload_expiration = time.time() + self.max_upload_time @@ -445,7 +439,7 @@ class ObjectController(object): request.headers['etag'].lower() != etag: return HTTPUnprocessableEntity(request=request) metadata = { - 'X-Timestamp': request.headers['x-timestamp'], + 'X-Timestamp': request.timestamp.internal, 'Content-Type': request.headers['content-type'], 'ETag': etag, 'Content-Length': str(upload_size), @@ -499,8 +493,7 @@ class ObjectController(object): with disk_file.open(): metadata = disk_file.get_metadata() obj_size = int(metadata['Content-Length']) - file_x_ts = metadata['X-Timestamp'] - file_x_ts_flt = float(file_x_ts) + file_x_ts = Timestamp(metadata['X-Timestamp']) keep_cache = (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)) @@ -514,19 +507,20 @@ class ObjectController(object): key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] - response.last_modified = math.ceil(file_x_ts_flt) + response.last_modified = math.ceil(float(file_x_ts)) response.content_length = obj_size try: response.content_encoding = metadata[ 'Content-Encoding'] except KeyError: pass - response.headers['X-Timestamp'] = file_x_ts + response.headers['X-Timestamp'] = file_x_ts.normal + response.headers['X-Backend-Timestamp'] = file_x_ts.internal resp = request.get_response(response) except (DiskFileNotExist, DiskFileQuarantined) as e: headers = {} if hasattr(e, 'timestamp'): - headers['X-Backend-Timestamp'] = e.timestamp + headers['X-Backend-Timestamp'] = e.timestamp.internal resp = HTTPNotFound(request=request, headers=headers, conditional_response=True) return resp @@ -548,7 +542,7 @@ class ObjectController(object): except (DiskFileNotExist, DiskFileQuarantined) as e: headers = {} if hasattr(e, 'timestamp'): - headers['X-Backend-Timestamp'] = e.timestamp + headers['X-Backend-Timestamp'] = e.timestamp.internal return HTTPNotFound(request=request, headers=headers, conditional_response=True) response = Response(request=request, conditional_response=True) @@ -559,10 +553,11 @@ class ObjectController(object): key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] - ts = metadata['X-Timestamp'] + ts = Timestamp(metadata['X-Timestamp']) response.last_modified = math.ceil(float(ts)) # Needed for container sync feature - response.headers['X-Timestamp'] = ts + response.headers['X-Timestamp'] = ts.normal + response.headers['X-Backend-Timestamp'] = ts.internal response.content_length = int(metadata['Content-Length']) try: response.content_encoding = metadata['Content-Encoding'] @@ -576,10 +571,7 @@ class ObjectController(object): """Handle HTTP DELETE requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) - if 'x-timestamp' not in request.headers or \ - not check_float(request.headers['x-timestamp']): - return HTTPBadRequest(body='Missing timestamp', request=request, - content_type='text/plain') + req_timestamp = valid_timestamp(request) try: disk_file = self.get_diskfile( device, partition, account, container, obj, @@ -601,8 +593,8 @@ class ObjectController(object): orig_metadata = {} response_class = HTTPNotFound else: - orig_timestamp = orig_metadata.get('X-Timestamp', 0) - if orig_timestamp < request.headers['x-timestamp']: + orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) + if orig_timestamp < req_timestamp: response_class = HTTPNoContent else: response_class = HTTPConflict @@ -633,12 +625,11 @@ class ObjectController(object): self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device, policy_idx) - req_timestamp = request.headers['X-Timestamp'] if orig_timestamp < req_timestamp: disk_file.delete(req_timestamp) self.container_update( 'DELETE', account, container, obj, request, - HeaderKeyDict({'x-timestamp': req_timestamp}), + HeaderKeyDict({'x-timestamp': req_timestamp.internal}), device, policy_idx) return response_class(request=request) diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 233935254d..5d65e3c783 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -36,7 +36,7 @@ from eventlet import sleep from eventlet.timeout import Timeout from swift.common.wsgi import make_pre_authed_env -from swift.common.utils import normalize_timestamp, config_true_value, \ +from swift.common.utils import Timestamp, config_true_value, \ public, split_path, list_from_csv, GreenthreadSafeIterator, \ quorum_size, GreenAsyncPile from swift.common.bufferedhttp import http_connect @@ -926,7 +926,7 @@ class Controller(object): headers = HeaderKeyDict(additional) if additional else HeaderKeyDict() if transfer: self.transfer_headers(orig_req.headers, headers) - headers.setdefault('x-timestamp', normalize_timestamp(time.time())) + headers.setdefault('x-timestamp', Timestamp(time.time()).internal) if orig_req: referer = orig_req.as_referer() else: @@ -1158,7 +1158,7 @@ class Controller(object): """ partition, nodes = self.app.account_ring.get_nodes(account) path = '/%s' % account - headers = {'X-Timestamp': normalize_timestamp(time.time()), + headers = {'X-Timestamp': Timestamp(time.time()).internal, 'X-Trans-Id': self.trans_id, 'Connection': 'close'} resp = self.make_requests(Request.blank('/v1' + path), diff --git a/swift/proxy/controllers/container.py b/swift/proxy/controllers/container.py index 69eb68bc46..5920561525 100644 --- a/swift/proxy/controllers/container.py +++ b/swift/proxy/controllers/container.py @@ -17,7 +17,7 @@ from swift import gettext_ as _ from urllib import unquote import time -from swift.common.utils import public, csv_append, normalize_timestamp +from swift.common.utils import public, csv_append, Timestamp from swift.common.constraints import check_metadata from swift.common import constraints from swift.common.http import HTTP_ACCEPTED @@ -209,7 +209,7 @@ class ContainerController(Controller): def _backend_requests(self, req, n_outgoing, account_partition, accounts, policy_index=None): - additional = {'X-Timestamp': normalize_timestamp(time.time())} + additional = {'X-Timestamp': Timestamp(time.time()).internal} if policy_index is None: additional['X-Backend-Storage-Policy-Default'] = \ int(POLICIES.default) diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 346238c419..63162f50ad 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -37,8 +37,8 @@ from eventlet.timeout import Timeout from swift.common.utils import ( clean_content_type, config_true_value, ContextPool, csv_append, - GreenAsyncPile, GreenthreadSafeIterator, json, - normalize_delete_at_timestamp, normalize_timestamp, public, quorum_size) + GreenAsyncPile, GreenthreadSafeIterator, json, Timestamp, + normalize_delete_at_timestamp, public, quorum_size) from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_metadata, check_object_creation, \ check_copy_from_header @@ -309,7 +309,7 @@ class ObjectController(Controller): req.headers[POLICY_INDEX] = policy_index partition, nodes = obj_ring.get_nodes( self.account_name, self.container_name, self.object_name) - req.headers['X-Timestamp'] = normalize_timestamp(time.time()) + req.headers['X-Timestamp'] = Timestamp(time.time()).internal headers = self._backend_requests( req, len(nodes), container_partition, containers, @@ -510,19 +510,18 @@ class ObjectController(Controller): # Used by container sync feature if 'x-timestamp' in req.headers: try: - req.headers['X-Timestamp'] = \ - normalize_timestamp(req.headers['x-timestamp']) + req_timestamp = Timestamp(req.headers['X-Timestamp']) if hresp.environ and 'swift_x_timestamp' in hresp.environ and \ - float(hresp.environ['swift_x_timestamp']) >= \ - float(req.headers['x-timestamp']): + hresp.environ['swift_x_timestamp'] >= req_timestamp: return HTTPAccepted(request=req) except ValueError: return HTTPBadRequest( request=req, content_type='text/plain', body='X-Timestamp should be a UNIX timestamp float value; ' 'was %r' % req.headers['x-timestamp']) + req.headers['X-Timestamp'] = req_timestamp.internal else: - req.headers['X-Timestamp'] = normalize_timestamp(time.time()) + req.headers['X-Timestamp'] = Timestamp(time.time()).internal # Sometimes the 'content-type' header exists, but is set to None. content_type_manually_set = True detect_content_type = \ @@ -554,7 +553,7 @@ class ObjectController(Controller): ts_source = time.mktime(time.strptime( hresp.headers['last-modified'], '%a, %d %b %Y %H:%M:%S GMT')) - new_ts = normalize_timestamp(ts_source) + new_ts = Timestamp(ts_source).internal vers_obj_name = lprefix + new_ts copy_headers = { 'Destination': '%s/%s' % (lcontainer, vers_obj_name)} @@ -766,7 +765,8 @@ class ObjectController(Controller): resp.headers['X-Copied-From-Last-Modified'] = \ source_resp.headers['last-modified'] copy_headers_into(req, resp) - resp.last_modified = math.ceil(float(req.headers['X-Timestamp'])) + resp.last_modified = math.ceil( + float(Timestamp(req.headers['X-Timestamp']))) return resp @public @@ -858,15 +858,15 @@ class ObjectController(Controller): # Used by container sync feature if 'x-timestamp' in req.headers: try: - req.headers['X-Timestamp'] = \ - normalize_timestamp(req.headers['x-timestamp']) + req_timestamp = Timestamp(req.headers['X-Timestamp']) except ValueError: return HTTPBadRequest( request=req, content_type='text/plain', body='X-Timestamp should be a UNIX timestamp float value; ' 'was %r' % req.headers['x-timestamp']) + req.headers['X-Timestamp'] = req_timestamp.internal else: - req.headers['X-Timestamp'] = normalize_timestamp(time.time()) + req.headers['X-Timestamp'] = Timestamp(time.time()).internal headers = self._backend_requests( req, len(nodes), container_partition, containers) diff --git a/test/probe/test_object_expirer.py b/test/probe/test_object_expirer.py index 4e336483c1..243e419578 100644 --- a/test/probe/test_object_expirer.py +++ b/test/probe/test_object_expirer.py @@ -21,6 +21,7 @@ from nose import SkipTest from swift.common.internal_client import InternalClient from swift.common.manager import Manager from swift.common.storage_policy import POLICIES +from swift.common.utils import Timestamp from test.probe.common import reset_environment, get_to_final_state from test.probe.test_container_merge_policy_index import BrainSplitter @@ -60,12 +61,12 @@ class TestObjectExpirer(unittest.TestCase): # create an expiring object and a container with the wrong policy self.brain.stop_primary_half() self.brain.put_container(int(old_policy)) - self.brain.put_object(headers={'X-Delete-After': 1}) + self.brain.put_object(headers={'X-Delete-After': 2}) # get the object timestamp metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, headers={'X-Backend-Storage-Policy-Index': int(old_policy)}) - create_timestamp = metadata['x-timestamp'] + create_timestamp = Timestamp(metadata['x-timestamp']) self.brain.start_primary_half() # get the expiring object updates in their queue, while we have all # the servers up @@ -89,7 +90,7 @@ class TestObjectExpirer(unittest.TestCase): acceptable_statuses=(4,), headers={'X-Backend-Storage-Policy-Index': int(old_policy)}) self.assert_('x-backend-timestamp' in metadata) - self.assertEqual(metadata['x-backend-timestamp'], + self.assertEqual(Timestamp(metadata['x-backend-timestamp']), create_timestamp) # but it is still in the listing @@ -124,8 +125,8 @@ class TestObjectExpirer(unittest.TestCase): (found_in_policy, policy)) found_in_policy = policy self.assert_('x-backend-timestamp' in metadata) - self.assert_(float(metadata['x-backend-timestamp']) > - float(create_timestamp)) + self.assert_(Timestamp(metadata['x-backend-timestamp']) > + create_timestamp) if __name__ == "__main__": unittest.main() diff --git a/test/unit/account/test_backend.py b/test/unit/account/test_backend.py index 5b47f102f1..eb0bccc8a4 100644 --- a/test/unit/account/test_backend.py +++ b/test/unit/account/test_backend.py @@ -29,7 +29,7 @@ from contextlib import contextmanager import random from swift.account.backend import AccountBroker -from swift.common.utils import normalize_timestamp +from swift.common.utils import Timestamp from test.unit import patch_policies, with_tempdir from swift.common.db import DatabaseConnectionError from swift.common.storage_policy import StoragePolicy, POLICIES @@ -57,7 +57,7 @@ class TestAccountBroker(unittest.TestCase): self.fail("Unexpected exception raised: %r" % e) else: self.fail("Expected a DatabaseConnectionError exception") - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) with broker.get() as conn: curs = conn.cursor() curs.execute('SELECT 1') @@ -67,7 +67,7 @@ class TestAccountBroker(unittest.TestCase): # Test AccountBroker throwing a conn away after exception first_conn = None broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) with broker.get() as conn: first_conn = conn try: @@ -81,20 +81,20 @@ class TestAccountBroker(unittest.TestCase): def test_empty(self): # Test AccountBroker.empty broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) self.assert_(broker.empty()) - broker.put_container('o', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('o', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) self.assert_(not broker.empty()) sleep(.00001) - broker.put_container('o', 0, normalize_timestamp(time()), 0, 0, + broker.put_container('o', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) self.assert_(broker.empty()) def test_reclaim(self): broker = AccountBroker(':memory:', account='test_account') - broker.initialize(normalize_timestamp('1')) - broker.put_container('c', normalize_timestamp(time()), 0, 0, 0, + broker.initialize(Timestamp('1').internal) + broker.put_container('c', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual(conn.execute( @@ -103,7 +103,7 @@ class TestAccountBroker(unittest.TestCase): self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) - broker.reclaim(normalize_timestamp(time() - 999), time()) + broker.reclaim(Timestamp(time() - 999).internal, time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " @@ -112,7 +112,7 @@ class TestAccountBroker(unittest.TestCase): "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) - broker.put_container('c', 0, normalize_timestamp(time()), 0, 0, + broker.put_container('c', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual(conn.execute( @@ -121,7 +121,7 @@ class TestAccountBroker(unittest.TestCase): self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1) - broker.reclaim(normalize_timestamp(time() - 999), time()) + broker.reclaim(Timestamp(time() - 999).internal, time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " @@ -130,7 +130,7 @@ class TestAccountBroker(unittest.TestCase): "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1) sleep(.00001) - broker.reclaim(normalize_timestamp(time()), time()) + broker.reclaim(Timestamp(time()).internal, time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " @@ -142,15 +142,15 @@ class TestAccountBroker(unittest.TestCase): broker.put_container('x', 0, 0, 0, 0, POLICIES.default.idx) broker.put_container('y', 0, 0, 0, 0, POLICIES.default.idx) broker.put_container('z', 0, 0, 0, 0, POLICIES.default.idx) - broker.reclaim(normalize_timestamp(time()), time()) + broker.reclaim(Timestamp(time()).internal, time()) # self.assertEqual(len(res), 2) # self.assert_(isinstance(res, tuple)) # containers, account_name = res # self.assert_(containers is None) # self.assert_(account_name is None) # Now delete the account - broker.delete_db(normalize_timestamp(time())) - broker.reclaim(normalize_timestamp(time()), time()) + broker.delete_db(Timestamp(time()).internal) + broker.reclaim(Timestamp(time()).internal, time()) # self.assertEqual(len(res), 2) # self.assert_(isinstance(res, tuple)) # containers, account_name = res @@ -162,34 +162,34 @@ class TestAccountBroker(unittest.TestCase): # self.assert_('a' not in containers) def test_delete_db_status(self): - start = int(time()) - ts = itertools.count(start) + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) + start = ts.next() broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp(ts.next())) + broker.initialize(start) info = broker.get_info() - self.assertEqual(info['put_timestamp'], normalize_timestamp(start)) - self.assert_(float(info['created_at']) >= start) + self.assertEqual(info['put_timestamp'], Timestamp(start).internal) + self.assert_(Timestamp(info['created_at']) >= start) self.assertEqual(info['delete_timestamp'], '0') if self.__class__ == TestAccountBrokerBeforeMetadata: self.assertEqual(info['status_changed_at'], '0') else: self.assertEqual(info['status_changed_at'], - normalize_timestamp(start)) + Timestamp(start).internal) # delete it - delete_timestamp = normalize_timestamp(ts.next()) + delete_timestamp = ts.next() broker.delete_db(delete_timestamp) info = broker.get_info() - self.assertEqual(info['put_timestamp'], normalize_timestamp(start)) - self.assert_(float(info['created_at']) >= start) + self.assertEqual(info['put_timestamp'], Timestamp(start).internal) + self.assert_(Timestamp(info['created_at']) >= start) self.assertEqual(info['delete_timestamp'], delete_timestamp) self.assertEqual(info['status_changed_at'], delete_timestamp) def test_delete_container(self): # Test AccountBroker.delete_container broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) - broker.put_container('o', normalize_timestamp(time()), 0, 0, 0, + broker.initialize(Timestamp('1').internal) + broker.put_container('o', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual(conn.execute( @@ -199,7 +199,7 @@ class TestAccountBroker(unittest.TestCase): "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) - broker.put_container('o', 0, normalize_timestamp(time()), 0, 0, + broker.put_container('o', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual(conn.execute( @@ -212,10 +212,10 @@ class TestAccountBroker(unittest.TestCase): def test_put_container(self): # Test AccountBroker.put_container broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) # Create initial container - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal broker.put_container('"{}"', timestamp, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: @@ -243,7 +243,7 @@ class TestAccountBroker(unittest.TestCase): # Put new event sleep(.00001) - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal broker.put_container('"{}"', timestamp, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: @@ -257,7 +257,7 @@ class TestAccountBroker(unittest.TestCase): "SELECT deleted FROM container").fetchone()[0], 0) # Put old event - otimestamp = normalize_timestamp(float(timestamp) - 1) + otimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal broker.put_container('"{}"', otimestamp, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: @@ -271,7 +271,7 @@ class TestAccountBroker(unittest.TestCase): "SELECT deleted FROM container").fetchone()[0], 0) # Put old delete event - dtimestamp = normalize_timestamp(float(timestamp) - 1) + dtimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal broker.put_container('"{}"', 0, dtimestamp, 0, 0, POLICIES.default.idx) with broker.get() as conn: @@ -289,7 +289,7 @@ class TestAccountBroker(unittest.TestCase): # Put new delete event sleep(.00001) - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal broker.put_container('"{}"', 0, timestamp, 0, 0, POLICIES.default.idx) with broker.get() as conn: @@ -304,7 +304,7 @@ class TestAccountBroker(unittest.TestCase): # Put new event sleep(.00001) - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal broker.put_container('"{}"', timestamp, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: @@ -320,46 +320,46 @@ class TestAccountBroker(unittest.TestCase): def test_get_info(self): # Test AccountBroker.get_info broker = AccountBroker(':memory:', account='test1') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) info = broker.get_info() self.assertEqual(info['account'], 'test1') self.assertEqual(info['hash'], '00000000000000000000000000000000') - self.assertEqual(info['put_timestamp'], normalize_timestamp(1)) + self.assertEqual(info['put_timestamp'], Timestamp(1).internal) self.assertEqual(info['delete_timestamp'], '0') if self.__class__ == TestAccountBrokerBeforeMetadata: self.assertEqual(info['status_changed_at'], '0') else: - self.assertEqual(info['status_changed_at'], normalize_timestamp(1)) + self.assertEqual(info['status_changed_at'], Timestamp(1).internal) info = broker.get_info() self.assertEqual(info['container_count'], 0) - broker.put_container('c1', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('c1', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 1) sleep(.00001) - broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('c2', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 2) sleep(.00001) - broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('c2', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 2) sleep(.00001) - broker.put_container('c1', 0, normalize_timestamp(time()), 0, 0, + broker.put_container('c1', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 1) sleep(.00001) - broker.put_container('c2', 0, normalize_timestamp(time()), 0, 0, + broker.put_container('c2', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 0) @@ -367,20 +367,20 @@ class TestAccountBroker(unittest.TestCase): def test_list_containers_iter(self): # Test AccountBroker.list_containers_iter broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) for cont1 in xrange(4): for cont2 in xrange(125): broker.put_container('%d-%04d' % (cont1, cont2), - normalize_timestamp(time()), 0, 0, 0, + Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) for cont in xrange(125): broker.put_container('2-0051-%04d' % cont, - normalize_timestamp(time()), 0, 0, 0, + Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) for cont in xrange(125): broker.put_container('3-%04d-0049' % cont, - normalize_timestamp(time()), 0, 0, 0, + Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) listing = broker.list_containers_iter(100, '', None, None, '') @@ -445,7 +445,7 @@ class TestAccountBroker(unittest.TestCase): '3-0047-', '3-0048', '3-0048-', '3-0049', '3-0049-', '3-0050']) - broker.put_container('3-0049-', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('3-0049-', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) listing = broker.list_containers_iter(10, '3-0048', None, None, None) self.assertEqual(len(listing), 10) @@ -470,26 +470,26 @@ class TestAccountBroker(unittest.TestCase): # Test AccountBroker.list_containers_iter for an # account that has an odd container with a trailing delimiter broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) - broker.put_container('a', normalize_timestamp(time()), 0, 0, 0, + broker.initialize(Timestamp('1').internal) + broker.put_container('a', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) - broker.put_container('a-', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('a-', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) - broker.put_container('a-a', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('a-a', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) - broker.put_container('a-a-a', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('a-a-a', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) - broker.put_container('a-a-b', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('a-a-b', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) - broker.put_container('a-b', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('a-b', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) - broker.put_container('b', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('b', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) - broker.put_container('b-a', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('b-a', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) - broker.put_container('b-b', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('b-b', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) - broker.put_container('c', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('c', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) listing = broker.list_containers_iter(15, None, None, None, None) self.assertEqual(len(listing), 10) @@ -510,27 +510,30 @@ class TestAccountBroker(unittest.TestCase): def test_chexor(self): broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) - broker.put_container('a', normalize_timestamp(1), - normalize_timestamp(0), 0, 0, + broker.initialize(Timestamp('1').internal) + broker.put_container('a', Timestamp(1).internal, + Timestamp(0).internal, 0, 0, POLICIES.default.idx) - broker.put_container('b', normalize_timestamp(2), - normalize_timestamp(0), 0, 0, + broker.put_container('b', Timestamp(2).internal, + Timestamp(0).internal, 0, 0, POLICIES.default.idx) hasha = hashlib.md5( - '%s-%s' % ('a', '0000000001.00000-0000000000.00000-0-0') + '%s-%s' % ('a', "%s-%s-%s-%s" % ( + Timestamp(1).internal, Timestamp(0).internal, 0, 0)) ).digest() hashb = hashlib.md5( - '%s-%s' % ('b', '0000000002.00000-0000000000.00000-0-0') + '%s-%s' % ('b', "%s-%s-%s-%s" % ( + Timestamp(2).internal, Timestamp(0).internal, 0, 0)) ).digest() hashc = \ ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEqual(broker.get_info()['hash'], hashc) - broker.put_container('b', normalize_timestamp(3), - normalize_timestamp(0), 0, 0, + broker.put_container('b', Timestamp(3).internal, + Timestamp(0).internal, 0, 0, POLICIES.default.idx) hashb = hashlib.md5( - '%s-%s' % ('b', '0000000003.00000-0000000000.00000-0-0') + '%s-%s' % ('b', "%s-%s-%s-%s" % ( + Timestamp(3).internal, Timestamp(0).internal, 0, 0)) ).digest() hashc = \ ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) @@ -538,12 +541,12 @@ class TestAccountBroker(unittest.TestCase): def test_merge_items(self): broker1 = AccountBroker(':memory:', account='a') - broker1.initialize(normalize_timestamp('1')) + broker1.initialize(Timestamp('1').internal) broker2 = AccountBroker(':memory:', account='a') - broker2.initialize(normalize_timestamp('1')) - broker1.put_container('a', normalize_timestamp(1), 0, 0, 0, + broker2.initialize(Timestamp('1').internal) + broker1.put_container('a', Timestamp(1).internal, 0, 0, 0, POLICIES.default.idx) - broker1.put_container('b', normalize_timestamp(2), 0, 0, 0, + broker1.put_container('b', Timestamp(2).internal, 0, 0, 0, POLICIES.default.idx) id = broker1.get_info()['id'] broker2.merge_items(broker1.get_items_since( @@ -551,7 +554,7 @@ class TestAccountBroker(unittest.TestCase): items = broker2.get_items_since(-1, 1000) self.assertEqual(len(items), 2) self.assertEqual(['a', 'b'], sorted([rec['name'] for rec in items])) - broker1.put_container('c', normalize_timestamp(3), 0, 0, 0, + broker1.put_container('c', Timestamp(3).internal, 0, 0, 0, POLICIES.default.idx) broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) @@ -567,14 +570,14 @@ class TestAccountBroker(unittest.TestCase): broker_path = os.path.join(tempdir, 'test-load-old.db') try: broker = AccountBroker(broker_path, account='real') - broker.initialize(normalize_timestamp(1)) + broker.initialize(Timestamp(1).internal) with open(broker_path + '.pending', 'a+b') as pending: pending.write(':') pending.write(pickle.dumps( # name, put_timestamp, delete_timestamp, object_count, # bytes_used, deleted - ('oldcon', normalize_timestamp(200), - normalize_timestamp(0), + ('oldcon', Timestamp(200).internal, + Timestamp(0).internal, 896, 9216695, 0)).encode('base64')) broker._commit_puts() @@ -593,9 +596,9 @@ class TestAccountBroker(unittest.TestCase): StoragePolicy(2, 'two', False), StoragePolicy(3, 'three', False)]) def test_get_policy_stats(self): - ts = itertools.count() + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp(ts.next())) + broker.initialize(ts.next()) # check empty policy_stats self.assertTrue(broker.empty()) policy_stats = broker.get_policy_stats() @@ -604,7 +607,7 @@ class TestAccountBroker(unittest.TestCase): # add some empty containers for policy in POLICIES: container_name = 'c-%s' % policy.name - put_timestamp = normalize_timestamp(ts.next()) + put_timestamp = ts.next() broker.put_container(container_name, put_timestamp, 0, 0, 0, @@ -618,7 +621,7 @@ class TestAccountBroker(unittest.TestCase): # update the containers object & byte count for policy in POLICIES: container_name = 'c-%s' % policy.name - put_timestamp = normalize_timestamp(ts.next()) + put_timestamp = ts.next() count = policy.idx * 100 # good as any integer broker.put_container(container_name, put_timestamp, 0, @@ -640,7 +643,7 @@ class TestAccountBroker(unittest.TestCase): # now delete the containers one by one for policy in POLICIES: container_name = 'c-%s' % policy.name - delete_timestamp = normalize_timestamp(ts.next()) + delete_timestamp = ts.next() broker.put_container(container_name, 0, delete_timestamp, 0, 0, @@ -654,9 +657,9 @@ class TestAccountBroker(unittest.TestCase): @patch_policies([StoragePolicy(0, 'zero', False), StoragePolicy(1, 'one', True)]) def test_policy_stats_tracking(self): - ts = itertools.count() + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp(ts.next())) + broker.initialize(ts.next()) # policy 0 broker.put_container('con1', ts.next(), 0, 12, 2798641, 0) @@ -731,7 +734,7 @@ def premetadata_create_account_stat_table(self, conn, put_timestamp): conn.execute(''' UPDATE account_stat SET account = ?, created_at = ?, id = ?, put_timestamp = ? - ''', (self.account, normalize_timestamp(time()), str(uuid4()), + ''', (self.account, Timestamp(time()).internal, str(uuid4()), put_timestamp)) @@ -764,7 +767,7 @@ class TestAccountBrokerBeforeMetadata(TestAccountBroker): AccountBroker.create_account_stat_table = \ premetadata_create_account_stat_table broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) exc = None with broker.get() as conn: try: @@ -777,7 +780,7 @@ class TestAccountBrokerBeforeMetadata(TestAccountBroker): AccountBroker.create_account_stat_table = \ self._imported_create_account_stat_table broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) with broker.get() as conn: conn.execute('SELECT metadata FROM account_stat') @@ -851,7 +854,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker): self._imported_initialize = AccountBroker._initialize AccountBroker._initialize = prespi_AccountBroker_initialize broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) exc = None with broker.get() as conn: try: @@ -872,7 +875,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker): self._imported_create_container_table AccountBroker._initialize = self._imported_initialize broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) with broker.get() as conn: conn.execute('SELECT storage_policy_index FROM container') @@ -882,7 +885,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker): # first init an acct DB without the policy_stat table present broker = AccountBroker(db_path, account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) with broker.get() as conn: try: conn.execute(''' @@ -900,7 +903,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker): self.assertEqual(len(stats), 0) # now do a PUT to create the table - broker.put_container('o', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('o', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) broker._commit_puts_stale_ok() @@ -918,7 +921,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker): # first init an acct DB without the policy_stat table present broker = AccountBroker(db_path, account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) with broker.get() as conn: try: conn.execute(''' @@ -940,7 +943,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker): delete_timestamp, object_count, bytes_used, deleted) VALUES (?, ?, ?, ?, ?, ?) - ''', ('test_name', normalize_timestamp(time()), 0, 1, 2, 0)) + ''', ('test_name', Timestamp(time()).internal, 0, 1, 2, 0)) conn.commit() # make sure we can iter containers without the migration @@ -955,7 +958,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker): # which will update the DB schema as well as update policy_stats # for legacy containers in the DB (those without an SPI) other_policy = [p for p in POLICIES if p.idx != 0][0] - broker.put_container('test_second', normalize_timestamp(time()), + broker.put_container('test_second', Timestamp(time()).internal, 0, 3, 4, other_policy.idx) broker._commit_puts_stale_ok() @@ -994,14 +997,15 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker): def test_half_upgraded_database(self, tempdir): db_path = os.path.join(tempdir, 'account.db') ts = itertools.count() + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = AccountBroker(db_path, account='a') - broker.initialize(normalize_timestamp(ts.next())) + broker.initialize(ts.next()) self.assertTrue(broker.empty()) # add a container (to pending file) - broker.put_container('c', normalize_timestamp(ts.next()), 0, 0, 0, + broker.put_container('c', ts.next(), 0, 0, 0, POLICIES.default.idx) real_get = broker.get diff --git a/test/unit/account/test_utils.py b/test/unit/account/test_utils.py new file mode 100644 index 0000000000..46f8835bff --- /dev/null +++ b/test/unit/account/test_utils.py @@ -0,0 +1,128 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +import time +import unittest + +import mock + +from swift.account import utils, backend +from swift.common.storage_policy import POLICIES +from swift.common.utils import Timestamp +from swift.common.swob import HeaderKeyDict + +from test.unit import patch_policies + + +class TestFakeAccountBroker(unittest.TestCase): + + def test_fake_broker_get_info(self): + broker = utils.FakeAccountBroker() + now = time.time() + with mock.patch('time.time', new=lambda: now): + info = broker.get_info() + timestamp = Timestamp(now) + expected = { + 'container_count': 0, + 'object_count': 0, + 'bytes_used': 0, + 'created_at': timestamp.internal, + 'put_timestamp': timestamp.internal, + } + self.assertEqual(info, expected) + + def test_fake_broker_list_containers_iter(self): + broker = utils.FakeAccountBroker() + self.assertEqual(broker.list_containers_iter(), []) + + def test_fake_broker_metadata(self): + broker = utils.FakeAccountBroker() + self.assertEqual(broker.metadata, {}) + + def test_fake_broker_get_policy_stats(self): + broker = utils.FakeAccountBroker() + self.assertEqual(broker.get_policy_stats(), {}) + + +class TestAccountUtils(unittest.TestCase): + + def test_get_response_headers_fake_broker(self): + broker = utils.FakeAccountBroker() + now = time.time() + expected = { + 'X-Account-Container-Count': 0, + 'X-Account-Object-Count': 0, + 'X-Account-Bytes-Used': 0, + 'X-Timestamp': Timestamp(now).normal, + 'X-PUT-Timestamp': Timestamp(now).normal, + } + with mock.patch('time.time', new=lambda: now): + resp_headers = utils.get_response_headers(broker) + self.assertEqual(resp_headers, expected) + + def test_get_response_headers_empty_memory_broker(self): + broker = backend.AccountBroker(':memory:', account='a') + now = time.time() + with mock.patch('time.time', new=lambda: now): + broker.initialize(Timestamp(now).internal) + expected = { + 'X-Account-Container-Count': 0, + 'X-Account-Object-Count': 0, + 'X-Account-Bytes-Used': 0, + 'X-Timestamp': Timestamp(now).normal, + 'X-PUT-Timestamp': Timestamp(now).normal, + } + resp_headers = utils.get_response_headers(broker) + self.assertEqual(resp_headers, expected) + + @patch_policies + def test_get_response_headers_with_data(self): + broker = backend.AccountBroker(':memory:', account='a') + now = time.time() + with mock.patch('time.time', new=lambda: now): + broker.initialize(Timestamp(now).internal) + # add some container data + ts = (Timestamp(t).internal for t in itertools.count(int(now))) + total_containers = 0 + total_objects = 0 + total_bytes = 0 + for policy in POLICIES: + delete_timestamp = ts.next() + put_timestamp = ts.next() + object_count = int(policy) + bytes_used = int(policy) * 10 + broker.put_container('c-%s' % policy.name, put_timestamp, + delete_timestamp, object_count, bytes_used, + int(policy)) + total_containers += 1 + total_objects += object_count + total_bytes += bytes_used + expected = HeaderKeyDict({ + 'X-Account-Container-Count': total_containers, + 'X-Account-Object-Count': total_objects, + 'X-Account-Bytes-Used': total_bytes, + 'X-Timestamp': Timestamp(now).normal, + 'X-PUT-Timestamp': Timestamp(now).normal, + }) + for policy in POLICIES: + prefix = 'X-Account-Storage-Policy-%s-' % policy.name + expected[prefix + 'Object-Count'] = int(policy) + expected[prefix + 'Bytes-Used'] = int(policy) * 10 + resp_headers = utils.get_response_headers(broker) + for key, value in resp_headers.items(): + expected_value = expected.pop(key) + self.assertEqual(expected_value, str(value), + 'value for %r was %r not %r' % ( + key, value, expected_value)) + self.assertFalse(expected) diff --git a/test/unit/cli/test_info.py b/test/unit/cli/test_info.py index d86c70354c..8a4387214b 100644 --- a/test/unit/cli/test_info.py +++ b/test/unit/cli/test_info.py @@ -115,10 +115,10 @@ class TestCliInfo(TestCliInfoBase): Account: acct Account Hash: dc5be2aa4347a22a0fee6bc7de505b47 Metadata: - Created at: 1970-01-01 00:01:40.100000 (100.1) - Put Timestamp: 1970-01-01 00:01:46.300000 (106.3) - Delete Timestamp: 1970-01-01 00:01:47.900000 (107.9) - Status Timestamp: 1970-01-01 00:01:48.300000 (108.3) + Created at: 1970-01-01T00:01:40.100000 (100.1) + Put Timestamp: 1970-01-01T00:01:46.300000 (106.3) + Delete Timestamp: 1970-01-01T00:01:47.900000 (107.9) + Status Timestamp: 1970-01-01T00:01:48.300000 (108.3) Container Count: 3 Object Count: 20 Bytes Used: 42 @@ -158,15 +158,15 @@ No system metadata found in db file Container: cont Container Hash: d49d0ecbb53be1fcc49624f2f7c7ccae Metadata: - Created at: 1970-01-01 00:01:40.100000 (0000000100.10000) - Put Timestamp: 1970-01-01 00:01:46.300000 (0000000106.30000) - Delete Timestamp: 1970-01-01 00:01:47.900000 (0000000107.90000) - Status Timestamp: 1970-01-01 00:01:48.300000 (0000000108.30000) + Created at: 1970-01-01T00:01:40.100000 (0000000100.10000) + Put Timestamp: 1970-01-01T00:01:46.300000 (0000000106.30000) + Delete Timestamp: 1970-01-01T00:01:47.900000 (0000000107.90000) + Status Timestamp: 1970-01-01T00:01:48.300000 (0000000108.30000) Object Count: 20 Bytes Used: 42 Storage Policy: %s (0) - Reported Put Timestamp: 1970-01-01 02:48:26.300000 (0000010106.30000) - Reported Delete Timestamp: 1970-01-01 02:48:27.900000 (0000010107.90000) + Reported Put Timestamp: 1970-01-01T02:48:26.300000 (0000010106.30000) + Reported Delete Timestamp: 1970-01-01T02:48:27.900000 (0000010107.90000) Reported Object Count: 20 Reported Bytes Used: 42 Chexor: abaddeadbeefcafe @@ -452,8 +452,9 @@ class TestPrintObjFullMeta(TestCliInfoBase): Object: dummy Object hash: 128fdf98bddd1b1e8695f4340e67a67a Content-Type: application/octet-stream -Timestamp: 1970-01-01 00:01:46.300000 (106.3) -User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' +Timestamp: 1970-01-01T00:01:46.300000 (%s) +User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' % ( + utils.Timestamp(106.3).internal) self.assertEquals(out.getvalue().strip(), exp_out) @@ -469,8 +470,9 @@ User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' print_obj_metadata(metadata) exp_out = '''Path: Not found in metadata Content-Type: application/octet-stream -Timestamp: 1970-01-01 00:01:46.300000 (106.3) -User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' +Timestamp: 1970-01-01T00:01:46.300000 (%s) +User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' % ( + utils.Timestamp(106.3).internal) self.assertEquals(out.getvalue().strip(), exp_out) @@ -485,8 +487,9 @@ User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' Object: dummy Object hash: 128fdf98bddd1b1e8695f4340e67a67a Content-Type: Not found in metadata -Timestamp: 1970-01-01 00:01:46.300000 (106.3) -User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' +Timestamp: 1970-01-01T00:01:46.300000 (%s) +User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' % ( + utils.Timestamp(106.3).internal) self.assertEquals(out.getvalue().strip(), exp_out) diff --git a/test/unit/common/test_constraints.py b/test/unit/common/test_constraints.py index c6539cf3d7..753f858de7 100644 --- a/test/unit/common/test_constraints.py +++ b/test/unit/common/test_constraints.py @@ -16,6 +16,7 @@ import unittest import mock import tempfile +import time from test import safe_repr from test.unit import MockTrue @@ -182,6 +183,20 @@ class TestConstraints(unittest.TestCase): self.assertFalse(constraints.check_float('')) self.assertTrue(constraints.check_float('0')) + def test_valid_timestamp(self): + self.assertRaises(HTTPException, + constraints.valid_timestamp, + Request.blank('/')) + self.assertRaises(HTTPException, + constraints.valid_timestamp, + Request.blank('/', headers={ + 'X-Timestamp': 'asdf'})) + timestamp = utils.Timestamp(time.time()) + req = Request.blank('/', headers={'X-Timestamp': timestamp.internal}) + self.assertEqual(timestamp, constraints.valid_timestamp(req)) + req = Request.blank('/', headers={'X-Timestamp': timestamp.normal}) + self.assertEqual(timestamp, constraints.valid_timestamp(req)) + def test_check_utf8(self): unicode_sample = u'\uc77c\uc601' valid_utf8_str = unicode_sample.encode('utf-8') diff --git a/test/unit/common/test_db.py b/test/unit/common/test_db.py index 461e869f6b..e0c164de9b 100644 --- a/test/unit/common/test_db.py +++ b/test/unit/common/test_db.py @@ -35,7 +35,7 @@ import swift.common.db from swift.common.db import chexor, dict_factory, get_db_connection, \ DatabaseBroker, DatabaseConnectionError, DatabaseAlreadyExists, \ GreenDBConnection, PICKLE_PROTOCOL -from swift.common.utils import normalize_timestamp, mkdirs, json +from swift.common.utils import normalize_timestamp, mkdirs, json, Timestamp from swift.common.exceptions import LockTimeout from test.unit import with_tempdir @@ -217,7 +217,7 @@ class ExampleBroker(DatabaseBroker): INSERT INTO test_stat ( created_at, put_timestamp, status_changed_at) VALUES (?, ?, ?); - """, (normalize_timestamp(time.time()), put_timestamp, + """, (Timestamp(time.time()).internal, put_timestamp, put_timestamp)) def merge_items(self, item_list): @@ -268,8 +268,8 @@ class ExampleBroker(DatabaseBroker): def _is_deleted(self, conn): info = conn.execute('SELECT * FROM test_stat').fetchone() return (info['test_count'] in (None, '', 0, '0')) and \ - (normalize_timestamp(info['delete_timestamp']) > - normalize_timestamp(info['put_timestamp'])) + (Timestamp(info['delete_timestamp']) > + Timestamp(info['put_timestamp'])) class TestExampleBroker(unittest.TestCase): @@ -282,7 +282,7 @@ class TestExampleBroker(unittest.TestCase): policy = 0 def test_merge_timestamps_simple_delete(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) put_timestamp = ts.next() broker = self.broker_class(':memory:', account='a', container='c') @@ -302,8 +302,7 @@ class TestExampleBroker(unittest.TestCase): self.assertEqual(info['created_at'], created_at) self.assertEqual(info['put_timestamp'], put_timestamp) self.assertEqual(info['delete_timestamp'], delete_timestamp) - self.assert_(info['status_changed_at'] > - normalize_timestamp(put_timestamp)) + self.assert_(info['status_changed_at'] > Timestamp(put_timestamp)) def put_item(self, broker, timestamp): broker.put_test('test', timestamp) @@ -312,7 +311,7 @@ class TestExampleBroker(unittest.TestCase): broker.delete_test('test', timestamp) def test_merge_timestamps_delete_with_objects(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) put_timestamp = ts.next() broker = self.broker_class(':memory:', account='a', container='c') @@ -345,7 +344,7 @@ class TestExampleBroker(unittest.TestCase): self.assert_(broker.is_deleted()) def test_merge_timestamps_simple_recreate(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) put_timestamp = ts.next() broker = self.broker_class(':memory:', account='a', container='c') @@ -361,7 +360,7 @@ class TestExampleBroker(unittest.TestCase): self.assertEqual(info['delete_timestamp'], delete_timestamp) orig_status_changed_at = info['status_changed_at'] self.assert_(orig_status_changed_at > - normalize_timestamp(virgin_status_changed_at)) + Timestamp(virgin_status_changed_at)) # recreate recreate_timestamp = ts.next() status_changed_at = time.time() @@ -375,7 +374,7 @@ class TestExampleBroker(unittest.TestCase): self.assert_(info['status_changed_at'], status_changed_at) def test_merge_timestamps_recreate_with_objects(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) put_timestamp = ts.next() broker = self.broker_class(':memory:', account='a', container='c') @@ -390,8 +389,8 @@ class TestExampleBroker(unittest.TestCase): self.assertEqual(info['put_timestamp'], put_timestamp) self.assertEqual(info['delete_timestamp'], delete_timestamp) orig_status_changed_at = info['status_changed_at'] - self.assert_(normalize_timestamp(orig_status_changed_at) >= - normalize_timestamp(put_timestamp)) + self.assert_(Timestamp(orig_status_changed_at) >= + Timestamp(put_timestamp)) # add object self.put_item(broker, ts.next()) count_key = '%s_count' % broker.db_contains_type @@ -411,7 +410,7 @@ class TestExampleBroker(unittest.TestCase): self.assertFalse(broker.is_deleted()) def test_merge_timestamps_update_put_no_status_change(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) put_timestamp = ts.next() broker = self.broker_class(':memory:', account='a', container='c') @@ -426,7 +425,7 @@ class TestExampleBroker(unittest.TestCase): self.assertEqual(orig_status_changed_at, info['status_changed_at']) def test_merge_timestamps_update_delete_no_status_change(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) put_timestamp = ts.next() broker = self.broker_class(':memory:', account='a', container='c') @@ -458,15 +457,15 @@ class TestExampleBroker(unittest.TestCase): broker = self.broker_class(':memory:', account='test', container='c') created_at = time.time() with patch('swift.common.db.time.time', new=lambda: created_at): - broker.initialize(normalize_timestamp(1), + broker.initialize(Timestamp(1).internal, storage_policy_index=int(self.policy)) info = broker.get_info() count_key = '%s_count' % broker.db_contains_type expected = { count_key: 0, - 'created_at': normalize_timestamp(created_at), - 'put_timestamp': normalize_timestamp(1), - 'status_changed_at': normalize_timestamp(1), + 'created_at': Timestamp(created_at).internal, + 'put_timestamp': Timestamp(1).internal, + 'status_changed_at': Timestamp(1).internal, 'delete_timestamp': '0', } for k, v in expected.items(): @@ -476,14 +475,14 @@ class TestExampleBroker(unittest.TestCase): def test_get_raw_metadata(self): broker = self.broker_class(':memory:', account='test', container='c') - broker.initialize(normalize_timestamp(0), + broker.initialize(Timestamp(0).internal, storage_policy_index=int(self.policy)) self.assertEqual(broker.metadata, {}) self.assertEqual(broker.get_raw_metadata(), '') key = u'test\u062a'.encode('utf-8') value = u'value\u062a' metadata = { - key: [value, normalize_timestamp(1)] + key: [value, Timestamp(1).internal] } broker.update_metadata(metadata) self.assertEqual(broker.metadata, metadata) @@ -491,7 +490,7 @@ class TestExampleBroker(unittest.TestCase): json.dumps(metadata)) def test_put_timestamp(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) broker = self.broker_class(':memory:', account='a', container='c') orig_put_timestamp = ts.next() @@ -514,7 +513,7 @@ class TestExampleBroker(unittest.TestCase): newer_put_timestamp) def test_status_changed_at(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) broker = self.broker_class(':memory:', account='test', container='c') put_timestamp = ts.next() @@ -525,7 +524,7 @@ class TestExampleBroker(unittest.TestCase): self.assertEquals(broker.get_info()['status_changed_at'], put_timestamp) self.assertEquals(broker.get_info()['created_at'], - normalize_timestamp(created_at)) + Timestamp(created_at).internal) status_changed_at = ts.next() broker.update_status_changed_at(status_changed_at) self.assertEqual(broker.get_info()['status_changed_at'], @@ -543,7 +542,7 @@ class TestExampleBroker(unittest.TestCase): def test_get_syncs(self): broker = self.broker_class(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp(time.time()), + broker.initialize(Timestamp(time.time()).internal, storage_policy_index=int(self.policy)) self.assertEqual([], broker.get_syncs()) broker.merge_syncs([{'sync_point': 1, 'remote_id': 'remote1'}]) @@ -557,7 +556,7 @@ class TestExampleBroker(unittest.TestCase): @with_tempdir def test_commit_pending(self, tempdir): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) broker = self.broker_class(os.path.join(tempdir, 'test.db'), account='a', container='c') diff --git a/test/unit/common/test_direct_client.py b/test/unit/common/test_direct_client.py index bd0e3d885c..90b6c1754f 100644 --- a/test/unit/common/test_direct_client.py +++ b/test/unit/common/test_direct_client.py @@ -25,7 +25,7 @@ import mock from swift.common import direct_client from swift.common.exceptions import ClientException -from swift.common.utils import json, normalize_timestamp +from swift.common.utils import json, Timestamp from swift.common.swob import HeaderKeyDict, RESPONSE_REASONS from swift.common.storage_policy import POLICY_INDEX, POLICIES @@ -116,9 +116,9 @@ class TestDirectClient(unittest.TestCase): now = time.time() headers = direct_client.gen_headers(add_ts=True) self.assertEqual(headers['user-agent'], stub_user_agent) - self.assert_(now - 1 < float(headers['x-timestamp']) < now + 1) + self.assert_(now - 1 < Timestamp(headers['x-timestamp']) < now + 1) self.assertEqual(headers['x-timestamp'], - normalize_timestamp(float(headers['x-timestamp']))) + Timestamp(headers['x-timestamp']).internal) self.assertEqual(2, len(headers)) headers = direct_client.gen_headers(hdrs_in={'foo-bar': '47'}) @@ -142,9 +142,9 @@ class TestDirectClient(unittest.TestCase): expected_header_count += 1 self.assertEqual( headers['x-timestamp'], - normalize_timestamp(float(headers['x-timestamp']))) + Timestamp(headers['x-timestamp']).internal) self.assert_( - now - 1 < float(headers['x-timestamp']) < now + 1) + now - 1 < Timestamp(headers['x-timestamp']) < now + 1) self.assertEqual(expected_header_count, len(headers)) def test_direct_get_account(self): @@ -275,7 +275,7 @@ class TestDirectClient(unittest.TestCase): self.assert_('HEAD' in str(err)) def test_direct_head_container_deleted(self): - important_timestamp = normalize_timestamp(time.time()) + important_timestamp = Timestamp(time.time()).internal headers = HeaderKeyDict({'X-Backend-Important-Timestamp': important_timestamp}) @@ -432,7 +432,7 @@ class TestDirectClient(unittest.TestCase): self.assert_('HEAD' in str(err)) def test_direct_head_object_not_found(self): - important_timestamp = normalize_timestamp(time.time()) + important_timestamp = Timestamp(time.time()).internal stub_headers = {'X-Backend-Important-Timestamp': important_timestamp} with mocked_http_conn(404, headers=stub_headers) as conn: try: diff --git a/test/unit/common/test_swob.py b/test/unit/common/test_swob.py index 5ced5b68d9..7a070b1883 100644 --- a/test/unit/common/test_swob.py +++ b/test/unit/common/test_swob.py @@ -23,6 +23,7 @@ from StringIO import StringIO from urllib import quote import swift.common.swob +from swift.common import utils, exceptions class TestHeaderEnvironProxy(unittest.TestCase): @@ -464,6 +465,25 @@ class TestRequest(unittest.TestCase): self.assertEquals(req.params['a'], 'b') self.assertEquals(req.params['c'], 'd') + def test_timestamp_missing(self): + req = swift.common.swob.Request.blank('/') + self.assertRaises(exceptions.InvalidTimestamp, + getattr, req, 'timestamp') + + def test_timestamp_invalid(self): + req = swift.common.swob.Request.blank( + '/', headers={'X-Timestamp': 'asdf'}) + self.assertRaises(exceptions.InvalidTimestamp, + getattr, req, 'timestamp') + + def test_timestamp(self): + req = swift.common.swob.Request.blank( + '/', headers={'X-Timestamp': '1402447134.13507_00000001'}) + expected = utils.Timestamp('1402447134.13507', offset=1) + self.assertEqual(req.timestamp, expected) + self.assertEqual(req.timestamp.normal, expected.normal) + self.assertEqual(req.timestamp.internal, expected.internal) + def test_path(self): req = swift.common.swob.Request.blank('/hi?a=b&c=d') self.assertEquals(req.path, '/hi') diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index f0b90a8f3d..4d874c4da8 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -148,6 +148,562 @@ def reset_loggers(): delattr(utils.get_logger, 'console_handler4logger') +class TestTimestamp(unittest.TestCase): + """Tests for swift.common.utils.Timestamp""" + + def test_invalid_input(self): + self.assertRaises(ValueError, utils.Timestamp, time.time(), offset=-1) + + def test_invalid_string_conversion(self): + t = utils.Timestamp(time.time()) + self.assertRaises(TypeError, str, t) + + def test_normal_format_no_offset(self): + expected = '1402436408.91203' + test_values = ( + '1402436408.91203', + '1402436408.91203_00000000', + '1402436408.912030000', + '1402436408.912030000_0000000000000', + '000001402436408.912030000', + '000001402436408.912030000_0000000000', + 1402436408.91203, + 1402436408.912029, + 1402436408.9120300000000000, + 1402436408.91202999999999999, + utils.Timestamp(1402436408.91203), + utils.Timestamp(1402436408.91203, offset=0), + utils.Timestamp(1402436408.912029), + utils.Timestamp(1402436408.912029, offset=0), + utils.Timestamp('1402436408.91203'), + utils.Timestamp('1402436408.91203', offset=0), + utils.Timestamp('1402436408.91203_00000000'), + utils.Timestamp('1402436408.91203_00000000', offset=0), + ) + for value in test_values: + timestamp = utils.Timestamp(value) + self.assertEqual(timestamp.normal, expected) + # timestamp instance can also compare to string or float + self.assertEqual(timestamp, expected) + self.assertEqual(timestamp, float(expected)) + self.assertEqual(timestamp, utils.normalize_timestamp(expected)) + + def test_isoformat(self): + expected = '2014-06-10T22:47:32.054580' + test_values = ( + '1402440452.05458', + '1402440452.054579', + '1402440452.05458_00000000', + '1402440452.054579_00000000', + '1402440452.054580000', + '1402440452.054579999', + '1402440452.054580000_0000000000000', + '1402440452.054579999_0000ff00', + '000001402440452.054580000', + '000001402440452.0545799', + '000001402440452.054580000_0000000000', + '000001402440452.054579999999_00000fffff', + 1402440452.05458, + 1402440452.054579, + 1402440452.0545800000000000, + 1402440452.054579999, + utils.Timestamp(1402440452.05458), + utils.Timestamp(1402440452.0545799), + utils.Timestamp(1402440452.05458, offset=0), + utils.Timestamp(1402440452.05457999999, offset=0), + utils.Timestamp(1402440452.05458, offset=100), + utils.Timestamp(1402440452.054579, offset=100), + utils.Timestamp('1402440452.05458'), + utils.Timestamp('1402440452.054579999'), + utils.Timestamp('1402440452.05458', offset=0), + utils.Timestamp('1402440452.054579', offset=0), + utils.Timestamp('1402440452.05458', offset=300), + utils.Timestamp('1402440452.05457999', offset=300), + utils.Timestamp('1402440452.05458_00000000'), + utils.Timestamp('1402440452.05457999_00000000'), + utils.Timestamp('1402440452.05458_00000000', offset=0), + utils.Timestamp('1402440452.05457999_00000aaa', offset=0), + utils.Timestamp('1402440452.05458_00000000', offset=400), + utils.Timestamp('1402440452.054579_0a', offset=400), + ) + for value in test_values: + self.assertEqual(utils.Timestamp(value).isoformat, expected) + expected = '1970-01-01T00:00:00.000000' + test_values = ( + '0', + '0000000000.00000', + '0000000000.00000_ffffffffffff', + 0, + 0.0, + ) + for value in test_values: + self.assertEqual(utils.Timestamp(value).isoformat, expected) + + def test_not_equal(self): + ts = '1402436408.91203_0000000000000001' + test_values = ( + utils.Timestamp('1402436408.91203_0000000000000002'), + utils.Timestamp('1402436408.91203'), + utils.Timestamp(1402436408.91203), + utils.Timestamp(1402436408.91204), + utils.Timestamp(1402436408.91203, offset=0), + utils.Timestamp(1402436408.91203, offset=2), + ) + for value in test_values: + self.assertTrue(value != ts) + + def test_no_force_internal_no_offset(self): + """Test that internal is the same as normal with no offset""" + with mock.patch('swift.common.utils.FORCE_INTERNAL', new=False): + self.assertEqual(utils.Timestamp(0).internal, '0000000000.00000') + self.assertEqual(utils.Timestamp(1402437380.58186).internal, + '1402437380.58186') + self.assertEqual(utils.Timestamp(1402437380.581859).internal, + '1402437380.58186') + self.assertEqual(utils.Timestamp(0).internal, + utils.normalize_timestamp(0)) + + def test_no_force_internal_with_offset(self): + """Test that internal always includes the offset if significant""" + with mock.patch('swift.common.utils.FORCE_INTERNAL', new=False): + self.assertEqual(utils.Timestamp(0, offset=1).internal, + '0000000000.00000_0000000000000001') + self.assertEqual( + utils.Timestamp(1402437380.58186, offset=16).internal, + '1402437380.58186_0000000000000010') + self.assertEqual( + utils.Timestamp(1402437380.581859, offset=240).internal, + '1402437380.58186_00000000000000f0') + self.assertEqual( + utils.Timestamp('1402437380.581859_00000001', + offset=240).internal, + '1402437380.58186_00000000000000f1') + + def test_force_internal(self): + """Test that internal always includes the offset if forced""" + with mock.patch('swift.common.utils.FORCE_INTERNAL', new=True): + self.assertEqual(utils.Timestamp(0).internal, + '0000000000.00000_0000000000000000') + self.assertEqual(utils.Timestamp(1402437380.58186).internal, + '1402437380.58186_0000000000000000') + self.assertEqual(utils.Timestamp(1402437380.581859).internal, + '1402437380.58186_0000000000000000') + self.assertEqual(utils.Timestamp(0, offset=1).internal, + '0000000000.00000_0000000000000001') + self.assertEqual( + utils.Timestamp(1402437380.58186, offset=16).internal, + '1402437380.58186_0000000000000010') + self.assertEqual( + utils.Timestamp(1402437380.581859, offset=16).internal, + '1402437380.58186_0000000000000010') + + def test_internal_format_no_offset(self): + expected = '1402436408.91203_0000000000000000' + test_values = ( + '1402436408.91203', + '1402436408.91203_00000000', + '1402436408.912030000', + '1402436408.912030000_0000000000000', + '000001402436408.912030000', + '000001402436408.912030000_0000000000', + 1402436408.91203, + 1402436408.9120300000000000, + 1402436408.912029, + 1402436408.912029999999999999, + utils.Timestamp(1402436408.91203), + utils.Timestamp(1402436408.91203, offset=0), + utils.Timestamp(1402436408.912029), + utils.Timestamp(1402436408.91202999999999999, offset=0), + utils.Timestamp('1402436408.91203'), + utils.Timestamp('1402436408.91203', offset=0), + utils.Timestamp('1402436408.912029'), + utils.Timestamp('1402436408.912029', offset=0), + utils.Timestamp('1402436408.912029999999999'), + utils.Timestamp('1402436408.912029999999999', offset=0), + ) + for value in test_values: + # timestamp instance is always equivalent + self.assertEqual(utils.Timestamp(value), expected) + if utils.FORCE_INTERNAL: + # the FORCE_INTERNAL flag makes the internal format always + # include the offset portion of the timestamp even when it's + # not significant and would be bad during upgrades + self.assertEqual(utils.Timestamp(value).internal, expected) + else: + # unless we FORCE_INTERNAL, when there's no offset the + # internal format is equivalent to the normalized format + self.assertEqual(utils.Timestamp(value).internal, + '1402436408.91203') + + def test_internal_format_with_offset(self): + expected = '1402436408.91203_00000000000000f0' + test_values = ( + '1402436408.91203_000000f0', + '1402436408.912030000_0000000000f0', + '1402436408.912029_000000f0', + '1402436408.91202999999_0000000000f0', + '000001402436408.912030000_000000000f0', + '000001402436408.9120299999_000000000f0', + utils.Timestamp(1402436408.91203, offset=240), + utils.Timestamp(1402436408.912029, offset=240), + utils.Timestamp('1402436408.91203', offset=240), + utils.Timestamp('1402436408.91203_00000000', offset=240), + utils.Timestamp('1402436408.91203_0000000f', offset=225), + utils.Timestamp('1402436408.9120299999', offset=240), + utils.Timestamp('1402436408.9120299999_00000000', offset=240), + utils.Timestamp('1402436408.9120299999_00000010', offset=224), + ) + for value in test_values: + timestamp = utils.Timestamp(value) + self.assertEqual(timestamp.internal, expected) + # can compare with offset if the string is internalized + self.assertEqual(timestamp, expected) + # if comparison value only includes the normalized portion and the + # timestamp includes an offset, it is considered greater + normal = utils.Timestamp(expected).normal + self.assertTrue(timestamp > normal, + '%r is not bigger than %r given %r' % ( + timestamp, normal, value)) + self.assertTrue(timestamp > float(normal), + '%r is not bigger than %f given %r' % ( + timestamp, float(normal), value)) + + def test_int(self): + expected = 1402437965 + test_values = ( + '1402437965.91203', + '1402437965.91203_00000000', + '1402437965.912030000', + '1402437965.912030000_0000000000000', + '000001402437965.912030000', + '000001402437965.912030000_0000000000', + 1402437965.91203, + 1402437965.9120300000000000, + 1402437965.912029, + 1402437965.912029999999999999, + utils.Timestamp(1402437965.91203), + utils.Timestamp(1402437965.91203, offset=0), + utils.Timestamp(1402437965.91203, offset=500), + utils.Timestamp(1402437965.912029), + utils.Timestamp(1402437965.91202999999999999, offset=0), + utils.Timestamp(1402437965.91202999999999999, offset=300), + utils.Timestamp('1402437965.91203'), + utils.Timestamp('1402437965.91203', offset=0), + utils.Timestamp('1402437965.91203', offset=400), + utils.Timestamp('1402437965.912029'), + utils.Timestamp('1402437965.912029', offset=0), + utils.Timestamp('1402437965.912029', offset=200), + utils.Timestamp('1402437965.912029999999999'), + utils.Timestamp('1402437965.912029999999999', offset=0), + utils.Timestamp('1402437965.912029999999999', offset=100), + ) + for value in test_values: + timestamp = utils.Timestamp(value) + self.assertEqual(int(timestamp), expected) + self.assertTrue(timestamp > expected) + + def test_float(self): + expected = 1402438115.91203 + test_values = ( + '1402438115.91203', + '1402438115.91203_00000000', + '1402438115.912030000', + '1402438115.912030000_0000000000000', + '000001402438115.912030000', + '000001402438115.912030000_0000000000', + 1402438115.91203, + 1402438115.9120300000000000, + 1402438115.912029, + 1402438115.912029999999999999, + utils.Timestamp(1402438115.91203), + utils.Timestamp(1402438115.91203, offset=0), + utils.Timestamp(1402438115.91203, offset=500), + utils.Timestamp(1402438115.912029), + utils.Timestamp(1402438115.91202999999999999, offset=0), + utils.Timestamp(1402438115.91202999999999999, offset=300), + utils.Timestamp('1402438115.91203'), + utils.Timestamp('1402438115.91203', offset=0), + utils.Timestamp('1402438115.91203', offset=400), + utils.Timestamp('1402438115.912029'), + utils.Timestamp('1402438115.912029', offset=0), + utils.Timestamp('1402438115.912029', offset=200), + utils.Timestamp('1402438115.912029999999999'), + utils.Timestamp('1402438115.912029999999999', offset=0), + utils.Timestamp('1402438115.912029999999999', offset=100), + ) + tolerance = 0.00001 + minimum = expected - tolerance + maximum = expected + tolerance + for value in test_values: + timestamp = utils.Timestamp(value) + self.assertTrue(float(timestamp) > minimum, + '%f is not bigger than %f given %r' % ( + timestamp, minimum, value)) + self.assertTrue(float(timestamp) < maximum, + '%f is not smaller than %f given %r' % ( + timestamp, maximum, value)) + # direct comparision of timestamp works too + self.assertTrue(timestamp > minimum, + '%s is not bigger than %f given %r' % ( + timestamp.normal, minimum, value)) + self.assertTrue(timestamp < maximum, + '%s is not smaller than %f given %r' % ( + timestamp.normal, maximum, value)) + # ... even against strings + self.assertTrue(timestamp > '%f' % minimum, + '%s is not bigger than %s given %r' % ( + timestamp.normal, minimum, value)) + self.assertTrue(timestamp < '%f' % maximum, + '%s is not smaller than %s given %r' % ( + timestamp.normal, maximum, value)) + + def test_false(self): + self.assertFalse(utils.Timestamp(0)) + self.assertFalse(utils.Timestamp(0, offset=0)) + self.assertFalse(utils.Timestamp('0')) + self.assertFalse(utils.Timestamp('0', offset=0)) + self.assertFalse(utils.Timestamp(0.0)) + self.assertFalse(utils.Timestamp(0.0, offset=0)) + self.assertFalse(utils.Timestamp('0.0')) + self.assertFalse(utils.Timestamp('0.0', offset=0)) + self.assertFalse(utils.Timestamp(00000000.00000000)) + self.assertFalse(utils.Timestamp(00000000.00000000, offset=0)) + self.assertFalse(utils.Timestamp('00000000.00000000')) + self.assertFalse(utils.Timestamp('00000000.00000000', offset=0)) + + def test_true(self): + self.assertTrue(utils.Timestamp(1)) + self.assertTrue(utils.Timestamp(1, offset=1)) + self.assertTrue(utils.Timestamp(0, offset=1)) + self.assertTrue(utils.Timestamp('1')) + self.assertTrue(utils.Timestamp('1', offset=1)) + self.assertTrue(utils.Timestamp('0', offset=1)) + self.assertTrue(utils.Timestamp(1.1)) + self.assertTrue(utils.Timestamp(1.1, offset=1)) + self.assertTrue(utils.Timestamp(0.0, offset=1)) + self.assertTrue(utils.Timestamp('1.1')) + self.assertTrue(utils.Timestamp('1.1', offset=1)) + self.assertTrue(utils.Timestamp('0.0', offset=1)) + self.assertTrue(utils.Timestamp(11111111.11111111)) + self.assertTrue(utils.Timestamp(11111111.11111111, offset=1)) + self.assertTrue(utils.Timestamp(00000000.00000000, offset=1)) + self.assertTrue(utils.Timestamp('11111111.11111111')) + self.assertTrue(utils.Timestamp('11111111.11111111', offset=1)) + self.assertTrue(utils.Timestamp('00000000.00000000', offset=1)) + + def test_greater_no_offset(self): + now = time.time() + older = now - 1 + timestamp = utils.Timestamp(now) + test_values = ( + 0, '0', 0.0, '0.0', '0000.0000', '000.000_000', + 1, '1', 1.1, '1.1', '1111.1111', '111.111_111', + 1402443112.213252, '1402443112.213252', '1402443112.213252_ffff', + older, '%f' % older, '%f_0000ffff' % older, + ) + for value in test_values: + other = utils.Timestamp(value) + self.assertNotEqual(timestamp, other) # sanity + self.assertTrue(timestamp > value, + '%r is not greater than %r given %r' % ( + timestamp, value, value)) + self.assertTrue(timestamp > other, + '%r is not greater than %r given %r' % ( + timestamp, other, value)) + self.assertTrue(timestamp > other.normal, + '%r is not greater than %r given %r' % ( + timestamp, other.normal, value)) + self.assertTrue(timestamp > other.internal, + '%r is not greater than %r given %r' % ( + timestamp, other.internal, value)) + self.assertTrue(timestamp > float(other), + '%r is not greater than %r given %r' % ( + timestamp, float(other), value)) + self.assertTrue(timestamp > int(other), + '%r is not greater than %r given %r' % ( + timestamp, int(other), value)) + + def test_greater_with_offset(self): + now = time.time() + older = now - 1 + test_values = ( + 0, '0', 0.0, '0.0', '0000.0000', '000.000_000', + 1, '1', 1.1, '1.1', '1111.1111', '111.111_111', + 1402443346.935174, '1402443346.93517', '1402443346.935169_ffff', + older, '%f' % older, '%f_0000ffff' % older, + now, '%f' % now, '%f_00000000' % now, + ) + for offset in range(1, 1000, 100): + timestamp = utils.Timestamp(now, offset=offset) + for value in test_values: + other = utils.Timestamp(value) + self.assertNotEqual(timestamp, other) # sanity + self.assertTrue(timestamp > value, + '%r is not greater than %r given %r' % ( + timestamp, value, value)) + self.assertTrue(timestamp > other, + '%r is not greater than %r given %r' % ( + timestamp, other, value)) + self.assertTrue(timestamp > other.normal, + '%r is not greater than %r given %r' % ( + timestamp, other.normal, value)) + self.assertTrue(timestamp > other.internal, + '%r is not greater than %r given %r' % ( + timestamp, other.internal, value)) + self.assertTrue(timestamp > float(other), + '%r is not greater than %r given %r' % ( + timestamp, float(other), value)) + self.assertTrue(timestamp > int(other), + '%r is not greater than %r given %r' % ( + timestamp, int(other), value)) + + def test_smaller_no_offset(self): + now = time.time() + newer = now + 1 + timestamp = utils.Timestamp(now) + test_values = ( + 9999999999.99999, '9999999999.99999', '9999999999.99999_ffff', + newer, '%f' % newer, '%f_0000ffff' % newer, + ) + for value in test_values: + other = utils.Timestamp(value) + self.assertNotEqual(timestamp, other) # sanity + self.assertTrue(timestamp < value, + '%r is not smaller than %r given %r' % ( + timestamp, value, value)) + self.assertTrue(timestamp < other, + '%r is not smaller than %r given %r' % ( + timestamp, other, value)) + self.assertTrue(timestamp < other.normal, + '%r is not smaller than %r given %r' % ( + timestamp, other.normal, value)) + self.assertTrue(timestamp < other.internal, + '%r is not smaller than %r given %r' % ( + timestamp, other.internal, value)) + self.assertTrue(timestamp < float(other), + '%r is not smaller than %r given %r' % ( + timestamp, float(other), value)) + self.assertTrue(timestamp < int(other), + '%r is not smaller than %r given %r' % ( + timestamp, int(other), value)) + + def test_smaller_with_offset(self): + now = time.time() + newer = now + 1 + test_values = ( + 9999999999.99999, '9999999999.99999', '9999999999.99999_ffff', + newer, '%f' % newer, '%f_0000ffff' % newer, + ) + for offset in range(1, 1000, 100): + timestamp = utils.Timestamp(now, offset=offset) + for value in test_values: + other = utils.Timestamp(value) + self.assertNotEqual(timestamp, other) # sanity + self.assertTrue(timestamp < value, + '%r is not smaller than %r given %r' % ( + timestamp, value, value)) + self.assertTrue(timestamp < other, + '%r is not smaller than %r given %r' % ( + timestamp, other, value)) + self.assertTrue(timestamp < other.normal, + '%r is not smaller than %r given %r' % ( + timestamp, other.normal, value)) + self.assertTrue(timestamp < other.internal, + '%r is not smaller than %r given %r' % ( + timestamp, other.internal, value)) + self.assertTrue(timestamp < float(other), + '%r is not smaller than %r given %r' % ( + timestamp, float(other), value)) + self.assertTrue(timestamp < int(other), + '%r is not smaller than %r given %r' % ( + timestamp, int(other), value)) + + def test_ordering(self): + given = [ + '1402444820.62590_000000000000000a', + '1402444820.62589_0000000000000001', + '1402444821.52589_0000000000000004', + '1402444920.62589_0000000000000004', + '1402444821.62589_000000000000000a', + '1402444821.72589_000000000000000a', + '1402444920.62589_0000000000000002', + '1402444820.62589_0000000000000002', + '1402444820.62589_000000000000000a', + '1402444820.62590_0000000000000004', + '1402444920.62589_000000000000000a', + '1402444820.62590_0000000000000002', + '1402444821.52589_0000000000000002', + '1402444821.52589_0000000000000000', + '1402444920.62589', + '1402444821.62589_0000000000000004', + '1402444821.72589_0000000000000001', + '1402444820.62590', + '1402444820.62590_0000000000000001', + '1402444820.62589_0000000000000004', + '1402444821.72589_0000000000000000', + '1402444821.52589_000000000000000a', + '1402444821.72589_0000000000000004', + '1402444821.62589', + '1402444821.52589_0000000000000001', + '1402444821.62589_0000000000000001', + '1402444821.62589_0000000000000002', + '1402444821.72589_0000000000000002', + '1402444820.62589', + '1402444920.62589_0000000000000001'] + expected = [ + '1402444820.62589', + '1402444820.62589_0000000000000001', + '1402444820.62589_0000000000000002', + '1402444820.62589_0000000000000004', + '1402444820.62589_000000000000000a', + '1402444820.62590', + '1402444820.62590_0000000000000001', + '1402444820.62590_0000000000000002', + '1402444820.62590_0000000000000004', + '1402444820.62590_000000000000000a', + '1402444821.52589', + '1402444821.52589_0000000000000001', + '1402444821.52589_0000000000000002', + '1402444821.52589_0000000000000004', + '1402444821.52589_000000000000000a', + '1402444821.62589', + '1402444821.62589_0000000000000001', + '1402444821.62589_0000000000000002', + '1402444821.62589_0000000000000004', + '1402444821.62589_000000000000000a', + '1402444821.72589', + '1402444821.72589_0000000000000001', + '1402444821.72589_0000000000000002', + '1402444821.72589_0000000000000004', + '1402444821.72589_000000000000000a', + '1402444920.62589', + '1402444920.62589_0000000000000001', + '1402444920.62589_0000000000000002', + '1402444920.62589_0000000000000004', + '1402444920.62589_000000000000000a', + ] + # less visual version + """ + now = time.time() + given = [ + utils.Timestamp(now + i, offset=offset).internal + for i in (0, 0.00001, 0.9, 1.0, 1.1, 100.0) + for offset in (0, 1, 2, 4, 10) + ] + expected = [t for t in given] + random.shuffle(given) + """ + self.assertEqual(len(given), len(expected)) # sanity + timestamps = [utils.Timestamp(t) for t in given] + # our expected values don't include insignificant offsets + with mock.patch('swift.common.utils.FORCE_INTERNAL', new=False): + self.assertEqual( + [t.internal for t in sorted(timestamps)], expected) + # string sorting works as well + self.assertEqual( + sorted([t.internal for t in timestamps]), expected) + + class TestUtils(unittest.TestCase): """Tests for swift.common.utils """ diff --git a/test/unit/container/test_backend.py b/test/unit/container/test_backend.py index 41b0c3f10c..c2d7208ba1 100644 --- a/test/unit/container/test_backend.py +++ b/test/unit/container/test_backend.py @@ -28,7 +28,7 @@ import sqlite3 import pickle from swift.container.backend import ContainerBroker -from swift.common.utils import normalize_timestamp +from swift.common.utils import Timestamp from swift.common.storage_policy import POLICIES import mock @@ -44,7 +44,7 @@ class TestContainerBroker(unittest.TestCase): # Test ContainerBroker.__init__ broker = ContainerBroker(':memory:', account='a', container='c') self.assertEqual(broker.db_file, ':memory:') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) with broker.get() as conn: curs = conn.cursor() curs.execute('SELECT 1') @@ -52,11 +52,11 @@ class TestContainerBroker(unittest.TestCase): @patch_policies def test_storage_policy_property(self): - ts = itertools.count(1) + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) for policy in POLICIES: broker = ContainerBroker(':memory:', account='a', container='policy_%s' % policy.name) - broker.initialize(normalize_timestamp(ts.next()), policy.idx) + broker.initialize(ts.next(), policy.idx) with broker.get() as conn: try: conn.execute('''SELECT storage_policy_index @@ -78,7 +78,7 @@ class TestContainerBroker(unittest.TestCase): # unhandled exception first_conn = None broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) with broker.get() as conn: first_conn = conn try: @@ -92,20 +92,20 @@ class TestContainerBroker(unittest.TestCase): def test_empty(self): # Test ContainerBroker.empty broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) self.assert_(broker.empty()) - broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', + broker.put_object('o', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') self.assert_(not broker.empty()) sleep(.00001) - broker.delete_object('o', normalize_timestamp(time())) + broker.delete_object('o', Timestamp(time()).internal) self.assert_(broker.empty()) def test_reclaim(self): broker = ContainerBroker(':memory:', account='test_account', container='test_container') - broker.initialize(normalize_timestamp('1'), 0) - broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', + broker.initialize(Timestamp('1').internal, 0) + broker.put_object('o', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') with broker.get() as conn: self.assertEquals(conn.execute( @@ -114,7 +114,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(conn.execute( "SELECT count(*) FROM object " "WHERE deleted = 1").fetchone()[0], 0) - broker.reclaim(normalize_timestamp(time() - 999), time()) + broker.reclaim(Timestamp(time() - 999).internal, time()) with broker.get() as conn: self.assertEquals(conn.execute( "SELECT count(*) FROM object " @@ -123,7 +123,7 @@ class TestContainerBroker(unittest.TestCase): "SELECT count(*) FROM object " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) - broker.delete_object('o', normalize_timestamp(time())) + broker.delete_object('o', Timestamp(time()).internal) with broker.get() as conn: self.assertEquals(conn.execute( "SELECT count(*) FROM object " @@ -131,7 +131,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(conn.execute( "SELECT count(*) FROM object " "WHERE deleted = 1").fetchone()[0], 1) - broker.reclaim(normalize_timestamp(time() - 999), time()) + broker.reclaim(Timestamp(time() - 999).internal, time()) with broker.get() as conn: self.assertEquals(conn.execute( "SELECT count(*) FROM object " @@ -140,7 +140,7 @@ class TestContainerBroker(unittest.TestCase): "SELECT count(*) FROM object " "WHERE deleted = 1").fetchone()[0], 1) sleep(.00001) - broker.reclaim(normalize_timestamp(time()), time()) + broker.reclaim(Timestamp(time()).internal, time()) with broker.get() as conn: self.assertEquals(conn.execute( "SELECT count(*) FROM object " @@ -149,21 +149,21 @@ class TestContainerBroker(unittest.TestCase): "SELECT count(*) FROM object " "WHERE deleted = 1").fetchone()[0], 0) # Test the return values of reclaim() - broker.put_object('w', normalize_timestamp(time()), 0, 'text/plain', + broker.put_object('w', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('x', normalize_timestamp(time()), 0, 'text/plain', + broker.put_object('x', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('y', normalize_timestamp(time()), 0, 'text/plain', + broker.put_object('y', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('z', normalize_timestamp(time()), 0, 'text/plain', + broker.put_object('z', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') # Test before deletion - broker.reclaim(normalize_timestamp(time()), time()) - broker.delete_db(normalize_timestamp(time())) + broker.reclaim(Timestamp(time()).internal, time()) + broker.delete_db(Timestamp(time()).internal) def test_get_info_is_deleted(self): start = int(time()) - ts = (normalize_timestamp(t) for t in itertools.count(start)) + ts = (Timestamp(t).internal for t in itertools.count(start)) broker = ContainerBroker(':memory:', account='test_account', container='test_container') # create it @@ -172,8 +172,8 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(is_deleted, broker.is_deleted()) self.assertEqual(is_deleted, False) # sanity self.assertEqual(info, broker.get_info()) - self.assertEqual(info['put_timestamp'], normalize_timestamp(start)) - self.assert_(float(info['created_at']) >= start) + self.assertEqual(info['put_timestamp'], Timestamp(start).internal) + self.assert_(Timestamp(info['created_at']) >= start) self.assertEqual(info['delete_timestamp'], '0') if self.__class__ in (TestContainerBrokerBeforeMetadata, TestContainerBrokerBeforeXSync, @@ -181,7 +181,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(info['status_changed_at'], '0') else: self.assertEqual(info['status_changed_at'], - normalize_timestamp(start)) + Timestamp(start).internal) # delete it delete_timestamp = ts.next() @@ -190,8 +190,8 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(is_deleted, True) # sanity self.assertEqual(is_deleted, broker.is_deleted()) self.assertEqual(info, broker.get_info()) - self.assertEqual(info['put_timestamp'], normalize_timestamp(start)) - self.assert_(float(info['created_at']) >= start) + self.assertEqual(info['put_timestamp'], Timestamp(start).internal) + self.assert_(Timestamp(info['created_at']) >= start) self.assertEqual(info['delete_timestamp'], delete_timestamp) self.assertEqual(info['status_changed_at'], delete_timestamp) @@ -202,16 +202,16 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(is_deleted, False) # sanity self.assertEqual(is_deleted, broker.is_deleted()) self.assertEqual(info, broker.get_info()) - self.assertEqual(info['put_timestamp'], normalize_timestamp(start)) - self.assert_(float(info['created_at']) >= start) + self.assertEqual(info['put_timestamp'], Timestamp(start).internal) + self.assert_(Timestamp(info['created_at']) >= start) self.assertEqual(info['delete_timestamp'], delete_timestamp) self.assertEqual(info['status_changed_at'], delete_timestamp) def test_delete_object(self): # Test ContainerBroker.delete_object broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) - broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', + broker.initialize(Timestamp('1').internal, 0) + broker.put_object('o', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') with broker.get() as conn: self.assertEquals(conn.execute( @@ -221,7 +221,7 @@ class TestContainerBroker(unittest.TestCase): "SELECT count(*) FROM object " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) - broker.delete_object('o', normalize_timestamp(time())) + broker.delete_object('o', Timestamp(time()).internal) with broker.get() as conn: self.assertEquals(conn.execute( "SELECT count(*) FROM object " @@ -233,10 +233,10 @@ class TestContainerBroker(unittest.TestCase): def test_put_object(self): # Test ContainerBroker.put_object broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) # Create initial object - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal broker.put_object('"{}"', timestamp, 123, 'application/x-test', '5af83e3196bf99f440f31f2e1a6c9afe') @@ -280,7 +280,7 @@ class TestContainerBroker(unittest.TestCase): # Put new event sleep(.00001) - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal broker.put_object('"{}"', timestamp, 124, 'application/x-test', 'aa0749bacbc79ec65fe206943d8fe449') @@ -302,7 +302,7 @@ class TestContainerBroker(unittest.TestCase): "SELECT deleted FROM object").fetchone()[0], 0) # Put old event - otimestamp = normalize_timestamp(float(timestamp) - 1) + otimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal broker.put_object('"{}"', otimestamp, 124, 'application/x-test', 'aa0749bacbc79ec65fe206943d8fe449') @@ -324,7 +324,7 @@ class TestContainerBroker(unittest.TestCase): "SELECT deleted FROM object").fetchone()[0], 0) # Put old delete event - dtimestamp = normalize_timestamp(float(timestamp) - 1) + dtimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal broker.put_object('"{}"', dtimestamp, 0, '', '', deleted=1) with broker.get() as conn: @@ -346,7 +346,7 @@ class TestContainerBroker(unittest.TestCase): # Put new delete event sleep(.00001) - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal broker.put_object('"{}"', timestamp, 0, '', '', deleted=1) with broker.get() as conn: @@ -360,7 +360,7 @@ class TestContainerBroker(unittest.TestCase): # Put new event sleep(.00001) - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal broker.put_object('"{}"', timestamp, 123, 'application/x-test', '5af83e3196bf99f440f31f2e1a6c9afe') @@ -383,12 +383,12 @@ class TestContainerBroker(unittest.TestCase): # We'll use this later sleep(.0001) - in_between_timestamp = normalize_timestamp(time()) + in_between_timestamp = Timestamp(time()).internal # New post event sleep(.0001) previous_timestamp = timestamp - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal with broker.get() as conn: self.assertEquals(conn.execute( "SELECT name FROM object").fetchone()[0], @@ -432,7 +432,7 @@ class TestContainerBroker(unittest.TestCase): @patch_policies def test_put_misplaced_object_does_not_effect_container_stats(self): policy = random.choice(list(POLICIES)) - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = ContainerBroker(':memory:', account='a', container='c') @@ -460,7 +460,7 @@ class TestContainerBroker(unittest.TestCase): @patch_policies def test_has_multiple_policies(self): policy = random.choice(list(POLICIES)) - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = ContainerBroker(':memory:', account='a', container='c') @@ -484,7 +484,7 @@ class TestContainerBroker(unittest.TestCase): @patch_policies def test_get_policy_info(self): policy = random.choice(list(POLICIES)) - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = ContainerBroker(':memory:', account='a', container='c') @@ -521,7 +521,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(policy_stats, expected) def test_policy_stat_tracking(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = ContainerBroker(':memory:', account='a', container='c') @@ -558,13 +558,13 @@ class TestContainerBroker(unittest.TestCase): # Test ContainerBroker.get_info broker = ContainerBroker(':memory:', account='test1', container='test2') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) info = broker.get_info() self.assertEquals(info['account'], 'test1') self.assertEquals(info['container'], 'test2') self.assertEquals(info['hash'], '00000000000000000000000000000000') - self.assertEqual(info['put_timestamp'], normalize_timestamp(1)) + self.assertEqual(info['put_timestamp'], Timestamp(1).internal) self.assertEqual(info['delete_timestamp'], '0') if self.__class__ in (TestContainerBrokerBeforeMetadata, TestContainerBrokerBeforeXSync, @@ -572,40 +572,40 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(info['status_changed_at'], '0') else: self.assertEqual(info['status_changed_at'], - normalize_timestamp(1)) + Timestamp(1).internal) info = broker.get_info() self.assertEquals(info['object_count'], 0) self.assertEquals(info['bytes_used'], 0) - broker.put_object('o1', normalize_timestamp(time()), 123, 'text/plain', + broker.put_object('o1', Timestamp(time()).internal, 123, 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') info = broker.get_info() self.assertEquals(info['object_count'], 1) self.assertEquals(info['bytes_used'], 123) sleep(.00001) - broker.put_object('o2', normalize_timestamp(time()), 123, 'text/plain', + broker.put_object('o2', Timestamp(time()).internal, 123, 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') info = broker.get_info() self.assertEquals(info['object_count'], 2) self.assertEquals(info['bytes_used'], 246) sleep(.00001) - broker.put_object('o2', normalize_timestamp(time()), 1000, + broker.put_object('o2', Timestamp(time()).internal, 1000, 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') info = broker.get_info() self.assertEquals(info['object_count'], 2) self.assertEquals(info['bytes_used'], 1123) sleep(.00001) - broker.delete_object('o1', normalize_timestamp(time())) + broker.delete_object('o1', Timestamp(time()).internal) info = broker.get_info() self.assertEquals(info['object_count'], 1) self.assertEquals(info['bytes_used'], 1000) sleep(.00001) - broker.delete_object('o2', normalize_timestamp(time())) + broker.delete_object('o2', Timestamp(time()).internal) info = broker.get_info() self.assertEquals(info['object_count'], 0) self.assertEquals(info['bytes_used'], 0) @@ -617,7 +617,7 @@ class TestContainerBroker(unittest.TestCase): def test_set_x_syncs(self): broker = ContainerBroker(':memory:', account='test1', container='test2') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) info = broker.get_info() self.assertEquals(info['x_container_sync_point1'], -1) @@ -631,7 +631,7 @@ class TestContainerBroker(unittest.TestCase): def test_get_report_info(self): broker = ContainerBroker(':memory:', account='test1', container='test2') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) info = broker.get_info() self.assertEquals(info['account'], 'test1') @@ -641,7 +641,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(info['reported_object_count'], 0) self.assertEquals(info['reported_bytes_used'], 0) - broker.put_object('o1', normalize_timestamp(time()), 123, 'text/plain', + broker.put_object('o1', Timestamp(time()).internal, 123, 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') info = broker.get_info() self.assertEquals(info['object_count'], 1) @@ -650,7 +650,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(info['reported_bytes_used'], 0) sleep(.00001) - broker.put_object('o2', normalize_timestamp(time()), 123, 'text/plain', + broker.put_object('o2', Timestamp(time()).internal, 123, 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') info = broker.get_info() self.assertEquals(info['object_count'], 2) @@ -659,7 +659,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(info['reported_bytes_used'], 0) sleep(.00001) - broker.put_object('o2', normalize_timestamp(time()), 1000, + broker.put_object('o2', Timestamp(time()).internal, 1000, 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') info = broker.get_info() self.assertEquals(info['object_count'], 2) @@ -667,9 +667,9 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(info['reported_object_count'], 0) self.assertEquals(info['reported_bytes_used'], 0) - put_timestamp = normalize_timestamp(time()) + put_timestamp = Timestamp(time()).internal sleep(.001) - delete_timestamp = normalize_timestamp(time()) + delete_timestamp = Timestamp(time()).internal broker.reported(put_timestamp, delete_timestamp, 2, 1123) info = broker.get_info() self.assertEquals(info['object_count'], 2) @@ -680,7 +680,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(info['reported_bytes_used'], 1123) sleep(.00001) - broker.delete_object('o1', normalize_timestamp(time())) + broker.delete_object('o1', Timestamp(time()).internal) info = broker.get_info() self.assertEquals(info['object_count'], 1) self.assertEquals(info['bytes_used'], 1000) @@ -688,7 +688,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(info['reported_bytes_used'], 1123) sleep(.00001) - broker.delete_object('o2', normalize_timestamp(time())) + broker.delete_object('o2', Timestamp(time()).internal) info = broker.get_info() self.assertEquals(info['object_count'], 0) self.assertEquals(info['bytes_used'], 0) @@ -698,20 +698,20 @@ class TestContainerBroker(unittest.TestCase): def test_list_objects_iter(self): # Test ContainerBroker.list_objects_iter broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) for obj1 in xrange(4): for obj2 in xrange(125): broker.put_object('%d/%04d' % (obj1, obj2), - normalize_timestamp(time()), 0, 'text/plain', + Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') for obj in xrange(125): broker.put_object('2/0051/%04d' % obj, - normalize_timestamp(time()), 0, 'text/plain', + Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') for obj in xrange(125): broker.put_object('3/%04d/0049' % obj, - normalize_timestamp(time()), 0, 'text/plain', + Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') listing = broker.list_objects_iter(100, '', None, None, '') @@ -777,7 +777,7 @@ class TestContainerBroker(unittest.TestCase): '3/0047/', '3/0048', '3/0048/', '3/0049', '3/0049/', '3/0050']) - broker.put_object('3/0049/', normalize_timestamp(time()), 0, + broker.put_object('3/0049/', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') listing = broker.list_objects_iter(10, '3/0048', None, None, None) self.assertEquals(len(listing), 10) @@ -817,20 +817,20 @@ class TestContainerBroker(unittest.TestCase): # Test ContainerBroker.list_objects_iter using a # delimiter that is not a slash broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) for obj1 in xrange(4): for obj2 in xrange(125): broker.put_object('%d:%04d' % (obj1, obj2), - normalize_timestamp(time()), 0, 'text/plain', + Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') for obj in xrange(125): broker.put_object('2:0051:%04d' % obj, - normalize_timestamp(time()), 0, 'text/plain', + Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') for obj in xrange(125): broker.put_object('3:%04d:0049' % obj, - normalize_timestamp(time()), 0, 'text/plain', + Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') listing = broker.list_objects_iter(100, '', None, None, '') @@ -895,7 +895,7 @@ class TestContainerBroker(unittest.TestCase): '3:0047:', '3:0048', '3:0048:', '3:0049', '3:0049:', '3:0050']) - broker.put_object('3:0049:', normalize_timestamp(time()), 0, + broker.put_object('3:0049:', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') listing = broker.list_objects_iter(10, '3:0048', None, None, None) self.assertEquals(len(listing), 10) @@ -934,25 +934,25 @@ class TestContainerBroker(unittest.TestCase): def test_list_objects_iter_prefix_delim(self): # Test ContainerBroker.list_objects_iter broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) broker.put_object( - '/pets/dogs/1', normalize_timestamp(0), 0, + '/pets/dogs/1', Timestamp(0).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object( - '/pets/dogs/2', normalize_timestamp(0), 0, + '/pets/dogs/2', Timestamp(0).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object( - '/pets/fish/a', normalize_timestamp(0), 0, + '/pets/fish/a', Timestamp(0).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object( - '/pets/fish/b', normalize_timestamp(0), 0, + '/pets/fish/b', Timestamp(0).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object( - '/pets/fish_info.txt', normalize_timestamp(0), 0, + '/pets/fish_info.txt', Timestamp(0).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object( - '/snakes', normalize_timestamp(0), 0, + '/snakes', Timestamp(0).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') #def list_objects_iter(self, limit, marker, prefix, delimiter, @@ -971,50 +971,50 @@ class TestContainerBroker(unittest.TestCase): # Test ContainerBroker.list_objects_iter for a # container that has an odd file with a trailing delimiter broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) - broker.put_object('a', normalize_timestamp(time()), 0, + broker.initialize(Timestamp('1').internal, 0) + broker.put_object('a', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a/', normalize_timestamp(time()), 0, + broker.put_object('a/', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a/a', normalize_timestamp(time()), 0, + broker.put_object('a/a', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a/a/a', normalize_timestamp(time()), 0, + broker.put_object('a/a/a', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a/a/b', normalize_timestamp(time()), 0, + broker.put_object('a/a/b', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a/b', normalize_timestamp(time()), 0, + broker.put_object('a/b', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b', normalize_timestamp(time()), 0, + broker.put_object('b', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b/a', normalize_timestamp(time()), 0, + broker.put_object('b/a', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b/b', normalize_timestamp(time()), 0, + broker.put_object('b/b', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('c', normalize_timestamp(time()), 0, + broker.put_object('c', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a/0', normalize_timestamp(time()), 0, + broker.put_object('a/0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0', normalize_timestamp(time()), 0, + broker.put_object('0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0/', normalize_timestamp(time()), 0, + broker.put_object('0/', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('00', normalize_timestamp(time()), 0, + broker.put_object('00', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0/0', normalize_timestamp(time()), 0, + broker.put_object('0/0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0/00', normalize_timestamp(time()), 0, + broker.put_object('0/00', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0/1', normalize_timestamp(time()), 0, + broker.put_object('0/1', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0/1/', normalize_timestamp(time()), 0, + broker.put_object('0/1/', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0/1/0', normalize_timestamp(time()), 0, + broker.put_object('0/1/0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('1', normalize_timestamp(time()), 0, + broker.put_object('1', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('1/', normalize_timestamp(time()), 0, + broker.put_object('1/', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('1/0', normalize_timestamp(time()), 0, + broker.put_object('1/0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') listing = broker.list_objects_iter(25, None, None, None, None) self.assertEquals(len(listing), 22) @@ -1051,50 +1051,50 @@ class TestContainerBroker(unittest.TestCase): # Test ContainerBroker.list_objects_iter for a # container that has an odd file with a trailing delimiter broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) - broker.put_object('a', normalize_timestamp(time()), 0, + broker.initialize(Timestamp('1').internal, 0) + broker.put_object('a', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a:', normalize_timestamp(time()), 0, + broker.put_object('a:', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a:a', normalize_timestamp(time()), 0, + broker.put_object('a:a', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a:a:a', normalize_timestamp(time()), 0, + broker.put_object('a:a:a', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a:a:b', normalize_timestamp(time()), 0, + broker.put_object('a:a:b', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a:b', normalize_timestamp(time()), 0, + broker.put_object('a:b', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b', normalize_timestamp(time()), 0, + broker.put_object('b', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b:a', normalize_timestamp(time()), 0, + broker.put_object('b:a', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b:b', normalize_timestamp(time()), 0, + broker.put_object('b:b', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('c', normalize_timestamp(time()), 0, + broker.put_object('c', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a:0', normalize_timestamp(time()), 0, + broker.put_object('a:0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0', normalize_timestamp(time()), 0, + broker.put_object('0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0:', normalize_timestamp(time()), 0, + broker.put_object('0:', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('00', normalize_timestamp(time()), 0, + broker.put_object('00', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0:0', normalize_timestamp(time()), 0, + broker.put_object('0:0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0:00', normalize_timestamp(time()), 0, + broker.put_object('0:00', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0:1', normalize_timestamp(time()), 0, + broker.put_object('0:1', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0:1:', normalize_timestamp(time()), 0, + broker.put_object('0:1:', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0:1:0', normalize_timestamp(time()), 0, + broker.put_object('0:1:0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('1', normalize_timestamp(time()), 0, + broker.put_object('1', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('1:', normalize_timestamp(time()), 0, + broker.put_object('1:', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('1:0', normalize_timestamp(time()), 0, + broker.put_object('1:0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') listing = broker.list_objects_iter(25, None, None, None, None) self.assertEquals(len(listing), 22) @@ -1129,19 +1129,19 @@ class TestContainerBroker(unittest.TestCase): def test_chexor(self): broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) - broker.put_object('a', normalize_timestamp(1), 0, + broker.initialize(Timestamp('1').internal, 0) + broker.put_object('a', Timestamp(1).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b', normalize_timestamp(2), 0, + broker.put_object('b', Timestamp(2).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - hasha = hashlib.md5('%s-%s' % ('a', '0000000001.00000')).digest() - hashb = hashlib.md5('%s-%s' % ('b', '0000000002.00000')).digest() + hasha = hashlib.md5('%s-%s' % ('a', Timestamp(1).internal)).digest() + hashb = hashlib.md5('%s-%s' % ('b', Timestamp(2).internal)).digest() hashc = ''.join( - ('%2x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) + ('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEquals(broker.get_info()['hash'], hashc) - broker.put_object('b', normalize_timestamp(3), 0, + broker.put_object('b', Timestamp(3).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - hashb = hashlib.md5('%s-%s' % ('b', '0000000003.00000')).digest() + hashb = hashlib.md5('%s-%s' % ('b', Timestamp(3).internal)).digest() hashc = ''.join( ('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEquals(broker.get_info()['hash'], hashc) @@ -1149,7 +1149,7 @@ class TestContainerBroker(unittest.TestCase): def test_newid(self): # test DatabaseBroker.newid broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) id = broker.get_info()['id'] broker.newid('someid') self.assertNotEquals(id, broker.get_info()['id']) @@ -1157,11 +1157,11 @@ class TestContainerBroker(unittest.TestCase): def test_get_items_since(self): # test DatabaseBroker.get_items_since broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) - broker.put_object('a', normalize_timestamp(1), 0, + broker.initialize(Timestamp('1').internal, 0) + broker.put_object('a', Timestamp(1).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') max_row = broker.get_replication_info()['max_row'] - broker.put_object('b', normalize_timestamp(2), 0, + broker.put_object('b', Timestamp(2).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') items = broker.get_items_since(max_row, 1000) self.assertEquals(len(items), 1) @@ -1170,9 +1170,9 @@ class TestContainerBroker(unittest.TestCase): def test_sync_merging(self): # exercise the DatabaseBroker sync functions a bit broker1 = ContainerBroker(':memory:', account='a', container='c') - broker1.initialize(normalize_timestamp('1'), 0) + broker1.initialize(Timestamp('1').internal, 0) broker2 = ContainerBroker(':memory:', account='a', container='c') - broker2.initialize(normalize_timestamp('1'), 0) + broker2.initialize(Timestamp('1').internal, 0) self.assertEquals(broker2.get_sync('12345'), -1) broker1.merge_syncs([{'sync_point': 3, 'remote_id': '12345'}]) broker2.merge_syncs(broker1.get_syncs()) @@ -1180,12 +1180,12 @@ class TestContainerBroker(unittest.TestCase): def test_merge_items(self): broker1 = ContainerBroker(':memory:', account='a', container='c') - broker1.initialize(normalize_timestamp('1'), 0) + broker1.initialize(Timestamp('1').internal, 0) broker2 = ContainerBroker(':memory:', account='a', container='c') - broker2.initialize(normalize_timestamp('1'), 0) - broker1.put_object('a', normalize_timestamp(1), 0, + broker2.initialize(Timestamp('1').internal, 0) + broker1.put_object('a', Timestamp(1).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker1.put_object('b', normalize_timestamp(2), 0, + broker1.put_object('b', Timestamp(2).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') id = broker1.get_info()['id'] broker2.merge_items(broker1.get_items_since( @@ -1193,7 +1193,7 @@ class TestContainerBroker(unittest.TestCase): items = broker2.get_items_since(-1, 1000) self.assertEquals(len(items), 2) self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) - broker1.put_object('c', normalize_timestamp(3), 0, + broker1.put_object('c', Timestamp(3).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) @@ -1205,17 +1205,17 @@ class TestContainerBroker(unittest.TestCase): def test_merge_items_overwrite(self): # test DatabaseBroker.merge_items broker1 = ContainerBroker(':memory:', account='a', container='c') - broker1.initialize(normalize_timestamp('1'), 0) + broker1.initialize(Timestamp('1').internal, 0) id = broker1.get_info()['id'] broker2 = ContainerBroker(':memory:', account='a', container='c') - broker2.initialize(normalize_timestamp('1'), 0) - broker1.put_object('a', normalize_timestamp(2), 0, + broker2.initialize(Timestamp('1').internal, 0) + broker1.put_object('a', Timestamp(2).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker1.put_object('b', normalize_timestamp(3), 0, + broker1.put_object('b', Timestamp(3).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) - broker1.put_object('a', normalize_timestamp(4), 0, + broker1.put_object('a', Timestamp(4).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) @@ -1223,24 +1223,24 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) for rec in items: if rec['name'] == 'a': - self.assertEquals(rec['created_at'], normalize_timestamp(4)) + self.assertEquals(rec['created_at'], Timestamp(4).internal) if rec['name'] == 'b': - self.assertEquals(rec['created_at'], normalize_timestamp(3)) + self.assertEquals(rec['created_at'], Timestamp(3).internal) def test_merge_items_post_overwrite_out_of_order(self): # test DatabaseBroker.merge_items broker1 = ContainerBroker(':memory:', account='a', container='c') - broker1.initialize(normalize_timestamp('1'), 0) + broker1.initialize(Timestamp('1').internal, 0) id = broker1.get_info()['id'] broker2 = ContainerBroker(':memory:', account='a', container='c') - broker2.initialize(normalize_timestamp('1'), 0) - broker1.put_object('a', normalize_timestamp(2), 0, + broker2.initialize(Timestamp('1').internal, 0) + broker1.put_object('a', Timestamp(2).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker1.put_object('b', normalize_timestamp(3), 0, + broker1.put_object('b', Timestamp(3).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) - broker1.put_object('a', normalize_timestamp(4), 0, + broker1.put_object('a', Timestamp(4).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) @@ -1248,18 +1248,18 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) for rec in items: if rec['name'] == 'a': - self.assertEquals(rec['created_at'], normalize_timestamp(4)) + self.assertEquals(rec['created_at'], Timestamp(4).internal) if rec['name'] == 'b': - self.assertEquals(rec['created_at'], normalize_timestamp(3)) + self.assertEquals(rec['created_at'], Timestamp(3).internal) self.assertEquals(rec['content_type'], 'text/plain') items = broker2.get_items_since(-1, 1000) self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) for rec in items: if rec['name'] == 'a': - self.assertEquals(rec['created_at'], normalize_timestamp(4)) + self.assertEquals(rec['created_at'], Timestamp(4).internal) if rec['name'] == 'b': - self.assertEquals(rec['created_at'], normalize_timestamp(3)) - broker1.put_object('b', normalize_timestamp(5), 0, + self.assertEquals(rec['created_at'], Timestamp(3).internal) + broker1.put_object('b', Timestamp(5).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) @@ -1267,13 +1267,13 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) for rec in items: if rec['name'] == 'a': - self.assertEquals(rec['created_at'], normalize_timestamp(4)) + self.assertEquals(rec['created_at'], Timestamp(4).internal) if rec['name'] == 'b': - self.assertEquals(rec['created_at'], normalize_timestamp(5)) + self.assertEquals(rec['created_at'], Timestamp(5).internal) self.assertEquals(rec['content_type'], 'text/plain') def test_set_storage_policy_index(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = ContainerBroker(':memory:', account='test_account', container='test_container') @@ -1329,7 +1329,7 @@ class TestContainerBroker(unittest.TestCase): # never-had-an-object container to make sure we handle it broker = ContainerBroker(':memory:', account='test_account', container='test_container') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) info = broker.get_info() self.assertEqual(0, info['storage_policy_index']) @@ -1340,14 +1340,14 @@ class TestContainerBroker(unittest.TestCase): def test_reconciler_sync(self): broker = ContainerBroker(':memory:', account='test_account', container='test_container') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) self.assertEquals(-1, broker.get_reconciler_sync()) broker.update_reconciler_sync(10) self.assertEquals(10, broker.get_reconciler_sync()) @with_tempdir def test_legacy_pending_files(self, tempdir): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) db_path = os.path.join(tempdir, 'container.db') @@ -1458,7 +1458,7 @@ def premetadata_create_container_info_table(self, conn, put_timestamp, :param put_timestamp: put timestamp """ if put_timestamp is None: - put_timestamp = normalize_timestamp(0) + put_timestamp = Timestamp(0).internal conn.executescript(''' CREATE TABLE container_stat ( account TEXT, @@ -1485,7 +1485,7 @@ def premetadata_create_container_info_table(self, conn, put_timestamp, UPDATE container_stat SET account = ?, container = ?, created_at = ?, id = ?, put_timestamp = ? - ''', (self.account, self.container, normalize_timestamp(time()), + ''', (self.account, self.container, Timestamp(time()).internal, str(uuid4()), put_timestamp)) @@ -1499,7 +1499,7 @@ class TestContainerBrokerBeforeMetadata(ContainerBrokerMigrationMixin, def setUp(self): super(TestContainerBrokerBeforeMetadata, self).setUp() broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) exc = None with broker.get() as conn: try: @@ -1511,7 +1511,7 @@ class TestContainerBrokerBeforeMetadata(ContainerBrokerMigrationMixin, def tearDown(self): super(TestContainerBrokerBeforeMetadata, self).tearDown() broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) with broker.get() as conn: conn.execute('SELECT metadata FROM container_stat') @@ -1529,7 +1529,7 @@ def prexsync_create_container_info_table(self, conn, put_timestamp, :param put_timestamp: put timestamp """ if put_timestamp is None: - put_timestamp = normalize_timestamp(0) + put_timestamp = Timestamp(0).internal conn.executescript(""" CREATE TABLE container_stat ( account TEXT, @@ -1557,7 +1557,7 @@ def prexsync_create_container_info_table(self, conn, put_timestamp, UPDATE container_stat SET account = ?, container = ?, created_at = ?, id = ?, put_timestamp = ? - ''', (self.account, self.container, normalize_timestamp(time()), + ''', (self.account, self.container, Timestamp(time()).internal, str(uuid4()), put_timestamp)) @@ -1573,7 +1573,7 @@ class TestContainerBrokerBeforeXSync(ContainerBrokerMigrationMixin, ContainerBroker.create_container_info_table = \ prexsync_create_container_info_table broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) exc = None with broker.get() as conn: try: @@ -1586,7 +1586,7 @@ class TestContainerBrokerBeforeXSync(ContainerBrokerMigrationMixin, def tearDown(self): super(TestContainerBrokerBeforeXSync, self).tearDown() broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) with broker.get() as conn: conn.execute('SELECT x_container_sync_point1 FROM container_stat') @@ -1641,7 +1641,7 @@ def prespi_create_container_info_table(self, conn, put_timestamp, :param put_timestamp: put timestamp """ if put_timestamp is None: - put_timestamp = normalize_timestamp(0) + put_timestamp = Timestamp(0).internal conn.executescript(""" CREATE TABLE container_stat ( account TEXT, @@ -1671,7 +1671,7 @@ def prespi_create_container_info_table(self, conn, put_timestamp, UPDATE container_stat SET account = ?, container = ?, created_at = ?, id = ?, put_timestamp = ? - ''', (self.account, self.container, normalize_timestamp(time()), + ''', (self.account, self.container, Timestamp(time()).internal, str(uuid4()), put_timestamp)) @@ -1688,7 +1688,7 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin, prespi_create_container_info_table broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) exc = None with broker.get() as conn: try: @@ -1701,7 +1701,7 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin, def tearDown(self): super(TestContainerBrokerBeforeSPI, self).tearDown() broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) with broker.get() as conn: conn.execute('SELECT storage_policy_index FROM container_stat') @@ -1712,7 +1712,7 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin, # initialize an un-migrated database broker = ContainerBroker(db_path, account='a', container='c') - put_timestamp = normalize_timestamp(int(time())) + put_timestamp = Timestamp(int(time())).internal broker.initialize(put_timestamp, None) with broker.get() as conn: try: @@ -1729,7 +1729,7 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin, 'from object table!') # manually insert an existing row to avoid automatic migration - obj_put_timestamp = normalize_timestamp(time()) + obj_put_timestamp = Timestamp(time()).internal with broker.get() as conn: conn.execute(''' INSERT INTO object (name, created_at, size, @@ -1767,7 +1767,7 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin, self.assertEqual(info[k], v, 'The value for %s was %r not %r' % ( k, info[k], v)) - self.assert_(float(info['created_at']) > float(put_timestamp)) + self.assert_(Timestamp(info['created_at']) > Timestamp(put_timestamp)) self.assertNotEqual(int(info['hash'], 16), 0) orig_hash = info['hash'] # get_replication_info @@ -1776,7 +1776,7 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin, expected['count'] = expected.pop('object_count') for k, v in expected.items(): self.assertEqual(info[k], v) - self.assert_(float(info['created_at']) > float(put_timestamp)) + self.assert_(Timestamp(info['created_at']) > Timestamp(put_timestamp)) self.assertEqual(info['hash'], orig_hash) self.assertEqual(info['max_row'], 1) self.assertEqual(info['metadata'], '') @@ -1839,7 +1839,7 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin, # now do a PUT with a different value for storage_policy_index # which will update the DB schema as well as update policy_stats # for legacy objects in the DB (those without an SPI) - second_object_put_timestamp = normalize_timestamp(time()) + second_object_put_timestamp = Timestamp(time()).internal other_policy = [p for p in POLICIES if p.idx != 0][0] broker.put_object('test_second', second_object_put_timestamp, 456, 'text/plain', diff --git a/test/unit/container/test_reconciler.py b/test/unit/container/test_reconciler.py index 42ad854ba3..fe1bc18a86 100644 --- a/test/unit/container/test_reconciler.py +++ b/test/unit/container/test_reconciler.py @@ -30,14 +30,15 @@ from swift.container import reconciler from swift.container.server import gen_resp_headers from swift.common.direct_client import ClientException from swift.common import swob -from swift.common.utils import split_path, normalize_timestamp +from swift.common.utils import split_path, Timestamp from test.unit import debug_logger, FakeRing, fake_http_connect from test.unit.common.middleware.helpers import FakeSwift def timestamp_to_last_modified(timestamp): - return datetime.fromtimestamp(timestamp).strftime('%Y-%m-%dT%H:%M:%S.%f') + return datetime.fromtimestamp( + float(Timestamp(timestamp))).strftime('%Y-%m-%dT%H:%M:%S.%f') def container_resp_headers(**kwargs): @@ -122,15 +123,16 @@ class FakeInternalClient(reconciler.InternalClient): # empty container continue obj_path = container_path + '/' + obj_name - headers = {'X-Timestamp': normalize_timestamp(timestamp)} + ts = Timestamp(timestamp) + headers = {'X-Timestamp': ts.normal, + 'X-Backend-Timestamp': ts.internal} # register object response self.app.storage_policy[storage_policy_index].register( 'GET', obj_path, swob.HTTPOk, headers) self.app.storage_policy[storage_policy_index].register( 'DELETE', obj_path, swob.HTTPNoContent, {}) # container listing entry - last_modified = timestamp_to_last_modified( - float(timestamp)) + last_modified = timestamp_to_last_modified(timestamp) obj_data = { 'bytes': 0, # listing data is unicode @@ -183,7 +185,7 @@ class TestReconcilerUtils(unittest.TestCase): def test_parse_raw_obj(self): got = reconciler.parse_raw_obj({ 'name': "2:/AUTH_bob/con/obj", - 'hash': normalize_timestamp(2017551.49350), + 'hash': Timestamp(2017551.49350).internal, 'last_modified': timestamp_to_last_modified(2017551.49352), 'content_type': 'application/x-delete', }) @@ -197,7 +199,7 @@ class TestReconcilerUtils(unittest.TestCase): got = reconciler.parse_raw_obj({ 'name': "1:/AUTH_bob/con/obj", - 'hash': normalize_timestamp(1234.20190), + 'hash': Timestamp(1234.20190).internal, 'last_modified': timestamp_to_last_modified(1234.20192), 'content_type': 'application/x-put', }) @@ -212,7 +214,7 @@ class TestReconcilerUtils(unittest.TestCase): # negative test obj_info = { 'name': "1:/AUTH_bob/con/obj", - 'hash': normalize_timestamp(1234.20190), + 'hash': Timestamp(1234.20190).internal, 'last_modified': timestamp_to_last_modified(1234.20192), } self.assertRaises(ValueError, reconciler.parse_raw_obj, obj_info) @@ -235,15 +237,15 @@ class TestReconcilerUtils(unittest.TestCase): mock_path = 'swift.container.reconciler.direct_head_container' stub_resp_headers = [ container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=0, ), container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=1, ), container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=0, ), ] @@ -278,7 +280,7 @@ class TestReconcilerUtils(unittest.TestCase): 'Container Server blew up', http_status=500, http_reason='Server Error', http_headers=container_resp_headers( - status_changed_at=normalize_timestamp(0), + status_changed_at=Timestamp(0).internal, storage_policy_index=0, ), ), @@ -295,11 +297,11 @@ class TestReconcilerUtils(unittest.TestCase): mock_path = 'swift.container.reconciler.direct_head_container' stub_resp_headers = [ container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=1, ), container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=0, ), socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED)), @@ -316,7 +318,7 @@ class TestReconcilerUtils(unittest.TestCase): mock_path = 'swift.container.reconciler.direct_head_container' stub_resp_headers = [ container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=0, ), socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED)), @@ -324,7 +326,7 @@ class TestReconcilerUtils(unittest.TestCase): 'Container Server blew up', http_status=500, http_reason='Server Error', http_headers=container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=1, ), ), @@ -339,7 +341,7 @@ class TestReconcilerUtils(unittest.TestCase): def test_get_container_policy_index_for_deleted(self): mock_path = 'swift.container.reconciler.direct_head_container' headers = container_resp_headers( - status_changed_at=normalize_timestamp(time.time()), + status_changed_at=Timestamp(time.time()).internal, storage_policy_index=1, ) stub_resp_headers = [ @@ -484,15 +486,15 @@ class TestReconcilerUtils(unittest.TestCase): mock_path = 'swift.container.reconciler.direct_head_container' stub_resp_headers = [ container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=0, ), container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=1, ), container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=0, ), ] @@ -536,8 +538,8 @@ class TestReconcilerUtils(unittest.TestCase): 'partition': partition, 'method': method, 'path': path, 'headers': headers, 'query_string': query_string}) - x_timestamp = normalize_timestamp(time.time()) - headers = {'x-timestamp': x_timestamp} + x_timestamp = Timestamp(time.time()) + headers = {'x-timestamp': x_timestamp.internal} fake_hc = fake_http_connect(200, 200, 200, give_connect=test_connect) with mock.patch(mock_path, fake_hc): reconciler.direct_delete_container_entry( @@ -620,7 +622,7 @@ class TestReconcilerUtils(unittest.TestCase): 'headers': headers, 'query_string': query_string}) fake_hc = fake_http_connect(200, 200, 200, give_connect=test_connect) - now = float(normalize_timestamp(time.time())) + now = time.time() with contextlib.nested( mock.patch(mock_path, fake_hc), mock.patch('swift.container.reconciler.time.time', @@ -639,7 +641,7 @@ class TestReconcilerUtils(unittest.TestCase): for args in connect_args: self.assertEqual(args['headers']['X-Timestamp'], - normalize_timestamp(now)) + Timestamp(now).internal) self.assertEqual(args['headers']['X-Etag'], '5948918.63946') self.assertEqual(args['path'], '/.misplaced_objects/5947200/17:/a/c/o') @@ -820,7 +822,7 @@ class TestReconciler(unittest.TestCase): # we DELETE the object from the wrong place with source_ts + offset 1 # timestamp to make sure the change takes effect self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp(3618.84188)) + Timestamp(3618.84187, offset=1).internal) # and pop the queue for that one self.assertEqual(self.reconciler.stats['pop_queue'], 1) self.assertEqual(deleted_container_entries, [( @@ -884,14 +886,14 @@ class TestReconciler(unittest.TestCase): put_headers = self.fake_swift.storage_policy[0].headers[1] # we PUT the object in the right place with q_ts + offset 2 self.assertEqual(put_headers.get('X-Timestamp'), - normalize_timestamp('3618.84189')) + Timestamp(3618.84187, offset=2)) # cleans up the old self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1) self.assertEqual(self.reconciler.stats['cleanup_success'], 1) # we DELETE the object from the wrong place with source_ts + offset 1 # timestamp to make sure the change takes effect self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp('3618.84188')) + Timestamp(3618.84187, offset=1)) # and when we're done, we pop the entry from the queue self.assertEqual(self.reconciler.stats['pop_queue'], 1) self.assertEqual(deleted_container_entries, @@ -931,14 +933,14 @@ class TestReconciler(unittest.TestCase): put_headers = self.fake_swift.storage_policy[1].headers[1] # we PUT the object in the right place with q_ts + offset 2 self.assertEqual(put_headers.get('X-Timestamp'), - normalize_timestamp('3618.84189')) + Timestamp(3618.84187, offset=2).internal) # cleans up the old self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1) self.assertEqual(self.reconciler.stats['cleanup_success'], 1) # we DELETE the object from the wrong place with source_ts + offset 1 # timestamp to make sure the change takes effect self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp('3618.84188')) + Timestamp(3618.84187, offset=1).internal) # and when we're done, we pop the entry from the queue self.assertEqual(self.reconciler.stats['pop_queue'], 1) self.assertEqual(deleted_container_entries, @@ -987,14 +989,14 @@ class TestReconciler(unittest.TestCase): put_headers = self.fake_swift.storage_policy[0].headers[1] # we PUT the object in the right place with q_ts + offset 2 self.assertEqual(put_headers.get('X-Timestamp'), - normalize_timestamp('3618.84189')) + Timestamp(3618.84187, offset=2).internal) # cleans up the old self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1) self.assertEqual(self.reconciler.stats['cleanup_success'], 1) # we DELETE the object from the wrong place with source_ts + offset 1 # timestamp to make sure the change takes effect self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp('3618.84188')) + Timestamp(3618.84187, offset=1).internal) self.assertEqual( delete_headers.get('X-Backend-Storage-Policy-Index'), '1') # and when we're done, we pop the entry from the queue @@ -1005,18 +1007,18 @@ class TestReconciler(unittest.TestCase): self.assertEqual(self.reconciler.stats['success'], 1) def test_object_delete(self): - q_ts = float(normalize_timestamp(time.time())) + q_ts = time.time() self._mock_listing({ (None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): ( - normalize_timestamp(q_ts), 'application/x-delete'), + Timestamp(q_ts).internal, 'application/x-delete'), # object exists in "correct" storage policy - slightly older - (0, "/AUTH_bob/c/o1"): normalize_timestamp(q_ts - 1), + (0, "/AUTH_bob/c/o1"): Timestamp(q_ts - 1).internal, }) self._mock_oldest_spi({'c': 0}) # the tombstone exists in the enqueued storage policy self.fake_swift.storage_policy[1].register( 'GET', '/v1/AUTH_bob/c/o1', swob.HTTPNotFound, - {'X-Backend-Timestamp': normalize_timestamp(q_ts)}) + {'X-Backend-Timestamp': Timestamp(q_ts).internal}) deleted_container_entries = self._run_once() # found a misplaced object @@ -1044,14 +1046,14 @@ class TestReconciler(unittest.TestCase): reconcile_headers = self.fake_swift.storage_policy[0].headers[1] # we DELETE the object in the right place with q_ts + offset 2 self.assertEqual(reconcile_headers.get('X-Timestamp'), - normalize_timestamp(q_ts + 0.00002)) + Timestamp(q_ts, offset=2).internal) # cleans up the old self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1) self.assertEqual(self.reconciler.stats['cleanup_success'], 1) # we DELETE the object from the wrong place with source_ts + offset 1 # timestamp to make sure the change takes effect self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp(q_ts + 0.00001)) + Timestamp(q_ts, offset=1)) # and when we're done, we pop the entry from the queue self.assertEqual(self.reconciler.stats['pop_queue'], 1) self.assertEqual(deleted_container_entries, @@ -1117,13 +1119,13 @@ class TestReconciler(unittest.TestCase): # .. with source timestamp + offset 2 put_headers = self.fake_swift.storage_policy[0].headers[1] self.assertEqual(put_headers.get('X-Timestamp'), - normalize_timestamp(3600.234587)) + Timestamp(3600.234567, offset=2)) # src object is cleaned up self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1) self.assertEqual(self.reconciler.stats['cleanup_success'], 1) # ... with q_ts + offset 1 self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp(3600.123466)) + Timestamp(3600.123456, offset=1)) # and queue is popped self.assertEqual(self.reconciler.stats['pop_queue'], 1) self.assertEqual(deleted_container_entries, @@ -1132,7 +1134,7 @@ class TestReconciler(unittest.TestCase): def test_object_move_src_object_older_than_queue_entry(self): # should be some sort of retry case - q_ts = float(normalize_timestamp(time.time())) + q_ts = time.time() container = str(int(q_ts // 3600 * 3600)) q_path = '.misplaced_objects/%s' % container self._mock_listing({ @@ -1169,7 +1171,7 @@ class TestReconciler(unittest.TestCase): def test_src_object_unavailable_with_slightly_newer_tombstone(self): # should be some sort of retry case - q_ts = float(normalize_timestamp(time.time())) + q_ts = float(Timestamp(time.time())) container = str(int(q_ts // 3600 * 3600)) q_path = '.misplaced_objects/%s' % container self._mock_listing({ @@ -1178,7 +1180,7 @@ class TestReconciler(unittest.TestCase): self._mock_oldest_spi({'c': 0}) self.fake_swift.storage_policy[1].register( 'GET', '/v1/AUTH_bob/c/o1', swob.HTTPNotFound, - {'X-Backend-Timestamp': normalize_timestamp(q_ts + 0.00002)}) + {'X-Backend-Timestamp': Timestamp(q_ts, offset=2).internal}) deleted_container_entries = self._run_once() # found a misplaced object @@ -1208,7 +1210,7 @@ class TestReconciler(unittest.TestCase): def test_src_object_unavailable_server_error(self): # should be some sort of retry case - q_ts = float(normalize_timestamp(time.time())) + q_ts = float(Timestamp(time.time())) container = str(int(q_ts // 3600 * 3600)) q_path = '.misplaced_objects/%s' % container self._mock_listing({ @@ -1248,7 +1250,7 @@ class TestReconciler(unittest.TestCase): # setup the cluster self._mock_listing({ (None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3600.123456, - (1, '/AUTH_bob/c/o1'): 3600.234567, # slightly newer + (1, '/AUTH_bob/c/o1'): 3600.123457, # slightly newer }) self._mock_oldest_spi({'c': 0}) # destination @@ -1283,12 +1285,12 @@ class TestReconciler(unittest.TestCase): # .. with source timestamp + offset 2 put_headers = self.fake_swift.storage_policy[0].headers[1] self.assertEqual(put_headers.get('X-Timestamp'), - normalize_timestamp(3600.234587)) + Timestamp(3600.123457, offset=2)) # we try to cleanup self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1) # ... with q_ts + offset 1 self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp(3600.123466)) + Timestamp(3600.12346, offset=1)) # but cleanup fails! self.assertEqual(self.reconciler.stats['cleanup_failed'], 1) # so the queue is not popped @@ -1366,7 +1368,7 @@ class TestReconciler(unittest.TestCase): self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1) self.assertEqual(self.reconciler.stats['cleanup_success'], 1) self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp(3679.20191)) + Timestamp(3679.2019, offset=1)) # and wipe our hands of it self.assertEqual(self.reconciler.stats['pop_queue'], 1) self.assertEqual(deleted_container_entries, @@ -1407,7 +1409,7 @@ class TestReconciler(unittest.TestCase): self.assertEqual(self.reconciler.stats['cleanup_success'], 1) self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp('3679.20191')) + Timestamp(3679.2019, offset=1)) # and since we cleaned up the old object, so this counts as done self.assertEqual(self.reconciler.stats['pop_queue'], 1) self.assertEqual(deleted_container_entries, @@ -1448,13 +1450,13 @@ class TestReconciler(unittest.TestCase): # ... with a q_ts + offset 2 put_headers = self.fake_swift.storage_policy[0].headers[1] self.assertEqual(put_headers.get('X-Timestamp'), - normalize_timestamp(36123.38395)) + Timestamp(36123.38393, offset=2)) # then clean the dark matter self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1) self.assertEqual(self.reconciler.stats['cleanup_success'], 1) # ... with a q_ts + offset 1 self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp(36123.38394)) + Timestamp(36123.38393, offset=1)) # and pop the queue self.assertEqual(self.reconciler.stats['pop_queue'], 1) @@ -1499,7 +1501,7 @@ class TestReconciler(unittest.TestCase): put_headers = self.fake_swift.storage_policy[0].headers[1] # ...with q_ts + offset 2 (20-microseconds) self.assertEqual(put_headers.get('X-Timestamp'), - normalize_timestamp(36123.383945)) + Timestamp(36123.383925, offset=2)) # but it failed self.assertEqual(self.reconciler.stats['copy_success'], 0) self.assertEqual(self.reconciler.stats['copy_failed'], 1) @@ -1550,7 +1552,7 @@ class TestReconciler(unittest.TestCase): put_headers = self.fake_swift.storage_policy[0].headers[1] # ...with q_ts + offset 2 (20-microseconds) self.assertEqual(put_headers.get('X-Timestamp'), - normalize_timestamp(36123.383945)) + Timestamp(36123.383925, offset=2)) # but it blows up hard self.assertEqual(self.reconciler.stats['unhandled_error'], 1) # so we don't cleanup @@ -1561,7 +1563,7 @@ class TestReconciler(unittest.TestCase): self.assertEqual(self.reconciler.stats['retry'], 1) def test_object_move_no_such_object_no_tombstone_recent(self): - q_ts = float(normalize_timestamp(time.time())) + q_ts = float(Timestamp(time.time())) container = str(int(q_ts // 3600 * 3600)) q_path = '.misplaced_objects/%s' % container @@ -1593,7 +1595,7 @@ class TestReconciler(unittest.TestCase): self.assertEqual(deleted_container_entries, []) def test_object_move_no_such_object_no_tombstone_ancient(self): - queue_ts = float(normalize_timestamp(time.time())) - \ + queue_ts = float(Timestamp(time.time())) - \ self.reconciler.reclaim_age * 1.1 container = str(int(queue_ts // 3600 * 3600)) @@ -1630,8 +1632,7 @@ class TestReconciler(unittest.TestCase): [('.misplaced_objects', container, '1:/AUTH_jeb/c/o1')]) def test_delete_old_empty_queue_containers(self): - ts = float(normalize_timestamp(time.time())) - \ - self.reconciler.reclaim_age * 1.1 + ts = time.time() - self.reconciler.reclaim_age * 1.1 container = str(int(ts // 3600 * 3600)) older_ts = ts - 3600 older_container = str(int(older_ts // 3600 * 3600)) diff --git a/test/unit/container/test_replicator.py b/test/unit/container/test_replicator.py index a1888c4031..74cf7734a4 100644 --- a/test/unit/container/test_replicator.py +++ b/test/unit/container/test_replicator.py @@ -26,7 +26,7 @@ from swift.common import db_replicator from swift.container import replicator, backend, server from swift.container.reconciler import ( MISPLACED_OBJECTS_ACCOUNT, get_reconciler_container_name) -from swift.common.utils import normalize_timestamp +from swift.common.utils import Timestamp from swift.common.storage_policy import POLICIES from test.unit.common import test_db_replicator @@ -44,18 +44,18 @@ class TestReplicator(unittest.TestCase): def test_report_up_to_date(self): repl = replicator.ContainerReplicator({}) - info = {'put_timestamp': normalize_timestamp(1), - 'delete_timestamp': normalize_timestamp(0), + info = {'put_timestamp': Timestamp(1).internal, + 'delete_timestamp': Timestamp(0).internal, 'object_count': 0, 'bytes_used': 0, - 'reported_put_timestamp': normalize_timestamp(1), - 'reported_delete_timestamp': normalize_timestamp(0), + 'reported_put_timestamp': Timestamp(1).internal, + 'reported_delete_timestamp': Timestamp(0).internal, 'reported_object_count': 0, 'reported_bytes_used': 0} self.assertTrue(repl.report_up_to_date(info)) - info['delete_timestamp'] = normalize_timestamp(2) + info['delete_timestamp'] = Timestamp(2).internal self.assertFalse(repl.report_up_to_date(info)) - info['reported_delete_timestamp'] = normalize_timestamp(2) + info['reported_delete_timestamp'] = Timestamp(2).internal self.assertTrue(repl.report_up_to_date(info)) info['object_count'] = 1 self.assertFalse(repl.report_up_to_date(info)) @@ -65,9 +65,9 @@ class TestReplicator(unittest.TestCase): self.assertFalse(repl.report_up_to_date(info)) info['reported_bytes_used'] = 1 self.assertTrue(repl.report_up_to_date(info)) - info['put_timestamp'] = normalize_timestamp(3) + info['put_timestamp'] = Timestamp(3).internal self.assertFalse(repl.report_up_to_date(info)) - info['reported_put_timestamp'] = normalize_timestamp(3) + info['reported_put_timestamp'] = Timestamp(3).internal self.assertTrue(repl.report_up_to_date(info)) @@ -328,21 +328,21 @@ class TestReplicatorSync(test_db_replicator.TestReplicatorSync): self.assertTrue(remote_broker.is_deleted()) info = broker.get_info() remote_info = remote_broker.get_info() - self.assert_(float(remote_info['status_changed_at']) > - float(remote_info['put_timestamp']), + self.assert_(Timestamp(remote_info['status_changed_at']) > + Timestamp(remote_info['put_timestamp']), 'remote status_changed_at (%s) is not ' 'greater than put_timestamp (%s)' % ( remote_info['status_changed_at'], remote_info['put_timestamp'])) - self.assert_(float(remote_info['status_changed_at']) > - float(info['status_changed_at']), + self.assert_(Timestamp(remote_info['status_changed_at']) > + Timestamp(info['status_changed_at']), 'remote status_changed_at (%s) is not ' 'greater than local status_changed_at (%s)' % ( remote_info['status_changed_at'], info['status_changed_at'])) def test_sync_bogus_db_quarantines(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) policy = random.choice(list(POLICIES)) @@ -667,22 +667,21 @@ class TestReplicatorSync(test_db_replicator.TestReplicatorSync): remote_broker.update_status_changed_at(remote_recreate_timestamp) def test_sync_to_remote_with_misplaced(self): - ts = itertools.count(int(time.time())) + ts = (Timestamp(t).internal for t in + itertools.count(int(time.time()))) # create "local" broker policy = random.choice(list(POLICIES)) broker = self._get_broker('a', 'c', node_index=0) - broker.initialize(normalize_timestamp(ts.next()), - policy.idx) + broker.initialize(ts.next(), policy.idx) # create "remote" broker remote_policy = random.choice([p for p in POLICIES if p is not policy]) remote_broker = self._get_broker('a', 'c', node_index=1) - remote_broker.initialize(normalize_timestamp(ts.next()), - remote_policy.idx) + remote_broker.initialize(ts.next(), remote_policy.idx) # add misplaced row to remote_broker remote_broker.put_object( - '/a/c/o', normalize_timestamp(ts.next()), 0, 'content-type', + '/a/c/o', ts.next(), 0, 'content-type', 'etag', storage_policy_index=remote_broker.storage_policy_index) # since this row matches policy index or remote, it shows up in count self.assertEqual(remote_broker.get_info()['object_count'], 1) @@ -716,19 +715,18 @@ class TestReplicatorSync(test_db_replicator.TestReplicatorSync): self.assertEqual(info[key], value) def test_misplaced_rows_replicate_and_enqueue(self): - ts = itertools.count(int(time.time())) + ts = (Timestamp(t).internal for t in + itertools.count(int(time.time()))) policy = random.choice(list(POLICIES)) broker = self._get_broker('a', 'c', node_index=0) - broker.initialize(normalize_timestamp(ts.next()), - policy.idx) + broker.initialize(ts.next(), policy.idx) remote_policy = random.choice([p for p in POLICIES if p is not policy]) remote_broker = self._get_broker('a', 'c', node_index=1) - remote_broker.initialize(normalize_timestamp(ts.next()), - remote_policy.idx) + remote_broker.initialize(ts.next(), remote_policy.idx) # add a misplaced row to *local* broker - obj_put_timestamp = normalize_timestamp(ts.next()) + obj_put_timestamp = ts.next() broker.put_object( 'o', obj_put_timestamp, 0, 'content-type', 'etag', storage_policy_index=remote_policy.idx) @@ -777,22 +775,21 @@ class TestReplicatorSync(test_db_replicator.TestReplicatorSync): self.assertEqual(broker.get_reconciler_sync(), 1) def test_multiple_out_sync_reconciler_enqueue_normalize(self): - ts = itertools.count(int(time.time())) + ts = (Timestamp(t).internal for t in + itertools.count(int(time.time()))) policy = random.choice(list(POLICIES)) broker = self._get_broker('a', 'c', node_index=0) - broker.initialize(normalize_timestamp(ts.next()), policy.idx) + broker.initialize(ts.next(), policy.idx) remote_policy = random.choice([p for p in POLICIES if p is not policy]) remote_broker = self._get_broker('a', 'c', node_index=1) - remote_broker.initialize(normalize_timestamp(ts.next()), - remote_policy.idx) + remote_broker.initialize(ts.next(), remote_policy.idx) # add some rows to brokers for db in (broker, remote_broker): for p in (policy, remote_policy): - db.put_object('o-%s' % p.name, normalize_timestamp(ts.next()), - 0, 'content-type', 'etag', - storage_policy_index=p.idx) + db.put_object('o-%s' % p.name, ts.next(), 0, 'content-type', + 'etag', storage_policy_index=p.idx) db._commit_puts() expected_policy_stats = { diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index 1823196300..757ce5ac97 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -35,8 +35,8 @@ from swift.common.swob import Request, HeaderKeyDict import swift.container from swift.container import server as container_server from swift.common import constraints -from swift.common.utils import (normalize_timestamp, mkdirs, public, - replication, lock_parent_directory) +from swift.common.utils import (Timestamp, mkdirs, public, replication, + lock_parent_directory, json) from test.unit import fake_http_connect from swift.common.storage_policy import (POLICY_INDEX, POLICIES, StoragePolicy) @@ -171,9 +171,9 @@ class TestContainerController(unittest.TestCase): def test_HEAD(self): start = int(time.time()) - ts = itertools.count(start) + ts = (Timestamp(t).internal for t in itertools.count(start)) req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'x-timestamp': normalize_timestamp(ts.next())}) + 'x-timestamp': ts.next()}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c', method='HEAD') response = req.get_response(self.controller) @@ -182,7 +182,7 @@ class TestContainerController(unittest.TestCase): self.assertEqual(response.headers['x-container-object-count'], '0') obj_put_request = Request.blank( '/sda1/p/a/c/o', method='PUT', headers={ - 'x-timestamp': normalize_timestamp(ts.next()), + 'x-timestamp': ts.next(), 'x-size': 42, 'x-content-type': 'text/plain', 'x-etag': 'x', @@ -196,20 +196,24 @@ class TestContainerController(unittest.TestCase): self.assertEqual(response.headers['x-container-bytes-used'], '42') self.assertEqual(response.headers['x-container-object-count'], '1') # created at time... - self.assert_(float(response.headers['x-timestamp']) >= start) + created_at_header = Timestamp(response.headers['x-timestamp']) + self.assertEqual(response.headers['x-timestamp'], + created_at_header.normal) + self.assert_(created_at_header >= start) self.assertEqual(response.headers['x-put-timestamp'], - normalize_timestamp(start)) + Timestamp(start).normal) # backend headers self.assertEqual(int(response.headers[POLICY_INDEX]), int(POLICIES.default)) - self.assert_(float(response.headers['x-backend-timestamp']) >= start) + self.assert_( + Timestamp(response.headers['x-backend-timestamp']) >= start) self.assertEqual(response.headers['x-backend-put-timestamp'], - normalize_timestamp(start)) + Timestamp(start).internal) self.assertEqual(response.headers['x-backend-delete-timestamp'], - normalize_timestamp(0)) + Timestamp(0).internal) self.assertEqual(response.headers['x-backend-status-changed-at'], - normalize_timestamp(start)) + Timestamp(start).internal) def test_HEAD_not_found(self): req = Request.blank('/sda1/p/a/c', method='HEAD') @@ -217,22 +221,23 @@ class TestContainerController(unittest.TestCase): self.assertEqual(resp.status_int, 404) self.assertEqual(int(resp.headers[POLICY_INDEX]), 0) self.assertEqual(resp.headers['x-backend-timestamp'], - normalize_timestamp(0)) + Timestamp(0).internal) self.assertEqual(resp.headers['x-backend-put-timestamp'], - normalize_timestamp(0)) + Timestamp(0).internal) self.assertEqual(resp.headers['x-backend-status-changed-at'], - normalize_timestamp(0)) + Timestamp(0).internal) self.assertEqual(resp.headers['x-backend-delete-timestamp'], - normalize_timestamp(0)) + Timestamp(0).internal) for header in ('x-container-object-count', 'x-container-bytes-used', 'x-timestamp', 'x-put-timestamp'): self.assertEqual(resp.headers[header], None) def test_deleted_headers(self): - ts = itertools.count(int(time.time())) + ts = (Timestamp(t).internal for t in + itertools.count(int(time.time()))) request_method_times = { - 'PUT': normalize_timestamp(ts.next()), - 'DELETE': normalize_timestamp(ts.next()), + 'PUT': ts.next(), + 'DELETE': ts.next(), } # setup a deleted container for method in ('PUT', 'DELETE'): @@ -249,8 +254,8 @@ class TestContainerController(unittest.TestCase): # backend headers self.assertEqual(int(resp.headers[POLICY_INDEX]), int(POLICIES.default)) - self.assert_(float(resp.headers['x-backend-timestamp']) >= - float(request_method_times['PUT'])) + self.assert_(Timestamp(resp.headers['x-backend-timestamp']) >= + Timestamp(request_method_times['PUT'])) self.assertEqual(resp.headers['x-backend-put-timestamp'], request_method_times['PUT']) self.assertEqual(resp.headers['x-backend-delete-timestamp'], @@ -357,7 +362,7 @@ class TestContainerController(unittest.TestCase): policy = random.choice(list(POLICIES)) # Set metadata header req = Request.blank('/sda1/p/a/c', method='PUT', - headers={'X-Timestamp': normalize_timestamp(1), + headers={'X-Timestamp': Timestamp(1).internal, POLICY_INDEX: policy.idx}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) @@ -370,7 +375,7 @@ class TestContainerController(unittest.TestCase): def test_PUT_no_policy_specified(self): # Set metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(1)}) + headers={'X-Timestamp': Timestamp(1).internal}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) @@ -383,18 +388,18 @@ class TestContainerController(unittest.TestCase): def test_PUT_bad_policy_specified(self): # Set metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(1), + headers={'X-Timestamp': Timestamp(1).internal, POLICY_INDEX: 'nada'}) resp = req.get_response(self.controller) # make sure we get bad response self.assertEquals(resp.status_int, 400) def test_PUT_no_policy_change(self): - ts = itertools.count(1) + ts = (Timestamp(t).internal for t in itertools.count(time.time())) policy = random.choice(list(POLICIES)) # Set metadata header req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), POLICY_INDEX: policy.idx}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) @@ -407,7 +412,7 @@ class TestContainerController(unittest.TestCase): # now try to update w/o changing the policy for method in ('POST', 'PUT'): req = Request.blank('/sda1/p/a/c', method=method, headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), POLICY_INDEX: policy.idx }) resp = req.get_response(self.controller) @@ -419,11 +424,11 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.headers.get(POLICY_INDEX), str(policy.idx)) def test_PUT_bad_policy_change(self): - ts = itertools.count(1) + ts = (Timestamp(t).internal for t in itertools.count(time.time())) policy = random.choice(list(POLICIES)) # Set metadata header req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), POLICY_INDEX: policy.idx}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) @@ -437,7 +442,7 @@ class TestContainerController(unittest.TestCase): for other_policy in other_policies: # now try to change it and make sure we get a conflict req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), POLICY_INDEX: other_policy.idx }) resp = req.get_response(self.controller) @@ -451,10 +456,10 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.headers.get(POLICY_INDEX), str(policy.idx)) def test_POST_ignores_policy_change(self): - ts = itertools.count(1) + ts = (Timestamp(t).internal for t in itertools.count(time.time())) policy = random.choice(list(POLICIES)) req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), POLICY_INDEX: policy.idx}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) @@ -468,7 +473,7 @@ class TestContainerController(unittest.TestCase): for other_policy in other_policies: # now try to change it and make sure we get a conflict req = Request.blank('/sda1/p/a/c', method='POST', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), POLICY_INDEX: other_policy.idx }) resp = req.get_response(self.controller) @@ -483,10 +488,11 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.headers.get(POLICY_INDEX), str(policy.idx)) def test_PUT_no_policy_for_existing_default(self): - ts = itertools.count(1) + ts = (Timestamp(t).internal for t in + itertools.count(int(time.time()))) # create a container with the default storage policy req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), }) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 201) # sanity check @@ -500,7 +506,7 @@ class TestContainerController(unittest.TestCase): # put again without specifying the storage policy req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), }) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 202) # sanity check @@ -517,10 +523,11 @@ class TestContainerController(unittest.TestCase): # during a config change restart across a multi node cluster. proxy_default = random.choice([p for p in POLICIES if not p.is_default]) - ts = itertools.count(1) + ts = (Timestamp(t).internal for t in + itertools.count(int(time.time()))) # create a container with the default storage policy req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), 'X-Backend-Storage-Policy-Default': int(proxy_default), }) resp = req.get_response(self.controller) @@ -535,7 +542,7 @@ class TestContainerController(unittest.TestCase): # put again without proxy specifying the different default req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), 'X-Backend-Storage-Policy-Default': int(POLICIES.default), }) resp = req.get_response(self.controller) @@ -549,11 +556,11 @@ class TestContainerController(unittest.TestCase): int(proxy_default)) def test_PUT_no_policy_for_existing_non_default(self): - ts = itertools.count(1) + ts = (Timestamp(t).internal for t in itertools.count(time.time())) non_default_policy = [p for p in POLICIES if not p.is_default][0] # create a container with the non-default storage policy req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), POLICY_INDEX: non_default_policy.idx, }) resp = req.get_response(self.controller) @@ -568,7 +575,7 @@ class TestContainerController(unittest.TestCase): # put again without specifiying the storage policy req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), }) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 202) # sanity check @@ -584,7 +591,7 @@ class TestContainerController(unittest.TestCase): # Set metadata header req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(1), + headers={'X-Timestamp': Timestamp(1).internal, 'X-Container-Meta-Test': 'Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) @@ -595,7 +602,7 @@ class TestContainerController(unittest.TestCase): # Set another metadata header, ensuring old one doesn't disappear req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(1), + headers={'X-Timestamp': Timestamp(1).internal, 'X-Container-Meta-Test2': 'Value2'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -607,7 +614,7 @@ class TestContainerController(unittest.TestCase): # Update metadata header req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(3), + headers={'X-Timestamp': Timestamp(3).internal, 'X-Container-Meta-Test': 'New Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 202) @@ -619,7 +626,7 @@ class TestContainerController(unittest.TestCase): # Send old update to metadata header req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(2), + headers={'X-Timestamp': Timestamp(2).internal, 'X-Container-Meta-Test': 'Old Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 202) @@ -631,7 +638,7 @@ class TestContainerController(unittest.TestCase): # Remove metadata header (by setting it to empty) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(4), + headers={'X-Timestamp': Timestamp(4).internal, 'X-Container-Meta-Test': ''}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 202) @@ -646,7 +653,7 @@ class TestContainerController(unittest.TestCase): key2 = '%sTest2' % prefix # Set metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(1), + headers={'X-Timestamp': Timestamp(1).internal, key: 'Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) @@ -656,7 +663,7 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.headers.get(key.lower()), 'Value') # Set another metadata header, ensuring old one doesn't disappear req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(1), + headers={'X-Timestamp': Timestamp(1).internal, key2: 'Value2'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -667,7 +674,7 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.headers.get(key2.lower()), 'Value2') # Update metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(3), + headers={'X-Timestamp': Timestamp(3).internal, key: 'New Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 202) @@ -678,7 +685,7 @@ class TestContainerController(unittest.TestCase): 'New Value') # Send old update to metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(2), + headers={'X-Timestamp': Timestamp(2).internal, key: 'Old Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 202) @@ -689,7 +696,7 @@ class TestContainerController(unittest.TestCase): 'New Value') # Remove metadata header (by setting it to empty) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(4), + headers={'X-Timestamp': Timestamp(4).internal, key: ''}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 202) @@ -725,13 +732,13 @@ class TestContainerController(unittest.TestCase): def test_POST_HEAD_metadata(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(1)}) + headers={'X-Timestamp': Timestamp(1).internal}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) # Set metadata header req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(1), + headers={'X-Timestamp': Timestamp(1).internal, 'X-Container-Meta-Test': 'Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -742,7 +749,7 @@ class TestContainerController(unittest.TestCase): # Update metadata header req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(3), + headers={'X-Timestamp': Timestamp(3).internal, 'X-Container-Meta-Test': 'New Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -754,7 +761,7 @@ class TestContainerController(unittest.TestCase): # Send old update to metadata header req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(2), + headers={'X-Timestamp': Timestamp(2).internal, 'X-Container-Meta-Test': 'Old Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -766,7 +773,7 @@ class TestContainerController(unittest.TestCase): # Remove metadata header (by setting it to empty) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(4), + headers={'X-Timestamp': Timestamp(4).internal, 'X-Container-Meta-Test': ''}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -779,12 +786,12 @@ class TestContainerController(unittest.TestCase): prefix = get_sys_meta_prefix('container') key = '%sTest' % prefix req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(1)}) + headers={'X-Timestamp': Timestamp(1).internal}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) # Set metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(1), + headers={'X-Timestamp': Timestamp(1).internal, key: 'Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -794,7 +801,7 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.headers.get(key.lower()), 'Value') # Update metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(3), + headers={'X-Timestamp': Timestamp(3).internal, key: 'New Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -805,7 +812,7 @@ class TestContainerController(unittest.TestCase): 'New Value') # Send old update to metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(2), + headers={'X-Timestamp': Timestamp(2).internal, key: 'Old Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -816,7 +823,7 @@ class TestContainerController(unittest.TestCase): 'New Value') # Remove metadata header (by setting it to empty) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(4), + headers={'X-Timestamp': Timestamp(4).internal, key: ''}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -962,11 +969,11 @@ class TestContainerController(unittest.TestCase): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': '0000000001.00000', + headers={'X-Timestamp': Timestamp(1).internal, 'X-Account-Host': '%s:%s' % bindsock.getsockname(), 'X-Account-Partition': '123', 'X-Account-Device': 'sda1'}) - event = spawn(accept, 201, '0000000001.00000') + event = spawn(accept, 201, Timestamp(1).internal) try: with Timeout(3): resp = req.get_response(self.controller) @@ -984,11 +991,11 @@ class TestContainerController(unittest.TestCase): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': '0000000003.00000', + headers={'X-Timestamp': Timestamp(3).internal, 'X-Account-Host': '%s:%s' % bindsock.getsockname(), 'X-Account-Partition': '123', 'X-Account-Device': 'sda1'}) - event = spawn(accept, 404, '0000000003.00000') + event = spawn(accept, 404, Timestamp(3).internal) try: with Timeout(3): resp = req.get_response(self.controller) @@ -1000,11 +1007,11 @@ class TestContainerController(unittest.TestCase): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': '0000000005.00000', + headers={'X-Timestamp': Timestamp(5).internal, 'X-Account-Host': '%s:%s' % bindsock.getsockname(), 'X-Account-Partition': '123', 'X-Account-Device': 'sda1'}) - event = spawn(accept, 503, '0000000005.00000') + event = spawn(accept, 503, Timestamp(5).internal) got_exc = False try: with Timeout(3): @@ -1125,9 +1132,9 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.status_int, 404) # sanity # backend headers expectations = { - 'x-backend-put-timestamp': normalize_timestamp(1), - 'x-backend-delete-timestamp': normalize_timestamp(2), - 'x-backend-status-changed-at': normalize_timestamp(2), + 'x-backend-put-timestamp': Timestamp(1).internal, + 'x-backend-delete-timestamp': Timestamp(2).internal, + 'x-backend-status-changed-at': Timestamp(2).internal, } for header, value in expectations.items(): self.assertEqual(resp.headers[header], value, @@ -1136,9 +1143,9 @@ class TestContainerController(unittest.TestCase): db = self.controller._get_container_broker('sda1', 'p', 'a', 'c') self.assertEqual(True, db.is_deleted()) info = db.get_info() - self.assertEquals(info['put_timestamp'], normalize_timestamp('1')) - self.assertEquals(info['delete_timestamp'], normalize_timestamp('2')) - self.assertEquals(info['status_changed_at'], normalize_timestamp('2')) + self.assertEquals(info['put_timestamp'], Timestamp('1').internal) + self.assertEquals(info['delete_timestamp'], Timestamp('2').internal) + self.assertEquals(info['status_changed_at'], Timestamp('2').internal) # recreate req = Request.blank(path, method='PUT', headers={'X-Timestamp': '4'}) @@ -1147,17 +1154,17 @@ class TestContainerController(unittest.TestCase): db = self.controller._get_container_broker('sda1', 'p', 'a', 'c') self.assertEqual(False, db.is_deleted()) info = db.get_info() - self.assertEquals(info['put_timestamp'], normalize_timestamp('4')) - self.assertEquals(info['delete_timestamp'], normalize_timestamp('2')) - self.assertEquals(info['status_changed_at'], normalize_timestamp('4')) + self.assertEquals(info['put_timestamp'], Timestamp('4').internal) + self.assertEquals(info['delete_timestamp'], Timestamp('2').internal) + self.assertEquals(info['status_changed_at'], Timestamp('4').internal) for method in ('GET', 'HEAD'): req = Request.blank(path) resp = req.get_response(self.controller) expectations = { - 'x-put-timestamp': normalize_timestamp(4), - 'x-backend-put-timestamp': normalize_timestamp(4), - 'x-backend-delete-timestamp': normalize_timestamp(2), - 'x-backend-status-changed-at': normalize_timestamp(4), + 'x-put-timestamp': Timestamp(4).normal, + 'x-backend-put-timestamp': Timestamp(4).internal, + 'x-backend-delete-timestamp': Timestamp(2).internal, + 'x-backend-status-changed-at': Timestamp(4).internal, } for header, expected in expectations.items(): self.assertEqual(resp.headers[header], expected, @@ -1217,8 +1224,8 @@ class TestContainerController(unittest.TestCase): [(exists, db.db_file) for exists in (False, True)]) # info was updated info = db.get_info() - self.assertEquals(info['put_timestamp'], normalize_timestamp('4')) - self.assertEquals(info['delete_timestamp'], normalize_timestamp('2')) + self.assertEquals(info['put_timestamp'], Timestamp('4').internal) + self.assertEquals(info['delete_timestamp'], Timestamp('2').internal) def test_DELETE_not_found(self): # Even if the container wasn't previously heard of, the container @@ -1231,7 +1238,7 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.status_int, 404) def test_change_storage_policy_via_DELETE_then_PUT(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) policy = random.choice(list(POLICIES)) req = Request.blank( @@ -1265,7 +1272,7 @@ class TestContainerController(unittest.TestCase): str(other_policy.idx)) def test_change_to_default_storage_policy_via_DELETE_then_PUT(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) non_default_policy = random.choice([p for p in POLICIES if not p.is_default]) @@ -1297,40 +1304,155 @@ class TestContainerController(unittest.TestCase): def test_DELETE_object(self): req = Request.blank( - '/sda1/p/a/c', - environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '2'}) + '/sda1/p/a/c', method='PUT', headers={ + 'X-Timestamp': Timestamp(2).internal}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank( - '/sda1/p/a/c/o', - environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0', - 'HTTP_X_SIZE': 1, 'HTTP_X_CONTENT_TYPE': 'text/plain', - 'HTTP_X_ETAG': 'x'}) + '/sda1/p/a/c/o', method='PUT', headers={ + 'X-Timestamp': Timestamp(0).internal, 'X-Size': 1, + 'X-Content-Type': 'text/plain', 'X-Etag': 'x'}) self._update_object_put_headers(req) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) - req = Request.blank( - '/sda1/p/a/c', - environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '3'}) + ts = (Timestamp(t).internal for t in + itertools.count(3)) + req = Request.blank('/sda1/p/a/c', method='DELETE', headers={ + 'X-Timestamp': ts.next()}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 409) - req = Request.blank( - '/sda1/p/a/c/o', - environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '4'}) + req = Request.blank('/sda1/p/a/c/o', method='DELETE', headers={ + 'X-Timestamp': ts.next()}) self._update_object_put_headers(req) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) - req = Request.blank( - '/sda1/p/a/c', - environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '5'}) + req = Request.blank('/sda1/p/a/c', method='DELETE', headers={ + 'X-Timestamp': ts.next()}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) - req = Request.blank( - '/sda1/p/a/c', - environ={'REQUEST_METHOD': 'GET'}, headers={'X-Timestamp': '6'}) + req = Request.blank('/sda1/p/a/c', method='GET', headers={ + 'X-Timestamp': ts.next()}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) + def test_object_update_with_offset(self): + ts = (Timestamp(t).internal for t in + itertools.count(int(time.time()))) + # create container + req = Request.blank('/sda1/p/a/c', method='PUT', headers={ + 'X-Timestamp': ts.next()}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 201) + # check status + req = Request.blank('/sda1/p/a/c', method='HEAD') + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assertEqual(int(resp.headers[POLICY_INDEX]), + int(POLICIES.default)) + # create object + obj_timestamp = ts.next() + req = Request.blank( + '/sda1/p/a/c/o', method='PUT', headers={ + 'X-Timestamp': obj_timestamp, 'X-Size': 1, + 'X-Content-Type': 'text/plain', 'X-Etag': 'x'}) + self._update_object_put_headers(req) + resp = req.get_response(self.controller) + self.assertEquals(resp.status_int, 201) + # check listing + req = Request.blank('/sda1/p/a/c', method='GET', + query_string='format=json') + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(int(resp.headers['X-Container-Object-Count']), 1) + self.assertEqual(int(resp.headers['X-Container-Bytes-Used']), 1) + listing_data = json.loads(resp.body) + self.assertEqual(1, len(listing_data)) + for obj in listing_data: + self.assertEqual(obj['name'], 'o') + self.assertEqual(obj['bytes'], 1) + self.assertEqual(obj['hash'], 'x') + self.assertEqual(obj['content_type'], 'text/plain') + # send an update with an offset + offset_timestamp = Timestamp(obj_timestamp, offset=1).internal + req = Request.blank( + '/sda1/p/a/c/o', method='PUT', headers={ + 'X-Timestamp': offset_timestamp, 'X-Size': 2, + 'X-Content-Type': 'text/html', 'X-Etag': 'y'}) + self._update_object_put_headers(req) + resp = req.get_response(self.controller) + self.assertEquals(resp.status_int, 201) + # check updated listing + req = Request.blank('/sda1/p/a/c', method='GET', + query_string='format=json') + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(int(resp.headers['X-Container-Object-Count']), 1) + self.assertEqual(int(resp.headers['X-Container-Bytes-Used']), 2) + listing_data = json.loads(resp.body) + self.assertEqual(1, len(listing_data)) + for obj in listing_data: + self.assertEqual(obj['name'], 'o') + self.assertEqual(obj['bytes'], 2) + self.assertEqual(obj['hash'], 'y') + self.assertEqual(obj['content_type'], 'text/html') + # now overwrite with a newer time + delete_timestamp = ts.next() + req = Request.blank( + '/sda1/p/a/c/o', method='DELETE', headers={ + 'X-Timestamp': delete_timestamp}) + self._update_object_put_headers(req) + resp = req.get_response(self.controller) + self.assertEquals(resp.status_int, 204) + # check empty listing + req = Request.blank('/sda1/p/a/c', method='GET', + query_string='format=json') + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(int(resp.headers['X-Container-Object-Count']), 0) + self.assertEqual(int(resp.headers['X-Container-Bytes-Used']), 0) + listing_data = json.loads(resp.body) + self.assertEqual(0, len(listing_data)) + # recreate with an offset + offset_timestamp = Timestamp(delete_timestamp, offset=1).internal + req = Request.blank( + '/sda1/p/a/c/o', method='PUT', headers={ + 'X-Timestamp': offset_timestamp, 'X-Size': 3, + 'X-Content-Type': 'text/enriched', 'X-Etag': 'z'}) + self._update_object_put_headers(req) + resp = req.get_response(self.controller) + self.assertEquals(resp.status_int, 201) + # check un-deleted listing + req = Request.blank('/sda1/p/a/c', method='GET', + query_string='format=json') + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(int(resp.headers['X-Container-Object-Count']), 1) + self.assertEqual(int(resp.headers['X-Container-Bytes-Used']), 3) + listing_data = json.loads(resp.body) + self.assertEqual(1, len(listing_data)) + for obj in listing_data: + self.assertEqual(obj['name'], 'o') + self.assertEqual(obj['bytes'], 3) + self.assertEqual(obj['hash'], 'z') + self.assertEqual(obj['content_type'], 'text/enriched') + # delete offset with newer offset + delete_timestamp = Timestamp(offset_timestamp, offset=1).internal + req = Request.blank( + '/sda1/p/a/c/o', method='DELETE', headers={ + 'X-Timestamp': delete_timestamp}) + self._update_object_put_headers(req) + resp = req.get_response(self.controller) + self.assertEquals(resp.status_int, 204) + # check empty listing + req = Request.blank('/sda1/p/a/c', method='GET', + query_string='format=json') + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(int(resp.headers['X-Container-Object-Count']), 0) + self.assertEqual(int(resp.headers['X-Container-Bytes-Used']), 0) + listing_data = json.loads(resp.body) + self.assertEqual(0, len(listing_data)) + def test_DELETE_account_update(self): bindsock = listen(('127.0.0.1', 0)) @@ -1365,11 +1487,11 @@ class TestContainerController(unittest.TestCase): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}, - headers={'X-Timestamp': '0000000002.00000', + headers={'X-Timestamp': Timestamp(2).internal, 'X-Account-Host': '%s:%s' % bindsock.getsockname(), 'X-Account-Partition': '123', 'X-Account-Device': 'sda1'}) - event = spawn(accept, 204, '0000000002.00000') + event = spawn(accept, 204, Timestamp(2).internal) try: with Timeout(3): resp = req.get_response(self.controller) @@ -1379,18 +1501,18 @@ class TestContainerController(unittest.TestCase): if err: raise Exception(err) req = Request.blank( - '/sda1/p/a/c', - environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '2'}) + '/sda1/p/a/c', method='PUT', headers={ + 'X-Timestamp': Timestamp(2).internal}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}, - headers={'X-Timestamp': '0000000003.00000', + headers={'X-Timestamp': Timestamp(3).internal, 'X-Account-Host': '%s:%s' % bindsock.getsockname(), 'X-Account-Partition': '123', 'X-Account-Device': 'sda1'}) - event = spawn(accept, 404, '0000000003.00000') + event = spawn(accept, 404, Timestamp(3).internal) try: with Timeout(3): resp = req.get_response(self.controller) @@ -1400,18 +1522,18 @@ class TestContainerController(unittest.TestCase): if err: raise Exception(err) req = Request.blank( - '/sda1/p/a/c', - environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '4'}) + '/sda1/p/a/c', method='PUT', headers={ + 'X-Timestamp': Timestamp(4).internal}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}, - headers={'X-Timestamp': '0000000005.00000', + headers={'X-Timestamp': Timestamp(5).internal, 'X-Account-Host': '%s:%s' % bindsock.getsockname(), 'X-Account-Partition': '123', 'X-Account-Device': 'sda1'}) - event = spawn(accept, 503, '0000000005.00000') + event = spawn(accept, 503, Timestamp(5).internal) got_exc = False try: with Timeout(3): @@ -1785,18 +1907,11 @@ class TestContainerController(unittest.TestCase): self.assertEquals(result, [u'\u2603', 'text/plain;charset="utf-8"']) def test_GET_accept_not_valid(self): - req = Request.blank( - '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', - 'HTTP_X_TIMESTAMP': '0'}) - req.get_response(self.controller) - req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Put-Timestamp': '1', - 'X-Delete-Timestamp': '0', - 'X-Object-Count': '0', - 'X-Bytes-Used': '0', - 'X-Timestamp': normalize_timestamp(0)}) - req.get_response(self.controller) - req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) + req = Request.blank('/sda1/p/a/c', method='PUT', headers={ + 'X-Timestamp': Timestamp(0).internal}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c', method='GET') req.accept = 'application/xml*' resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 406) @@ -2080,13 +2195,12 @@ class TestContainerController(unittest.TestCase): def test_params_format(self): req = Request.blank( - '/sda1/p/a/c', - headers={'X-Timestamp': normalize_timestamp(1)}, - environ={'REQUEST_METHOD': 'PUT'}) + '/sda1/p/a/c', method='PUT', + headers={'X-Timestamp': Timestamp(1).internal}) req.get_response(self.controller) for format in ('xml', 'json'): req = Request.blank('/sda1/p/a/c?format=%s' % format, - environ={'REQUEST_METHOD': 'GET'}) + method='GET') resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 200) @@ -2105,9 +2219,8 @@ class TestContainerController(unittest.TestCase): resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 412, "%d on param delimiter" % (resp.status_int)) - req = Request.blank('/sda1/p/a/c', - headers={'X-Timestamp': normalize_timestamp(1)}, - environ={'REQUEST_METHOD': 'PUT'}) + req = Request.blank('/sda1/p/a/c', method='PUT', + headers={'X-Timestamp': Timestamp(1).internal}) req.get_response(self.controller) # Good UTF8 sequence, ignored for limit, doesn't affect other queries for param in ('limit', 'marker', 'path', 'prefix', 'end_marker', @@ -2119,7 +2232,7 @@ class TestContainerController(unittest.TestCase): "%d on param %s" % (resp.status_int, param)) def test_put_auto_create(self): - headers = {'x-timestamp': normalize_timestamp(1), + headers = {'x-timestamp': Timestamp(1).internal, 'x-size': '0', 'x-content-type': 'text/plain', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e'} @@ -2149,7 +2262,7 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.status_int, 404) def test_delete_auto_create(self): - headers = {'x-timestamp': normalize_timestamp(1)} + headers = {'x-timestamp': Timestamp(1).internal} req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, @@ -2177,7 +2290,7 @@ class TestContainerController(unittest.TestCase): def test_content_type_on_HEAD(self): Request.blank('/sda1/p/a/o', - headers={'X-Timestamp': normalize_timestamp(1)}, + headers={'X-Timestamp': Timestamp(1).internal}, environ={'REQUEST_METHOD': 'PUT'}).get_response( self.controller) @@ -2267,7 +2380,7 @@ class TestContainerController(unittest.TestCase): 'x-bytes-used': 0, 'x-delete-timestamp': '0', 'x-object-count': 0, - 'x-put-timestamp': '0000012345.00000', + 'x-put-timestamp': Timestamp(12345).internal, POLICY_INDEX: '%s' % POLICIES.default.idx, 'referer': 'PUT http://localhost/sda1/p/a/c', 'user-agent': 'container-server %d' % os.getpid(), @@ -2285,7 +2398,7 @@ class TestContainerController(unittest.TestCase): 'x-bytes-used': 0, 'x-delete-timestamp': '0', 'x-object-count': 0, - 'x-put-timestamp': '0000012345.00000', + 'x-put-timestamp': Timestamp(12345).internal, POLICY_INDEX: '%s' % POLICIES.default.idx, 'referer': 'PUT http://localhost/sda1/p/a/c', 'user-agent': 'container-server %d' % os.getpid(), diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py index c5160c11f0..43bd85d55b 100644 --- a/test/unit/obj/test_diskfile.py +++ b/test/unit/obj/test_diskfile.py @@ -38,7 +38,7 @@ from test.unit import (FakeLogger, mock as unit_mock, temptree, from swift.obj import diskfile from swift.common import utils -from swift.common.utils import hash_path, mkdirs, normalize_timestamp +from swift.common.utils import hash_path, mkdirs, Timestamp from swift.common import ring from swift.common.exceptions import DiskFileNotExist, DiskFileQuarantined, \ DiskFileDeviceUnavailable, DiskFileDeleted, DiskFileNotOpen, \ @@ -254,7 +254,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): mkdirs(df._datadir) f = open( os.path.join(df._datadir, - normalize_timestamp(time() - 100) + '.ts'), + Timestamp(time() - 100).internal + '.ts'), 'wb') f.write('1234567890') f.close() @@ -272,7 +272,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): mkdirs(df._datadir) f = open( os.path.join(df._datadir, - normalize_timestamp(time() - 100) + '.ts'), + Timestamp(time() - 100).internal + '.ts'), 'wb') f.write('1234567890') f.close() @@ -304,7 +304,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): f = open( os.path.join( df._datadir, - normalize_timestamp(int(time()) - tdiff) + suff), + Timestamp(int(time()) - tdiff).internal + suff), 'wb') f.write('1234567890') f.close() @@ -330,7 +330,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): f = open( os.path.join( df._datadir, - normalize_timestamp(int(time()) - tdiff) + suff), + Timestamp(int(time()) - tdiff).internal + suff), 'wb') f.write('1234567890') f.close() @@ -412,7 +412,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): mkdirs(df._datadir) with open( os.path.join(df._datadir, - normalize_timestamp(time()) + '.ts'), + Timestamp(time()).internal + '.ts'), 'wb') as f: f.write('1234567890') part = os.path.join(self.objects, '0') @@ -442,7 +442,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): mkdirs(df._datadir) with open( os.path.join(df._datadir, - normalize_timestamp(time()) + '.ts'), + Timestamp(time()).internal + '.ts'), 'wb') as f: f.write('1234567890') part = os.path.join(self.objects, '0') @@ -462,7 +462,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): mkdirs(df._datadir) with open( os.path.join(df._datadir, - normalize_timestamp(time()) + '.ts'), + Timestamp(time()).internal + '.ts'), 'wb') as f: f.write('1234567890') part = os.path.join(self.objects, '0') @@ -479,7 +479,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): mkdirs(df._datadir) with open( os.path.join(df._datadir, - normalize_timestamp(time()) + '.ts'), + Timestamp(time()).internal + '.ts'), 'wb') as f: f.write('1234567890') part = os.path.join(self.objects, '0') @@ -516,7 +516,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): mkdirs(df._datadir) with open( os.path.join(df._datadir, - normalize_timestamp(time()) + '.ts'), + Timestamp(time()).internal + '.ts'), 'wb') as f: f.write('1234567890') part = os.path.join(self.objects, '0') @@ -554,89 +554,89 @@ class TestDiskFileModuleMethods(unittest.TestCase): def test_hash_cleanup_listdir_purge_data_newer_ts(self): # purge .data if there's a newer .ts - file1 = normalize_timestamp(time()) + '.data' - file2 = normalize_timestamp(time() + 1) + '.ts' + file1 = Timestamp(time()).internal + '.data' + file2 = Timestamp(time() + 1).internal + '.ts' file_list = [file1, file2] self.check_hash_cleanup_listdir(file_list, [file2]) def test_hash_cleanup_listdir_purge_ts_newer_data(self): # purge .ts if there's a newer .data - file1 = normalize_timestamp(time()) + '.ts' - file2 = normalize_timestamp(time() + 1) + '.data' + file1 = Timestamp(time()).internal + '.ts' + file2 = Timestamp(time() + 1).internal + '.data' file_list = [file1, file2] self.check_hash_cleanup_listdir(file_list, [file2]) def test_hash_cleanup_listdir_keep_meta_data_purge_ts(self): # keep .meta and .data if meta newer than data and purge .ts - file1 = normalize_timestamp(time()) + '.ts' - file2 = normalize_timestamp(time() + 1) + '.data' - file3 = normalize_timestamp(time() + 2) + '.meta' + file1 = Timestamp(time()).internal + '.ts' + file2 = Timestamp(time() + 1).internal + '.data' + file3 = Timestamp(time() + 2).internal + '.meta' file_list = [file1, file2, file3] self.check_hash_cleanup_listdir(file_list, [file3, file2]) def test_hash_cleanup_listdir_keep_one_ts(self): # keep only latest of multiple .ts files - file1 = normalize_timestamp(time()) + '.ts' - file2 = normalize_timestamp(time() + 1) + '.ts' - file3 = normalize_timestamp(time() + 2) + '.ts' + file1 = Timestamp(time()).internal + '.ts' + file2 = Timestamp(time() + 1).internal + '.ts' + file3 = Timestamp(time() + 2).internal + '.ts' file_list = [file1, file2, file3] self.check_hash_cleanup_listdir(file_list, [file3]) def test_hash_cleanup_listdir_keep_one_data(self): # keep only latest of multiple .data files - file1 = normalize_timestamp(time()) + '.data' - file2 = normalize_timestamp(time() + 1) + '.data' - file3 = normalize_timestamp(time() + 2) + '.data' + file1 = Timestamp(time()).internal + '.data' + file2 = Timestamp(time() + 1).internal + '.data' + file3 = Timestamp(time() + 2).internal + '.data' file_list = [file1, file2, file3] self.check_hash_cleanup_listdir(file_list, [file3]) def test_hash_cleanup_listdir_keep_one_meta(self): # keep only latest of multiple .meta files - file1 = normalize_timestamp(time()) + '.data' - file2 = normalize_timestamp(time() + 1) + '.meta' - file3 = normalize_timestamp(time() + 2) + '.meta' + file1 = Timestamp(time()).internal + '.data' + file2 = Timestamp(time() + 1).internal + '.meta' + file3 = Timestamp(time() + 2).internal + '.meta' file_list = [file1, file2, file3] self.check_hash_cleanup_listdir(file_list, [file3, file1]) def test_hash_cleanup_listdir_ignore_orphaned_ts(self): # A more recent orphaned .meta file will prevent old .ts files # from being cleaned up otherwise - file1 = normalize_timestamp(time()) + '.ts' - file2 = normalize_timestamp(time() + 1) + '.ts' - file3 = normalize_timestamp(time() + 2) + '.meta' + file1 = Timestamp(time()).internal + '.ts' + file2 = Timestamp(time() + 1).internal + '.ts' + file3 = Timestamp(time() + 2).internal + '.meta' file_list = [file1, file2, file3] self.check_hash_cleanup_listdir(file_list, [file3, file2]) def test_hash_cleanup_listdir_purge_old_data_only(self): # Oldest .data will be purge, .meta and .ts won't be touched - file1 = normalize_timestamp(time()) + '.data' - file2 = normalize_timestamp(time() + 1) + '.ts' - file3 = normalize_timestamp(time() + 2) + '.meta' + file1 = Timestamp(time()).internal + '.data' + file2 = Timestamp(time() + 1).internal + '.ts' + file3 = Timestamp(time() + 2).internal + '.meta' file_list = [file1, file2, file3] self.check_hash_cleanup_listdir(file_list, [file3, file2]) def test_hash_cleanup_listdir_purge_old_ts(self): # A single old .ts file will be removed - file1 = normalize_timestamp(time() - (diskfile.ONE_WEEK + 1)) + '.ts' + file1 = Timestamp(time() - (diskfile.ONE_WEEK + 1)).internal + '.ts' file_list = [file1] self.check_hash_cleanup_listdir(file_list, []) def test_hash_cleanup_listdir_meta_keeps_old_ts(self): # An orphaned .meta will not clean up a very old .ts - file1 = normalize_timestamp(time() - (diskfile.ONE_WEEK + 1)) + '.ts' - file2 = normalize_timestamp(time() + 2) + '.meta' + file1 = Timestamp(time() - (diskfile.ONE_WEEK + 1)).internal + '.ts' + file2 = Timestamp(time() + 2).internal + '.meta' file_list = [file1, file2] self.check_hash_cleanup_listdir(file_list, [file2, file1]) def test_hash_cleanup_listdir_keep_single_old_data(self): # A single old .data file will not be removed - file1 = normalize_timestamp(time() - (diskfile.ONE_WEEK + 1)) + '.data' + file1 = Timestamp(time() - (diskfile.ONE_WEEK + 1)).internal + '.data' file_list = [file1] self.check_hash_cleanup_listdir(file_list, [file1]) def test_hash_cleanup_listdir_keep_single_old_meta(self): # A single old .meta file will not be removed - file1 = normalize_timestamp(time() - (diskfile.ONE_WEEK + 1)) + '.meta' + file1 = Timestamp(time() - (diskfile.ONE_WEEK + 1)).internal + '.meta' file_list = [file1] self.check_hash_cleanup_listdir(file_list, [file1]) @@ -865,7 +865,7 @@ class TestDiskFileManager(unittest.TestCase): def test_pickle_async_update(self): self.df_mgr.logger.increment = mock.MagicMock() - ts = normalize_timestamp(10000.0) + ts = Timestamp(10000.0).internal with mock.patch('swift.obj.diskfile.write_pickle') as wp: self.df_mgr.pickle_async_update(self.existing_device1, 'a', 'c', 'o', @@ -981,11 +981,11 @@ class TestDiskFile(unittest.TestCase): mkdirs(df._datadir) if timestamp is None: timestamp = time() - timestamp = normalize_timestamp(timestamp) + timestamp = Timestamp(timestamp).internal if not metadata: metadata = {} if 'X-Timestamp' not in metadata: - metadata['X-Timestamp'] = normalize_timestamp(timestamp) + metadata['X-Timestamp'] = Timestamp(timestamp).internal if 'ETag' not in metadata: etag = md5() etag.update(data) @@ -1038,13 +1038,13 @@ class TestDiskFile(unittest.TestCase): def test_get_metadata(self): df = self._create_test_file('1234567890', timestamp=42) md = df.get_metadata() - self.assertEqual(md['X-Timestamp'], normalize_timestamp(42)) + self.assertEqual(md['X-Timestamp'], Timestamp(42).internal) def test_read_metadata(self): self._create_test_file('1234567890', timestamp=42) df = self._simple_get_diskfile() md = df.read_metadata() - self.assertEqual(md['X-Timestamp'], normalize_timestamp(42)) + self.assertEqual(md['X-Timestamp'], Timestamp(42).internal) def test_get_metadata_not_opened(self): df = self._simple_get_diskfile() @@ -1069,7 +1069,7 @@ class TestDiskFile(unittest.TestCase): self.assertEquals('1024', df._metadata['Content-Length']) # write some new metadata (fast POST, don't send orig meta, ts 42) df = self._simple_get_diskfile() - df.write_metadata({'X-Timestamp': normalize_timestamp(42), + df.write_metadata({'X-Timestamp': Timestamp(42).internal, 'X-Object-Meta-Key2': 'Value2'}) df = self._simple_get_diskfile() with df.open(): @@ -1240,7 +1240,7 @@ class TestDiskFile(unittest.TestCase): if ts: timestamp = ts else: - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal if prealloc: prealloc_size = fsize else: @@ -1578,7 +1578,7 @@ class TestDiskFile(unittest.TestCase): def test_write_metadata(self): df = self._create_test_file('1234567890') - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal metadata = {'X-Timestamp': timestamp, 'X-Object-Meta-test': 'data'} df.write_metadata(metadata) dl = os.listdir(df._datadir) @@ -1590,7 +1590,7 @@ class TestDiskFile(unittest.TestCase): df = self._get_open_disk_file() ts = time() df.delete(ts) - exp_name = '%s.ts' % str(normalize_timestamp(ts)) + exp_name = '%s.ts' % Timestamp(ts).internal dl = os.listdir(df._datadir) self.assertEquals(len(dl), 1) self.assertTrue(exp_name in set(dl)) @@ -1599,7 +1599,7 @@ class TestDiskFile(unittest.TestCase): df = self._get_open_disk_file() ts = time() df.delete(ts) - exp_name = '%s.ts' % str(normalize_timestamp(ts)) + exp_name = '%s.ts' % str(Timestamp(ts).internal) dl = os.listdir(df._datadir) self.assertEquals(len(dl), 1) self.assertTrue(exp_name in set(dl)) @@ -1610,7 +1610,7 @@ class TestDiskFile(unittest.TestCase): df = self._get_open_disk_file() ts = time() df.delete(ts) - exp_name = '%s.ts' % str(normalize_timestamp(ts)) + exp_name = '%s.ts' % str(Timestamp(ts).internal) dl = os.listdir(df._datadir) self.assertEquals(len(dl), 1) self.assertTrue(exp_name in set(dl)) @@ -1685,7 +1685,7 @@ class TestDiskFile(unittest.TestCase): try: df.open() except DiskFileDeleted as d: - self.assertEquals(d.timestamp, normalize_timestamp(10)) + self.assertEquals(d.timestamp, Timestamp(10).internal) else: self.fail("Expected DiskFileDeleted exception") @@ -1701,7 +1701,7 @@ class TestDiskFile(unittest.TestCase): try: df.open() except DiskFileDeleted as d: - self.assertEquals(d.timestamp, normalize_timestamp(8)) + self.assertEquals(d.timestamp, Timestamp(8).internal) else: self.fail("Expected DiskFileDeleted exception") @@ -1717,7 +1717,7 @@ class TestDiskFile(unittest.TestCase): with df.open(): self.assertTrue('X-Timestamp' in df._metadata) self.assertEquals(df._metadata['X-Timestamp'], - normalize_timestamp(10)) + Timestamp(10).internal) self.assertTrue('deleted' not in df._metadata) def test_ondisk_search_loop_data_meta_ts(self): @@ -1732,7 +1732,7 @@ class TestDiskFile(unittest.TestCase): with df.open(): self.assertTrue('X-Timestamp' in df._metadata) self.assertEquals(df._metadata['X-Timestamp'], - normalize_timestamp(10)) + Timestamp(10).internal) self.assertTrue('deleted' not in df._metadata) def test_ondisk_search_loop_wayward_files_ignored(self): @@ -1748,7 +1748,7 @@ class TestDiskFile(unittest.TestCase): with df.open(): self.assertTrue('X-Timestamp' in df._metadata) self.assertEquals(df._metadata['X-Timestamp'], - normalize_timestamp(10)) + Timestamp(10).internal) self.assertTrue('deleted' not in df._metadata) def test_ondisk_search_loop_listdir_error(self): @@ -1995,8 +1995,8 @@ class TestDiskFile(unittest.TestCase): suffixes=['456'])), []) def test_yield_hashes(self): - fresh_ts = normalize_timestamp(time() - 10) - fresher_ts = normalize_timestamp(time() - 1) + fresh_ts = Timestamp(time() - 10).internal + fresher_ts = Timestamp(time() - 1).internal def _listdir(path): if path.endswith('/dev/objects/9'): @@ -2037,8 +2037,8 @@ class TestDiskFile(unittest.TestCase): '9373a92d072897b136b3fc06595b7456', fresher_ts)]) def test_yield_hashes_suffixes(self): - fresh_ts = normalize_timestamp(time() - 10) - fresher_ts = normalize_timestamp(time() - 1) + fresh_ts = Timestamp(time() - 10).internal + fresher_ts = Timestamp(time() - 1).internal def _listdir(path): if path.endswith('/dev/objects/9'): @@ -2095,7 +2095,7 @@ class TestDiskFile(unittest.TestCase): df = self._get_open_disk_file() ts = time() df.delete(ts) - exp_name = '%s.ts' % str(normalize_timestamp(ts)) + exp_name = '%s.ts' % str(Timestamp(ts).internal) dl = os.listdir(df._datadir) self.assertEquals(len(dl), 1) self.assertTrue(exp_name in set(dl)) @@ -2127,7 +2127,7 @@ class TestDiskFile(unittest.TestCase): df = self._get_open_disk_file() ts = time() df.delete(ts) - exp_name = '%s.ts' % str(normalize_timestamp(ts)) + exp_name = '%s.ts' % str(Timestamp(ts).internal) dl = os.listdir(df._datadir) self.assertEquals(len(dl), 1) self.assertTrue(exp_name in set(dl)) @@ -2159,7 +2159,7 @@ class TestDiskFile(unittest.TestCase): df.delete(ts) except OSError: self.fail("OSError raised when it should have been swallowed") - exp_name = '%s.ts' % str(normalize_timestamp(ts)) + exp_name = '%s.ts' % str(Timestamp(ts).internal) dl = os.listdir(df._datadir) self.assertEquals(len(dl), 2) self.assertTrue(exp_name in set(dl)) diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index b9f9345159..a293ae99cf 100755 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -554,11 +554,11 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.data') + utils.Timestamp(timestamp).internal + '.data') self.assert_(os.path.isfile(objfile)) self.assertEquals(open(objfile).read(), 'VERIFY') self.assertEquals(diskfile.read_metadata(objfile), - {'X-Timestamp': timestamp, + {'X-Timestamp': utils.Timestamp(timestamp).internal, 'Content-Length': '6', 'ETag': '0b4c12d7e0a73840c1c4f148fda3b037', 'Content-Type': 'application/octet-stream', @@ -587,11 +587,11 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.data') + utils.Timestamp(timestamp).internal + '.data') self.assert_(os.path.isfile(objfile)) self.assertEquals(open(objfile).read(), 'VERIFY TWO') self.assertEquals(diskfile.read_metadata(objfile), - {'X-Timestamp': timestamp, + {'X-Timestamp': utils.Timestamp(timestamp).internal, 'Content-Length': '10', 'ETag': 'b381a4c5dab1eaa1eb9711fa647cd039', 'Content-Type': 'text/plain', @@ -622,11 +622,11 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.data') + utils.Timestamp(timestamp).internal + '.data') self.assertTrue(os.path.isfile(objfile)) self.assertEqual(open(objfile).read(), 'VERIFY TWO') self.assertEqual(diskfile.read_metadata(objfile), - {'X-Timestamp': timestamp, + {'X-Timestamp': utils.Timestamp(timestamp).internal, 'Content-Length': '10', 'ETag': 'b381a4c5dab1eaa1eb9711fa647cd039', 'Content-Type': 'text/plain', @@ -696,11 +696,11 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.data') + utils.Timestamp(timestamp).internal + '.data') self.assert_(os.path.isfile(objfile)) self.assertEquals(open(objfile).read(), 'VERIFY THREE') self.assertEquals(diskfile.read_metadata(objfile), - {'X-Timestamp': timestamp, + {'X-Timestamp': utils.Timestamp(timestamp).internal, 'Content-Length': '12', 'ETag': 'b114ab7b90d9ccac4bd5d99cc7ebb568', 'Content-Type': 'text/plain', @@ -843,7 +843,7 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.data') + utils.Timestamp(timestamp).internal + '.data') os.unlink(objfile) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) @@ -873,7 +873,8 @@ class TestObjectController(unittest.TestCase): environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) - self.assertEquals(resp.headers['X-Backend-Timestamp'], timestamp) + self.assertEquals(resp.headers['X-Backend-Timestamp'], + utils.Timestamp(timestamp).internal) def test_HEAD_quarantine_zbyte(self): # Test swift.obj.server.ObjectController.GET @@ -969,7 +970,7 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.data') + utils.Timestamp(timestamp).internal + '.data') os.unlink(objfile) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.object_controller) @@ -997,7 +998,8 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) - self.assertEquals(resp.headers['X-Backend-Timestamp'], timestamp) + self.assertEquals(resp.headers['X-Backend-Timestamp'], + utils.Timestamp(timestamp).internal) def test_GET_if_match(self): req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, @@ -1581,7 +1583,7 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.ts') + utils.Timestamp(timestamp).internal + '.ts') self.assertTrue(os.path.isfile(ts_1000_file)) # There should now be a 1000 ts file. self.assertEquals(len(os.listdir(os.path.dirname(ts_1000_file))), 1) @@ -1597,7 +1599,7 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.ts') + utils.Timestamp(timestamp).internal + '.ts') self.assertFalse(os.path.isfile(ts_999_file)) self.assertTrue(os.path.isfile(ts_1000_file)) self.assertEquals(len(os.listdir(os.path.dirname(ts_1000_file))), 1) @@ -1617,7 +1619,7 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.data') + utils.Timestamp(timestamp).internal + '.data') self.assertTrue(os.path.isfile(data_1002_file)) self.assertEquals(len(os.listdir(os.path.dirname(data_1002_file))), 1) @@ -1632,7 +1634,7 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.ts') + utils.Timestamp(timestamp).internal + '.ts') self.assertFalse(os.path.isfile(ts_1001_file)) self.assertTrue(os.path.isfile(data_1002_file)) self.assertEquals(len(os.listdir(os.path.dirname(ts_1001_file))), 1) @@ -1647,7 +1649,7 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.ts') + utils.Timestamp(timestamp).internal + '.ts') self.assertTrue(os.path.isfile(ts_1003_file)) self.assertEquals(len(os.listdir(os.path.dirname(ts_1003_file))), 1) @@ -1655,10 +1657,11 @@ class TestObjectController(unittest.TestCase): # Test swift.obj.server.ObjectController.DELETE and container # updates, making sure container update is called in the correct # state. - timestamp = normalize_timestamp(time()) + start = time() + timestamp = utils.Timestamp(start) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ - 'X-Timestamp': timestamp, + 'X-Timestamp': timestamp.internal, 'Content-Type': 'application/octet-stream', 'Content-Length': '4', }) @@ -1676,17 +1679,17 @@ class TestObjectController(unittest.TestCase): try: # The following request should return 409 (HTTP Conflict). A # tombstone file should not have been created with this timestamp. - timestamp = normalize_timestamp(float(timestamp) - 1) + timestamp = utils.Timestamp(start - 0.00001) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, - headers={'X-Timestamp': timestamp}) + headers={'X-Timestamp': timestamp.internal}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 409) objfile = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.ts') + utils.Timestamp(timestamp).internal + '.ts') self.assertFalse(os.path.isfile(objfile)) self.assertEquals(len(os.listdir(os.path.dirname(objfile))), 1) self.assertEquals(0, calls_made[0]) @@ -1695,18 +1698,17 @@ class TestObjectController(unittest.TestCase): # be truly deleted (container update is performed) because this # timestamp is newer. A tombstone file should have been created # with this timestamp. - sleep(.00001) - timestamp = normalize_timestamp(time()) + timestamp = utils.Timestamp(start + 0.00001) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, - headers={'X-Timestamp': timestamp}) + headers={'X-Timestamp': timestamp.internal}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 204) objfile = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.ts') + utils.Timestamp(timestamp).internal + '.ts') self.assert_(os.path.isfile(objfile)) self.assertEquals(1, calls_made[0]) self.assertEquals(len(os.listdir(os.path.dirname(objfile))), 1) @@ -1715,18 +1717,17 @@ class TestObjectController(unittest.TestCase): # already have been deleted, but it should have also performed a # container update because the timestamp is newer, and a tombstone # file should also exist with this timestamp. - sleep(.00001) - timestamp = normalize_timestamp(time()) + timestamp = utils.Timestamp(start + 0.00002) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, - headers={'X-Timestamp': timestamp}) + headers={'X-Timestamp': timestamp.internal}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) objfile = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.ts') + utils.Timestamp(timestamp).internal + '.ts') self.assert_(os.path.isfile(objfile)) self.assertEquals(2, calls_made[0]) self.assertEquals(len(os.listdir(os.path.dirname(objfile))), 1) @@ -1735,23 +1736,209 @@ class TestObjectController(unittest.TestCase): # already have been deleted, and it should not have performed a # container update because the timestamp is older, or created a # tombstone file with this timestamp. - timestamp = normalize_timestamp(float(timestamp) - 1) + timestamp = utils.Timestamp(start + 0.00001) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, - headers={'X-Timestamp': timestamp}) + headers={'X-Timestamp': timestamp.internal}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) objfile = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.ts') + utils.Timestamp(timestamp).internal + '.ts') self.assertFalse(os.path.isfile(objfile)) self.assertEquals(2, calls_made[0]) self.assertEquals(len(os.listdir(os.path.dirname(objfile))), 1) finally: self.object_controller.container_update = orig_cu + def test_object_update_with_offset(self): + ts = (utils.Timestamp(t).internal for t in + itertools.count(int(time()))) + container_updates = [] + + def capture_updates(ip, port, method, path, headers, *args, **kwargs): + container_updates.append((ip, port, method, path, headers)) + # create a new object + create_timestamp = ts.next() + req = Request.blank('/sda1/p/a/c/o', method='PUT', body='test1', + headers={'X-Timestamp': create_timestamp, + 'X-Container-Host': '10.0.0.1:8080', + 'X-Container-Device': 'sda1', + 'X-Container-Partition': 'p', + 'Content-Type': 'text/plain'}) + with mocked_http_conn( + 200, give_connect=capture_updates) as fake_conn: + resp = req.get_response(self.object_controller) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + self.assertEqual(resp.status_int, 201) + self.assertEquals(1, len(container_updates)) + for update in container_updates: + ip, port, method, path, headers = update + self.assertEqual(ip, '10.0.0.1') + self.assertEqual(port, '8080') + self.assertEqual(method, 'PUT') + self.assertEqual(path, '/sda1/p/a/c/o') + expected = { + 'X-Size': len('test1'), + 'X-Etag': md5('test1').hexdigest(), + 'X-Content-Type': 'text/plain', + 'X-Timestamp': create_timestamp, + } + for key, value in expected.items(): + self.assertEqual(headers[key], str(value)) + container_updates = [] # reset + # read back object + req = Request.blank('/sda1/p/a/c/o', method='GET') + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.headers['X-Timestamp'], + utils.Timestamp(create_timestamp).normal) + self.assertEqual(resp.headers['X-Backend-Timestamp'], + create_timestamp) + self.assertEqual(resp.body, 'test1') + # send an update with an offset + offset_timestamp = utils.Timestamp( + create_timestamp, offset=1).internal + req = Request.blank('/sda1/p/a/c/o', method='PUT', body='test2', + headers={'X-Timestamp': offset_timestamp, + 'X-Container-Host': '10.0.0.1:8080', + 'X-Container-Device': 'sda1', + 'X-Container-Partition': 'p', + 'Content-Type': 'text/html'}) + with mocked_http_conn( + 200, give_connect=capture_updates) as fake_conn: + resp = req.get_response(self.object_controller) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + self.assertEqual(resp.status_int, 201) + self.assertEquals(1, len(container_updates)) + for update in container_updates: + ip, port, method, path, headers = update + self.assertEqual(ip, '10.0.0.1') + self.assertEqual(port, '8080') + self.assertEqual(method, 'PUT') + self.assertEqual(path, '/sda1/p/a/c/o') + expected = { + 'X-Size': len('test2'), + 'X-Etag': md5('test2').hexdigest(), + 'X-Content-Type': 'text/html', + 'X-Timestamp': offset_timestamp, + } + for key, value in expected.items(): + self.assertEqual(headers[key], str(value)) + container_updates = [] # reset + # read back new offset + req = Request.blank('/sda1/p/a/c/o', method='GET') + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.headers['X-Timestamp'], + utils.Timestamp(offset_timestamp).normal) + self.assertEqual(resp.headers['X-Backend-Timestamp'], + offset_timestamp) + self.assertEqual(resp.body, 'test2') + # now overwrite with a newer time + overwrite_timestamp = ts.next() + req = Request.blank('/sda1/p/a/c/o', method='PUT', body='test3', + headers={'X-Timestamp': overwrite_timestamp, + 'X-Container-Host': '10.0.0.1:8080', + 'X-Container-Device': 'sda1', + 'X-Container-Partition': 'p', + 'Content-Type': 'text/enriched'}) + with mocked_http_conn( + 200, give_connect=capture_updates) as fake_conn: + resp = req.get_response(self.object_controller) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + self.assertEqual(resp.status_int, 201) + self.assertEquals(1, len(container_updates)) + for update in container_updates: + ip, port, method, path, headers = update + self.assertEqual(ip, '10.0.0.1') + self.assertEqual(port, '8080') + self.assertEqual(method, 'PUT') + self.assertEqual(path, '/sda1/p/a/c/o') + expected = { + 'X-Size': len('test3'), + 'X-Etag': md5('test3').hexdigest(), + 'X-Content-Type': 'text/enriched', + 'X-Timestamp': overwrite_timestamp, + } + for key, value in expected.items(): + self.assertEqual(headers[key], str(value)) + container_updates = [] # reset + # read back overwrite + req = Request.blank('/sda1/p/a/c/o', method='GET') + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.headers['X-Timestamp'], + utils.Timestamp(overwrite_timestamp).normal) + self.assertEqual(resp.headers['X-Backend-Timestamp'], + overwrite_timestamp) + self.assertEqual(resp.body, 'test3') + # delete with an offset + offset_delete = utils.Timestamp(overwrite_timestamp, + offset=1).internal + req = Request.blank('/sda1/p/a/c/o', method='DELETE', + headers={'X-Timestamp': offset_delete, + 'X-Container-Host': '10.0.0.1:8080', + 'X-Container-Device': 'sda1', + 'X-Container-Partition': 'p'}) + with mocked_http_conn( + 200, give_connect=capture_updates) as fake_conn: + resp = req.get_response(self.object_controller) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + self.assertEqual(resp.status_int, 204) + self.assertEquals(1, len(container_updates)) + for update in container_updates: + ip, port, method, path, headers = update + self.assertEqual(ip, '10.0.0.1') + self.assertEqual(port, '8080') + self.assertEqual(method, 'DELETE') + self.assertEqual(path, '/sda1/p/a/c/o') + expected = { + 'X-Timestamp': offset_delete, + } + for key, value in expected.items(): + self.assertEqual(headers[key], str(value)) + container_updates = [] # reset + # read back offset delete + req = Request.blank('/sda1/p/a/c/o', method='GET') + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 404) + self.assertEqual(resp.headers['X-Timestamp'], None) + self.assertEqual(resp.headers['X-Backend-Timestamp'], offset_delete) + # and one more delete with a newer timestamp + delete_timestamp = ts.next() + req = Request.blank('/sda1/p/a/c/o', method='DELETE', + headers={'X-Timestamp': delete_timestamp, + 'X-Container-Host': '10.0.0.1:8080', + 'X-Container-Device': 'sda1', + 'X-Container-Partition': 'p'}) + with mocked_http_conn( + 200, give_connect=capture_updates) as fake_conn: + resp = req.get_response(self.object_controller) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + self.assertEqual(resp.status_int, 404) + self.assertEquals(1, len(container_updates)) + for update in container_updates: + ip, port, method, path, headers = update + self.assertEqual(ip, '10.0.0.1') + self.assertEqual(port, '8080') + self.assertEqual(method, 'DELETE') + self.assertEqual(path, '/sda1/p/a/c/o') + expected = { + 'X-Timestamp': delete_timestamp, + } + for key, value in expected.items(): + self.assertEqual(headers[key], str(value)) + container_updates = [] # reset + # read back delete + req = Request.blank('/sda1/p/a/c/o', method='GET') + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 404) + self.assertEqual(resp.headers['X-Timestamp'], None) + self.assertEqual(resp.headers['X-Backend-Timestamp'], delete_timestamp) + def test_call_bad_request(self): # Test swift.obj.server.ObjectController.__call__ inbuf = StringIO() @@ -2208,7 +2395,7 @@ class TestObjectController(unittest.TestCase): 'x-content-type': 'application/burrito', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', - 'x-timestamp': '12345', + 'x-timestamp': utils.Timestamp('12345').internal, POLICY_INDEX: '37', 'referer': 'PUT http://localhost/sda1/p/a/c/o', 'user-agent': 'obj-server %d' % os.getpid(), @@ -2227,7 +2414,7 @@ class TestObjectController(unittest.TestCase): 'x-content-type': 'text/plain', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', - 'x-timestamp': '12345', + 'x-timestamp': utils.Timestamp('12345').internal, 'referer': 'PUT http://localhost/sda1/p/a/c/o', 'user-agent': 'obj-server %d' % os.getpid(), POLICY_INDEX: 0, # system account storage policy is 0 @@ -2245,7 +2432,7 @@ class TestObjectController(unittest.TestCase): 'x-content-type': 'text/plain', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', - 'x-timestamp': '12345', + 'x-timestamp': utils.Timestamp('12345').internal, 'referer': 'PUT http://localhost/sda1/p/a/c/o', 'user-agent': 'obj-server %d' % os.getpid(), POLICY_INDEX: 0, # system account storage policy is 0 @@ -2314,7 +2501,7 @@ class TestObjectController(unittest.TestCase): 'x-content-type': 'application/burrito', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', - 'x-timestamp': '12345', + 'x-timestamp': utils.Timestamp('12345').internal, POLICY_INDEX: '26', 'referer': 'PUT http://localhost/sda1/p/a/c/o', 'user-agent': 'obj-server %d' % os.getpid(), @@ -2332,7 +2519,7 @@ class TestObjectController(unittest.TestCase): 'x-content-type': 'application/burrito', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', - 'x-timestamp': '12345', + 'x-timestamp': utils.Timestamp('12345').internal, POLICY_INDEX: '26', 'referer': 'PUT http://localhost/sda1/p/a/c/o', 'user-agent': 'obj-server %d' % os.getpid(), @@ -2340,7 +2527,7 @@ class TestObjectController(unittest.TestCase): def test_object_delete_at_aysnc_update(self): policy = random.choice(list(POLICIES)) - ts = (normalize_timestamp(t) for t in + ts = (utils.Timestamp(t) for t in itertools.count(int(time()))) container_updates = [] @@ -2348,8 +2535,9 @@ class TestObjectController(unittest.TestCase): def capture_updates(ip, port, method, path, headers, *args, **kwargs): container_updates.append((ip, port, method, path, headers)) - put_timestamp = ts.next() - delete_at_timestamp = utils.normalize_delete_at_timestamp(ts.next()) + put_timestamp = ts.next().internal + delete_at_timestamp = utils.normalize_delete_at_timestamp( + ts.next().normal) delete_at_container = ( int(delete_at_timestamp) / self.object_controller.expiring_objects_container_divisor * @@ -2441,7 +2629,8 @@ class TestObjectController(unittest.TestCase): self.assertEquals( pickle.load(open(os.path.join( self.testdir, 'sda1', async_dir, 'a83', - '06fbf0b514e5199dfc4e00f42eb5ea83-0000000001.00000'))), + '06fbf0b514e5199dfc4e00f42eb5ea83-%s' % + utils.Timestamp(1).internal))), {'headers': {'x-timestamp': '1', 'x-out': 'set', 'user-agent': 'obj-server %s' % os.getpid(), POLICY_INDEX: policy.idx}, @@ -2480,7 +2669,8 @@ class TestObjectController(unittest.TestCase): self.assertEquals( pickle.load(open(os.path.join( self.testdir, 'sda1', async_dir, 'a83', - '06fbf0b514e5199dfc4e00f42eb5ea83-0000000001.00000'))), + '06fbf0b514e5199dfc4e00f42eb5ea83-%s' % + utils.Timestamp(1).internal))), {'headers': {'x-timestamp': '1', 'x-out': str(status), 'user-agent': 'obj-server %s' % os.getpid(), POLICY_INDEX: policy.idx}, @@ -2552,7 +2742,8 @@ class TestObjectController(unittest.TestCase): self.assertTrue( os.path.exists(os.path.join( self.testdir, 'sda1', async_dir, 'a83', - '06fbf0b514e5199dfc4e00f42eb5ea83-0000000001.00000'))) + '06fbf0b514e5199dfc4e00f42eb5ea83-%s' % + utils.Timestamp(1).internal))) finally: object_server.http_connect = orig_http_connect utils.HASH_PATH_PREFIX = _prefix @@ -2607,7 +2798,7 @@ class TestObjectController(unittest.TestCase): 'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-content-type': 'text/plain', - 'x-timestamp': '1', + 'x-timestamp': utils.Timestamp(1).internal, POLICY_INDEX: '0', # default when not given 'x-trans-id': '123', 'referer': 'PUT http://localhost/sda1/0/a/c/o'})) @@ -2639,14 +2830,14 @@ class TestObjectController(unittest.TestCase): self.assertEqual(account, 'a') self.assertEqual(container, 'c') self.assertEqual(obj, 'o') - self.assertEqual(timestamp, '1') + self.assertEqual(timestamp, utils.Timestamp(1).internal) self.assertEqual(policy_index, 0) self.assertEqual(data, { 'headers': HeaderKeyDict({ 'X-Size': '0', 'User-Agent': 'obj-server %s' % os.getpid(), 'X-Content-Type': 'text/plain', - 'X-Timestamp': '1', + 'X-Timestamp': utils.Timestamp(1).internal, 'X-Trans-Id': '123', 'Referer': 'PUT http://localhost/sda1/0/a/c/o', 'X-Backend-Storage-Policy-Index': '0', @@ -2710,7 +2901,7 @@ class TestObjectController(unittest.TestCase): '0000000002-a/c/o', None, None, None, HeaderKeyDict({ POLICY_INDEX: 0, - 'x-timestamp': '1', + 'x-timestamp': utils.Timestamp('1').internal, 'x-trans-id': '123', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1', 0]) @@ -2737,7 +2928,7 @@ class TestObjectController(unittest.TestCase): None, None, None, HeaderKeyDict({ POLICY_INDEX: 0, - 'x-timestamp': '1', + 'x-timestamp': utils.Timestamp('1').internal, 'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1', 0]) @@ -2764,7 +2955,7 @@ class TestObjectController(unittest.TestCase): None, None, None, HeaderKeyDict({ POLICY_INDEX: 0, - 'x-timestamp': '1', + 'x-timestamp': utils.Timestamp('1').internal, 'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1', 0]) @@ -2799,7 +2990,7 @@ class TestObjectController(unittest.TestCase): 'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-content-type': 'text/plain', - 'x-timestamp': '1', + 'x-timestamp': utils.Timestamp('1').internal, 'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1', 0]) @@ -2850,7 +3041,8 @@ class TestObjectController(unittest.TestCase): '0000000002-a/c/o', None, None, None, HeaderKeyDict({ POLICY_INDEX: 0, - 'x-timestamp': '1', 'x-trans-id': '1234', + 'x-timestamp': utils.Timestamp('1').internal, + 'x-trans-id': '1234', 'referer': 'DELETE http://localhost/v1/a/c/o'}), 'sda1', 0]) @@ -3061,7 +3253,7 @@ class TestObjectController(unittest.TestCase): resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) self.assertEquals(resp.headers['X-Backend-Timestamp'], - put_timestamp) + utils.Timestamp(put_timestamp)) finally: object_server.time.time = orig_time @@ -3131,7 +3323,7 @@ class TestObjectController(unittest.TestCase): resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) self.assertEquals(resp.headers['X-Backend-Timestamp'], - put_timestamp) + utils.Timestamp(put_timestamp)) finally: object_server.time.time = orig_time @@ -3251,7 +3443,7 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - test_timestamp + '.data') + utils.Timestamp(test_timestamp).internal + '.data') self.assert_(os.path.isfile(objfile)) # move time past expirery