Add UNIXEpoch formatter as a type for properties

Several properties in the object_store service use timestamp values that
are a count of seconds since the UNIX Epoch, or January 1, 1970. This
change adds a formatter that exposes this value to the user as a
datetime object, similar to the ISO8601 formatter. It is currently in
use by the transaction timestamp X-Timestamp header on all object_store
resources as well as the X-Delete-At header value on objects.

Change-Id: I3a14e3fae0e22d1e1ae411a5b119a08c6602c2b9

openstack/format.py

@@ -11,6 +11,10 @@
 # under the License.
 
 from datetime import datetime
+import numbers
+import time
+
+from iso8601 import iso8601
 import six
 
 from oslo_utils import timeutils
@@ -54,6 +58,57 @@ class ISO8601(Formatter):
             raise ValueError("Unable to deserialize ISO8601: %s" % value)
 
 
+class UNIXEpoch(Formatter):
+
+    EPOCH = datetime(1970, 1, 1, tzinfo=iso8601.UTC)
+
+    @classmethod
+    def serialize(cls, value):
+        """Convert a datetime to a UNIX epoch"""
+        if isinstance(value, datetime):
+            # Do not try to format using %s as it's platform dependent.
+            return (value - cls.EPOCH).total_seconds()
+        elif isinstance(value, numbers.Number):
+            return value
+        else:
+            raise ValueError("Unable to serialize UNIX epoch: %s" % value)
+
+    @classmethod
+    def deserialize(cls, value):
+        """Convert a UNIX epoch into a datetime object"""
+        try:
+            value = float(value)
+        except ValueError:
+            raise ValueError("Unable to deserialize UNIX epoch: %s" % value)
+
+        # gmtime doesn't respect microseconds so we need to parse them out
+        # if they're there and build the datetime appropriately with the
+        # proper precision.
+        # NOTES:
+        # 1. datetime.fromtimestamp sort of solves this but using localtime
+        #    instead of UTC, which we need.
+        # 2. On Python 2 we can't just str(value) as it truncates digits
+        #    that are significant to us.
+        parsed_value = "%000000f" % value
+        decimal = parsed_value.find(".")
+
+        if decimal == -1:
+            microsecond = 0
+        else:
+            # Some examples of these timestamps include less precision
+            # than the allowable 6 digits that can represent microseconds,
+            # so since we have a string we need to construct a real
+            # count of microseconds instead of just converting the
+            # stringified amount to an int.
+            fractional_second = float(parsed_value[decimal:]) * 1e6
+            microsecond = int(fractional_second)
+
+        gmt = time.gmtime(value)
+
+        return datetime(*gmt[:6], microsecond=microsecond,
+                        tzinfo=iso8601.UTC)
+
+
 class BoolStr(Formatter):
 
     # The behavior here primarily exists for the deserialize method

openstack/object_store/v1/account.py

@@ -11,6 +11,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from openstack import format
 from openstack.object_store.v1 import _base
 from openstack import resource
 
@@ -36,3 +37,5 @@ class Account(_base.BaseResource):
     #: A second secret key value for temporary URLs. If not set,
     #: this header is not returned by this operation.
     meta_temp_url_key_2 = resource.header("x-account-meta-temp-url-key-2")
+    #: The timestamp of the transaction.
+    timestamp = resource.header("x-timestamp", type=format.UNIXEpoch)

openstack/object_store/v1/container.py

@@ -11,6 +11,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from openstack import format
 from openstack.object_store.v1 import _base
 from openstack import resource
 
@@ -40,6 +41,8 @@ class Container(_base.BaseResource):
     object_count = resource.header("x-container-object-count", type=int)
     #: The count of bytes used in total.
     bytes_used = resource.header("x-container-bytes-used", type=int)
+    #: The timestamp of the transaction.
+    timestamp = resource.header("x-timestamp", type=format.UNIXEpoch)
 
     # Request headers (when id=None)
     #: If set to True, Object Storage queries all replicas to return the

openstack/object_store/v1/obj.py

@@ -11,6 +11,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from openstack import format
 from openstack.object_store import object_store_service
 from openstack import resource
 
@@ -117,13 +118,13 @@ class Object(resource.Resource):
     #: If set, the time when the object will be deleted by the system
     #: in the format of a UNIX Epoch timestamp.
     #: If not set, this header is not returned by this operation.
-    delete_at = resource.header("x-delete-at", type=int)
+    delete_at = resource.header("x-delete-at", type=format.UNIXEpoch)
     #: If set, to this is a dynamic large object manifest object.
     #: The value is the container and object name prefix of the
     #: segment objects in the form container/prefix.
     object_manifest = resource.header("x-object-manifest")
-    #: The UNIX timestamp of the transaction.
+    #: The timestamp of the transaction.
-    timestamp = resource.header("x-timestamp")
+    timestamp = resource.header("x-timestamp", type=format.UNIXEpoch)
 
     # Headers for PUT and POST requests
     #: Set to chunked to enable chunked transfer encoding. If used,

openstack/tests/unit/object_store/v1/test_account.py

@@ -10,6 +10,9 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from datetime import datetime
+import iso8601
+
 import testtools
 
 from openstack.object_store.v1 import account
@@ -24,7 +27,8 @@ ACCOUNT_EXAMPLE = {
     'x-account-bytes-used': '12345',
     'x-account-container-count': '678',
     'content-type': 'text/plain; charset=utf-8',
-    'x-account-object-count': '98765'
+    'x-account-object-count': '98765',
+    'x-timestamp': '1453413555.88937'
 }
 
 
@@ -52,3 +56,6 @@ class TestAccount(testtools.TestCase):
                          sot.account_container_count)
         self.assertEqual(int(ACCOUNT_EXAMPLE['x-account-object-count']),
                          sot.account_object_count)
+        self.assertEqual(datetime(2016, 1, 21, 21, 59, 15, 889370,
+                                  tzinfo=iso8601.UTC),
+                         sot.timestamp)

openstack/tests/unit/object_store/v1/test_container.py

@@ -10,6 +10,9 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from datetime import datetime
+import iso8601
+
 import mock
 import testtools
 
@@ -36,7 +39,8 @@ HEAD_EXAMPLE = {
     'x-container-sync-key': 'sync-key',
     'x-container-bytes-used': '630666',
     'x-versions-location': 'versions-location',
-    'content-type': 'application/json; charset=utf-8'
+    'content-type': 'application/json; charset=utf-8',
+    'x-timestamp': '1453414055.48672'
 }
 
 LIST_EXAMPLE = [
@@ -112,6 +116,9 @@ class TestContainer(testtools.TestCase):
                          sot.sync_key)
         self.assertEqual(HEAD_EXAMPLE['x-versions-location'],
                          sot.versions_location)
+        self.assertEqual(datetime(2016, 1, 21, 22, 7, 35, 486720,
+                                  tzinfo=iso8601.UTC),
+                         sot.timestamp)
 
     @mock.patch("openstack.resource.Resource.list")
     def test_list(self, fake_list):

openstack/tests/unit/object_store/v1/test_obj.py

@@ -10,6 +10,9 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from datetime import datetime
+import iso8601
+
 import mock
 import testtools
 
@@ -46,10 +49,11 @@ DICT_EXAMPLE = {
         'accept-ranges': 'bytes',
         'last-modified': 'Sun, 13 Jul 2014 18:41:04 GMT',
         'etag': '243f87b91224d85722564a80fd3cb1f1',
-        'x-timestamp': '1405276863.31924',
+        'x-timestamp': '1453414256.28112',
         'date': 'Thu, 28 Aug 2014 14:41:59 GMT',
         'content-type': 'application/octet-stream',
-        'id': 'tx5fb5ad4f4d0846c6b2bc7-0053ff3fb7'
+        'id': 'tx5fb5ad4f4d0846c6b2bc7-0053ff3fb7',
+        'x-delete-at': '1453416226.16744'
     }
 }
 
@@ -94,9 +98,14 @@ class TestObject(testtools.TestCase):
         self.assertEqual(headers['accept-ranges'], sot.accept_ranges)
         self.assertEqual(headers['last-modified'], sot.last_modified)
         self.assertEqual(headers['etag'], sot.etag)
-        self.assertEqual(headers['x-timestamp'], sot.timestamp)
+        self.assertEqual(datetime(2016, 1, 21, 22, 10, 56, 281120,
+                                  tzinfo=iso8601.UTC),
+                         sot.timestamp)
         self.assertEqual(headers['date'], sot.date)
         self.assertEqual(headers['content-type'], sot.content_type)
+        self.assertEqual(datetime(2016, 1, 21, 22, 43, 46, 167440,
+                                  tzinfo=iso8601.UTC),
+                         sot.delete_at)
 
     def test_get(self):
         sot = obj.Object.new(container=CONTAINER_NAME, name=OBJECT_NAME)

openstack/tests/unit/test_format.py

@@ -74,3 +74,32 @@ class TestISO8601Formatter(testtools.TestCase):
                 datetime(2015, 8, 27, 9, 49, 58,
                          tzinfo=iso8601.FixedOffset(-5, 0, "-05:00"))),
             "2015-08-27T09:49:58-05:00")
+
+
+class TestUNIXEpochFormatter(testtools.TestCase):
+
+    def test_deserialize(self):
+        self.assertEqual(format.UNIXEpoch.deserialize(1453412616.02406),
+                         datetime(2016, 1, 21, 21, 43, 36, 24060,
+                                  tzinfo=iso8601.UTC))
+        self.assertEqual(format.UNIXEpoch.deserialize(1389453423.35964),
+                         datetime(2014, 1, 11, 15, 17, 3, 359640,
+                                  tzinfo=iso8601.UTC))
+        self.assertEqual(format.UNIXEpoch.deserialize(1389453423),
+                         datetime(2014, 1, 11, 15, 17, 3, tzinfo=iso8601.UTC))
+        self.assertRaises(ValueError, format.UNIXEpoch.deserialize, "lol")
+
+    def test_serialize(self):
+        self.assertEqual(
+            format.UNIXEpoch.serialize(
+                datetime(2016, 1, 21, 21, 43, 36, 24060, tzinfo=iso8601.UTC)),
+            1453412616.02406)
+        self.assertEqual(
+            format.UNIXEpoch.serialize(
+                datetime(2014, 1, 11, 15, 17, 3, 359640, tzinfo=iso8601.UTC)),
+            1389453423.35964)
+        self.assertEqual(
+            format.UNIXEpoch.serialize(
+                datetime(2014, 1, 11, 15, 17, 3, tzinfo=iso8601.UTC)),
+            1389453423)
+        self.assertRaises(ValueError, format.UNIXEpoch.serialize, "lol")

requirements.txt

@@ -7,3 +7,4 @@ stevedore>=1.5.0 # Apache-2.0
 oslo.utils>=3.4.0 # Apache-2.0
 os-client-config>=1.13.1 # Apache-2.0
 keystoneauth1>=2.1.0 # Apache-2.0
+iso8601>=0.1.9 # MIT

test-requirements.txt

@@ -16,4 +16,3 @@ sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 # BSD
 testrepository>=0.0.18 # Apache-2.0/BSD
 testscenarios>=0.4 # Apache-2.0/BSD
 testtools>=1.4.0 # MIT
-iso8601>=0.1.9 # MIT