Merge pull request #476 from datastax/python-452

PYTHON-452 - Fix Cython deserializer date overflow with large timestamp

cassandra/cython_utils.pyx

@@ -35,10 +35,15 @@ from cassandra.util import is_little_endian
 
 import_datetime()
 
+DEF DAY_IN_SECONDS = 86400
+
 DATETIME_EPOC = datetime.datetime(1970, 1, 1)
 
 
 cdef datetime_from_timestamp(double timestamp):
-    cdef int seconds = <int> timestamp
-    cdef int microseconds = (<int64_t> (timestamp * 1000000)) % 1000000
-    return DATETIME_EPOC + timedelta_new(0, seconds, microseconds)
+    cdef int days = <int> (timestamp / DAY_IN_SECONDS)
+    cdef int64_t days_in_seconds = (<int64_t> days) * DAY_IN_SECONDS
+    cdef int seconds = <int> (timestamp - days_in_seconds)
+    cdef int microseconds = <int> ((timestamp - days_in_seconds - seconds) * 1000000)
+
+    return DATETIME_EPOC + timedelta_new(days, seconds, microseconds)

tests/integration/cqlengine/columns/test_validation.py

@@ -70,24 +70,41 @@ class TestDatetime(BaseCassEngTestCase):
                 return None
 
         now = datetime(1982, 1, 1, tzinfo=TZ())
-        dt = self.DatetimeTest.objects.create(test_id=0, created_at=now)
-        dt2 = self.DatetimeTest.objects(test_id=0).first()
+        dt = self.DatetimeTest.objects.create(test_id=1, created_at=now)
+        dt2 = self.DatetimeTest.objects(test_id=1).first()
         assert dt2.created_at.timetuple()[:6] == (now + timedelta(hours=1)).timetuple()[:6]
 
     def test_datetime_date_support(self):
         today = date.today()
-        self.DatetimeTest.objects.create(test_id=0, created_at=today)
-        dt2 = self.DatetimeTest.objects(test_id=0).first()
+        self.DatetimeTest.objects.create(test_id=2, created_at=today)
+        dt2 = self.DatetimeTest.objects(test_id=2).first()
         assert dt2.created_at.isoformat() == datetime(today.year, today.month, today.day).isoformat()
 
     def test_datetime_none(self):
-        dt = self.DatetimeTest.objects.create(test_id=1, created_at=None)
-        dt2 = self.DatetimeTest.objects(test_id=1).first()
+        dt = self.DatetimeTest.objects.create(test_id=3, created_at=None)
+        dt2 = self.DatetimeTest.objects(test_id=3).first()
         assert dt2.created_at is None
 
-        dts = self.DatetimeTest.objects.filter(test_id=1).values_list('created_at')
+        dts = self.DatetimeTest.objects.filter(test_id=3).values_list('created_at')
         assert dts[0][0] is None
 
+    def test_datetime_invalid(self):
+        dt_value= 'INVALID'
+        with self.assertRaises(TypeError):
+            self.DatetimeTest.objects.create(test_id=4, created_at=dt_value)
+
+    def test_datetime_timestamp(self):
+        dt_value = 1454520554
+        self.DatetimeTest.objects.create(test_id=5, created_at=dt_value)
+        dt2 = self.DatetimeTest.objects(test_id=5).first()
+        assert dt2.created_at == datetime.utcfromtimestamp(dt_value)
+
+    def test_datetime_large(self):
+        dt_value = datetime(2038, 12, 31, 10, 10, 10, 123000)
+        self.DatetimeTest.objects.create(test_id=6, created_at=dt_value)
+        dt2 = self.DatetimeTest.objects(test_id=6).first()
+        assert dt2.created_at == dt_value
+
 
 class TestBoolDefault(BaseCassEngTestCase):
     class BoolDefaultValueTest(Model):

tests/unit/cython/test_types.py

@@ -0,0 +1,28 @@
+# Copyright 2013-2016 DataStax, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from tests.unit.cython.utils import cyimport, cythontest
+types_testhelper = cyimport('tests.unit.cython.types_testhelper')
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest  # noqa
+
+
+class TypesTest(unittest.TestCase):
+
+    @cythontest
+    def test_datetype(self):
+        types_testhelper.test_datetype(self.assertEqual)

tests/unit/cython/test_utils.py

@@ -0,0 +1,29 @@
+# Copyright 2013-2016 DataStax, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from tests.unit.cython.utils import cyimport, cythontest
+utils_testhelper = cyimport('tests.unit.cython.utils_testhelper')
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest  # noqa
+
+
+class UtilsTest(unittest.TestCase):
+    """Test Cython Utils functions"""
+
+    @cythontest
+    def test_datetime_from_timestamp(self):
+        utils_testhelper.test_datetime_from_timestamp(self.assertEqual)

tests/unit/cython/types_testhelper.pyx

@@ -0,0 +1,72 @@
+# Copyright 2013-2016 DataStax, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import time
+import datetime
+
+include '../../../cassandra/ioutils.pyx'
+
+import io
+
+from cassandra.cqltypes import DateType
+from cassandra.protocol import write_value
+from cassandra.deserializers import find_deserializer
+from cassandra.bytesio cimport BytesIOReader
+from cassandra.buffer cimport Buffer
+from cassandra.deserializers cimport from_binary, Deserializer
+
+
+def test_datetype(assert_equal):
+
+    cdef Deserializer des = find_deserializer(DateType)
+
+    def deserialize(timestamp):
+        """Serialize a datetime and deserialize it using the cython deserializer"""
+
+        cdef BytesIOReader reader
+        cdef Buffer buf
+
+        dt = datetime.datetime.utcfromtimestamp(timestamp)
+
+        bytes = io.BytesIO()
+        write_value(bytes, DateType.serialize(dt, 0))
+        bytes.seek(0)
+        reader = BytesIOReader(bytes.read())
+        get_buf(reader, &buf)
+        deserialized_dt = from_binary(des, &buf, 0)
+
+        return deserialized_dt
+
+    # deserialize
+    # epoc
+    expected = 0
+    assert_equal(deserialize(expected), datetime.datetime.utcfromtimestamp(expected))
+
+    # beyond 32b
+    expected = 2 ** 33
+    assert_equal(deserialize(expected), datetime.datetime(2242, 3, 16, 12, 56, 32))
+
+    # less than epoc (PYTHON-119)
+    expected = -770172256
+    assert_equal(deserialize(expected), datetime.datetime(1945, 8, 5, 23, 15, 44))
+
+    # work around rounding difference among Python versions (PYTHON-230)
+    # This wont pass with the cython extension until we fix the microseconds alignment with CPython
+    #expected = 1424817268.274
+    #assert_equal(deserialize(expected), datetime.datetime(2015, 2, 24, 22, 34, 28, 274000))
+
+    # Large date overflow (PYTHON-452)
+    expected = 2177403010.123
+    assert_equal(deserialize(expected), datetime.datetime(2038, 12, 31, 10, 10, 10, 123000))

tests/unit/cython/utils_testhelper.pyx

@@ -0,0 +1,23 @@
+# Copyright 2013-2016 DataStax, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+
+from cassandra.cython_utils cimport datetime_from_timestamp
+
+
+def test_datetime_from_timestamp(assert_equal):
+    assert_equal(datetime_from_timestamp(1454781157.123456), datetime.datetime(2016, 2, 6, 17, 52, 37, 123456))
+    # PYTHON-452
+    assert_equal(datetime_from_timestamp(2177403010.123456), datetime.datetime(2038, 12, 31, 10, 10, 10, 123456))

tests/unit/test_time_util.py

@@ -36,6 +36,8 @@ class TimeUtilTest(unittest.TestCase):
 
         self.assertEqual(util.datetime_from_timestamp(0.123456), datetime.datetime(1970, 1, 1, 0, 0, 0, 123456))
 
+        self.assertEqual(util.datetime_from_timestamp(2177403010.123456), datetime.datetime(2038, 12, 31, 10, 10, 10, 123456))
+
     def test_times_from_uuid1(self):
         node = uuid.getnode()
         now = time.time()

tests/unit/test_types.py

@@ -204,6 +204,10 @@ class TypeTests(unittest.TestCase):
         expected = 1424817268.274
         self.assertEqual(DateType.deserialize(int64_pack(int(1000 * expected)), 0), datetime.datetime(2015, 2, 24, 22, 34, 28, 274000))
 
+        # Large date overflow (PYTHON-452)
+        expected = 2177403010.123
+        self.assertEqual(DateType.deserialize(int64_pack(int(1000 * expected)), 0), datetime.datetime(2038, 12, 31, 10, 10, 10, 123000))
+
     def test_write_read_string(self):
         with tempfile.TemporaryFile() as f:
             value = u'test'