From 713bfe42231a17cac6d1622c18301a31bf53b689 Mon Sep 17 00:00:00 2001 From: Alan Boudreault Date: Mon, 8 Feb 2016 11:12:12 -0500 Subject: [PATCH 1/3] PYTHON-452 - Fix Cython deserializer date overflow with large timestamp --- cassandra/cython_utils.pyx | 11 +++++-- .../cqlengine/columns/test_validation.py | 17 +++++++++++ tests/unit/cython/test_utils.py | 29 +++++++++++++++++++ tests/unit/cython/utils_testhelper.pyx | 23 +++++++++++++++ tests/unit/test_time_util.py | 2 ++ 5 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 tests/unit/cython/test_utils.py create mode 100644 tests/unit/cython/utils_testhelper.pyx diff --git a/cassandra/cython_utils.pyx b/cassandra/cython_utils.pyx index ad72f689..c9caf01f 100644 --- a/cassandra/cython_utils.pyx +++ b/cassandra/cython_utils.pyx @@ -35,10 +35,15 @@ from cassandra.util import is_little_endian import_datetime() +DEF DAY_IN_SECONDS = 86400 + DATETIME_EPOC = datetime.datetime(1970, 1, 1) cdef datetime_from_timestamp(double timestamp): - cdef int seconds = timestamp - cdef int microseconds = ( (timestamp * 1000000)) % 1000000 - return DATETIME_EPOC + timedelta_new(0, seconds, microseconds) + cdef int days = (timestamp / DAY_IN_SECONDS) + cdef int64_t days_in_seconds = ( days) * DAY_IN_SECONDS + cdef int seconds = (timestamp - days_in_seconds) + cdef int microseconds = ((timestamp - days_in_seconds - seconds) * 1000000) + + return DATETIME_EPOC + timedelta_new(days, seconds, microseconds) diff --git a/tests/integration/cqlengine/columns/test_validation.py b/tests/integration/cqlengine/columns/test_validation.py index 4609cca1..496bc42e 100644 --- a/tests/integration/cqlengine/columns/test_validation.py +++ b/tests/integration/cqlengine/columns/test_validation.py @@ -88,6 +88,23 @@ class TestDatetime(BaseCassEngTestCase): dts = self.DatetimeTest.objects.filter(test_id=1).values_list('created_at') assert dts[0][0] is None + def test_datetime_invalid(self): + dt_value= 'INVALID' + with self.assertRaises(TypeError): + self.DatetimeTest.objects.create(test_id=2, created_at=dt_value) + + def test_datetime_timestamp(self): + dt_value = 1454520554 + self.DatetimeTest.objects.create(test_id=2, created_at=dt_value) + dt2 = self.DatetimeTest.objects(test_id=2).first() + assert dt2.created_at == datetime.utcfromtimestamp(dt_value) + + def test_datetime_large(self): + dt_value = datetime(2038, 12, 31, 10, 10, 10, 123000) + self.DatetimeTest.objects.create(test_id=2, created_at=dt_value) + dt2 = self.DatetimeTest.objects(test_id=2).first() + assert dt2.created_at == dt_value + class TestBoolDefault(BaseCassEngTestCase): class BoolDefaultValueTest(Model): diff --git a/tests/unit/cython/test_utils.py b/tests/unit/cython/test_utils.py new file mode 100644 index 00000000..209056f6 --- /dev/null +++ b/tests/unit/cython/test_utils.py @@ -0,0 +1,29 @@ +# Copyright 2013-2016 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tests.unit.cython.utils import cyimport, cythontest +utils_testhelper = cyimport('tests.unit.cython.utils_testhelper') + +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa + + +class UtilsTest(unittest.TestCase): + """Test Cython Utils functions""" + + @cythontest + def test_datetime_from_timestamp(self): + utils_testhelper.test_datetime_from_timestamp(self.assertEqual) \ No newline at end of file diff --git a/tests/unit/cython/utils_testhelper.pyx b/tests/unit/cython/utils_testhelper.pyx new file mode 100644 index 00000000..32816d3a --- /dev/null +++ b/tests/unit/cython/utils_testhelper.pyx @@ -0,0 +1,23 @@ +# Copyright 2013-2016 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime + +from cassandra.cython_utils cimport datetime_from_timestamp + + +def test_datetime_from_timestamp(assert_equal): + assert_equal(datetime_from_timestamp(1454781157.123456), datetime.datetime(2016, 2, 6, 17, 52, 37, 123456)) + # PYTHON-452 + assert_equal(datetime_from_timestamp(2177403010.123456), datetime.datetime(2038, 12, 31, 10, 10, 10, 123456)) diff --git a/tests/unit/test_time_util.py b/tests/unit/test_time_util.py index 8e767acc..4455e588 100644 --- a/tests/unit/test_time_util.py +++ b/tests/unit/test_time_util.py @@ -36,6 +36,8 @@ class TimeUtilTest(unittest.TestCase): self.assertEqual(util.datetime_from_timestamp(0.123456), datetime.datetime(1970, 1, 1, 0, 0, 0, 123456)) + self.assertEqual(util.datetime_from_timestamp(2177403010.123456), datetime.datetime(2038, 12, 31, 10, 10, 10, 123456)) + def test_times_from_uuid1(self): node = uuid.getnode() now = time.time() From d64bcd753e558802a366f3ed45dac527097fdce7 Mon Sep 17 00:00:00 2001 From: Alan Boudreault Date: Tue, 9 Feb 2016 17:33:17 -0500 Subject: [PATCH 2/3] Add tests for Cython DateType deserialization --- tests/unit/cython/test_types.py | 28 +++++++++++ tests/unit/cython/types_testhelper.pyx | 65 ++++++++++++++++++++++++++ tests/unit/test_types.py | 4 ++ 3 files changed, 97 insertions(+) create mode 100644 tests/unit/cython/test_types.py create mode 100644 tests/unit/cython/types_testhelper.pyx diff --git a/tests/unit/cython/test_types.py b/tests/unit/cython/test_types.py new file mode 100644 index 00000000..0286450b --- /dev/null +++ b/tests/unit/cython/test_types.py @@ -0,0 +1,28 @@ +# Copyright 2013-2016 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tests.unit.cython.utils import cyimport, cythontest +types_testhelper = cyimport('tests.unit.cython.types_testhelper') + +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa + + +class TypesTest(unittest.TestCase): + + @cythontest + def test_datetype(self): + types_testhelper.test_datetype(self.assertEqual) diff --git a/tests/unit/cython/types_testhelper.pyx b/tests/unit/cython/types_testhelper.pyx new file mode 100644 index 00000000..b27a2f21 --- /dev/null +++ b/tests/unit/cython/types_testhelper.pyx @@ -0,0 +1,65 @@ +# Copyright 2013-2016 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import time +import datetime + +include '../../../cassandra/ioutils.pyx' + +from cassandra.cqltypes import DateType +from cassandra.deserializers import find_deserializer +from cassandra.bytesio cimport BytesIOReader +from cassandra.buffer cimport Buffer +from cassandra.deserializers cimport from_binary, Deserializer + + +def test_datetype(assert_equal): + + cdef Deserializer des = find_deserializer(DateType) + + def deserialize(timestamp): + """Serialize a datetime and deserialize it using the cython deserializer""" + + cdef BytesIOReader reader + cdef Buffer buf + + dt = datetime.datetime.utcfromtimestamp(timestamp) + reader = BytesIOReader(b'\x00\x00\x00\x08' + DateType.serialize(dt, 0)) + get_buf(reader, &buf) + deserialized_dt = from_binary(des, &buf, 0) + + return deserialized_dt + + # deserialize + # epoc + expected = 0 + assert_equal(deserialize(expected), datetime.datetime.utcfromtimestamp(expected)) + + # beyond 32b + expected = 2 ** 33 + assert_equal(deserialize(expected), datetime.datetime(2242, 3, 16, 12, 56, 32)) + + # less than epoc (PYTHON-119) + expected = -770172256 + assert_equal(deserialize(expected), datetime.datetime(1945, 8, 5, 23, 15, 44)) + + # work around rounding difference among Python versions (PYTHON-230) + # This wont pass with the cython extension until we fix the microseconds alignment with CPython + #expected = 1424817268.274 + #assert_equal(deserialize(expected), datetime.datetime(2015, 2, 24, 22, 34, 28, 274000)) + + # Large date overflow (PYTHON-452) + expected = 2177403010.123 + assert_equal(deserialize(expected), datetime.datetime(2038, 12, 31, 10, 10, 10, 123000)) diff --git a/tests/unit/test_types.py b/tests/unit/test_types.py index f6f618de..d8774b02 100644 --- a/tests/unit/test_types.py +++ b/tests/unit/test_types.py @@ -204,6 +204,10 @@ class TypeTests(unittest.TestCase): expected = 1424817268.274 self.assertEqual(DateType.deserialize(int64_pack(int(1000 * expected)), 0), datetime.datetime(2015, 2, 24, 22, 34, 28, 274000)) + # Large date overflow (PYTHON-452) + expected = 2177403010.123 + self.assertEqual(DateType.deserialize(int64_pack(int(1000 * expected)), 0), datetime.datetime(2038, 12, 31, 10, 10, 10, 123000)) + def test_write_read_string(self): with tempfile.TemporaryFile() as f: value = u'test' From 6bfa507f96a653f27277803414e1ece68e8096c9 Mon Sep 17 00:00:00 2001 From: Alan Boudreault Date: Wed, 10 Feb 2016 10:19:31 -0500 Subject: [PATCH 3/3] Fix all test_id values in TestDateTime --- .../cqlengine/columns/test_validation.py | 24 +++++++++---------- tests/unit/cython/types_testhelper.pyx | 9 ++++++- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/tests/integration/cqlengine/columns/test_validation.py b/tests/integration/cqlengine/columns/test_validation.py index 496bc42e..7b51080b 100644 --- a/tests/integration/cqlengine/columns/test_validation.py +++ b/tests/integration/cqlengine/columns/test_validation.py @@ -70,39 +70,39 @@ class TestDatetime(BaseCassEngTestCase): return None now = datetime(1982, 1, 1, tzinfo=TZ()) - dt = self.DatetimeTest.objects.create(test_id=0, created_at=now) - dt2 = self.DatetimeTest.objects(test_id=0).first() + dt = self.DatetimeTest.objects.create(test_id=1, created_at=now) + dt2 = self.DatetimeTest.objects(test_id=1).first() assert dt2.created_at.timetuple()[:6] == (now + timedelta(hours=1)).timetuple()[:6] def test_datetime_date_support(self): today = date.today() - self.DatetimeTest.objects.create(test_id=0, created_at=today) - dt2 = self.DatetimeTest.objects(test_id=0).first() + self.DatetimeTest.objects.create(test_id=2, created_at=today) + dt2 = self.DatetimeTest.objects(test_id=2).first() assert dt2.created_at.isoformat() == datetime(today.year, today.month, today.day).isoformat() def test_datetime_none(self): - dt = self.DatetimeTest.objects.create(test_id=1, created_at=None) - dt2 = self.DatetimeTest.objects(test_id=1).first() + dt = self.DatetimeTest.objects.create(test_id=3, created_at=None) + dt2 = self.DatetimeTest.objects(test_id=3).first() assert dt2.created_at is None - dts = self.DatetimeTest.objects.filter(test_id=1).values_list('created_at') + dts = self.DatetimeTest.objects.filter(test_id=3).values_list('created_at') assert dts[0][0] is None def test_datetime_invalid(self): dt_value= 'INVALID' with self.assertRaises(TypeError): - self.DatetimeTest.objects.create(test_id=2, created_at=dt_value) + self.DatetimeTest.objects.create(test_id=4, created_at=dt_value) def test_datetime_timestamp(self): dt_value = 1454520554 - self.DatetimeTest.objects.create(test_id=2, created_at=dt_value) - dt2 = self.DatetimeTest.objects(test_id=2).first() + self.DatetimeTest.objects.create(test_id=5, created_at=dt_value) + dt2 = self.DatetimeTest.objects(test_id=5).first() assert dt2.created_at == datetime.utcfromtimestamp(dt_value) def test_datetime_large(self): dt_value = datetime(2038, 12, 31, 10, 10, 10, 123000) - self.DatetimeTest.objects.create(test_id=2, created_at=dt_value) - dt2 = self.DatetimeTest.objects(test_id=2).first() + self.DatetimeTest.objects.create(test_id=6, created_at=dt_value) + dt2 = self.DatetimeTest.objects(test_id=6).first() assert dt2.created_at == dt_value diff --git a/tests/unit/cython/types_testhelper.pyx b/tests/unit/cython/types_testhelper.pyx index b27a2f21..0574d406 100644 --- a/tests/unit/cython/types_testhelper.pyx +++ b/tests/unit/cython/types_testhelper.pyx @@ -18,7 +18,10 @@ import datetime include '../../../cassandra/ioutils.pyx' +import io + from cassandra.cqltypes import DateType +from cassandra.protocol import write_value from cassandra.deserializers import find_deserializer from cassandra.bytesio cimport BytesIOReader from cassandra.buffer cimport Buffer @@ -36,7 +39,11 @@ def test_datetype(assert_equal): cdef Buffer buf dt = datetime.datetime.utcfromtimestamp(timestamp) - reader = BytesIOReader(b'\x00\x00\x00\x08' + DateType.serialize(dt, 0)) + + bytes = io.BytesIO() + write_value(bytes, DateType.serialize(dt, 0)) + bytes.seek(0) + reader = BytesIOReader(bytes.read()) get_buf(reader, &buf) deserialized_dt = from_binary(des, &buf, 0)