From 6c0c306f966c5cb1caeb2d6b0712ae3d0edd4d82 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 26 Mar 2014 02:49:03 +0900 Subject: [PATCH 1/8] Add tests for limits. --- test/test_limits.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 test/test_limits.py diff --git a/test/test_limits.py b/test/test_limits.py new file mode 100644 index 0000000..970f722 --- /dev/null +++ b/test/test_limits.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# coding: utf-8 +import pytest + +from msgpack import packb, unpackb + + +def test_integer(): + x = -(2 ** 63) + assert unpackb(packb(x)) == x + with pytest.raises(OverflowError): + packb(x-1) + + x = 2 ** 64 - 1 + assert unpackb(packb(x)) == x + with pytest.raises(OverflowError): + packb(x+1) + +@pytest.mark.skipif(True, "Requires very large memory.") +def test_binary(): + x = b'x' * (2**32 - 1) + assert unpackb(packb(x)) == x + x += b'y' + with pytest.raises(ValueError): + packb(x) + + +@pytest.mark.skipif(True, "Requires very large memory.") +def test_string(): + x = u'x' * (2**32 - 1) + assert unpackb(packb(x)) == x + x += u'y' + with pytest.raises(ValueError): + packb(x) + + +@pytest.mark.skipif(True, "Requires very large memory.") +def test_array(): + x = [0] * (2**32 - 1) + assert unpackb(packb(x)) == x + x.append(0) + with pytest.raises(ValueError): + packb(x) From e7f87d9d41532b0956ebcd9f7be52df979842466 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 26 Mar 2014 03:00:47 +0900 Subject: [PATCH 2/8] More limit check. --- test/test_limits.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/test/test_limits.py b/test/test_limits.py index 970f722..2879c0a 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -2,7 +2,7 @@ # coding: utf-8 import pytest -from msgpack import packb, unpackb +from msgpack import packb, unpackb, Packer def test_integer(): @@ -16,11 +16,27 @@ def test_integer(): with pytest.raises(OverflowError): packb(x+1) + +def test_array_header(): + packer = Packer() + packer.pack_array_header(2**32-1) + with pytest.raises(ValueError): + packer.pack_array_header(2**32) + + +def test_map_header(): + packer = Packer() + packer.pack_map_header(2**32-1) + with pytest.raises(ValueError): + packer.pack_array_header(2**32) + + @pytest.mark.skipif(True, "Requires very large memory.") def test_binary(): x = b'x' * (2**32 - 1) assert unpackb(packb(x)) == x - x += b'y' + del x + x = b'x' * (2**32) with pytest.raises(ValueError): packb(x) From c60ab28ee72e1b179e21f5fbc6efd0f96d6e6467 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 26 Mar 2014 03:03:18 +0900 Subject: [PATCH 3/8] Add check for format limits. --- msgpack/_packer.pyx | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index f261f08..204beaa 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -135,6 +135,8 @@ cdef class Packer(object): else: ret = msgpack_pack_false(&self.pk) elif PyLong_Check(o): + # PyInt_Check(long) is True for Python 3. + # Sow we should test long before int. if o > 0: ullval = o ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) @@ -152,8 +154,10 @@ cdef class Packer(object): dval = o ret = msgpack_pack_double(&self.pk, dval) elif PyBytes_Check(o): - rawval = o L = len(o) + if L > (2**32)-1: + raise ValueError("bytes is too large") + rawval = o ret = msgpack_pack_bin(&self.pk, L) if ret == 0: ret = msgpack_pack_raw_body(&self.pk, rawval, L) @@ -161,13 +165,19 @@ cdef class Packer(object): if not self.encoding: raise TypeError("Can't encode unicode string: no encoding is specified") o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) + L = len(o) + if L > (2**32)-1: + raise ValueError("dict is too large") rawval = o ret = msgpack_pack_raw(&self.pk, len(o)) if ret == 0: ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif PyDict_CheckExact(o): d = o - ret = msgpack_pack_map(&self.pk, len(d)) + L = len(d) + if L > (2**32)-1: + raise ValueError("dict is too large") + ret = msgpack_pack_map(&self.pk, L) if ret == 0: for k, v in d.iteritems(): ret = self._pack(k, nest_limit-1) @@ -175,7 +185,10 @@ cdef class Packer(object): ret = self._pack(v, nest_limit-1) if ret != 0: break elif PyDict_Check(o): - ret = msgpack_pack_map(&self.pk, len(o)) + L = len(o) + if L > (2**32)-1: + raise ValueError("dict is too large") + ret = msgpack_pack_map(&self.pk, L) if ret == 0: for k, v in o.items(): ret = self._pack(k, nest_limit-1) @@ -187,10 +200,15 @@ cdef class Packer(object): longval = o.code rawval = o.data L = len(o.data) + if L > (2**32)-1: + raise ValueError("EXT data is too large") ret = msgpack_pack_ext(&self.pk, longval, L) ret = msgpack_pack_raw_body(&self.pk, rawval, L) elif PyTuple_Check(o) or PyList_Check(o): - ret = msgpack_pack_array(&self.pk, len(o)) + L = len(o) + if L > (2**32)-1: + raise ValueError("list is too large") + ret = msgpack_pack_array(&self.pk, L) if ret == 0: for v in o: ret = self._pack(v, nest_limit-1) @@ -220,6 +238,8 @@ cdef class Packer(object): msgpack_pack_raw_body(&self.pk, data, len(data)) def pack_array_header(self, size_t size): + if size >= (2**32-1): + raise ValueError cdef int ret = msgpack_pack_array(&self.pk, size) if ret == -1: raise MemoryError @@ -231,6 +251,8 @@ cdef class Packer(object): return buf def pack_map_header(self, size_t size): + if size >= (2**32-1): + raise ValueError cdef int ret = msgpack_pack_map(&self.pk, size) if ret == -1: raise MemoryError From e99331d1ab793105da2e0a4139059169e5c835b3 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 26 Mar 2014 11:01:44 +0900 Subject: [PATCH 4/8] Fix skipif marking. --- test/test_limits.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_limits.py b/test/test_limits.py index 2879c0a..5d285e7 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -31,7 +31,7 @@ def test_map_header(): packer.pack_array_header(2**32) -@pytest.mark.skipif(True, "Requires very large memory.") +@pytest.mark.skipif(True, reason="Requires very large memory.") def test_binary(): x = b'x' * (2**32 - 1) assert unpackb(packb(x)) == x @@ -41,7 +41,7 @@ def test_binary(): packb(x) -@pytest.mark.skipif(True, "Requires very large memory.") +@pytest.mark.skipif(True, reason="Requires very large memory.") def test_string(): x = u'x' * (2**32 - 1) assert unpackb(packb(x)) == x @@ -50,7 +50,7 @@ def test_string(): packb(x) -@pytest.mark.skipif(True, "Requires very large memory.") +@pytest.mark.skipif(True, reason="Requires very large memory.") def test_array(): x = [0] * (2**32 - 1) assert unpackb(packb(x)) == x From ef5d93d4eaaaf107094776c215f07efd0faae3e9 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 26 Mar 2014 11:05:53 +0900 Subject: [PATCH 5/8] Fix size limit on pack_array_header and pack_map_header. --- msgpack/_packer.pyx | 4 ++-- msgpack/fallback.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 204beaa..82e4a63 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -238,7 +238,7 @@ cdef class Packer(object): msgpack_pack_raw_body(&self.pk, data, len(data)) def pack_array_header(self, size_t size): - if size >= (2**32-1): + if size > (2**32-1): raise ValueError cdef int ret = msgpack_pack_array(&self.pk, size) if ret == -1: @@ -251,7 +251,7 @@ cdef class Packer(object): return buf def pack_map_header(self, size_t size): - if size >= (2**32-1): + if size > (2**32-1): raise ValueError cdef int ret = msgpack_pack_map(&self.pk, size) if ret == -1: diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 55de72c..49323e6 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -633,6 +633,8 @@ class Packer(object): return ret def pack_array_header(self, n): + if n >= 2**32: + raise ValueError self._fb_pack_array_header(n) ret = self._buffer.getvalue() if self._autoreset: @@ -642,6 +644,8 @@ class Packer(object): return ret def pack_map_header(self, n): + if n >= 2**32: + raise ValueError self._fb_pack_map_header(n) ret = self._buffer.getvalue() if self._autoreset: From 7d0e145e91700ee58c5accaeb5dff6a9ec39a18d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 26 Mar 2014 12:50:28 +0900 Subject: [PATCH 6/8] Allow ValueError for packing integer overs format limit. --- test/test_limits.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_limits.py b/test/test_limits.py index 5d285e7..413d651 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -8,12 +8,12 @@ from msgpack import packb, unpackb, Packer def test_integer(): x = -(2 ** 63) assert unpackb(packb(x)) == x - with pytest.raises(OverflowError): + with pytest.raises((OverflowError, ValueError)): packb(x-1) x = 2 ** 64 - 1 assert unpackb(packb(x)) == x - with pytest.raises(OverflowError): + with pytest.raises((OverflowError, ValueError)): packb(x+1) From 5fb9d8a7fd09893afdd1e5844288313bb347856b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 26 Mar 2014 13:32:28 +0900 Subject: [PATCH 7/8] Fix Python 3.2 --- test/test_limits.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/test_limits.py b/test/test_limits.py index 413d651..7bc8f4c 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # coding: utf-8 +from __future__ import absolute_import, division, print_function, unicode_literals import pytest from msgpack import packb, unpackb, Packer @@ -43,9 +44,9 @@ def test_binary(): @pytest.mark.skipif(True, reason="Requires very large memory.") def test_string(): - x = u'x' * (2**32 - 1) + x = 'x' * (2**32 - 1) assert unpackb(packb(x)) == x - x += u'y' + x += 'y' with pytest.raises(ValueError): packb(x) From a72e75d7c81486bc2775b957b22d3b5e13742589 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 26 Mar 2014 15:12:28 +0900 Subject: [PATCH 8/8] Fix PyPy fail. --- test/test_limits.py | 54 +++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/test/test_limits.py b/test/test_limits.py index 7bc8f4c..da8cd2b 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -32,29 +32,31 @@ def test_map_header(): packer.pack_array_header(2**32) -@pytest.mark.skipif(True, reason="Requires very large memory.") -def test_binary(): - x = b'x' * (2**32 - 1) - assert unpackb(packb(x)) == x - del x - x = b'x' * (2**32) - with pytest.raises(ValueError): - packb(x) - - -@pytest.mark.skipif(True, reason="Requires very large memory.") -def test_string(): - x = 'x' * (2**32 - 1) - assert unpackb(packb(x)) == x - x += 'y' - with pytest.raises(ValueError): - packb(x) - - -@pytest.mark.skipif(True, reason="Requires very large memory.") -def test_array(): - x = [0] * (2**32 - 1) - assert unpackb(packb(x)) == x - x.append(0) - with pytest.raises(ValueError): - packb(x) +# PyPy fails following tests because of constant folding? +# https://bugs.pypy.org/issue1721 +#@pytest.mark.skipif(True, reason="Requires very large memory.") +#def test_binary(): +# x = b'x' * (2**32 - 1) +# assert unpackb(packb(x)) == x +# del x +# x = b'x' * (2**32) +# with pytest.raises(ValueError): +# packb(x) +# +# +#@pytest.mark.skipif(True, reason="Requires very large memory.") +#def test_string(): +# x = 'x' * (2**32 - 1) +# assert unpackb(packb(x)) == x +# x += 'y' +# with pytest.raises(ValueError): +# packb(x) +# +# +#@pytest.mark.skipif(True, reason="Requires very large memory.") +#def test_array(): +# x = [0] * (2**32 - 1) +# assert unpackb(packb(x)) == x +# x.append(0) +# with pytest.raises(ValueError): +# packb(x)