From b46f59b9339bee0562d22ca0631bf592b1da6b67 Mon Sep 17 00:00:00 2001 From: Hanno Schlichting Date: Tue, 11 Aug 2015 15:07:22 +0200 Subject: [PATCH 01/11] Address encoding issues with mixed str/bytes in executemany and with iterables. Refs #321, #364. --- pymysql/cursors.py | 41 ++++++++++++++++-------------- pymysql/tests/test_issues.py | 49 ++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 19 deletions(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 8d10f00..b79643b 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import print_function, absolute_import +from functools import partial import re import warnings @@ -93,14 +94,32 @@ class Cursor(object): def nextset(self): return self._nextset(False) + def _ensure_bytes(self, x, encoding=None): + if not PY2: + return x + if isinstance(x, unicode): + x = x.encode(encoding) + elif isinstance(x, (tuple, list)): + x = type(x)(self._ensure_bytes(v, encoding=encoding) for v in x) + return x + def _escape_args(self, args, conn): + ensure_bytes = partial(self._ensure_bytes, encoding=conn.encoding) + if isinstance(args, (tuple, list)): + if PY2: + args = tuple(map(ensure_bytes, args)) return tuple(conn.escape(arg) for arg in args) elif isinstance(args, dict): + if PY2: + args = dict((ensure_bytes(key), ensure_bytes(val)) for + (key, val) in args.items()) return dict((key, conn.escape(val)) for (key, val) in args.items()) else: - #If it's not a dictionary let's try escaping it anyways. - #Worst case it will throw a Value error + # If it's not a dictionary let's try escaping it anyways. + # Worst case it will throw a Value error + if PY2: + ensure_bytes(args) return conn.escape(args) def mogrify(self, query, args=None): @@ -111,24 +130,8 @@ class Cursor(object): This method follows the extension to the DB API 2.0 followed by Psycopg. """ conn = self._get_db() - if PY2: # Use bytes on Python 2 always - encoding = conn.encoding - - def ensure_bytes(x): - if isinstance(x, unicode): - x = x.encode(encoding) - return x - - query = ensure_bytes(query) - - if args is not None: - if isinstance(args, (tuple, list)): - args = tuple(map(ensure_bytes, args)) - elif isinstance(args, dict): - args = dict((ensure_bytes(key), ensure_bytes(val)) for (key, val) in args.items()) - else: - args = ensure_bytes(args) + query = self._ensure_bytes(query, encoding=conn.encoding) if args is not None: query = query % self._escape_args(args, conn) diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 50574c9..3c8a07c 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -377,3 +377,52 @@ class TestGitHubIssues(base.PyMySQLTestCase): warnings.filterwarnings("ignore") cur.execute('drop table if exists test_field_count') + def test_issue_321(self): + """ Test iterable as query argument. """ + conn = pymysql.connect(charset="utf8", **self.databases[0]) + self.safe_create_table( + conn, "issue321", + "create table issue321 (value_1 varchar(1), value_2 varchar(1))") + + sql_insert = "insert into issue321 (value_1, value_2) values (%s, %s)" + sql_dict_insert = ("insert into issue321 (value_1, value_2) " + "values (%(value_1)s, %(value_2)s)") + sql_select = ("select * from issue321 where " + "value_1 in %s and value_2=%s") + data = [ + [(u"a", ), u"\u0430"], + [[u"b"], u"\u0430"], + {"value_1": [[u"c"]], "value_2": u"\u0430"} + ] + cur = conn.cursor() + self.assertEqual(cur.execute(sql_insert, data[0]), 1) + self.assertEqual(cur.execute(sql_insert, data[1]), 1) + self.assertEqual(cur.execute(sql_dict_insert, data[2]), 1) + self.assertEqual( + cur.execute(sql_select, [(u"a", u"b", u"c"), u"\u0430"]), 3) + self.assertEqual(cur.fetchone(), (u"a", u"\u0430")) + self.assertEqual(cur.fetchone(), (u"b", u"\u0430")) + self.assertEqual(cur.fetchone(), (u"c", u"\u0430")) + + def test_issue_364(self): + """ Test mixed unicode/binary arguments in executemany. """ + conn = pymysql.connect(charset="utf8", **self.databases[0]) + self.safe_create_table( + conn, "issue363", + "create table issue363 (value_1 binary(3), value_2 varchar(3)) " + "engine=InnoDB default charset=utf8") + + sql = "insert into issue363 (value_1, value_2) values (%s, %s)" + values = [b"\x00\xff\x00", u"\xe4\xf6\xfc"] + + # test single insert and select + cur = conn.cursor() + cur.execute(sql, args=values) + cur.execute("select * from issue363") + self.assertEqual(cur.fetchone(), tuple(values)) + + # test multi insert and select + cur.executemany(sql, args=[values, values, values]) + cur.execute("select * from issue363") + for row in cur.fetchall(): + self.assertEqual(row, tuple(values)) From 8ba83623986b2d9f043836f8d9f2ca77b7b8a6b3 Mon Sep 17 00:00:00 2001 From: Hanno Schlichting Date: Tue, 11 Aug 2015 16:24:15 +0200 Subject: [PATCH 02/11] Extend the fix for #363 by handling explicit unicode queries. SQLAlchemy provides the queries as explicit unicode, even if they happen to only contain ascii characters. --- pymysql/cursors.py | 2 ++ pymysql/tests/test_issues.py | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index b79643b..6b0cad7 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -176,6 +176,8 @@ class Cursor(object): escape = self._escape_args if isinstance(prefix, text_type): prefix = prefix.encode(encoding) + if PY2 and isinstance(values, text_type): + values = values.encode(encoding) if isinstance(postfix, text_type): postfix = postfix.encode(encoding) sql = bytearray(prefix) diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 3c8a07c..7e774de 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -413,6 +413,7 @@ class TestGitHubIssues(base.PyMySQLTestCase): "engine=InnoDB default charset=utf8") sql = "insert into issue363 (value_1, value_2) values (%s, %s)" + usql = u"insert into issue363 (value_1, value_2) values (%s, %s)" values = [b"\x00\xff\x00", u"\xe4\xf6\xfc"] # test single insert and select @@ -421,8 +422,14 @@ class TestGitHubIssues(base.PyMySQLTestCase): cur.execute("select * from issue363") self.assertEqual(cur.fetchone(), tuple(values)) + # test single insert unicode query + cur.execute(usql, args=values) + # test multi insert and select - cur.executemany(sql, args=[values, values, values]) + cur.executemany(sql, args=(values, values, values)) cur.execute("select * from issue363") for row in cur.fetchall(): self.assertEqual(row, tuple(values)) + + # test multi insert with unicode query + cur.executemany(usql, args=(values, values, values)) From 3eeee0efe3aab0198571438077f98c57e45e6f9b Mon Sep 17 00:00:00 2001 From: Hanno Schlichting Date: Tue, 11 Aug 2015 22:10:21 +0200 Subject: [PATCH 03/11] Remove unnecessary PY2 check and replace unicode with text_type. --- pymysql/cursors.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 6b0cad7..266e137 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -95,9 +95,7 @@ class Cursor(object): return self._nextset(False) def _ensure_bytes(self, x, encoding=None): - if not PY2: - return x - if isinstance(x, unicode): + if isinstance(x, text_type): x = x.encode(encoding) elif isinstance(x, (tuple, list)): x = type(x)(self._ensure_bytes(v, encoding=encoding) for v in x) From 0635e6bdc50f6e1cbe497ceb0c2e8316fed98ed9 Mon Sep 17 00:00:00 2001 From: Hanno Schlichting Date: Sat, 15 Aug 2015 14:28:21 +0200 Subject: [PATCH 04/11] Classify Geometry as a text type, refs #363. --- pymysql/connections.py | 3 ++- pymysql/tests/test_issues.py | 48 +++++++++++++++++++++++++++++++----- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 4f66681..dbf209a 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -91,7 +91,8 @@ TEXT_TYPES = set([ FIELD_TYPE.STRING, FIELD_TYPE.TINY_BLOB, FIELD_TYPE.VAR_STRING, - FIELD_TYPE.VARCHAR]) + FIELD_TYPE.VARCHAR, + FIELD_TYPE.GEOMETRY]) sha_new = partial(hashlib.new, 'sha1') diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 7e774de..98d8eb3 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -408,18 +408,18 @@ class TestGitHubIssues(base.PyMySQLTestCase): """ Test mixed unicode/binary arguments in executemany. """ conn = pymysql.connect(charset="utf8", **self.databases[0]) self.safe_create_table( - conn, "issue363", - "create table issue363 (value_1 binary(3), value_2 varchar(3)) " + conn, "issue364", + "create table issue364 (value_1 binary(3), value_2 varchar(3)) " "engine=InnoDB default charset=utf8") - sql = "insert into issue363 (value_1, value_2) values (%s, %s)" - usql = u"insert into issue363 (value_1, value_2) values (%s, %s)" + sql = "insert into issue364 (value_1, value_2) values (%s, %s)" + usql = u"insert into issue364 (value_1, value_2) values (%s, %s)" values = [b"\x00\xff\x00", u"\xe4\xf6\xfc"] # test single insert and select cur = conn.cursor() cur.execute(sql, args=values) - cur.execute("select * from issue363") + cur.execute("select * from issue364") self.assertEqual(cur.fetchone(), tuple(values)) # test single insert unicode query @@ -427,9 +427,45 @@ class TestGitHubIssues(base.PyMySQLTestCase): # test multi insert and select cur.executemany(sql, args=(values, values, values)) - cur.execute("select * from issue363") + cur.execute("select * from issue364") for row in cur.fetchall(): self.assertEqual(row, tuple(values)) # test multi insert with unicode query cur.executemany(usql, args=(values, values, values)) + + def test_issue_363(self): + """ Test binary / geometry types. """ + conn = pymysql.connect(charset="utf8", **self.databases[0]) + self.safe_create_table( + conn, "issue363", + "CREATE TABLE issue363 ( " + "id INTEGER PRIMARY KEY, geom LINESTRING NOT NULL, " + "SPATIAL KEY geom (geom)) " + "ENGINE=MyISAM default charset=utf8") + + cur = conn.cursor() + cur.execute("INSERT INTO issue363 (id, geom) VALUES (" + "1998, GeomFromText('LINESTRING(1.1 1.1,2.2 2.2)'))") + + # select WKT + cur.execute("SELECT AsText(geom) FROM issue363") + row = cur.fetchone() + self.assertEqual(row, ("LINESTRING(1.1 1.1,2.2 2.2)", )) + + # select WKB + cur.execute("SELECT AsBinary(geom) FROM issue363") + row = cur.fetchone() + self.assertEqual(row, + (b"\x01\x02\x00\x00\x00\x02\x00\x00\x00" + b"\x9a\x99\x99\x99\x99\x99\xf1?" + b"\x9a\x99\x99\x99\x99\x99\xf1?" + b"\x9a\x99\x99\x99\x99\x99\x01@" + b"\x9a\x99\x99\x99\x99\x99\x01@", )) + + # select internal binary + cur.execute("SELECT geom FROM issue363") + row = cur.fetchone() + # don't assert the exact internal binary value, as it could + # vary across implementations + self.assertTrue(isinstance(row[0], bytes)) From 2552b6312cc23845e66320321d5b3aa3141fca0e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 26 Aug 2015 20:27:48 +1000 Subject: [PATCH 05/11] Add coveralls --- .coveragerc | 14 ++++++++++++++ .travis.yml | 6 +++++- tox.ini | 3 ++- 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..f4e21b5 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,14 @@ +[run] +branch = True +source = + pymysql + + +[report] +exclude_lines = + pragma: no cover + except ImportError: + if DEBUG: + def __repr__ + def __str__ + raise NotImplementedError diff --git a/.travis.yml b/.travis.yml index 5589182..0e5d703 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,12 +23,16 @@ matrix: env: TOX_ENV=py34 install: - - pip install -U tox + - pip install -U tox coveralls before_script: - "mysql -e 'create database test_pymysql DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;'" - "mysql -e 'create database test_pymysql2 DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;'" - "mysql -e 'select VERSION();'" - cp .travis.databases.json pymysql/tests/databases.json + - export COVERALLS_PARALLEL=true + +after_success: + - coveralls script: tox -e $TOX_ENV diff --git a/tox.ini b/tox.ini index 008c00e..dd67cd4 100644 --- a/tox.ini +++ b/tox.ini @@ -2,5 +2,6 @@ envlist = py26,py27,py33,py34,pypy,pypy3 [testenv] -commands = ./runtests.py +commands = coverage run ./runtests.py deps = unittest2 + coverage From 5db6913b30693e0741307f65d1195380bce4bc43 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 26 Aug 2015 20:31:00 +1000 Subject: [PATCH 06/11] Add coveralls badge --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index db1d94e..f84d2c5 100644 --- a/README.rst +++ b/README.rst @@ -5,6 +5,8 @@ PyMySQL .. image:: https://travis-ci.org/PyMySQL/PyMySQL.svg?branch=master :target: https://travis-ci.org/PyMySQL/PyMySQL +.. image:: https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=coveralls&service=github :target: https://coveralls.io/github/PyMySQL/PyMySQL?branch=coveralls + .. contents:: This package contains a pure-Python MySQL client library. The goal of PyMySQL From 0d8e1f68ca57338bf1ae561de99643797ec61843 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 27 Aug 2015 16:21:40 +1000 Subject: [PATCH 07/11] exclude dump_package from coverage - debug only --- pymysql/connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index dbf209a..4ac0a4f 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -107,7 +107,7 @@ DEFAULT_CHARSET = 'latin1' MAX_PACKET_LEN = 2**24-1 -def dump_packet(data): +def dump_packet(data): # pragma: no cover def is_ascii(data): if 65 <= byte2int(data) <= 122: if isinstance(data, int): From 96996db3785b5fa95509c137cf84187529f77f75 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 27 Aug 2015 16:22:10 +1000 Subject: [PATCH 08/11] exclude raise ValueError - usually when validating server protocol --- .coveragerc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.coveragerc b/.coveragerc index f4e21b5..e5bd9b5 100644 --- a/.coveragerc +++ b/.coveragerc @@ -12,3 +12,5 @@ exclude_lines = def __repr__ def __str__ raise NotImplementedError + def __getattr__ + raise ValueError From 47b29ef21cf28cf1efae9c026e3a9d925b68a890 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 27 Aug 2015 16:22:52 +1000 Subject: [PATCH 09/11] ignore coverage data --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 417ea07..236fc37 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.pyc *.pyo __pycache__ +.coverage /dist /PyMySQL.egg-info /.tox From d3e7d10407faa9c244d6fd2c298b66440834fcce Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 27 Aug 2015 19:17:58 +1000 Subject: [PATCH 10/11] Fix url for coveralls badge - wrong branch --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index f84d2c5..124a817 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,7 @@ PyMySQL .. image:: https://travis-ci.org/PyMySQL/PyMySQL.svg?branch=master :target: https://travis-ci.org/PyMySQL/PyMySQL -.. image:: https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=coveralls&service=github :target: https://coveralls.io/github/PyMySQL/PyMySQL?branch=coveralls +.. image:: https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=master&service=github :target: https://coveralls.io/github/PyMySQL/PyMySQL?branch=master .. contents:: From 1be420bb9aadb8580617332936ce418cc77db5d2 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 27 Aug 2015 22:18:49 +1000 Subject: [PATCH 11/11] add test test_no_delay_warning --- pymysql/tests/test_connection.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index c1e1934..a68e7de 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -1,7 +1,9 @@ import datetime import decimal -import pymysql import time +import sys +import unittest2 +import pymysql from pymysql.tests import base @@ -74,6 +76,13 @@ class TestConnection(base.PyMySQLTestCase): self.assertEqual(('foobar',), c.fetchone()) conn.close() + @unittest2.skipUnless(sys.version_info[0:2] >= (3,2), "required py-3.2") + def test_no_delay_warning(self): + current_db = self.databases[0].copy() + current_db['no_delay'] = True + with self.assertWarns(DeprecationWarning) as cm: + conn = pymysql.connect(**current_db) + # A custom type and function to escape it class Foo(object):