From 9d93b53701737cd53685a36711f65bcb786f8584 Mon Sep 17 00:00:00 2001 From: dev Date: Wed, 17 Feb 2016 20:09:55 +0300 Subject: [PATCH 1/6] bug with parameters view as %(name)s --- pymysql/cursors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index ff00d7b..18b6a25 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -12,7 +12,7 @@ from . import err #: Regular expression for :meth:`Cursor.executemany`. #: executemany only suports simple bulk insert. #: You can use it to load large dataset. -RE_INSERT_VALUES = re.compile(r"""(INSERT\s.+\sVALUES\s+)(\(\s*%s\s*(?:,\s*%s\s*)*\))(\s*(?:ON DUPLICATE.*)?)\Z""", +RE_INSERT_VALUES = re.compile(r"""(INSERT\s.+\sVALUES\s+)(\(\s*(?:%s|%\(.+\)s)\s*(?:,\s*(?:%s|%\(.+\)s)\s*)*\))(\s*(?:ON DUPLICATE.*)?)\Z""", re.IGNORECASE | re.DOTALL) From c1d11d6fa1086b4244aae3e2405085d9672102a4 Mon Sep 17 00:00:00 2001 From: dev Date: Fri, 19 Feb 2016 19:55:50 +0300 Subject: [PATCH 2/6] Add autodetect max_allowed_packet from server for best perfomance for executemany --- pymysql/connections.py | 11 +++++++++++ pymysql/cursors.py | 1 + 2 files changed, 12 insertions(+) diff --git a/pymysql/connections.py b/pymysql/connections.py index f571915..1029317 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -768,6 +768,15 @@ class Connection(object): result.read() return result.rows + def show_variable(self, name, default=0): + """ Get settings from server. only int type""" + self._execute_command(COMMAND.COM_QUERY,"show variables where variable_name = %s" % self.escape(name)) + result = MySQLResult(self) + result.read() + if result.rows: + return int(result.rows[0][1]) + return default + def select_db(self, db): '''Set current db''' self._execute_command(COMMAND.COM_INIT_DB, db) @@ -899,6 +908,8 @@ class Connection(object): c.close() self.commit() + self.max_allowed_packet = self.show_variable('max_allowed_packet') + if self.autocommit_mode is not None: self.autocommit(self.autocommit_mode) except BaseException as e: diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 18b6a25..af87c1e 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -40,6 +40,7 @@ class Cursor(object): self._executed = None self._result = None self._rows = None + self.max_stmt_length = connection.max_allowed_packet - 10000 # I don'n known packet_header_size def close(self): ''' From 6bcb9d26e65918b055c9b90ef1aa0526d76d6350 Mon Sep 17 00:00:00 2001 From: Stoyanov Evgeniy Date: Sat, 20 Feb 2016 23:07:52 +0300 Subject: [PATCH 3/6] Add test for executemany --- pymysql/tests/test_cursor.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index f900774..e5d1af4 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -69,3 +69,33 @@ class CursorTest(base.PyMySQLTestCase): ) self.assertIsNone(c2.fetchone()) + def test_executemany(self): + conn = self.test_connection + cursor = conn.cursor(pymysql.cursors.Cursor) + + m = pymysql.cursors.RE_INSERT_VALUES.match("INSERT INTO TEST (ID, NAME) VALUES (%s, %s)") + self.assertIsNotNone(m, 'error parse %s') + self.assertEqual(m.group(3), '', 'group 3 not blank, bug in RE_INSERT_VALUES?') + + m = pymysql.cursors.RE_INSERT_VALUES.match("INSERT INTO TEST (ID, NAME) VALUES (%(id)s, %(name)s)") + self.assertIsNotNone(m, 'error parse %(name)s') + self.assertEqual(m.group(3), '', 'group 3 not blank, bug in RE_INSERT_VALUES?') + + m = pymysql.cursors.RE_INSERT_VALUES.match("INSERT INTO TEST (ID, NAME) VALUES (%(id_name)s, %(name)s)") + self.assertIsNotNone(m, 'error parse %(id_name)s') + self.assertEqual(m.group(3), '', 'group 3 not blank, bug in RE_INSERT_VALUES?') + + m = pymysql.cursors.RE_INSERT_VALUES.match("INSERT INTO TEST (ID, NAME) VALUES (%(id_name)s, %(name)s) ON duplicate update") + self.assertIsNotNone(m, 'error parse %(id_name)s') + self.assertEqual(m.group(3), ' ON duplicate update', 'group 3 not ON duplicate update, bug in RE_INSERT_VALUES?') + + # cursor._executed myst bee "insert into test (data) values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9)" + # list args + data = xrange(10) + cursor.executemany("insert into test (data) values (%s)", data) + self.assertTrue(cursor._executed.endswith(",(7),(8),(9)"), 'execute many with %s not in one query') + + # dict args + data_dict = [{'data': i} for i in xrange(10)] + cursor.executemany("insert into test (data) values (%(data)s)", data_dict) + self.assertTrue(cursor._executed.endswith(",(7),(8),(9)"), 'execute many with %(data)s not in one query') \ No newline at end of file From 72c891f747a87db6c25658236632886b8dfa3f63 Mon Sep 17 00:00:00 2001 From: Stoyanov Evgeniy Date: Sun, 21 Feb 2016 11:42:52 +0300 Subject: [PATCH 4/6] Get max_allowed_packet only for executemany --- pymysql/connections.py | 14 ++++++++------ pymysql/cursors.py | 4 +++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 1029317..93136a4 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -768,13 +768,17 @@ class Connection(object): result.read() return result.rows - def show_variable(self, name, default=0): - """ Get settings from server. only int type""" - self._execute_command(COMMAND.COM_QUERY,"show variables where variable_name = %s" % self.escape(name)) + def get_system_variable(self, name, default=None): + """Get settings from server""" + self._execute_command(COMMAND.COM_QUERY, "SHOW VARIABLES WHERE VARIABLE_NAME = %s" % self.escape(name)) result = MySQLResult(self) result.read() if result.rows: - return int(result.rows[0][1]) + variable = result.rows[0][1] + if variable.isdigit(): + return int(variable) + else: + return variable return default def select_db(self, db): @@ -908,8 +912,6 @@ class Connection(object): c.close() self.commit() - self.max_allowed_packet = self.show_variable('max_allowed_packet') - if self.autocommit_mode is not None: self.autocommit(self.autocommit_mode) except BaseException as e: diff --git a/pymysql/cursors.py b/pymysql/cursors.py index af87c1e..bc946bf 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -26,6 +26,7 @@ class Cursor(object): #: Max size of allowed statement is max_allowed_packet - packet_header_size. #: Default value of max_allowed_packet is 1048576. max_stmt_length = 1024000 + packet_header_size = 4 def __init__(self, connection): ''' @@ -40,7 +41,6 @@ class Cursor(object): self._executed = None self._result = None self._rows = None - self.max_stmt_length = connection.max_allowed_packet - 10000 # I don'n known packet_header_size def close(self): ''' @@ -159,6 +159,8 @@ class Cursor(object): m = RE_INSERT_VALUES.match(query) if m: + if self.max_stmt_length == Cursor.max_stmt_length: + self.max_stmt_length = self.connection.get_system_variable("max_allowed_packet") - self.packet_header_size q_prefix = m.group(1) q_values = m.group(2).rstrip() q_postfix = m.group(3) or '' From c3e0f2f40e6fee16b47efeff3c4708fa5e14f152 Mon Sep 17 00:00:00 2001 From: dev Date: Wed, 24 Feb 2016 12:13:08 +0300 Subject: [PATCH 5/6] use server max_allowed_packet with limit for memory save. --- pymysql/connections.py | 3 +++ pymysql/cursors.py | 5 ++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 93136a4..369435a 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -902,6 +902,9 @@ class Connection(object): self._get_server_information() self._request_authentication() + self.max_allowed_packet = min(self.max_allowed_packet, + self.get_system_variable('max_allowed_packet', self.max_allowed_packet)) + if self.sql_mode is not None: c = self.cursor() c.execute("SET sql_mode=%s", (self.sql_mode,)) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index bc946bf..7070591 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -25,7 +25,7 @@ class Cursor(object): #: #: Max size of allowed statement is max_allowed_packet - packet_header_size. #: Default value of max_allowed_packet is 1048576. - max_stmt_length = 1024000 + max_stmt_length = 1024000 * 6 packet_header_size = 4 def __init__(self, connection): @@ -41,6 +41,7 @@ class Cursor(object): self._executed = None self._result = None self._rows = None + self.max_stmt_length = min(Cursor.max_stmt_length, connection.max_allowed_packet - Cursor.packet_header_size) def close(self): ''' @@ -159,8 +160,6 @@ class Cursor(object): m = RE_INSERT_VALUES.match(query) if m: - if self.max_stmt_length == Cursor.max_stmt_length: - self.max_stmt_length = self.connection.get_system_variable("max_allowed_packet") - self.packet_header_size q_prefix = m.group(1) q_values = m.group(2).rstrip() q_postfix = m.group(3) or '' From 07a7be71c93a2622168aab26a2e09e789bc036fa Mon Sep 17 00:00:00 2001 From: dev Date: Wed, 24 Feb 2016 12:39:06 +0300 Subject: [PATCH 6/6] fix use %(name)s in executemany --- pymysql/connections.py | 16 ---------------- pymysql/cursors.py | 4 +--- pymysql/tests/test_cursor.py | 2 +- 3 files changed, 2 insertions(+), 20 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 369435a..f571915 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -768,19 +768,6 @@ class Connection(object): result.read() return result.rows - def get_system_variable(self, name, default=None): - """Get settings from server""" - self._execute_command(COMMAND.COM_QUERY, "SHOW VARIABLES WHERE VARIABLE_NAME = %s" % self.escape(name)) - result = MySQLResult(self) - result.read() - if result.rows: - variable = result.rows[0][1] - if variable.isdigit(): - return int(variable) - else: - return variable - return default - def select_db(self, db): '''Set current db''' self._execute_command(COMMAND.COM_INIT_DB, db) @@ -902,9 +889,6 @@ class Connection(object): self._get_server_information() self._request_authentication() - self.max_allowed_packet = min(self.max_allowed_packet, - self.get_system_variable('max_allowed_packet', self.max_allowed_packet)) - if self.sql_mode is not None: c = self.cursor() c.execute("SET sql_mode=%s", (self.sql_mode,)) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 7070591..18b6a25 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -25,8 +25,7 @@ class Cursor(object): #: #: Max size of allowed statement is max_allowed_packet - packet_header_size. #: Default value of max_allowed_packet is 1048576. - max_stmt_length = 1024000 * 6 - packet_header_size = 4 + max_stmt_length = 1024000 def __init__(self, connection): ''' @@ -41,7 +40,6 @@ class Cursor(object): self._executed = None self._result = None self._rows = None - self.max_stmt_length = min(Cursor.max_stmt_length, connection.max_allowed_packet - Cursor.packet_header_size) def close(self): ''' diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index e5d1af4..34bcaa6 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -98,4 +98,4 @@ class CursorTest(base.PyMySQLTestCase): # dict args data_dict = [{'data': i} for i in xrange(10)] cursor.executemany("insert into test (data) values (%(data)s)", data_dict) - self.assertTrue(cursor._executed.endswith(",(7),(8),(9)"), 'execute many with %(data)s not in one query') \ No newline at end of file + self.assertTrue(cursor._executed.endswith(",(7),(8),(9)"), 'execute many with %(data)s not in one query')