merge from master

.coveragerc

@@ -2,10 +2,9 @@
 branch = True
 source =
     pymysql
-omit = pymysql/test/*
+omit = pymysql/tests/*
        pymysql/tests/thirdparty/test_MySQLdb/*
 
-
 [report]
 exclude_lines =
     pragma: no cover

.travis.databases.json

@@ -1,4 +1,4 @@
 [
     {"host": "localhost", "unix_socket": "/var/run/mysqld/mysqld.sock", "user": "root", "passwd": "", "db": "test_pymysql",  "use_unicode": true, "local_infile": true},
-    {"host": "localhost", "port": 3306, "user": "root", "passwd": "", "db": "test_pymysql2" }
+    {"host": "127.0.0.1", "port": 3306, "user": "travis_pymysql2", "password": "some password", "db": "test_pymysql2" }
 ]

.travis.initialize.db.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+#debug
+set -x
+#verbose
+set -v
+
+if [ ! -z "${DB}" ]; then
+    # disable existing database server in case of accidential connection
+    mysql -u root -e 'drop user travis@localhost; drop user root@localhost; drop user travis; create user super@localhost; grant all on *.* to super@localhost with grant option'
+    mysql -u super -e 'drop user root'
+    F=mysql-${DB}-linux-glibc2.5-x86_64
+    mkdir -p ${HOME}/mysql
+    P=${HOME}/mysql/${F} 
+    if [ ! -d "${P}" ]; then
+        wget http://cdn.mysql.com/Downloads/MySQL-${DB%.*}/${F}.tar.gz -O - | tar -zxf - --directory=${HOME}/mysql 
+    fi
+    if [ -f "${P}"/my.cnf ]; then
+        O="--defaults-file=${P}/my.cnf" 
+    fi
+    if [ -x "${P}"/scripts/mysql_install_db ]; then
+        I=${P}/scripts/mysql_install_db 
+        O="--defaults-file=${P}/my.cnf" 
+    else
+        I=${P}/bin/mysqld
+        IO=" --initialize " 
+        O="--no-defaults " 
+    fi
+    ${I} ${O} ${IO} --basedir=${P} --datadir=${HOME}/db-"${DB}" --log-error=/tmp/mysql.err
+    PWLINE=$(grep 'A temporary password is generated for root@localhost:' /tmp/mysql.err)
+    PASSWD=${PWLINE##* }
+    if [ -x ${P}/bin/mysql_ssl_rsa_setup ]; then
+        ${P}/bin/mysql_ssl_rsa_setup --datadir=${HOME}/db-"${DB}"
+    fi
+     # sha256 password auth keys:
+     openssl genrsa -out "${P}"/private_key.pem 2048
+     openssl rsa -in "${P}"/private_key.pem -pubout -out "${P}"/public_key.pem
+    ${P}/bin/mysqld_safe ${O} --ledir=/ --mysqld=${P}/bin/mysqld  --datadir=${HOME}/db-${DB} --socket=/tmp/mysql.sock --port 3307 --innodb-buffer-pool-size=200M  --lc-messages-dir=${P}/share --plugin-dir=${P}/lib/plugin/ --log-error=/tmp/mysql.err &
+    while [ ! -S /tmp/mysql.sock  ]; do
+       sleep 2
+    done
+    cat /tmp/mysql.err
+    if [ ! -z "${PASSWD}" ]; then
+        ${P}/bin/mysql -S /tmp/mysql.sock -u root -p"${PASSWD}" --connect-expired-password -e "SET PASSWORD = PASSWORD('')"
+    fi
+    mysql -S /tmp/mysql.sock -u root -e "create user ${USER}@localhost; create user ${USER}@'%'; grant all on *.* to  ${USER}@localhost WITH GRANT OPTION;grant all on *.* to  ${USER}@'%' WITH GRANT OPTION;"
+    sed -e 's/3306/3307/g' -e 's:/var/run/mysqld/mysqld.sock:/tmp/mysql.sock:g' .travis.databases.json > pymysql/tests/databases.json
+    echo -e "[client]\nsocket = /tmp/mysql.sock\n" > "${HOME}"/.my.cnf
+else
+    cp .travis.databases.json pymysql/tests/databases.json
+fi

.travis.yml

@@ -2,7 +2,7 @@ sudo: false
 language: python
 python: "3.4"
 cache:
-    - pip
+    pip: true
 
 env:
     matrix:
@@ -41,15 +41,27 @@ matrix:
               apt:
                  packages:
                      - libaio-dev
+          python: 3.3
           cache:
               directories:
-                  - mysql
+                  - ${HOME}/mysql
-# really only need libaio1 however libaio-dev is whitelisted
+        - env:
-#
+             - TOX_ENV=py34
-# http://dev.mysql.com/downloads/mysql/5.7.html
+             - DB=5.7.8-rc
-#        - env:
+          addons:
-#             - TOX_ENV=py34
+              apt:
-#             - DB=5.7.8-rc
+                 packages:
+                     - libaio-dev
+          python: 3.4
+          cache:
+              directories:
+                  - ${HOME}/mysql
+
+# different py version from 5.6 and 5.7 as cache seems to be based on py version
+
+# http://dev.mysql.com/downloads/mysql/5.7.html has latest development release version
+
+# really only need libaio1 for DB builds however libaio-dev is whitelisted for container builds and liaio1 isn't
 
 install:
     - if [ -n "${EXTRAPKG}" ]; then
@@ -62,33 +74,16 @@ install:
     - pip install -U tox coveralls
 
 before_script:
-    - if [ ! -z "${DB}" ]; then
+    - ./.travis.initialize.db.sh;
-          F=mysql-${DB}-linux-glibc2.5-x86_64;
+    - mysql -e 'create database test_pymysql  DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;'
-          mkdir -p ${HOME}/mysql;
+    - mysql -e 'create database test_pymysql2 DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;'
-          P=${HOME}/mysql/${F} ;
+    - mysql -u root -e "create user travis_pymysql2 identified by 'some password'; grant all on test_pymysql2.* to travis_pymysql2;"
-          if [ ! -d "${P}" ]; then
+    - mysql -u root -e "create user travis_pymysql2@localhost identified by 'some password'; grant all on test_pymysql2.* to travis_pymysql2@localhost;"
-              wget http://cdn.mysql.com/Downloads/MySQL-${DB%.*}/${F}.tar.gz -O - | tar -zxf - --directory=${HOME}/mysql ;
+    - mysql -e 'select VERSION();'
-          fi;
+    - rm -f ~/.my.cnf # set in .travis.initialize.db.sh for the above commands - we should be using database.json however
-          openssl genrsa -out "${P}"/private_key.pem 2048;
-          openssl rsa -in "${P}"/private_key.pem -pubout -out "${P}"/public_key.pem;
-          ${P}/scripts/mysql_install_db  --defaults-file=${P}/my.cnf --basedir=${P} --datadir=${HOME}/db-"${DB}" --log-error=/tmp/mysql.err;
-          ${P}/bin/mysqld_safe --defaults-file=${P}/my.cnf --ledir=/ --mysqld=${P}/bin/mysqld  --datadir=${HOME}/db-${DB} --socket=/tmp/mysql.sock --port 3307 --innodb-buffer-pool-size=200M  --lc-messages-dir=${P}/share --plugin-dir=${P}/lib/plugin/ --log-error=/tmp/mysql.err &
-          sleep 5; cat /tmp/mysql.err; df -h;
-          mysql -S /tmp/mysql.sock "create user ${USER}@localhost; create user ${USER}@'%'; grant all on *.* to  ${USER}@localhost WITH GRANT OPTION;grant all on *.* to  ${USER}@'%' WITH GRANT OPTION;";
-          echo 'check we are talking about the right sed' 1>&2 ;
-          sed -e 's/3306/3307/g' -e 's:/var/run/mysqld/mysqld.sock:/tmp/mysql.sock:g' .travis.databases.json > pymysql/tests/databases.json;
-          echo -e '[client]\nsocket = /tmp/mysql.sock\n' > "${HOME}"/.my.cnf ;
-      else
-          cp .travis.databases.json pymysql/tests/databases.json;
-      fi
-    - "mysql -e 'create database test_pymysql  DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;'"
-    - "mysql -e 'create database test_pymysql2 DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;'"
-    - "mysql -e 'select VERSION();'"
     - export COVERALLS_PARALLEL=true
 
 script:
-    - export PAMSERVICE=chfn
-    - export PASSWORD=travis
     - tox -e $TOX_ENV
 
 after_success:

pymysql/connections.py

@@ -671,7 +671,9 @@ class Connection(object):
         self.init_command = init_command
         self.max_allowed_packet = max_allowed_packet
         self.plugin_map = plugin_map
-        if not defer_connect:
+        if defer_connect:
+            self.socket = None
+        else:
             self.connect()
 
     def close(self):

pymysql/tests/test_SSCursor.py

@@ -35,6 +35,7 @@ class TestSSCursor(base.PyMySQLTestCase):
                 'zone VARCHAR(64),'
                 'name VARCHAR(64))'))
 
+            conn.begin()
             # Test INSERT
             for i in data:
                 cursor.execute('INSERT INTO tz_data VALUES (%s, %s, %s)', i)

pymysql/tests/test_connection.py

@@ -412,6 +412,60 @@ class TestConnection(base.PyMySQLTestCase):
         c.execute('select "foobar";')
         self.assertEqual(('foobar',), c.fetchone())
         conn.close()
+        with self.assertRaises(pymysql.err.Error):
+            conn.ping(reconnect=False)
+
+    def test_read_default_group(self):
+        conn = pymysql.connect(
+            read_default_group='client',
+            **self.databases[0]
+        )
+        self.assertTrue(conn.open)
+
+    def test_context(self):
+        with self.assertRaises(ValueError):
+            c = pymysql.connect(**self.databases[0])
+            with c as cur:
+                cur.execute('create table test ( a int )')
+                c.begin()
+                cur.execute('insert into test values ((1))')
+                raise ValueError('pseudo abort')
+                c.commit()
+        c = pymysql.connect(**self.databases[0])
+        with c as cur:
+            cur.execute('select count(*) from test')
+            self.assertEqual(0, cur.fetchone()[0])
+            cur.execute('insert into test values ((1))')
+        with c as cur:
+            cur.execute('select count(*) from test')
+            self.assertEqual(1,cur.fetchone()[0])
+            cur.execute('drop table test')
+
+    def test_set_charset(self):
+        c = pymysql.connect(**self.databases[0])
+        c.set_charset('utf8')
+        # TODO validate setting here
+
+    def test_defer_connect(self):
+        import socket
+        for db in self.databases:
+            d = db.copy()
+            try:
+                sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+                sock.connect(d['unix_socket'])
+            except KeyError:
+                sock = socket.create_connection(
+                                (d.get('host', 'localhost'), d.get('port', 3306)))
+            for k in ['unix_socket', 'host', 'port']:
+                try:
+                    del d[k]
+                except KeyError:
+                    pass
+
+            c = pymysql.connect(defer_connect=True, **d)
+            self.assertFalse(c.open)
+            c.connect(sock)
+            c.close()
 
     @unittest2.skipUnless(sys.version_info[0:2] >= (3,2), "required py-3.2")
     def test_no_delay_warning(self):
@@ -436,7 +490,8 @@ class TestEscape(base.PyMySQLTestCase):
         cur = con.cursor()
 
         self.assertEqual(con.escape("foo'bar"), "'foo\\'bar'")
-        cur.execute("SET sql_mode='NO_BACKSLASH_ESCAPES'")
+        # added NO_AUTO_CREATE_USER as not including it in 5.7 generates warnings
+        cur.execute("SET sql_mode='NO_BACKSLASH_ESCAPES,NO_AUTO_CREATE_USER'")
         self.assertEqual(con.escape("foo'bar"), "'foo''bar'")
 
     def test_escape_builtin_encoders(self):

pymysql/tests/test_issues.py

@@ -1,6 +1,7 @@
 import datetime
 import time
 import warnings
+import sys
 
 import pymysql
 from pymysql.tests import base
@@ -76,7 +77,7 @@ class TestOldIssues(base.PyMySQLTestCase):
             warnings.filterwarnings("ignore")
             c.execute("drop table if exists test")
         c.execute("""CREATE TABLE `test` (`station` int(10) NOT NULL DEFAULT '0', `dh`
-datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `echeance` int(1) NOT NULL
+datetime NOT NULL DEFAULT '2015-01-01 00:00:00', `echeance` int(1) NOT NULL
 DEFAULT '0', `me` double DEFAULT NULL, `mo` double DEFAULT NULL, PRIMARY
 KEY (`station`,`dh`,`echeance`)) ENGINE=MyISAM DEFAULT CHARSET=latin1;""")
         try:
@@ -198,9 +199,9 @@ class TestNewIssues(base.PyMySQLTestCase):
             self.assertEqual(2013, e.args[0])
 
     def test_issue_36(self):
-        conn = self.connections[0]
+        # connection 0 is super user, connection 1 isn't
+        conn = self.connections[1]
         c = conn.cursor()
-        # kill connections[0]
         c.execute("show processlist")
         kill_id = None
         for row in c.fetchall():
@@ -211,7 +212,7 @@ class TestNewIssues(base.PyMySQLTestCase):
                 break
         self.assertEqual(kill_id, conn.thread_id())
         # now nuke the connection
-        self.connections[1].kill(kill_id)
+        self.connections[0].kill(kill_id)
         # make sure this connection has broken
         try:
             c.execute("show tables")
@@ -226,12 +227,12 @@ class TestNewIssues(base.PyMySQLTestCase):
             # Wait since Travis-CI sometimes fail this test.
             time.sleep(0.1)
 
-            c = self.connections[1].cursor()
+            c = self.connections[0].cursor()
             c.execute("show processlist")
             ids = [row[0] for row in c.fetchall()]
             self.assertFalse(kill_id in ids)
         finally:
-            del self.connections[0]
+            del self.connections[1]
 
     def test_issue_37(self):
         conn = self.connections[0]
@@ -412,8 +413,8 @@ class TestGitHubIssues(base.PyMySQLTestCase):
             "create table issue364 (value_1 binary(3), value_2 varchar(3)) "
             "engine=InnoDB default charset=utf8")
 
-        sql = "insert into issue364 (value_1, value_2) values (%s, %s)"
+        sql = "insert into issue364 (value_1, value_2) values (_binary%s, _binary%s)"
-        usql = u"insert into issue364 (value_1, value_2) values (%s, %s)"
+        usql = u"insert into issue364 (value_1, value_2) values (_binary%s, _binary%s)"
         values = [b"\x00\xff\x00", u"\xe4\xf6\xfc"]
 
         # test single insert and select
@@ -445,15 +446,29 @@ class TestGitHubIssues(base.PyMySQLTestCase):
             "ENGINE=MyISAM default charset=utf8")
 
         cur = conn.cursor()
+        # FYI - not sure of 5.7.0 version
+        if sys.version_info[0:2] >= (3,2) and self.mysql_server_is(conn, (5, 7, 0)):
+            with self.assertWarns(pymysql.err.Warning) as cm:
+                cur.execute("INSERT INTO issue363 (id, geom) VALUES ("
+                            "1998, GeomFromText('LINESTRING(1.1 1.1,2.2 2.2)'))")
+        else:
             cur.execute("INSERT INTO issue363 (id, geom) VALUES ("
                         "1998, GeomFromText('LINESTRING(1.1 1.1,2.2 2.2)'))")
 
         # select WKT
+        if sys.version_info[0:2] >= (3,2) and self.mysql_server_is(conn, (5, 7, 0)):
+            with self.assertWarns(pymysql.err.Warning) as cm:
+                cur.execute("SELECT AsText(geom) FROM issue363")
+        else:
             cur.execute("SELECT AsText(geom) FROM issue363")
         row = cur.fetchone()
         self.assertEqual(row, ("LINESTRING(1.1 1.1,2.2 2.2)", ))
 
         # select WKB
+        if sys.version_info[0:2] >= (3,2) and self.mysql_server_is(conn, (5, 7, 0)):
+            with self.assertWarns(pymysql.err.Warning) as cm:
+                cur.execute("SELECT AsBinary(geom) FROM issue363")
+        else:
             cur.execute("SELECT AsBinary(geom) FROM issue363")
         row = cur.fetchone()
         self.assertEqual(row,