BUG27364914: Cursor prepared statements do not convert strings

Using the cursor with prepared statements, the returned values from
queries are converted from binaries values to their correspond python
values, like int and datetime types, but not for strings types which
bytearrays are returned.

This patch changes this behavior to return text values as string types.

A test was added for regression.

v5

Author: isrgomez

lib/mysql/connector/connection.py

@@ -469,8 +469,11 @@ def get_rows(self, count=None, binary=False, columns=None):
 
         try:
             if binary:
+                charset = self.charset
+                if charset == 'utf8mb4':
+                    charset = 'utf8'
                 rows = self._protocol.read_binary_result(
-                    self._socket, columns, count)
+                    self._socket, columns, count, charset)
             else:
                 rows = self._protocol.read_text_result(self._socket,
                                                        self._server_version,

lib/mysql/connector/conversion.py

@@ -561,7 +561,7 @@ def _STRING_to_python(self, value, dsc=None):  # pylint: disable=C0103
             if dsc[7] & FieldFlag.BINARY:
                 try:
                     return value.decode(self.charset)
-                except LookupError:
+                except (LookupError, UnicodeDecodeError):
                     return value
 
         if self.charset == 'binary':

lib/mysql/connector/cursor.py

@@ -1169,7 +1169,10 @@ def execute(self, operation, params=(), multi=False):  # multi is unused
             self._executed = operation
             try:
                 if not isinstance(operation, bytes):
-                    operation = operation.encode(self._connection.charset)
+                    charset = self._connection.charset
+                    if charset == 'utf8mb4':
+                        charset = 'utf8'
+                    operation = operation.encode(charset)
             except (UnicodeDecodeError, UnicodeEncodeError) as err:
                 raise errors.ProgrammingError(str(err))
 

lib/mysql/connector/protocol.py

@@ -418,7 +418,7 @@ def _parse_binary_time(self, packet, field):
 
         return (packet[length + 1:], tmp)
 
-    def _parse_binary_values(self, fields, packet):
+    def _parse_binary_values(self, fields, packet, charset='utf-8'):
         """Parse values from a binary result packet"""
         null_bitmap_length = (len(fields) + 7 + 2) // 8
         null_bitmap = [int(i) for i in packet[0:null_bitmap_length]]
@@ -446,11 +446,11 @@ def _parse_binary_values(self, fields, packet):
                 values.append(value)
             else:
                 (packet, value) = utils.read_lc_string(packet)
-                values.append(value)
+                values.append(value.decode(charset))
 
         return tuple(values)
 
-    def read_binary_result(self, sock, columns, count=1):
+    def read_binary_result(self, sock, columns, count=1, charset='utf-8'):
         """Read MySQL binary protocol result
 
         Reads all or given number of binary resultset rows from the socket.
@@ -470,7 +470,7 @@ def read_binary_result(self, sock, columns, count=1):
                 values = None
             elif packet[4] == 0:
                 eof = None
-                values = self._parse_binary_values(columns, packet[5:])
+                values = self._parse_binary_values(columns, packet[5:], charset)
             if eof is None and values is not None:
                 rows.append(values)
             elif eof is None and values is None:
@@ -626,6 +626,8 @@ def make_stmt_execute(self, statement_id, data=(), parameters=(),
         values = []
         types = []
         packed = b''
+        if charset == 'utf8mb4':
+            charset = 'utf8'
         if long_data_used is None:
             long_data_used = {}
         if parameters and data:

tests/test_bugs.py

@@ -2500,8 +2500,7 @@ def test_prepared_statement(self):
         stmt = "INSERT INTO {0} VALUES (?,?,?)".format(
             self.table)
         data = [(1, b'bytes', '1234'), (2, u'aaaаффф', '1111')]
-        exp = [(1, b'bytes', b'1234'),
-               (2, u'aaaаффф'.encode('cp1251'), b'1111')]
+        exp = [(1, 'bytes', '1234'), (2, u'aaaаффф', '1111')]
         cur.execute(stmt, data[0])
         self.cnx.commit()
         cur.execute("SELECT * FROM {0}".format(self.table))
@@ -3078,8 +3077,8 @@ def test_null(self):
                "VALUES (?, ?, ?, ?, ?, ?, ?)".format(self.tbl))
         params = (100, None, 'foo', None, datetime(2014, 8, 4, 9, 11, 14),
                   10, 'bar')
-        exp = (100, None, bytearray(b'foo'), None,
-               datetime(2014, 8, 4, 9, 11, 14), 10, bytearray(b'bar'))
+        exp = (100, None, 'foo', None,
+               datetime(2014, 8, 4, 9, 11, 14), 10, 'bar')
         cur.execute(sql, params)
 
         sql = "SELECT * FROM {0}".format(self.tbl)
@@ -4549,3 +4548,84 @@ def test_execute_get_json_type_as_str(self):
             self.assertTrue(isinstance(col, STRING_TYPES),
                             "{} is type {} and not the expected type "
                             "string".format(col, type(col)))
+
+
+class BugOra27364914(tests.MySQLConnectorTests):
+    """BUG#27364914: CURSOR PREPARED STATEMENTS DO NOT CONVERT STRINGS
+    """
+    charsets_list = ('gbk', 'sjis', 'big5', 'utf8', 'utf8mb4', 'latin1')
+
+    def setUp(self):
+        cnx = connection.MySQLConnection(**tests.get_mysql_config())
+        cur = cnx.cursor(cursor_class=cursor.MySQLCursorPrepared)
+
+        for charset in self.charsets_list:
+            tablename = '{0}_ps_test'.format(charset)
+            cur.execute("DROP TABLE IF EXISTS {0}".format(tablename))
+            table = (
+                "CREATE TABLE {table} ("
+                "  id INT AUTO_INCREMENT KEY,"
+                "  c1 VARCHAR(40),"
+                "  val2 datetime"
+                ") CHARACTER SET '{charset}'"
+            ).format(table=tablename, charset=charset)
+            cur.execute(table)
+            cnx.commit()
+        cur.close()
+        cnx.close()
+
+    def tearDown(self):
+        cnx = connection.MySQLConnection(**tests.get_mysql_config())
+        for charset in self.charsets_list:
+            tablename = '{0}_ps_test'.format(charset)
+            cnx.cmd_query("DROP TABLE IF EXISTS {0}".format(tablename))
+        cnx.close()
+
+    def _test_charset(self, charset, data):
+        config = tests.get_mysql_config()
+        config['charset'] = charset
+        config['use_unicode'] = True
+        self.cnx = connection.MySQLConnection(**tests.get_mysql_config())
+        cur = self.cnx.cursor(cursor_class=cursor.MySQLCursorPrepared)
+
+        tablename = '{0}_ps_test'.format(charset)
+        cur.execute("TRUNCATE {0}".format(tablename))
+        self.cnx.commit()
+
+        insert = "INSERT INTO {0} (c1) VALUES (%s)".format(tablename)
+        for value in data:
+            cur.execute(insert, (value,))
+        self.cnx.commit()
+
+        cur.execute("SELECT id, c1 FROM {0} ORDER BY id".format(tablename))
+        for row in cur:
+            self.assertTrue(isinstance(row[1], STRING_TYPES),
+                            "The value is expected to be a string")
+            self.assertEqual(data[row[0] - 1], row[1])
+
+        cur.close()
+        self.cnx.close()
+
+    @foreach_cnx()
+    def test_cursor_prepared_statement_with_charset_gbk(self):
+        self._test_charset('gbk', [u'赵孟頫', u'赵\孟\頫\\', u'遜'])
+
+    @foreach_cnx()
+    def test_cursor_prepared_statement_with_charset_sjis(self):
+        self._test_charset('sjis', ['\u005c'])
+
+    @foreach_cnx()
+    def test_cursor_prepared_statement_with_charset_big5(self):
+        self._test_charset('big5', ['\u5C62'])
+
+    @foreach_cnx()
+    def test_cursor_prepared_statement_with_charset_utf8mb4(self):
+        self._test_charset('utf8mb4', ['\u5C62'])
+
+    @foreach_cnx()
+    def test_cursor_prepared_statement_with_charset_utf8(self):
+        self._test_charset('utf8', [u'データベース', u'데이터베이스'])
+
+    @foreach_cnx()
+    def test_cursor_prepared_statement_with_charset_latin1(self):
+        self._test_charset('latin1', [u'ñ', u'Ñ'])

tests/test_cursor.py

@@ -1232,7 +1232,7 @@ def test_executemany(self):
         self.assertEqual(3, cur.rowcount)
 
         cur.execute(stmt_select)
-        self.assertEqual([(1, b'100'), (2, b'200'), (3, b'300')],
+        self.assertEqual([(1, '100'), (2, '200'), (3, '300')],
                          cur.fetchall(), "Multi insert test failed")
 
         data = [(2,), (3,)]

tests/test_protocol.py

@@ -408,9 +408,9 @@ def test__parse_binary_values(self):
                            b'\x00\x0a\x00\x00\x00\x10\x0f\x1e\x70\x82\x03\x00')
 
         # float/double are returned as DECIMAL by MySQL
-        exp = (bytearray(b'abc'),
-               bytearray(b'3.14'),
-               bytearray(b'-3.14159'),
+        exp = ('abc',
+               '3.14',
+               '-3.14159',
                datetime.date(2003, 1, 31),
                datetime.datetime(1977, 6, 14, 21, 33, 14),
                datetime.timedelta(10, 58530, 230000),