WL14689: Fallback conversion to str for types incompatible with MySQL

Currently, if one tries to insert a value of some type that is not
compatible with the default Connector/Python converter, an error is
raised.

This patch introduces a new connection option 'converter_str_fallback',
that allows the conversion to str of values types not supported by the
Connector/Python default converter class, if set to True.

In addition, this patch enables the usage of a custom class when the
C extension is used.

Author: nmariz

lib/mysql/connector/abstracts.py

@@ -123,6 +123,7 @@ def __init__(self, **kwargs):
         self._pool_config_version = None
         self.converter = None
         self._converter_class = None
+        self._converter_str_fallback = False
         self._compress = False
 
         self._consume_results = False
@@ -1173,6 +1174,7 @@ def set_converter_class(self, convclass):
             charset_name = CharacterSet.get_info(self._charset_id)[0]
             self._converter_class = convclass
             self.converter = convclass(charset_name, self._use_unicode)
+            self.converter.str_fallback = self._converter_str_fallback
         else:
             raise TypeError("Converter class should be a subclass "
                             "of conversion.MySQLConverterBase.")

lib/mysql/connector/connection.py

@@ -1368,11 +1368,13 @@ def cmd_stmt_execute(self, statement_id, data=(), parameters=(), flags=0):
         if self._client_flags & ClientFlag.CLIENT_QUERY_ATTRIBUTES:
             execute_packet = self._protocol.make_stmt_execute(
                 statement_id, data, tuple(parameters), flags,
-                long_data_used, self.charset, self._query_attrs)
+                long_data_used, self.charset, self._query_attrs,
+                self._converter_str_fallback)
         else:
             execute_packet = self._protocol.make_stmt_execute(
                 statement_id, data, tuple(parameters), flags,
-                long_data_used, self.charset)
+                long_data_used, self.charset,
+                converter_str_fallback=self._converter_str_fallback)
         packet = self._send_cmd(ServerCmd.STMT_EXECUTE, packet=execute_packet)
         result = self._handle_binary_result(packet)
         return result

lib/mysql/connector/connection_cext.py

@@ -237,6 +237,9 @@ def _open_connection(self):
 
         try:
             self._cmysql.connect(**cnx_kwargs)
+            self._cmysql.converter_str_fallback = self._converter_str_fallback
+            if self.converter:
+                self.converter.str_fallback = self._converter_str_fallback
         except MySQLInterfaceError as exc:
             raise errors.get_mysql_exception(msg=exc.msg, errno=exc.errno,
                                              sqlstate=exc.sqlstate)
@@ -437,6 +440,7 @@ def fetch_eof_columns(self, prep_stmt=None):
                 None,
                 None,
                 None,
+                None,
                 ~int(col[9]) & FieldFlag.NOT_NULL,
                 int(col[9])
             ))
@@ -468,7 +472,9 @@ def cmd_stmt_prepare(self, statement):
             raise errors.OperationalError("MySQL Connection not available")
 
         try:
-            return self._cmysql.stmt_prepare(statement)
+            stmt = self._cmysql.stmt_prepare(statement)
+            stmt.converter_str_fallback = self._converter_str_fallback
+            return stmt
         except MySQLInterfaceError as err:
             raise errors.InterfaceError(str(err))
 
@@ -650,11 +656,28 @@ def prepare_for_mysql(self, params):
         Returns dict.
         """
         if isinstance(params, (list, tuple)):
-            result = self._cmysql.convert_to_mysql(*params)
+            if self.converter:
+                result = [
+                    self.converter.quote(
+                        self.converter.escape(
+                            self.converter.to_mysql(value)
+                        )
+                    ) for value in params
+                ]
+            else:
+                result = self._cmysql.convert_to_mysql(*params)
         elif isinstance(params, dict):
             result = {}
-            for key, value in params.items():
-                result[key] = self._cmysql.convert_to_mysql(value)[0]
+            if self.converter:
+                for key, value in params.items():
+                    result[key] = self.converter.quote(
+                        self.converter.escape(
+                            self.converter.to_mysql(value)
+                        )
+                    )
+            else:
+                for key, value in params.items():
+                    result[key] = self._cmysql.convert_to_mysql(value)[0]
         else:
             raise ValueError("Could not process parameters")
 

lib/mysql/connector/constants.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2009, 2020, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2009, 2021, Oracle and/or its affiliates. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License, version 2.0, as
@@ -53,6 +53,7 @@
     'charset': 'utf8mb4',
     'collation': None,
     'converter_class': None,
+    'converter_str_fallback': False,
     'autocommit': False,
     'time_zone': None,
     'sql_mode': None,

lib/mysql/connector/conversion.py

@@ -48,14 +48,15 @@ class MySQLConverterBase(object):
     be a subclass of this class.
     """
 
-    def __init__(self, charset='utf8', use_unicode=True):
+    def __init__(self, charset='utf8', use_unicode=True, str_fallback=False):
         self.python_types = None
         self.mysql_types = None
         self.charset = None
         self.charset_id = 0
         self.use_unicode = None
         self.set_charset(charset)
-        self.set_unicode(use_unicode)
+        self.use_unicode = use_unicode
+        self.str_fallback = str_fallback
         self._cache_field_types = {}
 
     def set_charset(self, charset):
@@ -126,8 +127,8 @@ class MySQLConverter(MySQLConverterBase):
 
     """
 
-    def __init__(self, charset=None, use_unicode=True):
-        MySQLConverterBase.__init__(self, charset, use_unicode)
+    def __init__(self, charset=None, use_unicode=True, str_fallback=False):
+        MySQLConverterBase.__init__(self, charset, use_unicode, str_fallback)
         self._cache_field_types = {}
 
     def escape(self, value):
@@ -179,6 +180,8 @@ def to_mysql(self, value):
         try:
             return getattr(self, "_{0}_to_mysql".format(type_name))(value)
         except AttributeError:
+            if self.str_fallback:
+                return str(value).encode()
             raise TypeError("Python '{0}' cannot be converted to a "
                             "MySQL type".format(type_name))
 

lib/mysql/connector/protocol.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2009, 2020, Oracle and/or its affiliates.
+# Copyright (c) 2009, 2021, Oracle and/or its affiliates.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License, version 2.0, as
@@ -660,7 +660,7 @@ def _prepare_stmt_send_long_data(self, statement, param, data):
 
     def make_stmt_execute(self, statement_id, data=(), parameters=(),
                           flags=0, long_data_used=None, charset='utf8',
-                          query_attrs=None):
+                          query_attrs=None, converter_str_fallback=False):
         """Make a MySQL packet with the Statement Execute command"""
         iteration_count = 1
         null_bitmap = [0] * ((len(data) + 7) // 8)
@@ -725,6 +725,10 @@ def make_stmt_execute(self, statement_id, data=(), parameters=(),
                 elif isinstance(value, (datetime.timedelta, datetime.time)):
                     (packed, field_type) = self._prepare_binary_time(value)
                     values.append(packed)
+                elif converter_str_fallback:
+                    value = str(value).encode(charset)
+                    values.append(utils.lc_int(len(value)) + value)
+                    field_type = FieldType.STRING
                 else:
                     raise errors.ProgrammingError(
                         "MySQL binary protocol can not handle "

src/include/mysql_capi.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, 2020, Oracle and/or its affiliates.
+ * Copyright (c) 2014, 2021, Oracle and/or its affiliates.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License, version 2.0, as
@@ -61,6 +61,7 @@ typedef struct {
     PyObject *fields;
     PyObject *auth_plugin;
     PyObject *plugin_dir;
+    PyObject *converter_str_fallback;
     MY_CHARSET_INFO cs;
     unsigned int connection_timeout;
     // class members
@@ -276,6 +277,7 @@ typedef struct {
     struct column_info *cols;
     PyObject *have_result_set;
     PyObject *fields;
+    PyObject *converter_str_fallback;
     MY_CHARSET_INFO cs;
 } MySQLPrepStmt;
 

src/mysql_capi.c

@@ -342,6 +342,7 @@ MySQL_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 	self->use_unicode=          1;
 	self->auth_plugin=          PyUnicode_FromString("mysql_native_password");
 	self->plugin_dir=           PyUnicode_FromString(".");
+	self->converter_str_fallback = Py_False;
 
 	return (PyObject *)self;
 }
@@ -1885,6 +1886,14 @@ MySQL_convert_to_mysql(MySQL *self, PyObject *args)
         {
             new_value= pytomy_decimal(value);
         }
+        else if (self->converter_str_fallback == Py_True)
+        {
+            PyObject *str= PyObject_Str(value);
+            new_value= PyBytes_FromString(
+                (const char *)PyUnicode_1BYTE_DATA(str)
+            );
+            Py_DECREF(str);
+        }
         else
         {
             PyOS_snprintf(error, 100,
@@ -3332,6 +3341,15 @@ MySQLPrepStmt_execute(MySQLPrepStmt *self, PyObject *args)
             pbind->str_value= pytomy_decimal(value);
             mbind[i].buffer_type= MYSQL_TYPE_DECIMAL;
         }
+        else if (self->converter_str_fallback == Py_True)
+        {
+            PyObject *str= PyObject_Str(value);
+            pbind->str_value= PyBytes_FromString(
+                (const char *)PyUnicode_1BYTE_DATA(str)
+            );
+            mbind->buffer_type= MYSQL_TYPE_STRING;
+            Py_DECREF(str);
+        }
         else
         {
             retval= PyErr_Format(MySQLInterfaceError,

src/mysql_connector.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, 2020, Oracle and/or its affiliates.
+ * Copyright (c) 2014, 2021, Oracle and/or its affiliates.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License, version 2.0, as
@@ -71,6 +71,8 @@ static PyMemberDef MySQL_members[]=
 {
     {"have_result_set", T_OBJECT, offsetof(MySQL, have_result_set), 0,
      "True if current session has result set"},
+    {"converter_str_fallback", T_OBJECT, offsetof(MySQL, converter_str_fallback), 0,
+     "True for fallback to converting unsupported types to str"},
     {NULL}  /* Sentinel */
 };
 
@@ -306,6 +308,8 @@ static PyMemberDef MySQLPrepStmt_members[]=
      "True if statement has result set"},
     {"param_count", T_LONG, offsetof(MySQLPrepStmt, param_count), 1,
      "Returns the number of parameter markers present in the prepared statement"},
+    {"converter_str_fallback", T_OBJECT, offsetof(MySQLPrepStmt, converter_str_fallback), 0,
+     "True for fallback to converting unsupported types to str"},
     {NULL}  /* Sentinel */
 };
 

tests/cext/test_cext_connection.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2014, 2020, Oracle and/or its affiliates.
+# Copyright (c) 2014, 2021, Oracle and/or its affiliates.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License, version 2.0, as
@@ -94,13 +94,13 @@ def test_cmd_query(self):
         exp = {
             'eof': {'status_flag': 32, 'warning_count': 0},
             'columns': [
-                ['Variable_name', 253, None, None, None, None, 0, 1],
-                ('Value', 253, None, None, None, None, 1, 0)
+                ['Variable_name', 253, None, None, None, None, None, 0, 1],
+                ('Value', 253, None, None, None, None, None, 1, 0)
             ]
         }
 
         if tests.MYSQL_VERSION >= (5, 7, 10):
-            exp['columns'][0][7] = 4097
+            exp['columns'][0][8] = 4097
             exp['eof']['status_flag'] = 16385
 
         exp['columns'][0] = tuple(exp['columns'][0])

tests/test_connection.py

@@ -1062,18 +1062,14 @@ def test_config(self):
 
         # Test converter class
         class TestConverter(MySQLConverterBase):
-
-            def __init__(self, charset=None, unicode=True):
-                pass
+            ...
 
         self.cnx.config(converter_class=TestConverter)
         self.assertTrue(isinstance(self.cnx.converter, TestConverter))
         self.assertEqual(self.cnx._converter_class, TestConverter)
 
         class TestConverterWrong(object):
-
-            def __init__(self, charset, unicode):
-                pass
+            ...
 
         self.assertRaises(AttributeError,
                           self.cnx.config, converter_class=TestConverterWrong)
@@ -1316,15 +1312,13 @@ def test_set_converter_class(self):
         """Set the converter class"""
 
         class TestConverterWrong(object):
-            def __init__(self, charset, unicode):
-                pass
+            ...
 
         self.assertRaises(TypeError,
                           self.cnx.set_converter_class, TestConverterWrong)
 
         class TestConverter(MySQLConverterBase):
-            def __init__(self, charset, unicode):
-                pass
+            ...
 
         self.cnx.set_converter_class(TestConverter)
         self.assertTrue(isinstance(self.cnx.converter, TestConverter))