WL#16164: Implement support for new vector data type

This work log adds support for the new MySQL `VECTOR` type.

Change-Id: Id73305c9eaf2a76c1ba10fea7916bf5f319dd324

Author: oscpache

CHANGES.txt

@@ -13,6 +13,7 @@ v8.4.0
 
 - WL#16203: GPL License Exception Update
 - WL#16173: Update allowed cipher and cipher-suite lists
+- WL#16164: Implement support for new vector data type
 - WL#16053: Support GSSAPI/Kerberos authentication on Windows using authentication_ldap_sasl_client plug-in for C-extension
 
 v8.3.0

mysql-connector-python/lib/mysql/connector/aio/charsets.py

@@ -91,7 +91,7 @@ def set_mysql_major_version(self, version: int) -> None:
         self._name_store.clear()
 
         charsets_tuple: Sequence[Tuple[int, str, str, bool]] = None
-        if version == 8:
+        if version >= 8:
             charsets_tuple = MYSQL_8_CHARSETS
         elif version == 5:
             charsets_tuple = MYSQL_5_CHARSETS

mysql-connector-python/lib/mysql/connector/aio/protocol.py

@@ -305,13 +305,13 @@ async def read_text_result(  # type: ignore[override]
                     datas.append(packet[4:])
                     packet = await sock.read()
                 datas.append(packet[4:])
-                rowdata = read_lc_string_list(bytearray(b"").join(datas))
+                rowdata = read_lc_string_list(b"".join(datas))
             elif packet[4] == 254 and packet[0] < 7:
                 eof = self.parse_eof(packet)
                 rowdata = None
             else:
                 eof = None
-                rowdata = read_lc_string_list(packet[4:])
+                rowdata = read_lc_string_list(bytes(packet[4:]))
             if eof is None and rowdata is not None:
                 rows.append(rowdata)
             elif eof is None and rowdata is None:

mysql-connector-python/lib/mysql/connector/connection_cext.py

@@ -50,7 +50,7 @@
 
 from . import version
 from .abstracts import CMySQLPrepStmt, MySQLConnectionAbstract
-from .constants import ClientFlag, FieldFlag, ServerFlag, ShutdownType
+from .constants import ClientFlag, FieldFlag, FieldType, ServerFlag, ShutdownType
 from .conversion import MySQLConverter
 from .errors import (
     InterfaceError,
@@ -473,18 +473,36 @@ def get_rows(
                 # convert the values. This can be accomplished by setting
                 # the raw option to True.
                 self._cmysql.raw(True)
+
             row = fetch_row()
             while row:
+                row = list(row)
+
+                if not self._cmysql.raw() and not raw:
+                    # `not _cmysql.raw()` means the c-ext conversion layer will happen.
+                    # `not raw` means the caller wants conversion to happen.
+                    # For a VECTOR type, the c-ext conversion layer cannot return
+                    # an array.array type since such a type isn't part of the Python/C
+                    # API. Therefore, the c-ext will treat VECTOR types as if they
+                    # were BLOB types - be returned as `bytes` always.
+                    # Hence, a VECTOR type must be cast to an array.array type using the
+                    # built-in python conversion layer.
+                    # pylint: disable=protected-access
+                    for i, dsc in enumerate(self._columns):
+                        if dsc[1] == FieldType.VECTOR:
+                            row[i] = MySQLConverter._vector_to_python(row[i])
+
                 if not self._raw and self.converter:
-                    row = list(row)
                     for i, _ in enumerate(row):
                         if not raw:
                             row[i] = self.converter.to_python(self._columns[i], row[i])
-                    row = tuple(row)
-                rows.append(row)
+
+                rows.append(tuple(row))
                 counter += 1
+
                 if count and counter == count:
                     break
+
                 row = fetch_row()
             if not row:
                 _eof: Optional[CextEofPacketType] = self.fetch_eof_columns(prep_stmt)[

mysql-connector-python/lib/mysql/connector/constants.py

@@ -39,8 +39,12 @@
 
 NET_BUFFER_LENGTH: int = 8192
 MAX_MYSQL_TABLE_COLUMNS: int = 4096
-# Flag used to send the Query Attributes with 0 (or more) parameters.
 PARAMETER_COUNT_AVAILABLE: int = 8
+"""Flag used to send the Query Attributes with 0 (or more) parameters."""
+MYSQL_VECTOR_TYPE_CODE = "f"
+"""Expected `typecode` when decoding VECTOR values from
+MySQL (blob) to Python (array.array).
+"""
 
 DEFAULT_CONFIGURATION: Dict[str, Optional[Union[str, bool, int]]] = {
     "database": None,
@@ -235,6 +239,7 @@ class FieldType(_Constants):
     NEWDATE: int = 0x0E
     VARCHAR: int = 0x0F
     BIT: int = 0x10
+    VECTOR: int = 0xF2
     JSON: int = 0xF5
     NEWDECIMAL: int = 0xF6
     ENUM: int = 0xF7
@@ -248,34 +253,35 @@ class FieldType(_Constants):
     GEOMETRY: int = 0xFF
 
     desc: Dict[str, Tuple[int, str]] = {
-        "DECIMAL": (0x00, "DECIMAL"),
-        "TINY": (0x01, "TINY"),
-        "SHORT": (0x02, "SHORT"),
-        "LONG": (0x03, "LONG"),
-        "FLOAT": (0x04, "FLOAT"),
-        "DOUBLE": (0x05, "DOUBLE"),
-        "NULL": (0x06, "NULL"),
-        "TIMESTAMP": (0x07, "TIMESTAMP"),
-        "LONGLONG": (0x08, "LONGLONG"),
-        "INT24": (0x09, "INT24"),
-        "DATE": (0x0A, "DATE"),
-        "TIME": (0x0B, "TIME"),
-        "DATETIME": (0x0C, "DATETIME"),
-        "YEAR": (0x0D, "YEAR"),
-        "NEWDATE": (0x0E, "NEWDATE"),
-        "VARCHAR": (0x0F, "VARCHAR"),
-        "BIT": (0x10, "BIT"),
-        "JSON": (0xF5, "JSON"),
-        "NEWDECIMAL": (0xF6, "NEWDECIMAL"),
-        "ENUM": (0xF7, "ENUM"),
-        "SET": (0xF8, "SET"),
-        "TINY_BLOB": (0xF9, "TINY_BLOB"),
-        "MEDIUM_BLOB": (0xFA, "MEDIUM_BLOB"),
-        "LONG_BLOB": (0xFB, "LONG_BLOB"),
-        "BLOB": (0xFC, "BLOB"),
-        "VAR_STRING": (0xFD, "VAR_STRING"),
-        "STRING": (0xFE, "STRING"),
-        "GEOMETRY": (0xFF, "GEOMETRY"),
+        "DECIMAL": (DECIMAL, "DECIMAL"),
+        "TINY": (TINY, "TINY"),
+        "SHORT": (SHORT, "SHORT"),
+        "LONG": (LONG, "LONG"),
+        "FLOAT": (FLOAT, "FLOAT"),
+        "DOUBLE": (DOUBLE, "DOUBLE"),
+        "NULL": (NULL, "NULL"),
+        "TIMESTAMP": (TIMESTAMP, "TIMESTAMP"),
+        "LONGLONG": (LONGLONG, "LONGLONG"),
+        "INT24": (INT24, "INT24"),
+        "DATE": (DATE, "DATE"),
+        "TIME": (TIME, "TIME"),
+        "DATETIME": (DATETIME, "DATETIME"),
+        "YEAR": (YEAR, "YEAR"),
+        "NEWDATE": (NEWDATE, "NEWDATE"),
+        "VARCHAR": (VARCHAR, "VARCHAR"),
+        "BIT": (BIT, "BIT"),
+        "VECTOR": (VECTOR, "VECTOR"),
+        "JSON": (JSON, "JSON"),
+        "NEWDECIMAL": (NEWDECIMAL, "NEWDECIMAL"),
+        "ENUM": (ENUM, "ENUM"),
+        "SET": (SET, "SET"),
+        "TINY_BLOB": (TINY_BLOB, "TINY_BLOB"),
+        "MEDIUM_BLOB": (MEDIUM_BLOB, "MEDIUM_BLOB"),
+        "LONG_BLOB": (LONG_BLOB, "LONG_BLOB"),
+        "BLOB": (BLOB, "BLOB"),
+        "VAR_STRING": (VAR_STRING, "VAR_STRING"),
+        "STRING": (STRING, "STRING"),
+        "GEOMETRY": (GEOMETRY, "GEOMETRY"),
     }
 
     @classmethod

mysql-connector-python/lib/mysql/connector/conversion.py

@@ -29,6 +29,7 @@
 """Converting MySQL and Python types
 """
 
+import array
 import datetime
 import math
 import struct
@@ -37,7 +38,7 @@
 from decimal import Decimal
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
 
-from .constants import CharacterSet, FieldFlag, FieldType
+from .constants import MYSQL_VECTOR_TYPE_CODE, CharacterSet, FieldFlag, FieldType
 from .custom_types import HexLiteral
 from .types import (
     DescriptionType,
@@ -740,6 +741,22 @@ def _blob_to_python(
                 return bytes(value)
         return self._string_to_python(value, dsc)
 
+    @staticmethod
+    def _vector_to_python(
+        value: Optional[bytes], desc: Optional[DescriptionType] = None
+    ) -> Optional[array.array]:
+        """
+        Converts a MySQL VECTOR value to a Python array.array type.
+
+        Returns an array of floats if `value` isn't `None`, otherwise `None`.
+        """
+        if value is None or isinstance(value, array.array):
+            return value
+        elif isinstance(value, (bytes, bytearray)):
+            return array.array(MYSQL_VECTOR_TYPE_CODE, value)
+        else:
+            raise TypeError(f"Got unsupported type {value.__class__.__name__}")
+
     _long_blob_to_python = _blob_to_python
     _medium_blob_to_python = _blob_to_python
     _tiny_blob_to_python = _blob_to_python

mysql-connector-python/lib/mysql/connector/protocol.py

@@ -54,6 +54,7 @@
     FieldType,
     ServerCmd,
 )
+from .conversion import MySQLConverter
 from .errors import DatabaseError, InterfaceError, ProgrammingError, get_exception
 from .logger import logger
 from .plugins import MySQLAuthPlugin, get_auth_plugin
@@ -798,6 +799,10 @@ def _parse_binary_values(
             elif field[1] == FieldType.TIME:
                 (packet, value) = self._parse_binary_time(packet)
                 values.append(value)
+            elif field[1] == FieldType.VECTOR:
+                # pylint: disable=protected-access
+                (packet, value) = utils.read_lc_string(packet)
+                values.append(MySQLConverter._vector_to_python(value))
             elif field[7] == FieldFlag.BINARY or field[8] == 63:  # "binary" charset
                 (packet, value) = utils.read_lc_string(packet)
                 values.append(value)

mysql-connector-python/src/mysql_capi.c

@@ -2695,7 +2695,7 @@ MySQL_fetch_row(MySQL *self)
             }
             else {
                 PyTuple_SET_ITEM(result_row, i,
-                                 PyByteArray_FromStringAndSize(row[i], field_lengths[i]));
+                                 PyBytes_FromStringAndSize(row[i], field_lengths[i]));
             }
             continue;
         }
@@ -2782,7 +2782,12 @@ MySQL_fetch_row(MySQL *self)
         else if (field_type == MYSQL_TYPE_BIT) {
             PyTuple_SET_ITEM(result_row, i, mytopy_bit(row[i], field_lengths[i]));
         }
-        else if (field_type == MYSQL_TYPE_BLOB) {
+    #if MYSQL_VERSION_ID >= 90000
+        else if (field_type == MYSQL_TYPE_BLOB || field_type == MYSQL_TYPE_VECTOR)
+    #else
+        else if (field_type == MYSQL_TYPE_BLOB)
+    #endif
+        {
             if ((field_flags & BLOB_FLAG) &&
                 (field_flags & BINARY_FLAG) && field_charsetnr == 63) {
                 value = PyBytes_FromStringAndSize(row[i], field_lengths[i]);
@@ -2795,7 +2800,7 @@ MySQL_fetch_row(MySQL *self)
         }
         else if (field_type == MYSQL_TYPE_GEOMETRY) {
             PyTuple_SET_ITEM(result_row, i,
-                             PyByteArray_FromStringAndSize(row[i], field_lengths[i]));
+                             PyBytes_FromStringAndSize(row[i], field_lengths[i]));
         }
         else {
             // Do our best to convert whatever we got from MySQL to a str/bytes
@@ -3740,8 +3745,8 @@ MySQLPrepStmt_fetch_row(MySQLPrepStmt *self)
                     Py_XDECREF(obj);
                 }
                 else if (field->type == MYSQL_TYPE_GEOMETRY) {
-                    obj = PyByteArray_FromStringAndSize(NULL, self->cols[i].length);
-                    self->bind[i].buffer = PyByteArray_AsString(obj);
+                    obj = PyBytes_FromStringAndSize(NULL, self->cols[i].length);
+                    self->bind[i].buffer = PyBytes_AsString(obj);
                     self->bind[i].buffer_length = self->cols[i].length;
 
                     Py_BEGIN_ALLOW_THREADS

mysql-connector-python/src/mysql_capi_conversion.c

@@ -671,13 +671,13 @@ mytopy_string(const char *data, enum_field_types field_type,
     }
 
     if (strcmp(charset, "binary") == 0) {
-        return PyByteArray_FromStringAndSize(data, field_length);
+        return PyBytes_FromStringAndSize(data, field_length);
     }
 
     /* 'binary' charset = 63 */
     if (use_unicode && (field_type == MYSQL_TYPE_JSON || field_charsetnr != 63)) {
         return PyUnicode_Decode(data, field_length, charset, "replace");
     }
 
-    return PyByteArray_FromStringAndSize(data, field_length);
+    return PyBytes_FromStringAndSize(data, field_length);
 }

mysql-connector-python/tests/cext/test_cext_api.py

@@ -833,8 +833,8 @@ def test_query(self):
 
         self.assertTrue(cmy.query("SELECT 'ham', 'spam', 5", raw=True))
         row = cmy.fetch_row()
-        self.assertTrue(isinstance(row[0], bytearray))
-        self.assertEqual(bytearray(b"spam"), row[1])
+        self.assertTrue(isinstance(row[0], bytes))
+        self.assertEqual(b"spam", row[1])
         self.assertEqual(None, cmy.fetch_row())
         cmy.free_result()