Merge branch 'github/patches/24.2.0' into github/patches/24.2.1

Author: msimacek

ci.jsonnet

@@ -1 +1 @@
-{ "overlay": "16d0e566573ebdcf380bf6f309c02724baf4172b" }
+{ "overlay": "ea6007b968048485d43905914816208f870ac88b" }

graalpython/lib-graalpython/patches/jq-1.8.0.patch

@@ -0,0 +1,59 @@
+--- a/pyproject.toml
++++ b/pyproject.toml
+@@ -2,13 +2,10 @@
+ requires = [
+     "setuptools>=43",
+     "wheel",
++    "Cython==3.0.10",
+ ]
+ build-backend = "setuptools.build_meta"
+ 
+ [tool.cibuildwheel]
+-before-build = [
+-    "pip install cython==3.0.10",
+-    "cython {project}/jq.pyx",
+-]
+ test-requires = "-r test-requirements.txt"
+ test-command = "pytest {project}/tests"
+diff --git a/setup.py b/setup.py
+index 0b97097..54ed7b3 100644
+--- a/setup.py
++++ b/setup.py
+@@ -94,15 +94,27 @@ else:
+         os.path.join(jq_lib_dir, "modules/oniguruma/src/.libs/libonig.a"),
+     ]
+ 
++
++try:
++    # Follow recommendation from https://cython.readthedocs.io/en/latest/src/userguide/source_files_and_compilation.html#distributing-cython-modules
++    from Cython.Build import cythonize
++except ImportError:
++    cythonize = lambda o: o
++    ext = ".c"
++else:
++    ext = ".pyx"
++
++
+ jq_extension = Extension(
+     "jq",
+-    sources=["jq.c"],
++    sources=[_path_in_dir(f"jq{ext}")],
+     define_macros=[("MS_WIN64" , 1)] if os.name == "nt" and sys.maxsize > 2**32  else None, # https://github.com/cython/cython/issues/2670
+     include_dirs=[os.path.join(jq_lib_dir, "src")],
+     extra_link_args=["-lm"] + (["-Wl,-Bstatic", "-lpthread", "-lshlwapi", "-static-libgcc"] if os.name == 'nt' else []) + link_args_deps,
+     extra_objects=extra_objects,
+ )
+ 
++
+ setup(
+     name='jq',
+     version='1.8.0',
+@@ -112,7 +124,7 @@ setup(
+     url='https://github.com/mwilliamson/jq.py',
+     python_requires='>=3.6',
+     license='BSD 2-Clause',
+-    ext_modules = [jq_extension],
++    ext_modules = cythonize([jq_extension]),
+     cmdclass={"build_ext": jq_build_ext},
+     classifiers=[
+         'Development Status :: 5 - Production/Stable',

graalpython/lib-graalpython/patches/metadata.toml

@@ -200,6 +200,15 @@ version = '<=1.3.2'
 patch = 'joblib-1.3.2.patch'
 license = 'BSD-3-Clause'
 
+[[jq.rules]]
+version = '==1.8.0'
+patch = 'jq-1.8.0.patch'
+license = 'BSD-2-Clause'
+
+[[jq.add-sources]]
+version = '1.8.0'
+url = 'https://github.com/mwilliamson/jq.py/archive/refs/tags/1.8.0.tar.gz'
+
 [[jupyter_server.rules]]
 patch = 'jupyter_server.patch'
 license = 'BSD-3-Clause'
@@ -358,6 +367,12 @@ patch = 'pandas-2.2.2.patch'
 license = 'BSD-3-Clause'
 dist-type = 'sdist'
 
+[[pandas.rules]]
+version = '== 2.2.3'
+patch = 'pandas-2.2.3.patch'
+license = 'BSD-3-Clause'
+dist-type = 'sdist'
+
 [[pandas.rules]]
 version = '== 2.0.3'
 patch = 'pandas-2.0.3.patch'
@@ -394,11 +409,23 @@ patch = 'protobuf-3.21.9.patch'
 license = 'BSD-3-Clause'
 
 [[psycopg2.rules]]
-patch = "psycopg2.patch"
+version = '<= 2.9.9'
+patch = "psycopg2-2.9.9.patch"
+license = "LGPL-3.0-or-later WITH openssl-exception"
+
+[[psycopg2.rules]]
+version = '>= 2.9.10'
+patch = "psycopg2-2.9.10.patch"
 license = "LGPL-3.0-or-later WITH openssl-exception"
 
 [[psycopg2-binary.rules]]
-patch = "psycopg2-binary.patch"
+version = '<= 2.9.9'
+patch = "psycopg2-2.9.9.patch"
+license = "LGPL-3.0-or-later WITH openssl-exception"
+
+[[psycopg2-binary.rules]]
+version = '>= 2.9.10'
+patch = "psycopg2-2.9.10.patch"
 license = "LGPL-3.0-or-later WITH openssl-exception"
 
 [[py4j.rules]]
@@ -416,6 +443,11 @@ version = '== 12.0.0'
 patch = 'pyarrow-12.0.0.patch'
 license = 'Apache-2.0'
 
+[[pyarrow.rules]]
+version = '== 19.0.0'
+patch = 'pyarrow-19.0.0.patch'
+license = 'Apache-2.0'
+
 [[pybind11.rules]]
 # Upstreamed
 install-priority = 0
@@ -726,6 +758,10 @@ version = '== 3.*'
 # transformers tends to depend on a specific version of tokenizers. Pin it to avoid pulling unpatched tokenizers
 version = '== 4.33.3'
 
+[[trio.rules]]
+patch = 'trio.patch'
+license = 'Apache-2.0 OR MIT'
+
 [[typing_extensions.rules]]
 patch = 'typing_extensions.patch'
 license = 'PSF-2.0'
@@ -741,6 +777,7 @@ version = '>= 2, < 2.0.3'
 install-priority = 0
 
 [[uvloop.rules]]
+version = '<= 0.19.0'
 patch = 'uvloop.patch'
 license = 'MIT'
 

graalpython/lib-graalpython/patches/pandas-2.2.3.patch

@@ -0,0 +1,52 @@
+diff --git a/pandas/_libs/include/pandas/vendored/klib/khash_python.h b/pandas/_libs/include/pandas/vendored/klib/khash_python.h
+index 5a933b4..f579fc6 100644
+--- a/pandas/_libs/include/pandas/vendored/klib/khash_python.h
++++ b/pandas/_libs/include/pandas/vendored/klib/khash_python.h
+@@ -173,13 +173,15 @@ static inline int floatobject_cmp(PyFloatObject *a, PyFloatObject *b) {
+ // PyObject_RichCompareBool for complexobjects has a different behavior
+ // needs to be replaced
+ static inline int complexobject_cmp(PyComplexObject *a, PyComplexObject *b) {
+-  return (Py_IS_NAN(a->cval.real) && Py_IS_NAN(b->cval.real) &&
+-          Py_IS_NAN(a->cval.imag) && Py_IS_NAN(b->cval.imag)) ||
+-         (Py_IS_NAN(a->cval.real) && Py_IS_NAN(b->cval.real) &&
+-          a->cval.imag == b->cval.imag) ||
+-         (a->cval.real == b->cval.real && Py_IS_NAN(a->cval.imag) &&
+-          Py_IS_NAN(b->cval.imag)) ||
+-         (a->cval.real == b->cval.real && a->cval.imag == b->cval.imag);
++  Py_complex a_cval = PyComplex_AsCComplex((PyObject*)a);
++  Py_complex b_cval = PyComplex_AsCComplex((PyObject*)b);
++  return (Py_IS_NAN(a_cval.real) && Py_IS_NAN(b_cval.real) &&
++          Py_IS_NAN(a_cval.imag) && Py_IS_NAN(b_cval.imag)) ||
++         (Py_IS_NAN(a_cval.real) && Py_IS_NAN(b_cval.real) &&
++          a_cval.imag == b_cval.imag) ||
++         (a_cval.real == b_cval.real && Py_IS_NAN(a_cval.imag) &&
++          Py_IS_NAN(b_cval.imag)) ||
++         (a_cval.real == b_cval.real && a_cval.imag == b_cval.imag);
+ }
+ 
+ static inline int pyobject_cmp(PyObject *a, PyObject *b);
+@@ -250,8 +252,9 @@ static inline Py_hash_t floatobject_hash(PyFloatObject *key) {
+ 
+ // replaces _Py_HashDouble with _Pandas_HashDouble
+ static inline Py_hash_t complexobject_hash(PyComplexObject *key) {
+-  Py_uhash_t realhash = (Py_uhash_t)_Pandas_HashDouble(key->cval.real);
+-  Py_uhash_t imaghash = (Py_uhash_t)_Pandas_HashDouble(key->cval.imag);
++  Py_complex cval = PyComplex_AsCComplex((PyObject*)key);
++  Py_uhash_t realhash = (Py_uhash_t)_Pandas_HashDouble(cval.real);
++  Py_uhash_t imaghash = (Py_uhash_t)_Pandas_HashDouble(cval.imag);
+   if (realhash == (Py_uhash_t)-1 || imaghash == (Py_uhash_t)-1) {
+     return -1;
+   }
+diff --git a/pyproject.toml b/pyproject.toml
+index db9f055..c191232 100644
+--- a/pyproject.toml
++++ b/pyproject.toml
+@@ -5,7 +5,7 @@ requires = [
+     "meson-python==0.13.1",
+     "meson==1.2.1",
+     "wheel",
+-    "Cython~=3.0.5",  # Note: sync with setup.py, environment.yml and asv.conf.json
++    "Cython==3.0.10",  # Note: sync with setup.py, environment.yml and asv.conf.json
+     # Force numpy higher than 2.0, so that built wheels are compatible
+     # with both numpy 1 and 2
+     "numpy>=2.0",

graalpython/lib-graalpython/patches/psycopg2.patch renamed to graalpython/lib-graalpython/patches/psycopg2-2.9.10.patch

@@ -1,15 +1,15 @@
 diff --git a/psycopg/utils.c b/psycopg/utils.c
-index 16be906..c78a24b 100644
+index 1dfb87d..28cc6c1 100644
 --- a/psycopg/utils.c
 +++ b/psycopg/utils.c
 @@ -392,7 +392,9 @@ psyco_set_error(PyObject *exc, cursorObject *curs, const char *msg)
  static int
  psyco_is_main_interp(void)
  {
--#if PY_VERSION_HEX >= 0x03080000
+-#if PY_VERSION_HEX >= 0x030d0000
 +#if GRAALVM_PYTHON
 +    return 1;
-+#elif PY_VERSION_HEX >= 0x03080000
-     /* tested with Python 3.8.0a2 */
-     return _PyInterpreterState_Get() == PyInterpreterState_Main();
- #else
++#elif PY_VERSION_HEX >= 0x030d0000
+     /* tested with Python 3.13.0a6 */
+     return PyInterpreterState_Get() == PyInterpreterState_Main();
+ #elif PY_VERSION_HEX >= 0x03080000

graalpython/lib-graalpython/patches/psycopg2-binary.patch renamed to graalpython/lib-graalpython/patches/psycopg2-2.9.9.patch

@@ -0,0 +1,164 @@
+diff --git a/pyarrow/error.pxi b/pyarrow/error.pxi
+index cbe2552..8d0d9d9 100644
+--- a/pyarrow/error.pxi
++++ b/pyarrow/error.pxi
+@@ -248,7 +248,7 @@ cdef class SignalStopHandler:
+             if exc_value.signum:
+                 # Re-emit the exact same signal. We restored the Python signal
+                 # handler above, so it should receive it.
+-                if os.name == 'nt':
++                if os.name == 'nt' or sys.implementation.name == 'graalpy':
+                     SendSignal(exc_value.signum)
+                 else:
+                     SendSignalToThread(exc_value.signum,
+diff --git a/pyarrow/memory.pxi b/pyarrow/memory.pxi
+index 1ddcb01..6805e42 100644
+--- a/pyarrow/memory.pxi
++++ b/pyarrow/memory.pxi
+@@ -20,6 +20,10 @@
+ # cython: embedsignature = True
+ 
+ 
++cdef extern from "Python.h":
++    void Py_INCREF(object)
++
++
+ cdef class MemoryPool(_Weakrefable):
+     """
+     Base class for memory allocation.
+@@ -35,6 +39,13 @@ cdef class MemoryPool(_Weakrefable):
+ 
+     cdef void init(self, CMemoryPool* pool):
+         self.pool = pool
++        # GraalPy change: pyarrow doesn't maintain python references from
++        # buffers to pools, but they dereference the pointer to the pool in the
++        # destructor. They just assume buffers will get GC'ed before their
++        # pools. You can easily get a segfault even on CPython if you make
++        # a buffer outlive its pool. Since we can't guarantee destruction
++        # order, we just leak the pool.
++        Py_INCREF(self)
+ 
+     def release_unused(self):
+         """
+diff --git a/pyarrow_build_backend.py b/pyarrow_build_backend.py
+new file mode 100644
+index 0000000..0929c5f
+--- /dev/null
++++ b/pyarrow_build_backend.py
+@@ -0,0 +1,93 @@
++import os
++import re
++import sys
++import tarfile
++import subprocess
++import tempfile
++import shutil
++import tarfile
++import urllib.request
++from pathlib import Path
++
++VERSION = '19.0.0'
++
++
++def build_sdist(sdist_directory, config_settings=None):
++    nv = f'pyarrow-{VERSION}'
++    srcdir = Path(__file__).parent
++    archive_path = Path(sdist_directory) / f'{nv}.tar.gz'
++
++    def tarfilter(info):
++        if re.match(r'\./(?:.git|venv|[^-/]+-venv|dist)', info.name):
++            return None
++        info.name = f'./{nv}/{info.name}'
++        return info
++
++    with tarfile.open(archive_path, 'w:gz') as tar:
++        tar.add('.', filter=tarfilter)
++    return archive_path.name
++
++
++def build_wheel(wheel_directory, config_settings=None, metadata_directory=None):
++    wheel_directory = Path(wheel_directory).absolute()
++    with tempfile.TemporaryDirectory() as tmpdir:
++        tmpdir = Path(tmpdir).absolute()
++        tarname = f'apache-arrow-{VERSION}.tar.gz'
++        tarpath = tmpdir / tarname
++        urllib.request.urlretrieve(f"https://github.com/apache/arrow/archive/refs/tags/{tarname}", tarpath)
++        with tarfile.open(tarpath) as tar:
++            tar.extractall(tmpdir)
++        arrow_dir = tmpdir / f'arrow-apache-arrow-{VERSION}'
++        assert arrow_dir.is_dir()
++        arrow_dist = tmpdir / 'arrow-dist'
++        build_dir = tmpdir / 'arrow-build'
++        subprocess.check_call([
++            'cmake', '-S', str(arrow_dir / 'cpp'), '-B', str(build_dir),
++            '-DCMAKE_INSTALL_LIBDIR=lib',
++            f'-DCMAKE_INSTALL_PREFIX={arrow_dist}',
++            '-DCMAKE_BUILD_TYPE=Release',
++            '-DARROW_RPATH_ORIGIN=ON',
++            '-DARROW_BUILD_TESTS=OFF',
++            '-DARROW_BUILD_SHARED=ON',
++            '-DARROW_BUILD_STATIC=OFF',
++            # Features
++            '-DARROW_COMPUTE=ON',
++            '-DARROW_CSV=ON',
++            '-DARROW_JSON=ON',
++            '-DARROW_FILESYSTEM=ON',
++            '-DARROW_DATASET=ON',
++            '-DARROW_PARQUET=ON',
++            '-DPARQUET_REQUIRE_ENCRYPTION=ON',
++            '-DARROW_GANDIVA=ON',
++            '-DARROW_WITH_BZ2=ON',
++            '-DARROW_WITH_ZLIB=ON',
++            '-DARROW_WITH_ZSTD=ON',
++            '-DARROW_WITH_LZ4=ON',
++            '-DARROW_WITH_SNAPPY=ON',
++            '-DARROW_WITH_BROTLI=ON',
++        ])
++        subprocess.check_call([
++            'cmake', '--build', str(build_dir), '--parallel',
++        ])
++        subprocess.check_call([
++            'cmake', '--install', str(build_dir),
++        ])
++        env = os.environ.copy()
++        env['ARROW_HOME'] = str(arrow_dist)
++        env['CMAKE_PREFIX_PATH'] = str(arrow_dist)
++        env['PYARROW_WITH_DATASET'] = '1'
++        env['PYARROW_WITH_PARQUET'] = '1'
++        env['PYARROW_WITH_PARQUET_ENCRYPTION'] = '1'
++        env['PYARROW_WITH_GANDIVA'] = '1'
++        env['PYARROW_BUNDLE_ARROW_CPP'] = '1'
++        env['PYARROW_BUNDLE_CYTHON_CPP'] = '1'
++        subprocess.run(
++            [sys.executable, 'setup.py', 'bdist_wheel'],
++            env=env,
++            check=True,
++        )
++        wheels = list(Path('dist').glob('*.whl'))
++        assert len(wheels) == 1, f"Expected 1 wheel, found {len(wheels)}"
++        wheel = wheels[0]
++        shutil.copyfile(wheel, wheel_directory / wheel.name)
++        return str(wheel.name)
+diff --git a/pyproject.toml b/pyproject.toml
+index ef2043f..cb08a86 100644
+--- a/pyproject.toml
++++ b/pyproject.toml
+@@ -17,7 +17,7 @@
+ 
+ [build-system]
+ requires = [
+-    "cython >= 0.29.31",
++    "cython >= 0.29.31, < 3",
+     # Starting with NumPy 1.25, NumPy is (by default) as far back compatible
+     # as oldest-support-numpy was (customizable with a NPY_TARGET_VERSION
+     # define).  For older Python versions (where NumPy 1.25 is not yet available)
+@@ -29,7 +29,8 @@ requires = [
+     "setuptools_scm[toml]>=8",
+     "setuptools>=64",
+ ]
+-build-backend = "setuptools.build_meta"
++build-backend = "pyarrow_build_backend"
++backend-path = ["."]
+ 
+ [project]
+ name = "pyarrow"