Integration tests for reading parquet w/ Variants

.github/workflows/arrow.yml

@@ -68,7 +68,10 @@ jobs:
       - name: Test arrow-schema
         run: cargo test -p arrow-schema --all-features
       - name: Test arrow-array
-        run: cargo test -p arrow-array --all-features
+        run: |
+          cargo test -p arrow-array --all-features
+          # Disable feature `force_validate`
+          cargo test -p arrow-array --features=ffi
       - name: Test arrow-select
         run: cargo test -p arrow-select --all-features
       - name: Test arrow-cast

.github/workflows/arrow_flight.yml

@@ -60,7 +60,7 @@ jobs:
           cargo test -p arrow-flight --all-features
       - name: Test --examples
         run: |
-          cargo test -p arrow-flight  --features=flight-sql,tls --examples
+          cargo test -p arrow-flight  --features=flight-sql,tls-ring --examples
 
   vendor:
     name: Verify Vendored Code

.github/workflows/docs.yml

@@ -79,7 +79,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Download crate docs
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v5
         with:
           name: crate-docs
           path: website/build

.github/workflows/parquet-variant.yml

@@ -31,6 +31,8 @@ on:
   pull_request:
     paths:
       - parquet-variant/**
+      - parquet-variant-json/**
+      - parquet-variant-compute/**
       - .github/**
 
 jobs:
@@ -50,6 +52,8 @@ jobs:
         run: cargo test -p parquet-variant
       - name: Test parquet-variant-json
         run: cargo test -p parquet-variant-json
+      - name: Test parquet-variant-compute
+        run: cargo test -p parquet-variant-compute
 
   # test compilation
   linux-features:
@@ -63,10 +67,12 @@ jobs:
           submodules: true
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
-      - name: Check compilation
+      - name: Check compilation (parquet-variant)
         run: cargo check -p parquet-variant
-      - name: Check compilation
+      - name: Check compilation (parquet-variant-json)
         run: cargo check -p parquet-variant-json
+      - name: Check compilation (parquet-variant-compute)
+        run: cargo check -p parquet-variant-compute
 
   clippy:
     name: Clippy
@@ -79,7 +85,9 @@ jobs:
         uses: ./.github/actions/setup-builder
       - name: Setup Clippy
         run: rustup component add clippy
-      - name: Run clippy
+      - name: Run clippy (parquet-variant)
         run: cargo clippy -p parquet-variant --all-targets --all-features -- -D warnings
-      - name: Run clippy
+      - name: Run clippy (parquet-variant-json)
         run: cargo clippy -p parquet-variant-json --all-targets --all-features -- -D warnings
+      - name: Run clippy (parquet-variant-compute)
+        run: cargo clippy -p parquet-variant-compute --all-targets --all-features -- -D warnings

CONTRIBUTING.md

@@ -89,8 +89,7 @@ You can also use rust's official docker image:
 docker run --rm -v $(pwd):/arrow-rs -it rust /bin/bash -c "cd /arrow-rs && rustup component add rustfmt && cargo build"
 ```
 
-The command above assumes that are in the root directory of the project, not in the same
-directory as this README.md.
+The command above assumes that are in the root directory of the project.
 
 You can also compile specific workspaces:
 

Cargo.toml

@@ -67,7 +67,7 @@ exclude = [
 ]
 
 [workspace.package]
-version = "55.2.0"
+version = "56.0.0"
 homepage = "https://github.com/apache/arrow-rs"
 repository = "https://github.com/apache/arrow-rs"
 authors = ["Apache Arrow <[email protected]>"]
@@ -84,22 +84,22 @@ edition = "2021"
 rust-version = "1.84"
 
 [workspace.dependencies]
-arrow = { version = "55.2.0", path = "./arrow", default-features = false }
-arrow-arith = { version = "55.2.0", path = "./arrow-arith" }
-arrow-array = { version = "55.2.0", path = "./arrow-array" }
-arrow-buffer = { version = "55.2.0", path = "./arrow-buffer" }
-arrow-cast = { version = "55.2.0", path = "./arrow-cast" }
-arrow-csv = { version = "55.2.0", path = "./arrow-csv" }
-arrow-data = { version = "55.2.0", path = "./arrow-data" }
-arrow-ipc = { version = "55.2.0", path = "./arrow-ipc" }
-arrow-json = { version = "55.2.0", path = "./arrow-json" }
-arrow-ord = { version = "55.2.0", path = "./arrow-ord" }
-arrow-pyarrow = { version = "55.2.0", path = "./arrow-pyarrow" }
-arrow-row = { version = "55.2.0", path = "./arrow-row" }
-arrow-schema = { version = "55.2.0", path = "./arrow-schema" }
-arrow-select = { version = "55.2.0", path = "./arrow-select" }
-arrow-string = { version = "55.2.0", path = "./arrow-string" }
-parquet = { version = "55.2.0", path = "./parquet", default-features = false }
+arrow = { version = "56.0.0", path = "./arrow", default-features = false }
+arrow-arith = { version = "56.0.0", path = "./arrow-arith" }
+arrow-array = { version = "56.0.0", path = "./arrow-array" }
+arrow-buffer = { version = "56.0.0", path = "./arrow-buffer" }
+arrow-cast = { version = "56.0.0", path = "./arrow-cast" }
+arrow-csv = { version = "56.0.0", path = "./arrow-csv" }
+arrow-data = { version = "56.0.0", path = "./arrow-data" }
+arrow-ipc = { version = "56.0.0", path = "./arrow-ipc" }
+arrow-json = { version = "56.0.0", path = "./arrow-json" }
+arrow-ord = { version = "56.0.0", path = "./arrow-ord" }
+arrow-pyarrow = { version = "56.0.0", path = "./arrow-pyarrow" }
+arrow-row = { version = "56.0.0", path = "./arrow-row" }
+arrow-schema = { version = "56.0.0", path = "./arrow-schema" }
+arrow-select = { version = "56.0.0", path = "./arrow-select" }
+arrow-string = { version = "56.0.0", path = "./arrow-string" }
+parquet = { version = "56.0.0", path = "./parquet", default-features = false }
 
 # These crates have not yet been released and thus do not use the workspace version
 parquet-variant = { version = "0.1.0", path = "./parquet-variant" }

README.md

@@ -108,7 +108,7 @@ The deprecated version is the next version which will be released (please
 consult the list above). To mark the API as deprecated, use the
 `#[deprecated(since = "...", note = "...")]` attribute.
 
-Foe example
+For example
 
 ```rust
 #[deprecated(since = "51.0.0", note = "Use `date_part` instead")]

arrow-array/src/builder/mod.rs

@@ -447,6 +447,7 @@ pub fn make_builder(datatype: &DataType, capacity: usize) -> Box<dyn ArrayBuilde
         DataType::Float64 => Box::new(Float64Builder::with_capacity(capacity)),
         DataType::Binary => Box::new(BinaryBuilder::with_capacity(capacity, 1024)),
         DataType::LargeBinary => Box::new(LargeBinaryBuilder::with_capacity(capacity, 1024)),
+        DataType::BinaryView => Box::new(BinaryViewBuilder::with_capacity(capacity)),
         DataType::FixedSizeBinary(len) => {
             Box::new(FixedSizeBinaryBuilder::with_capacity(capacity, *len))
         }
@@ -464,6 +465,7 @@ pub fn make_builder(datatype: &DataType, capacity: usize) -> Box<dyn ArrayBuilde
         ),
         DataType::Utf8 => Box::new(StringBuilder::with_capacity(capacity, 1024)),
         DataType::LargeUtf8 => Box::new(LargeStringBuilder::with_capacity(capacity, 1024)),
+        DataType::Utf8View => Box::new(StringViewBuilder::with_capacity(capacity)),
         DataType::Date32 => Box::new(Date32Builder::with_capacity(capacity)),
         DataType::Date64 => Box::new(Date64Builder::with_capacity(capacity)),
         DataType::Time32(TimeUnit::Second) => {

arrow-array/src/ffi.rs

@@ -408,7 +408,17 @@ impl ImportedArrowArray<'_> {
             .map(|index| {
                 let len = self.buffer_len(index, variadic_buffer_lens, &self.data_type)?;
                 match unsafe { create_buffer(self.owner.clone(), self.array, index, len) } {
-                    Some(buf) => Ok(buf),
+                    Some(buf) => {
+                        // External libraries may use a dangling pointer for a buffer with length 0.
+                        // We respect the array length specified in the C Data Interface. Actually,
+                        // if the length is incorrect, we cannot create a correct buffer even if
+                        // the pointer is valid.
+                        if buf.is_empty() {
+                            Ok(MutableBuffer::new(0).into())
+                        } else {
+                            Ok(buf)
+                        }
+                    }
                     None if len == 0 => {
                         // Null data buffer, which Rust doesn't allow. So create
                         // an empty buffer.
@@ -515,7 +525,7 @@ impl ImportedArrowArray<'_> {
         unsafe { create_buffer(self.owner.clone(), self.array, 0, buffer_len) }
     }
 
-    fn dictionary(&self) -> Result<Option<ImportedArrowArray>> {
+    fn dictionary(&self) -> Result<Option<ImportedArrowArray<'_>>> {
         match (self.array.dictionary(), &self.data_type) {
             (Some(array), DataType::Dictionary(_, value_type)) => Ok(Some(ImportedArrowArray {
                 array,
@@ -1296,9 +1306,15 @@ mod tests_to_then_from_ffi {
 
 #[cfg(test)]
 mod tests_from_ffi {
+    #[cfg(not(feature = "force_validate"))]
+    use std::ptr::NonNull;
     use std::sync::Arc;
 
+    #[cfg(feature = "force_validate")]
     use arrow_buffer::{bit_util, buffer::Buffer};
+    #[cfg(not(feature = "force_validate"))]
+    use arrow_buffer::{bit_util, buffer::Buffer, ScalarBuffer};
+
     use arrow_data::transform::MutableArrayData;
     use arrow_data::ArrayData;
     use arrow_schema::{DataType, Field};
@@ -1660,6 +1676,25 @@ mod tests_from_ffi {
         }
     }
 
+    #[test]
+    #[cfg(not(feature = "force_validate"))]
+    fn test_utf8_view_ffi_from_dangling_pointer() {
+        let empty = GenericByteViewBuilder::<StringViewType>::new().finish();
+        let buffers = empty.data_buffers().to_vec();
+        let nulls = empty.nulls().cloned();
+
+        // Create a dangling pointer to a view buffer with zero length.
+        let alloc = Arc::new(1);
+        let buffer = unsafe { Buffer::from_custom_allocation(NonNull::<u8>::dangling(), 0, alloc) };
+        let views = unsafe { ScalarBuffer::new_unchecked(buffer) };
+
+        let str_view: GenericByteViewArray<StringViewType> =
+            unsafe { GenericByteViewArray::new_unchecked(views, buffers, nulls) };
+        let imported = roundtrip_byte_view_array(str_view);
+        assert_eq!(imported.len(), 0);
+        assert_eq!(&imported, &empty);
+    }
+
     #[test]
     fn test_round_trip_byte_view() {
         fn test_case<T>()

arrow-avro/Cargo.toml

@@ -55,21 +55,32 @@ zstd = { version = "0.13", default-features = false, optional = true }
 bzip2 = { version = "0.6.0", optional = true }
 xz = { version = "0.1", default-features = false, optional = true }
 crc = { version = "3.0", optional = true }
+strum_macros = "0.27"
 uuid = "1.17"
+indexmap = "2.10"
+
 
 [dev-dependencies]
+arrow-data = { workspace = true }
 rand = { version = "0.9.1", default-features = false, features = [
     "std",
     "std_rng",
     "thread_rng",
 ] }
-criterion = { version = "0.6.0", default-features = false }
+criterion = { version = "0.7.0", default-features = false }
 tempfile = "3.3"
 arrow = { workspace = true }
 futures = "0.3.31"
 bytes = "1.10.1"
 async-stream = "0.3.6"
+apache-avro = "0.14.0"
+num-bigint = "0.4"
+once_cell = "1.21.3"
 
 [[bench]]
 name = "avro_reader"
 harness = false
+
+[[bench]]
+name = "decoder"
+harness = false