Merge pull request #306 from benfleis/issue-303--partitioned-scan-drops

issue 303 - bump kernel version to fix partitioned scans (with a checkpoint) missing data

Partition equality checks over checkpointed data broke with the kernel v0.20.0 update

    add breaking test (breaks with v0.20.0, passes with v0.21.0)
    update kernel and API usage, test passes

Addresses #303 #304 (appears to dup even if descriptions vary slightly)

Author: benfleis

CMakeLists.txt

@@ -145,7 +145,7 @@ set(CARGO_PROFILE "$<IF:$<CONFIG:Debug>,dev,release>")
 ExternalProject_Add(
   ${KERNEL_NAME}
   GIT_REPOSITORY "https://github.com/delta-io/delta-kernel-rs"
-  GIT_TAG v0.20.0
+  GIT_TAG v0.21.0
   # Prints the env variables passed to the cargo build to the terminal, useful
   # in debugging because passing them through CMake is an error-prone mess
   CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${RUST_UNSET_ENV_VARS}
@@ -155,8 +155,15 @@ ExternalProject_Add(
   # Build debug build
   BUILD_COMMAND
     ${CMAKE_COMMAND} -E env ${RUST_UNSET_ENV_VARS} ${RUST_ENV_VARS} cargo build
-      --package delta_kernel_ffi --workspace --profile=${CARGO_PROFILE} --all-features
+      --package delta_kernel_ffi --profile=${CARGO_PROFILE}
+      --features "default-engine-rustls,arrow,test-ffi,delta-kernel-unity-catalog,tracing"
       ${RUST_PLATFORM_PARAM}
+  # acceptance depends on delta_kernel_benchmarks (regular dep, not dev), whose build.rs
+  # downloads benchmark workload data we don't need. Pre-create the .done marker to skip it.
+  COMMAND ${CMAKE_COMMAND} -E make_directory
+    "${CMAKE_BINARY_DIR}/rust/src/delta_kernel/benchmarks/workloads"
+  COMMAND ${CMAKE_COMMAND} -E touch
+    "${CMAKE_BINARY_DIR}/rust/src/delta_kernel/benchmarks/workloads/.done"
   # Build DATs
   COMMAND
     ${CMAKE_COMMAND} -E env ${RUST_UNSET_ENV_VARS} ${RUST_ENV_VARS} cargo build

Makefile

@@ -4,6 +4,8 @@ PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 EXT_NAME=deltatable
 EXT_CONFIG=${PROJ_DIR}extension_config.cmake
 
+PIP=python3 -m pip
+
 ifeq ($(SANITIZER_MODE), thread)
 	EXT_DEBUG_FLAGS:=-DENABLE_THREAD_SANITIZER=1
 endif
@@ -47,7 +49,7 @@ include benchmark/benchmark.Makefile
 #   export JAVA_HOME=/opt/homebrew/Cellar/openjdk@11/11.0.27/libexec/openjdk.jdk/Contents/Home
 generate-data:
 	# NOTE: @benfleis - for now pin versions that work, since unversioned/HEAD caused a big JVM stack trace that I couldn't trivially track down;
-	python3 -m pip install delta-spark==4.0.0 deltalake==1.2.1 duckdb==1.4.4 pandas==2.3.3 pyarrow==22.0.0 pyspark==4.0.1 typing-extensions==4.15.0
+	${PIP} install delta-spark==4.0.0 deltalake==1.2.1 duckdb==1.4.4 pandas==2.3.3 pyarrow==22.0.0 pyspark==4.0.1 typing-extensions==4.15.0
 	python3 scripts/data_generator/generate_test_data.py
 	# avoid footguns -- make outputs read only
 	find data/generated -mindepth 1 -print0 | xargs -0 -n 1000 chmod a-w

data/inlined/issue_303_partitioned/delta_lake/_delta_log/00000000000000000000.json

@@ -0,0 +1,4 @@
+{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}
+{"metaData":{"id":"95cf8a2e-12e7-45f1-b500-36475b7992f3","name":null,"description":null,"format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"i\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"part\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["part"],"createdTime":1776087838136,"configuration":{}}}
+{"add":{"path":"part=42/part-00000-11caa99d-eb7e-4e4d-8115-163749c9c310-c000.snappy.parquet","partitionValues":{"part":"42"},"size":512,"modificationTime":1776087838199,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"i\":0},\"maxValues\":{\"i\":4},\"nullCount\":{\"i\":0}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}}
+{"commitInfo":{"timestamp":1776087838201,"operation":"WRITE","operationParameters":{"partitionBy":"[\"part\"]","mode":"ErrorIfExists"},"engineInfo":"delta-rs:py-1.2.1","operationMetrics":{"execution_time_ms":68,"num_added_files":1,"num_added_rows":5,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.2.1"}}

data/inlined/issue_303_partitioned/delta_lake/_delta_log/00000000000000000001.json

@@ -0,0 +1,2 @@
+{"add":{"path":"part=42/part-00000-b3e88156-0bdb-4686-8bcf-a4417eff97c1-c000.snappy.parquet","partitionValues":{"part":"42"},"size":484,"modificationTime":1776088842353,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"i\":0},\"maxValues\":{\"i\":0},\"nullCount\":{\"i\":0}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}}
+{"commitInfo":{"timestamp":1776088842355,"operation":"WRITE","operationParameters":{"partitionBy":"[\"part\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.2.1","clientVersion":"delta-rs.py-1.2.1","operationMetrics":{"execution_time_ms":72,"num_added_files":1,"num_added_rows":1,"num_partitions":0,"num_removed_files":0}}}

data/inlined/issue_303_partitioned/delta_lake/_delta_log/00000000000000000002.json

@@ -0,0 +1,2 @@
+{"add":{"path":"part=42/part-00000-1285e5ec-5dcd-4d75-a622-f0a154872ff8-c000.snappy.parquet","partitionValues":{"part":"42"},"size":484,"modificationTime":1776088842362,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"i\":1},\"maxValues\":{\"i\":1},\"nullCount\":{\"i\":0}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}}
+{"commitInfo":{"timestamp":1776088842362,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part\"]"},"engineInfo":"delta-rs:py-1.2.1","operationMetrics":{"execution_time_ms":0,"num_added_files":1,"num_added_rows":1,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.2.1"}}

data/inlined/issue_303_partitioned/delta_lake/_delta_log/00000000000000000003.json

@@ -0,0 +1,2 @@
+{"add":{"path":"part=42/part-00000-abcc7a43-2e0b-4c27-a603-0fec0979d55b-c000.snappy.parquet","partitionValues":{"part":"42"},"size":484,"modificationTime":1776088842365,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"i\":2},\"maxValues\":{\"i\":2},\"nullCount\":{\"i\":0}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}}
+{"commitInfo":{"timestamp":1776088842365,"operation":"WRITE","operationParameters":{"partitionBy":"[\"part\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.2.1","clientVersion":"delta-rs.py-1.2.1","operationMetrics":{"execution_time_ms":0,"num_added_files":1,"num_added_rows":1,"num_partitions":0,"num_removed_files":0}}}

data/inlined/issue_303_partitioned/delta_lake/_delta_log/00000000000000000004.json

@@ -0,0 +1,2 @@
+{"add":{"path":"part=42/part-00000-fae64a23-6a7c-44a8-97f2-5a5a508d1272-c000.snappy.parquet","partitionValues":{"part":"42"},"size":484,"modificationTime":1776088842367,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"i\":3},\"maxValues\":{\"i\":3},\"nullCount\":{\"i\":0}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}}
+{"commitInfo":{"timestamp":1776088842367,"operation":"WRITE","operationParameters":{"partitionBy":"[\"part\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.2.1","operationMetrics":{"execution_time_ms":0,"num_added_files":1,"num_added_rows":1,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.2.1"}}

data/inlined/issue_303_partitioned/delta_lake/_delta_log/00000000000000000005.json

@@ -0,0 +1,2 @@
+{"add":{"path":"part=42/part-00000-776f3463-38c1-400b-93a8-f8d22fb86289-c000.snappy.parquet","partitionValues":{"part":"42"},"size":484,"modificationTime":1776088842369,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"i\":4},\"maxValues\":{\"i\":4},\"nullCount\":{\"i\":0}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}}
+{"commitInfo":{"timestamp":1776088842369,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part\"]"},"engineInfo":"delta-rs:py-1.2.1","operationMetrics":{"execution_time_ms":0,"num_added_files":1,"num_added_rows":1,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.2.1"}}

data/inlined/issue_303_partitioned/delta_lake/_delta_log/00000000000000000006.json

@@ -0,0 +1,2 @@
+{"add":{"path":"part=42/part-00000-d4bdd985-affe-4a52-bcf4-a84b66e482cc-c000.snappy.parquet","partitionValues":{"part":"42"},"size":484,"modificationTime":1776088842372,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"i\":5},\"maxValues\":{\"i\":5},\"nullCount\":{\"i\":0}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}}
+{"commitInfo":{"timestamp":1776088842372,"operation":"WRITE","operationParameters":{"partitionBy":"[\"part\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.2.1","operationMetrics":{"execution_time_ms":0,"num_added_files":1,"num_added_rows":1,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.2.1"}}

data/inlined/issue_303_partitioned/delta_lake/_delta_log/00000000000000000007.json

@@ -0,0 +1,2 @@
+{"add":{"path":"part=42/part-00000-87daa17e-840e-4408-9347-9e29fbeb8e32-c000.snappy.parquet","partitionValues":{"part":"42"},"size":484,"modificationTime":1776088842374,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"i\":6},\"maxValues\":{\"i\":6},\"nullCount\":{\"i\":0}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}}
+{"commitInfo":{"timestamp":1776088842374,"operation":"WRITE","operationParameters":{"partitionBy":"[\"part\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.2.1","clientVersion":"delta-rs.py-1.2.1","operationMetrics":{"execution_time_ms":0,"num_added_files":1,"num_added_rows":1,"num_partitions":0,"num_removed_files":0}}}