Merge branch 'dev' of https://github.com/yayen-lin/sirius into sirius-array-type

Author: yayen-lin

.claude/skills/log-analyzer/SKILL.md

@@ -198,4 +198,33 @@ After you've localized the issue, recommend the right next step:
 | Re-materialized inputs after a downgrade (per-task view) | `<folder>/memory_history.csv -> peak_bytes_to_materialize_input > 0` |
 | Per-pipeline summary across one query | filter `_pipeline_aggregates.csv` by `query_begin_ts` |
 | How well work was balanced across GPUs (multi-GPU runs) | group `task_outputs.csv` by `gpu_id` (per pipeline or per operator type); see `references/single_query.md` § "GPU balance" |
+| Host (pinned) pool memory at query start/end | `<folder>/query_meta.json -> pool_stats.host` |
+| GPU device memory pool at query start/end (per GPU) | `<folder>/query_meta.json -> pool_stats.gpu[]` |
 | Why my parser run looks weird | `_summary.json -> format_warnings`, then `tools/log_analyzer/README.md` |
+
+## Pool stats: detecting memory leaks across queries
+
+`query_meta.json` contains a `pool_stats` section that captures memory state at the query boundary:
+
+```json
+{
+  "pool_stats": {
+    "host": {
+      "begin": {"allocated_bytes": 5242880, "peak_bytes": 307232768, "free_blocks": 5115},
+      "end":   {"allocated_bytes": 5242880, "peak_bytes": 307232768, "free_blocks": 5115}
+    },
+    "gpu": [
+      {
+        "device_id": 0,
+        "begin": {"allocated_bytes": 0, "peak_bytes": 104857600, "reserved_bytes": 107374182400},
+        "end":   {"allocated_bytes": 0, "peak_bytes": 104857600, "reserved_bytes": 107374182400}
+      }
+    ]
+  }
+}
+```
+
+- **Host pool** (`[query_pool]` log tag): uses a `fixed_size_host_memory_resource` slab allocator. `free_blocks` drops during the query and should return to the QueryBegin value at QueryEnd. A persistent `end.allocated_bytes > begin.allocated_bytes` is a pinned-host memory leak.
+- **GPU pool** (`[gpu_pool]` log tag, one line per GPU): uses a `reservation_aware_resource_adaptor`. `allocated_bytes` reflects live GPU allocations across all streams; `reserved_bytes` reflects the capacity claimed by active reservations. A persistent `end.allocated_bytes > begin.allocated_bytes` is a GPU device memory leak.
+
+The two pool tags are distinguishable in the raw log: host stats use `[query_pool]`, GPU stats use `[gpu_pool]`.

.github/workflows/check.yml

@@ -146,6 +146,24 @@ jobs:
         env:
           CUDAARCHS: "100"
 
+      # Rust bindings (sirius-sys + sirius) — fmt/clippy + build/link the test.
+      # Run on a single representative entry. `cargo test --no-run` compiles the
+      # cxx glue against the Sirius FFI header and links the proof-of-life test
+      # against the single Sirius library built above (build.rs discovers it under
+      # build/release). It is NOT run here: constructing a context does GPU
+      # bring-up, and these are CPU runners — the actual run belongs on a GPU
+      # runner (test workflow).
+      - name: Rust bindings checks
+        if: >-
+          matrix.build.arch == 'x64' && matrix.build.compiler == 'gcc'
+          && matrix.build.build-type == 'release' && matrix.build.environment == 'default'
+          && matrix.build.cudf-channel == 'stable' && matrix.build.variant == 'super'
+        run: |
+          set -euo pipefail
+          pixi run cargo fmt --manifest-path rust/Cargo.toml --package sirius --package sirius-sys --check
+          pixi run cargo clippy --manifest-path rust/Cargo.toml --package sirius --package sirius-sys --all-targets -- -D warnings
+          pixi run cargo test --no-run --manifest-path rust/Cargo.toml --package sirius --package sirius-sys
+
       - name: Build (vcpkg)
         if: matrix.build.environment == 'vcpkg'
         run: |

.github/workflows/experimental.yml

@@ -0,0 +1,70 @@
+# NOTICE - Actions permissions policy (Settings > Actions > General):
+#   - "Allow sirius-db, and select non-sirius-db, actions and reusable workflows"
+#     - Allow actions created by GitHub: YES (covers actions/*)
+#     - Allow actions by Marketplace verified creators: NO
+#     - Explicitly allowed third-party actions:
+#       prefix-dev/setup-pixi@*
+#       mozilla-actions/sccache-action@*
+#     - Require actions to be pinned to a full-length commit SHA: NO
+#
+# Adding a new action? You must:
+#   1. Pin it to a full-length commit SHA (e.g. uses: owner/repo@<40-char-sha> # vX.Y.Z)
+#   2. If it is not from sirius-db, GitHub, or the explicitly allowed third-party actions above, add it to the
+#      project-level allowlist above before merging
+name: Experimental
+
+on:
+  pull_request:
+    paths:
+      - 'experimental/**'
+      - '.github/workflows/experimental.yml'
+  merge_group:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  starrocks:
+    name: starrocks
+    runs-on: ubuntu-24.04
+    defaults:
+      run:
+        working-directory: experimental/starrocks
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          submodules: false
+
+      - name: Init starrocks submodule
+        working-directory: ${{ github.workspace }}
+        run: git submodule update --init --depth=1 experimental/starrocks/starrocks
+
+      - uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6
+        with:
+          cache: false
+          pixi-version: v0.70.2
+          manifest-path: experimental/starrocks/pixi.toml
+          environments: cn
+
+      - name: fmt
+        run: pixi run -e cn cargo fmt --all -- --check
+
+      - name: clippy
+        run: pixi run -e cn cargo clippy --all-targets -- -D warnings
+
+      - name: test
+        run: pixi run -e cn cargo test --workspace
+
+  experimental:
+    name: experimental
+    runs-on: ubuntu-24.04
+    needs: [starrocks]
+    if: always()
+    steps:
+      - name: All checks ok
+        if: ${{ needs.starrocks.result == 'success' }}
+        run: exit 0
+      - name: Some checks failed
+        if: ${{ needs.starrocks.result != 'success' }}
+        run: exit 1

.gitignore

@@ -48,7 +48,8 @@ log/
 
 .planning/
 
-# Generated S3 integration-test fixtures (regenerated by test/cpp/integration/s3/fixtures.sh)
+# Generated S3 integration-test fixtures (regenerated by generate_fixtures.py;
+# at test time the harness writes them under a temp dir instead)
 /test/cpp/integration/s3/fixtures/local/
 /test/cpp/integration/s3/fixtures/generated/
 /test/cpp/integration/s3/certs/

.pre-commit-config.yaml

@@ -13,7 +13,9 @@
 # See https://github.com/pre-commit/pre-commit
 
 # Global exclusions, csv are excluded because they are often test answers, and experimental is out of lint scope.
-exclude: \.csv$|^legacy/queries/.*|^experimental/.*
+# .patch files are verbatim diffs: formatters (trailing-whitespace, end-of-file-fixer)
+# strip the leading context space on blank lines and corrupt `git apply`.
+exclude: \.csv$|\.patch$|^legacy/queries/.*|^experimental/.*
 
 repos:
   # Standard hooks

CMakeLists.txt

@@ -28,6 +28,14 @@ endif()
 option(ENABLE_STREAM_CHECK
        "Enable stream check library for debugging default stream usage" OFF)
 
+# The [s3] integration tests launch MinIO via a patched build of upstream
+# testcontainers-native, fetched + patched at configure time and built as a Go
+# c-archive — so it needs a Go toolchain and network at configure/build time. ON
+# by default (the pixi env provides Go); turn OFF for offline / Go-less /
+# release / package builds, which then skip the S3 container harness entirely.
+option(SIRIUS_BUILD_S3_TESTS
+       "Build the testcontainers-backed S3 integration test harness" ON)
+
 # In the vcpkg build, vcpkg headers must take priority over: 1) conda's CCCL
 # 3.1.4 (injected via CXXFLAGS as -I.../include/cccl) 2) DuckDB's bundled fmt
 # 6.1.2 (uses duckdb_fmt namespace, incompatible with spdlog) Fix: strip conda
@@ -139,6 +147,15 @@ endif()
 # Rust telemetry instrumentation (C++ FFI via Corrosion)
 add_subdirectory(rust/crates/telemetry/bridge)
 
+# Upstream testcontainers-native, fetched + patched at configure time (see
+# cmake/testcontainers_native.cmake), used by the S3 integration test harness to
+# start MinIO containers from the test binary. Builds a Go c-archive, so a Go
+# toolchain (provided by pixi) and network access on the first configure/build
+# are required — hence gated behind SIRIUS_BUILD_S3_TESTS.
+if(SIRIUS_BUILD_S3_TESTS)
+  include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/testcontainers_native.cmake")
+endif()
+
 # cmake-format: off
 set(EXTENSION_SOURCES
     src/config.cpp
@@ -150,8 +167,8 @@ set(EXTENSION_SOURCES
     src/data/host_parquet_representation_converters.cpp
     src/downgrade/downgrade_executor.cpp
     src/expression/aggregate_id.cpp
-    src/expression/ast/clone.cpp
     src/expression/ast/node.cpp
+    src/expression/ast/utils.cpp
     src/expression/from_duckdb.cpp
     src/expression/function_id.cpp
     src/expression/join_condition.cpp
@@ -196,20 +213,15 @@ set(EXTENSION_SOURCES
     src/op/partition/gpu_partition_impl.cpp
     src/op/result/host_table_chunk_reader.cpp
     src/op/scan/cached_ranges.cpp
-    src/op/scan/duckdb_scan_executor.cpp
-    src/op/scan/cpu_source_task.cpp
     src/op/scan/duckdb_native_metadata.cpp
     src/op/scan/duckdb_native_decoder.cpp
-    src/op/scan/duckdb_scan_task.cpp
     src/op/scan/equality_delete_filter.cpp
     src/op/scan/hive_partition.cpp
     src/op/scan/iceberg_avro_reader.cpp
     src/op/scan/iceberg_delete_pipeline.cpp
     src/op/scan/iceberg_metadata_reader.cpp
-    src/op/scan/iceberg_scan_task.cpp
     src/op/scan/positional_delete_filter.cpp
     src/op/scan/puffin_reader.cpp
-    src/op/scan/parquet_scan_task.cpp
     src/op/scan/parquet_schema_mapping.cpp
     src/op/scan/prefetched_data_source.cpp
     src/op/scan/scan_plan.cpp
@@ -275,17 +287,20 @@ set(EXTENSION_SOURCES
     src/planner/sirius_plan_limit.cpp
     src/planner/sirius_plan_order.cpp
     src/planner/sirius_plan_projection.cpp
+    src/planner/sirius_plan_projection_utils.cpp
     src/planner/sirius_plan_recursive_cte.cpp
     src/planner/sirius_plan_top_n.cpp
     src/pin_table.cpp
     src/scan_manager/gpu_ingestible_factory.cpp
+    src/scan_manager/round_robin_strategy.cpp
     src/scan_manager/sirius_scan_manager.cpp
     src/scan_manager/split_connector.cpp
     src/scan_manager/split_provider.cpp
     src/sirius_config.cpp
     src/sirius_context.cpp
     src/sirius_engine.cpp
     src/sirius_extension.cpp
+    src/sirius_ffi.cpp
     src/sirius_interface.cpp
     src/sirius_sql_rewrite.cpp
     src/telemetry/telemetry_context.cpp
@@ -372,6 +387,11 @@ target_link_libraries(
 target_link_libraries(sirius_loadable_extension PkgConfig::LIBURING
                       PkgConfig::CURL OpenSSL::Crypto)
 
+# The sirius-sys + sirius Rust crates are built by cargo, not CMake (unlike the
+# telemetry bridge above, which CMake drives via Corrosion). Their build.rs
+# discovers the Sirius headers (repo + conda) and links the libsirius artifact
+# this build produces under build/<preset>/; see rust/crates/sirius-sys.
+
 # cucascade upstream PRs #126/#128/#130 moved cucascade to static-CUDA linkage
 # and pulled the NVML *static stub* (libnvidia-ml.a) into libcucascade.a via the
 # `CUDA::nvml_static` imported target. Upstream's
@@ -445,6 +465,7 @@ set(TEST_SOURCES
     test/cpp/exec/test_interruptible_mpmc.cpp
     test/cpp/expression/test_ast_aggregate.cpp
     test/cpp/expression/test_ast_clone.cpp
+    test/cpp/expression/test_ast_substitute.cpp
     test/cpp/expression/test_ast_from_duckdb.cpp
     test/cpp/expression/test_ast_scaffold.cpp
     test/cpp/expression/test_ast_to_duckdb.cpp
@@ -461,11 +482,9 @@ set(TEST_SOURCES
     test/cpp/integration/test_gpu_execution_array.cpp
     test/cpp/integration/test_gpu_execution_locality.cpp
     test/cpp/integration/test_gpu_execution_multi_format.cpp
-    test/cpp/integration/test_gpu_execution_order_nulls.cpp
     test/cpp/integration/test_gpu_execution_tpch.cpp
     test/cpp/integration/test_gpu_execution_tpch_mgpu_audit.cpp
     test/cpp/integration/test_table_gpu_cache_warm_mgpu.cpp
-    test/cpp/integration/test_transparent_execution.cpp
     test/cpp/io/s3/test_sigv4.cpp
     test/cpp/io/s3/test_s3_default_visibility_guard.cpp
     test/cpp/io/s3/test_s3_blocking_ioctx.cpp
@@ -484,7 +503,6 @@ set(TEST_SOURCES
     test/cpp/operator/test_physical_hash_join_mgpu.cpp
     test/cpp/operator/test_physical_order_mgpu.cpp
     test/cpp/operator/test_mgpu_stress.cpp
-    test/cpp/memory/test_host_table_utils.cpp
     test/cpp/memory/test_multiple_blocks_allocation_accessor.cpp
     test/cpp/operator/aggregate/test_gpu_merge_impl.cpp
     test/cpp/operator/aggregate/test_physical_grouped_aggregate.cpp
@@ -509,31 +527,29 @@ set(TEST_SOURCES
     test/cpp/operator/test_sirius_dynamic_filter.cpp
     test/cpp/parallel/test_task_executor.cpp
     test/cpp/planner/test_distinct_hash_join_detection.cpp
+    test/cpp/planner/test_projection_fold.cpp
     test/cpp/pipeline/test_gpu_pipeline_executor.cpp
     test/cpp/pipeline/test_oom_reschedule.cpp
     test/cpp/pipeline/test_pipeline_memory_history.cpp
     test/cpp/pipeline/test_gpu_pipeline_task_history.cpp
     test/cpp/pipeline/test_gpu_pipeline_disk_readback.cpp
     test/cpp/pipeline/test_get_next_ports_after_sink.cpp
-    test/cpp/pipeline/test_plan_printer.cpp
     test/cpp/pipeline/test_repository_wiring_materializer.cpp
     test/cpp/pipeline/test_task_scheduler.cpp
     test/cpp/scan/bench_decode_codecs.cpp
-    test/cpp/scan/test_column_builder.cpp
     test/cpp/scan/test_duckdb_block_layout.cpp
+    test/cpp/scan/test_duckdb_native_batch_coalescer.cpp
+    test/cpp/scan/test_duckdb_native_gpu_ingestible.cpp
     test/cpp/scan/test_duckdb_native_walker.cpp
     test/cpp/scan/test_gpu_decode_alp.cpp
     test/cpp/scan/test_gpu_decode_bitpacking.cpp
     test/cpp/scan/test_gpu_decode_rle.cpp
     test/cpp/scan/test_gpu_decode_strings.cpp
     test/cpp/scan/test_gpu_native_decode.cpp
     test/cpp/scan/test_parquet_schema_mapping.cpp
-    test/cpp/scan/test_parquet_scan_task.cpp
     test/cpp/scan/test_parquet_gpu_ingestible.cpp
     test/cpp/scan/test_pinned_table_gpu_ingestible.cpp
-    test/cpp/scan/test_scan_executor.cpp
     test/cpp/scan/test_split_connector.cpp
-    test/cpp/scan_manager/test_sirius_scan_manager_s3.cpp
     test/cpp/scan_manager/test_describe_parquet_s3.cpp
     test/cpp/scan_manager/test_pin_table_multi_gpu.cpp
     test/cpp/sirius_extension/test_sirius_read_parquet_cardinality.cpp
@@ -547,6 +563,12 @@ if(SIRIUS_LEGACY_TEST_SOURCES)
   list(APPEND TEST_SOURCES ${SIRIUS_LEGACY_TEST_SOURCES})
 endif()
 
+# The testcontainers-backed S3 harness only compiles when its dependency is
+# built; unittest.cpp guards its use behind SIRIUS_HAVE_TESTCONTAINERS.
+if(SIRIUS_BUILD_S3_TESTS)
+  list(APPEND TEST_SOURCES test/cpp/utils/s3_container.cpp)
+endif()
+
 add_executable(sirius_unittest ${TEST_SOURCES})
 
 # TPC-DS integration test fixture, generated at build time (opt-in target). The
@@ -583,6 +605,16 @@ target_include_directories(
 
 target_link_libraries(sirius_unittest sirius_extension duckdb_static)
 link_extension_libraries(sirius_unittest "")
+
+# S3 container harness: the testcontainers-native bridge plus libcurl for
+# host-side fixture upload (SigV4 signing comes from sirius_extension). Gated so
+# offline/Go-less builds skip it; the harness calls in unittest.cpp are guarded
+# by SIRIUS_HAVE_TESTCONTAINERS.
+if(SIRIUS_BUILD_S3_TESTS)
+  target_link_libraries(sirius_unittest testcontainers_native PkgConfig::CURL)
+  target_compile_definitions(sirius_unittest
+                             PRIVATE SIRIUS_HAVE_TESTCONTAINERS=1)
+endif()
 # DuckDB v1.5.0 unity builds emit strong symbols for static constexpr members
 # that conflict with inline definitions from headers included in test files.
 target_link_options(sirius_unittest PRIVATE