Merge upstream/main into split/delta-dv-java-scan-handoff-pr-clean

Resolved conflicts in:
- gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaScanTransformer.scala
- gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaPostTransformRules.scala

Accepted upstream changes which include:
- New scanFilters override for Delta column-mapping support
- New doValidateInternal override for deletion vector validation
- Updated transformColumnMappingPlan with improved documentation and logic

Author: Mohammad Linjawi

.github/workflows/flink.yml

@@ -59,10 +59,18 @@ jobs:
         run: |
           source /opt/rh/gcc-toolset-11/enable
           sudo dnf install -y patchelf
-          sudo yum install https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/tzdata-2025a-1.el9.noarch.rpm -y
+          sudo yum install https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/tzdata-2026a-1.el9.noarch.rpm -y
           sudo .github/workflows/util/install-flink-deps.sh
+          # Remove system fmt v11 from the CI image to avoid header conflicts with
+          # velox bundled fmt 10.x during folly/velox native compilation.
+          sudo rm -rf /usr/local/include/fmt
+          sudo rm -rf /usr/local/lib/cmake/fmt
+          sudo rm -f /usr/local/lib/libfmt* /usr/local/lib64/libfmt*
+          export VELOX_DEPENDENCY_SOURCE=BUNDLED
+          export fmt_SOURCE=BUNDLED
+          export folly_SOURCE=BUNDLED
           git clone -b gluten-0530 https://github.com/bigo-sg/velox4j.git
-          cd velox4j && git reset --hard 889bafcf2fa04e8c31a30edbdf40fe203ef58484
+          cd velox4j && git reset --hard 115edf79d265a61c30d45dfcc6ce932ad92378ca
           git apply $GITHUB_WORKSPACE/gluten-flink/patches/fix-velox4j.patch
           $GITHUB_WORKSPACE/build/mvn clean install -DskipTests -Dgpg.skip -Dspotless.skip=true
           cd ..

.github/workflows/util/install-spark-resources.sh

@@ -118,7 +118,7 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
   4.0)
       # Spark-4.0, scala 2.12 // using 2.12 as a hack as 4.0 does not have 2.13 suffix
       cd ${INSTALL_DIR} && \
-      install_spark "4.0.1" "3" "2.12"
+      install_spark "4.0.2" "3" "2.12"
       mv /opt/shims/spark40/spark_home/assembly/target/scala-2.12 /opt/shims/spark40/spark_home/assembly/target/scala-2.13
       ;;
   4.1)

.github/workflows/velox_backend_ansi.yml

@@ -124,8 +124,6 @@ jobs:
             ccache -sz
             bash dev/ci-velox-buildstatic-centos-7.sh
             ccache -s
-            mkdir -p /work/.m2/repository/org/apache/arrow/
-            cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/
           "
       - name: Save ccache
         if: always()
@@ -138,11 +136,6 @@ jobs:
           name: velox-native-lib-ansi-${{github.sha}}
           path: ./cpp/build/
           if-no-files-found: error
-      - uses: actions/upload-artifact@v4
-        with:
-          name: arrow-jars-ansi-${{github.sha}}
-          path: .m2/repository/org/apache/arrow/
-          if-no-files-found: error
 
   spark-test-backends-velox-ansi41:
     needs: build-native-lib
@@ -159,11 +152,6 @@ jobs:
         with:
           name: velox-native-lib-ansi-${{github.sha}}
           path: ./cpp/build/
-      - name: Download Arrow Jars
-        uses: actions/download-artifact@v4
-        with:
-          name: arrow-jars-ansi-${{github.sha}}
-          path: /root/.m2/repository/org/apache/arrow/
       - name: Prepare
         run: |
           dnf install -y python3.11 python3.11-pip python3.11-devel && \
@@ -236,11 +224,6 @@ jobs:
         with:
           name: velox-native-lib-ansi-${{github.sha}}
           path: ./cpp/build/
-      - name: Download Arrow Jars
-        uses: actions/download-artifact@v4
-        with:
-          name: arrow-jars-ansi-${{github.sha}}
-          path: /root/.m2/repository/org/apache/arrow/
       - name: Prepare
         run: |
           dnf install -y python3.11 python3.11-pip python3.11-devel && \
@@ -323,11 +306,6 @@ jobs:
         with:
           name: velox-native-lib-ansi-${{github.sha}}
           path: ./cpp/build/
-      - name: Download Arrow Jars
-        uses: actions/download-artifact@v4
-        with:
-          name: arrow-jars-ansi-${{github.sha}}
-          path: /root/.m2/repository/org/apache/arrow/
       - name: Prepare
         run: |
           dnf install -y python3.11 python3.11-pip python3.11-devel && \
@@ -400,11 +378,6 @@ jobs:
         with:
           name: velox-native-lib-ansi-${{github.sha}}
           path: ./cpp/build/
-      - name: Download Arrow Jars
-        uses: actions/download-artifact@v4
-        with:
-          name: arrow-jars-ansi-${{github.sha}}
-          path: /root/.m2/repository/org/apache/arrow/
       - name: Prepare
         run: |
           dnf install -y python3.11 python3.11-pip python3.11-devel && \

.github/workflows/velox_backend_arm.yml

@@ -68,8 +68,6 @@ jobs:
           df -a
           bash dev/ci-velox-buildstatic-centos-9.sh
           ccache -s
-          mkdir -p $GITHUB_WORKSPACE/.m2/repository/org/apache/arrow/
-          cp -r /root/.m2/repository/org/apache/arrow/* $GITHUB_WORKSPACE/.m2/repository/org/apache/arrow/
       - name: "Save ccache"
         uses: actions/cache/save@v4
         id: ccache
@@ -81,11 +79,6 @@ jobs:
           name: velox-native-lib-centos-8-${{github.sha}}
           path: ./cpp/build/releases/
           if-no-files-found: error
-      - uses: actions/upload-artifact@v4
-        with:
-          name: arrow-jars-centos-8-${{github.sha}}
-          path: .m2/repository/org/apache/arrow/
-          if-no-files-found: error
 
   tpc-test-centos8:
     needs: build-native-lib-centos-8
@@ -104,11 +97,6 @@ jobs:
         with:
           name: velox-native-lib-centos-8-${{github.sha}}
           path: ./cpp/build/releases/
-      - name: Download All Arrow Jar Artifacts
-        uses: actions/download-artifact@v4
-        with:
-          name: arrow-jars-centos-8-${{github.sha}}
-          path: /root/.m2/repository/org/apache/arrow/
       - name: Update mirror list
         run: |
           sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
@@ -150,7 +138,6 @@ jobs:
       - name: Build Gluten native libraries
         run: |
           df -a
-          sed -i "s|gflags_static|gflags_shared|g" /usr/local/lib/cmake/folly/folly-targets.cmake # TODO: remove after upgrading folly to 2024.09.30 or later which has fixed the gflags linkage issue
           bash dev/ci-velox-buildshared-centos-9.sh
           ccache -s
       - name: Run CPP unit test

.github/workflows/velox_backend_enhanced.yml

@@ -75,8 +75,6 @@ jobs:
             mkdir -p /work/.ccache
             bash dev/ci-velox-buildstatic-centos-7-enhanced-features.sh
             ccache -s
-            mkdir -p /work/.m2/repository/org/apache/arrow/
-            cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/
           "
 
       - name: "Save ccache"
@@ -90,11 +88,6 @@ jobs:
           name: velox-native-lib-enhanced-centos-7-${{github.sha}}
           path: ./cpp/build/releases/
           if-no-files-found: error
-      - uses: actions/upload-artifact@v4
-        with:
-          name: arrow-jars-enhanced-centos-7-${{github.sha}}
-          path: .m2/repository/org/apache/arrow/
-          if-no-files-found: error
 
   spark-test-spark34:
     needs: build-native-lib-centos-7
@@ -107,11 +100,6 @@ jobs:
         with:
           name: velox-native-lib-enhanced-centos-7-${{github.sha}}
           path: ./cpp/build/releases
-      - name: Download Arrow Jars
-        uses: actions/download-artifact@v4
-        with:
-          name: arrow-jars-enhanced-centos-7-${{github.sha}}
-          path: /root/.m2/repository/org/apache/arrow/
       - name: Prepare spark.test.home for Spark 3.4.4 (other tests)
         run: |
           dnf module -y install python39 && \
@@ -165,11 +153,6 @@ jobs:
         with:
           name: velox-native-lib-enhanced-centos-7-${{github.sha}}
           path: ./cpp/build/releases
-      - name: Download Arrow Jars
-        uses: actions/download-artifact@v4
-        with:
-          name: arrow-jars-enhanced-centos-7-${{github.sha}}
-          path: /root/.m2/repository/org/apache/arrow/
       - name: Prepare spark.test.home for Spark 3.5.5 (other tests)
         run: |
           dnf module -y install python39 && \
@@ -226,11 +209,6 @@ jobs:
         with:
           name: velox-native-lib-enhanced-centos-7-${{github.sha}}
           path: ./cpp/build/releases
-      - name: Download Arrow Jars
-        uses: actions/download-artifact@v4
-        with:
-          name: arrow-jars-enhanced-centos-7-${{github.sha}}
-          path: /root/.m2/repository/org/apache/arrow/
       - name: Prepare Spark Resources for Spark 3.5.5 #TODO remove after image update
         run: |
           rm -rf /opt/shims/spark35
@@ -277,11 +255,6 @@ jobs:
         with:
           name: velox-native-lib-enhanced-centos-7-${{github.sha}}
           path: ./cpp/build/releases/
-      - name: Download Arrow Jars
-        uses: actions/download-artifact@v4
-        with:
-          name: arrow-jars-enhanced-centos-7-${{github.sha}}
-          path: /root/.m2/repository/org/apache/arrow/
       - name: Prepare
         run: |
           dnf module -y install python39 && \