ci: ANSI mode baseline workflow for Spark 4.1 #4
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one or more | |
| # contributor license agreements. See the NOTICE file distributed with | |
| # this work for additional information regarding copyright ownership. | |
| # The ASF licenses this file to You under the Apache License, Version 2.0 | |
| # (the "License"); you may not use this file except in compliance with | |
| # the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: Velox Backend ANSI Mode | |
| on: | |
| pull_request: | |
| paths: | |
| - '.github/workflows/velox_backend_ansi.yml' | |
| - '.github/workflows/util/install-resources.sh' | |
| - 'pom.xml' | |
| - 'backends-velox/**' | |
| - 'gluten-core/**' | |
| - 'gluten-substrait/**' | |
| - 'gluten-arrow/**' | |
| - 'gluten-ut/**' | |
| - 'shims/**' | |
| - 'ep/build-velox/**' | |
| - 'cpp/**' | |
| - 'dev/**' | |
| - 'build/mvn' | |
| env: | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| MVN_CMD: 'build/mvn -ntp' | |
| WGET_CMD: 'wget -nv' | |
| CCACHE_DIR: "${{ github.workspace }}/.ccache" | |
| SPARK_ANSI_SQL_MODE: true | |
| concurrency: | |
| group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} | |
| cancel-in-progress: true | |
| jobs: | |
| build-native-lib: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get Ccache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: '${{ env.CCACHE_DIR }}' | |
| key: ccache-centos7-release-default-${{github.sha}} | |
| restore-keys: | | |
| ccache-centos7-release-default | |
| - name: Build Gluten native libraries | |
| run: | | |
| docker pull apache/gluten:vcpkg-centos-7 | |
| docker run -v $GITHUB_WORKSPACE:/work -w /work apache/gluten:vcpkg-centos-7 bash -c " | |
| set -e | |
| yum install tzdata -y | |
| cd /work | |
| export CCACHE_DIR=/work/.ccache | |
| export CCACHE_MAXSIZE=1G | |
| mkdir -p /work/.ccache | |
| ccache -sz | |
| bash dev/ci-velox-buildstatic-centos-7.sh | |
| ccache -s | |
| mkdir -p /work/.m2/repository/org/apache/arrow/ | |
| cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/ | |
| " | |
| - name: Save ccache | |
| if: always() | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: '${{ env.CCACHE_DIR }}' | |
| key: ccache-centos7-release-default-${{github.sha}} | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: velox-native-lib-ansi-${{github.sha}} | |
| path: ./cpp/build/ | |
| if-no-files-found: error | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: arrow-jars-ansi-${{github.sha}} | |
| path: .m2/repository/org/apache/arrow/ | |
| if-no-files-found: error | |
| spark-test-backends-velox-ansi: | |
| needs: build-native-lib | |
| runs-on: ubuntu-22.04 | |
| env: | |
| SPARK_TESTING: true | |
| container: apache/gluten:centos-9-jdk17 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download Native Lib | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: velox-native-lib-ansi-${{github.sha}} | |
| path: ./cpp/build/ | |
| - name: Download Arrow Jars | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: arrow-jars-ansi-${{github.sha}} | |
| path: /root/.m2/repository/org/apache/arrow/ | |
| - name: Prepare | |
| run: | | |
| dnf install -y python3.11 python3.11-pip python3.11-devel && \ | |
| alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \ | |
| alternatives --set python3 /usr/bin/python3.11 && \ | |
| pip3 install setuptools==77.0.3 && \ | |
| pip3 install pyspark==3.5.5 cython && \ | |
| pip3 install pandas==2.2.3 pyarrow==20.0.0 | |
| - name: Prepare Spark Resources | |
| run: | | |
| rm -rf /opt/shims/spark41 | |
| bash .github/workflows/util/install-resources.sh 4.1 | |
| mv /opt/shims/spark41/spark_home/assembly/target/scala-2.12 /opt/shims/spark41/spark_home/assembly/target/scala-2.13 | |
| - name: "Spark 4.1 backends-velox Tests (ANSI ON)" | |
| run: | | |
| set -o pipefail | |
| cd $GITHUB_WORKSPACE/ | |
| export SPARK_SCALA_VERSION=2.13 | |
| yum install -y java-17-openjdk-devel | |
| export JAVA_HOME=/usr/lib/jvm/java-17-openjdk | |
| export PATH=$JAVA_HOME/bin:$PATH | |
| echo "SPARK_ANSI_SQL_MODE=$SPARK_ANSI_SQL_MODE" | |
| java -version | |
| $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox \ | |
| -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/ -Dspark.gluten.sql.ansiFallback.enabled=false" \ | |
| -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \ | |
| 2>&1 | tee backends-velox-test-output.log | |
| - name: "Parse test results" | |
| if: always() | |
| run: | | |
| echo "=========================================" | |
| echo " backends-velox (Spark 4.1, ANSI=$SPARK_ANSI_SQL_MODE)" | |
| echo "=========================================" | |
| echo "" | |
| echo "--- Test Summary ---" | |
| grep -E "Tests run:.*Failures:|BUILD " backends-velox-test-output.log || echo "(no summary found)" | |
| echo "" | |
| echo "--- Failed Tests ---" | |
| grep -B1 "<<< FAIL!" backends-velox-test-output.log || echo "(no failures)" | |
| echo "" | |
| echo "--- Error Tests ---" | |
| grep -B1 "<<< ERROR!" backends-velox-test-output.log || echo "(no errors)" | |
| - name: Upload test report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: spark-test-backends-velox-ansi-report | |
| path: | | |
| **/surefire-reports/TEST-*.xml | |
| backends-velox-test-output.log | |
| - name: Upload log files | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: spark-test-backends-velox-ansi-logs | |
| path: | | |
| **/target/*.log | |
| spark-test-spark-ut-ansi: | |
| needs: build-native-lib | |
| runs-on: ubuntu-22.04 | |
| env: | |
| SPARK_TESTING: true | |
| container: apache/gluten:centos-9-jdk17 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download Native Lib | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: velox-native-lib-ansi-${{github.sha}} | |
| path: ./cpp/build/ | |
| - name: Download Arrow Jars | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: arrow-jars-ansi-${{github.sha}} | |
| path: /root/.m2/repository/org/apache/arrow/ | |
| - name: Prepare | |
| run: | | |
| dnf install -y python3.11 python3.11-pip python3.11-devel && \ | |
| alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \ | |
| alternatives --set python3 /usr/bin/python3.11 && \ | |
| pip3 install setuptools==77.0.3 && \ | |
| pip3 install pyspark==3.5.5 cython && \ | |
| pip3 install pandas==2.2.3 pyarrow==20.0.0 | |
| - name: Prepare Spark Resources | |
| run: | | |
| rm -rf /opt/shims/spark41 | |
| bash .github/workflows/util/install-resources.sh 4.1 | |
| mv /opt/shims/spark41/spark_home/assembly/target/scala-2.12 /opt/shims/spark41/spark_home/assembly/target/scala-2.13 | |
| - name: "Spark 4.1 spark-ut Tests (ANSI ON)" | |
| run: | | |
| set -o pipefail | |
| cd $GITHUB_WORKSPACE/ | |
| export SPARK_SCALA_VERSION=2.13 | |
| yum install -y java-17-openjdk-devel | |
| export JAVA_HOME=/usr/lib/jvm/java-17-openjdk | |
| export PATH=$JAVA_HOME/bin:$PATH | |
| echo "SPARK_ANSI_SQL_MODE=$SPARK_ANSI_SQL_MODE" | |
| java -version | |
| $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut \ | |
| -DwildcardSuites='org.apache.spark.' -Dtest=none -DfailIfNoTests=false \ | |
| -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/ -Dspark.gluten.sql.ansiFallback.enabled=false" \ | |
| -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \ | |
| 2>&1 | tee spark-ut-test-output.log | |
| - name: "Parse test results" | |
| if: always() | |
| run: | | |
| echo "=========================================" | |
| echo " spark-ut (Spark 4.1, ANSI=$SPARK_ANSI_SQL_MODE)" | |
| echo "=========================================" | |
| echo "" | |
| echo "--- Test Summary ---" | |
| grep -E "Tests run:.*Failures:|BUILD " spark-ut-test-output.log || echo "(no summary found)" | |
| echo "" | |
| echo "--- Failed Tests ---" | |
| grep -B1 "<<< FAIL!" spark-ut-test-output.log || echo "(no failures)" | |
| echo "" | |
| echo "--- Error Tests ---" | |
| grep -B1 "<<< ERROR!" spark-ut-test-output.log || echo "(no errors)" | |
| - name: Upload test report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: spark-test-spark-ut-ansi-report | |
| path: | | |
| **/surefire-reports/TEST-*.xml | |
| spark-ut-test-output.log | |
| - name: Upload log files | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: spark-test-spark-ut-ansi-logs | |
| path: | | |
| **/target/*.log | |
| **/gluten-ut/**/hs_err_*.log | |
| **/gluten-ut/**/core.* |