Skip to content

Commit a2b07b8

Browse files
baibaichenclaude
baibaichen
and
claude
committed
ci(ansi): simplify CastWithAnsiOffSuite + add Spark 4.0 ANSI CI jobs
- Remove all redundant overrides from GlutenCastWithAnsiOffSuite (cast(), testGluten rewrites, UDT registration) for both spark40 and spark41, keeping only beforeAll + GlutenTestsTrait mixin - Remove corresponding excludes from VeloxTestSettings for both versions - Enable GlutenCastWithAnsiOnSuite for spark40 - Add --jvm-arg support to run-scala-test.sh - Add Spark 4.0 backends-velox and spark-ut ANSI CI jobs to workflow Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 652a2fb commit a2b07b8

6 files changed

Lines changed: 170 additions & 561 deletions

File tree

‎.github/workflows/velox_backend_ansi.yml‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,3 +242,158 @@ jobs:
242242
**/target/*.log
243243
**/gluten-ut/**/hs_err_*.log
244244
**/gluten-ut/**/core.*
245+
246+
spark-test-backends-velox-ansi-spark40:
247+
needs: build-native-lib
248+
runs-on: ubuntu-22.04
249+
env:
250+
SPARK_TESTING: true
251+
container: apache/gluten:centos-9-jdk17
252+
steps:
253+
- uses: actions/checkout@v4
254+
- name: Download Native Lib
255+
uses: actions/download-artifact@v4
256+
with:
257+
name: velox-native-lib-ansi-${{github.sha}}
258+
path: ./cpp/build/
259+
- name: Download Arrow Jars
260+
uses: actions/download-artifact@v4
261+
with:
262+
name: arrow-jars-ansi-${{github.sha}}
263+
path: /root/.m2/repository/org/apache/arrow/
264+
- name: Prepare
265+
run: |
266+
dnf install -y python3.11 python3.11-pip python3.11-devel && \
267+
alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \
268+
alternatives --set python3 /usr/bin/python3.11 && \
269+
pip3 install setuptools==77.0.3 && \
270+
pip3 install pyspark==3.5.5 cython && \
271+
pip3 install pandas==2.2.3 pyarrow==20.0.0
272+
- name: Prepare Spark Resources
273+
run: |
274+
rm -rf /opt/shims/spark40
275+
bash .github/workflows/util/install-resources.sh 4.0
276+
mv /opt/shims/spark40/spark_home/assembly/target/scala-2.12 /opt/shims/spark40/spark_home/assembly/target/scala-2.13
277+
- name: "Spark 4.0 backends-velox Tests (ANSI ON)"
278+
run: |
279+
set -o pipefail
280+
cd $GITHUB_WORKSPACE/
281+
export SPARK_SCALA_VERSION=2.13
282+
yum install -y java-17-openjdk-devel
283+
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
284+
export PATH=$JAVA_HOME/bin:$PATH
285+
echo "SPARK_ANSI_SQL_MODE=$SPARK_ANSI_SQL_MODE"
286+
java -version
287+
$MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox \
288+
-DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/ -Dspark.gluten.sql.ansiFallback.enabled=false" \
289+
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \
290+
2>&1 | tee backends-velox-spark40-test-output.log
291+
- name: "Parse test results"
292+
if: always()
293+
run: |
294+
echo "========================================="
295+
echo " backends-velox (Spark 4.0, ANSI=$SPARK_ANSI_SQL_MODE)"
296+
echo "========================================="
297+
echo ""
298+
echo "--- Test Summary ---"
299+
grep -E "Tests run:.*Failures:|BUILD " backends-velox-spark40-test-output.log || echo "(no summary found)"
300+
echo ""
301+
echo "--- Failed Tests ---"
302+
grep -B1 "<<< FAIL!" backends-velox-spark40-test-output.log || echo "(no failures)"
303+
echo ""
304+
echo "--- Error Tests ---"
305+
grep -B1 "<<< ERROR!" backends-velox-spark40-test-output.log || echo "(no errors)"
306+
- name: Upload test report
307+
if: always()
308+
uses: actions/upload-artifact@v4
309+
with:
310+
name: spark-test-backends-velox-ansi-spark40-report
311+
path: |
312+
**/surefire-reports/TEST-*.xml
313+
backends-velox-spark40-test-output.log
314+
- name: Upload log files
315+
if: always()
316+
uses: actions/upload-artifact@v4
317+
with:
318+
name: spark-test-backends-velox-ansi-spark40-logs
319+
path: |
320+
**/target/*.log
321+
322+
spark-test-spark-ut-ansi-spark40:
323+
needs: build-native-lib
324+
runs-on: ubuntu-22.04
325+
env:
326+
SPARK_TESTING: true
327+
container: apache/gluten:centos-9-jdk17
328+
steps:
329+
- uses: actions/checkout@v4
330+
- name: Download Native Lib
331+
uses: actions/download-artifact@v4
332+
with:
333+
name: velox-native-lib-ansi-${{github.sha}}
334+
path: ./cpp/build/
335+
- name: Download Arrow Jars
336+
uses: actions/download-artifact@v4
337+
with:
338+
name: arrow-jars-ansi-${{github.sha}}
339+
path: /root/.m2/repository/org/apache/arrow/
340+
- name: Prepare
341+
run: |
342+
dnf install -y python3.11 python3.11-pip python3.11-devel && \
343+
alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \
344+
alternatives --set python3 /usr/bin/python3.11 && \
345+
pip3 install setuptools==77.0.3 && \
346+
pip3 install pyspark==3.5.5 cython && \
347+
pip3 install pandas==2.2.3 pyarrow==20.0.0
348+
- name: Prepare Spark Resources
349+
run: |
350+
rm -rf /opt/shims/spark40
351+
bash .github/workflows/util/install-resources.sh 4.0
352+
mv /opt/shims/spark40/spark_home/assembly/target/scala-2.12 /opt/shims/spark40/spark_home/assembly/target/scala-2.13
353+
- name: "Spark 4.0 spark-ut Tests (ANSI ON)"
354+
run: |
355+
set -o pipefail
356+
cd $GITHUB_WORKSPACE/
357+
export SPARK_SCALA_VERSION=2.13
358+
yum install -y java-17-openjdk-devel
359+
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
360+
export PATH=$JAVA_HOME/bin:$PATH
361+
echo "SPARK_ANSI_SQL_MODE=$SPARK_ANSI_SQL_MODE"
362+
java -version
363+
$MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut \
364+
-DwildcardSuites='org.apache.spark.' -Dtest=none -DfailIfNoTests=false \
365+
-DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/ -Dspark.gluten.sql.ansiFallback.enabled=false" \
366+
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \
367+
2>&1 | tee spark-ut-spark40-test-output.log
368+
- name: "Parse test results"
369+
if: always()
370+
run: |
371+
echo "========================================="
372+
echo " spark-ut (Spark 4.0, ANSI=$SPARK_ANSI_SQL_MODE)"
373+
echo "========================================="
374+
echo ""
375+
echo "--- Test Summary ---"
376+
grep -E "Tests run:.*Failures:|BUILD " spark-ut-spark40-test-output.log || echo "(no summary found)"
377+
echo ""
378+
echo "--- Failed Tests ---"
379+
grep -B1 "<<< FAIL!" spark-ut-spark40-test-output.log || echo "(no failures)"
380+
echo ""
381+
echo "--- Error Tests ---"
382+
grep -B1 "<<< ERROR!" spark-ut-spark40-test-output.log || echo "(no errors)"
383+
- name: Upload test report
384+
if: always()
385+
uses: actions/upload-artifact@v4
386+
with:
387+
name: spark-test-spark-ut-ansi-spark40-report
388+
path: |
389+
**/surefire-reports/TEST-*.xml
390+
spark-ut-spark40-test-output.log
391+
- name: Upload log files
392+
if: always()
393+
uses: actions/upload-artifact@v4
394+
with:
395+
name: spark-test-spark-ut-ansi-spark40-logs
396+
path: |
397+
**/target/*.log
398+
**/gluten-ut/**/hs_err_*.log
399+
**/gluten-ut/**/core.*

‎dev/run-scala-test.sh‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,7 @@ Optional:
215215
--force Force Maven rebuild, bypass build cache
216216
--profile Enable Maven profiler (reports in .profiler/)
217217
--export-only Export classpath and exit (no test execution)
218+
--jvm-arg <arg> Pass extra JVM argument to test process (repeatable)
218219
--help Show this help message
219220
220221
Examples:
@@ -234,6 +235,12 @@ Examples:
234235
-pl gluten-ut/spark40 \\
235236
-s org.apache.spark.sql.GlutenTPCDSV1_4_PlanStabilitySuite \\
236237
-s org.apache.spark.sql.GlutenTPCHPlanStabilitySuite
238+
239+
# Run with ANSI mode (disable fallback via --jvm-arg)
240+
$0 --jvm-arg "-Dspark.gluten.sql.ansiFallback.enabled=false" \\
241+
-Pjava-17,spark-4.1,scala-2.13,backends-velox,hadoop-3.3,spark-ut \\
242+
-pl gluten-ut/spark41 \\
243+
-s org.apache.spark.sql.catalyst.expressions.GlutenCastWithAnsiOffSuite
237244
EOF
238245
}
239246

@@ -348,6 +355,7 @@ EXPORT_ONLY=false
348355
ENABLE_CLEAN=false
349356
FORCE_BUILD=false
350357
USE_MVND=false
358+
EXTRA_JVM_ARGS=()
351359

352360
while [[ $# -gt 0 ]]; do
353361
case $1 in
@@ -387,6 +395,10 @@ while [[ $# -gt 0 ]]; do
387395
EXPORT_ONLY=true
388396
shift
389397
;;
398+
--jvm-arg)
399+
EXTRA_JVM_ARGS+=("$2")
400+
shift 2
401+
;;
390402
--help)
391403
print_usage
392404
exit 0
@@ -684,6 +696,7 @@ SPARK_TEST_HOME_ARG=""
684696

685697
JAVA_ARGS=(
686698
${JVM_ARGS}
699+
"${EXTRA_JVM_ARGS[@]}"
687700
"-Dlog4j.configurationFile=file:${GLUTEN_HOME}/${MODULE}/src/test/resources/log4j2.properties"
688701
${SPARK_TEST_HOME_ARG}
689702
-cp "${PATHING_JAR}"

‎gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -102,16 +102,6 @@ class VeloxTestSettings extends BackendTestSettings {
102102
.exclude("SPARK-45786: Decimal multiply, divide, remainder, quot")
103103
enableSuite[GlutenBitwiseExpressionsSuite]
104104
enableSuite[GlutenCastWithAnsiOffSuite]
105-
.exclude(
106-
"Process Infinity, -Infinity, NaN in case insensitive manner" // +inf not supported in folly.
107-
)
108-
// Set timezone through config.
109-
.exclude("data type casting")
110-
// Revised by setting timezone through config and commented unsupported cases.
111-
.exclude("cast string to timestamp")
112-
.exclude("cast from timestamp II")
113-
.exclude("SPARK-36286: invalid string cast to timestamp")
114-
.exclude("SPARK-39749: cast Decimal to string")
115105
enableSuite[GlutenTryCastSuite]
116106
.exclude(
117107
"Process Infinity, -Infinity, NaN in case insensitive manner" // +inf not supported in folly.
@@ -217,7 +207,7 @@ class VeloxTestSettings extends BackendTestSettings {
217207
enableSuite[GlutenBitmapExpressionUtilsSuite]
218208
enableSuite[GlutenCallMethodViaReflectionSuite]
219209
enableSuite[GlutenCanonicalizeSuite]
220-
// TODO: 4.x enableSuite[GlutenCastWithAnsiOnSuite] // 4 failures
210+
enableSuite[GlutenCastWithAnsiOnSuite]
221211
enableSuite[GlutenCodeGenerationSuite]
222212
enableSuite[GlutenCodeGeneratorWithInterpretedFallbackSuite]
223213
enableSuite[GlutenCollationExpressionSuite]

0 commit comments

Comments
 (0)