Skip to content

Commit 9b05fd7

Browse files
author
Chong Gao
committed
Spark 411 shim: XFAIL null_list.parquet because of Spark 411 changed the infer type
Signed-off-by: Chong Gao <res_life@163.com>
1 parent fe3a7f9 commit 9b05fd7

1 file changed

Lines changed: 8 additions & 2 deletions

File tree

integration_tests/src/main/python/parquet_testing_test.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2023-2025, NVIDIA CORPORATION.
1+
# Copyright (c) 2023-2026, NVIDIA CORPORATION.
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
@@ -22,7 +22,7 @@
2222
from marks import allow_non_gpu
2323
from pathlib import Path
2424
import pytest
25-
from spark_session import is_before_spark_330, is_spark_350_or_later
25+
from spark_session import is_before_spark_330, is_spark_350_or_later, is_spark_411_or_later
2626
import warnings
2727

2828
_rebase_confs = {
@@ -73,6 +73,12 @@
7373
_error_files["lz4_raw_compressed.parquet"] = "Exception"
7474
_error_files["lz4_raw_compressed_larger.parquet"] = "Exception"
7575

76+
if is_spark_411_or_later():
77+
# XFAIL null_list.parquet for Spark 4.1.0+ due to https://github.com/NVIDIA/spark-rapids/issues/14242
78+
# Before Spark 4.1.0, infers array<int> type for the null_list.parquet,
79+
# After Spark 4.1.0+, infers array<void> type for the null_list.parquet
80+
_xfail_files["null_list.parquet"] = "https://github.com/NVIDIA/spark-rapids/issues/14242"
81+
7682
def hdfs_glob(path_str, pattern):
7783
"""
7884
Finds hdfs files by checking the input path with glob pattern

0 commit comments

Comments
 (0)