Skip to content

Commit c5a606a

Browse files
igorpeshanskyigorpeshansky
authored
Merge branch 'main' into regex-inline-flags
2 parents 05e53ac + 22b85ef commit c5a606a

10 files changed

Lines changed: 365 additions & 3 deletions

‎integration_tests/src/main/python/delta_lake_test.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@
2222
from parquet_test import reader_opt_confs_no_native
2323
from parquet_test_utils import parquet_row_group_midpoints
2424
from spark_session import with_cpu_session, with_gpu_session, is_databricks_runtime, \
25-
is_spark_320_or_later, is_spark_340_or_later, supports_delta_lake_deletion_vectors, is_spark_401_or_later, \
25+
is_spark_320_or_later, is_spark_340_or_later, is_spark_40x, \
26+
supports_delta_lake_deletion_vectors, is_spark_401_or_later, \
2627
gpu_supports_delta_dv_scan, is_before_spark_353, is_databricks173_or_later
2728

2829
_conf = {'spark.rapids.sql.explain': 'ALL'}
@@ -852,8 +853,9 @@ def do_delete_and_disable_DV(spark, data_path):
852853
reason="Deletion vector scan is not supported on Databricks")
853854
@pytest.mark.skipif(is_before_spark_353(),
854855
reason="Spark-RAPIDS supports scan with deletion vectors starting in Spark 3.5.3")
855-
@pytest.mark.skipif(is_spark_401_or_later(),
856-
reason="REORG is not supported in Spark 4.0.1+ (https://github.com/delta-io/delta/issues/5690)")
856+
@pytest.mark.skipif(is_spark_40x() and is_spark_401_or_later(),
857+
reason="Delta 4.0.0 REORG is incompatible with Spark 4.0.1+ in Spark 4.0.x "
858+
"profiles (https://github.com/delta-io/delta/issues/5690)")
857859
def test_delta_scan_split_with_DV_enabled_after_DVs_materialized(spark_tmp_path):
858860
def do_delete_and_reorg(spark, data_path):
859861
num_deleted = spark.sql(f"DELETE FROM delta.`{data_path}` WHERE a = 0").collect()[0][0]

‎tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsBucketedWriteSuite.scala‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*
2+
* Copyright (c) 2026, NVIDIA CORPORATION.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
/*** spark-rapids-shim-json-lines
18+
{"spark": "330"}
19+
spark-rapids-shim-json-lines ***/
20+
package org.apache.spark.sql.rapids.suites
21+
22+
import org.apache.spark.sql.rapids.utils.RapidsSQLTestsTrait
23+
import org.apache.spark.sql.sources.BucketedWriteWithoutHiveSupportSuite
24+
25+
class RapidsBucketedWriteWithoutHiveSupportSuite
26+
extends BucketedWriteWithoutHiveSupportSuite
27+
with RapidsSQLTestsTrait

‎tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsCreateTableAsSelectSuite.scala‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Copyright (c) 2026, NVIDIA CORPORATION.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
/*** spark-rapids-shim-json-lines
18+
{"spark": "330"}
19+
spark-rapids-shim-json-lines ***/
20+
package org.apache.spark.sql.rapids.suites
21+
22+
import java.nio.file.Files
23+
24+
import org.scalatest.Ignore
25+
26+
import org.apache.spark.sql.rapids.utils.RapidsSQLTestsTrait
27+
import org.apache.spark.sql.sources.CreateTableAsSelectSuite
28+
import org.apache.spark.util.Utils
29+
30+
class RapidsCreateTableAsSelectSuite
31+
extends CreateTableAsSelectSuite with RapidsSQLTestsTrait {
32+
33+
private val writePermissionTest =
34+
"CREATE TABLE USING AS SELECT based on the file without write permission"
35+
36+
private lazy val enforcesWritePermissions: Boolean = {
37+
val probeDir = Utils.createTempDir()
38+
try {
39+
probeDir.setWritable(false) && !Files.isWritable(probeDir.toPath)
40+
} finally {
41+
probeDir.setWritable(true)
42+
Utils.deleteRecursively(probeDir)
43+
}
44+
}
45+
46+
override def tags: Map[String, Set[String]] = {
47+
if (enforcesWritePermissions) {
48+
super.tags
49+
} else {
50+
val inheritedTags = super.tags
51+
inheritedTags.updated(
52+
writePermissionTest,
53+
inheritedTags.getOrElse(writePermissionTest, Set.empty) + classOf[Ignore].getName)
54+
}
55+
}
56+
}

‎tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsDataSourceV2DataFrameSuite.scala‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Copyright (c) 2026, NVIDIA CORPORATION.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
/*** spark-rapids-shim-json-lines
18+
{"spark": "330"}
19+
spark-rapids-shim-json-lines ***/
20+
package org.apache.spark.sql.rapids.suites
21+
22+
import org.apache.spark.sql.connector.DataSourceV2DataFrameSuite
23+
import org.apache.spark.sql.rapids.utils.RapidsSQLTestsTrait
24+
25+
class RapidsDataSourceV2DataFrameSuite
26+
extends DataSourceV2DataFrameSuite with RapidsSQLTestsTrait

‎tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsFileDataSourceV2FallBackSuite.scala‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/*
2+
* Copyright (c) 2026, NVIDIA CORPORATION.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
/*** spark-rapids-shim-json-lines
18+
{"spark": "330"}
19+
spark-rapids-shim-json-lines ***/
20+
package org.apache.spark.sql.rapids.suites
21+
22+
import scala.collection.mutable.ArrayBuffer
23+
24+
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
25+
import org.apache.spark.sql.connector.FileDataSourceV2FallBackSuite
26+
import org.apache.spark.sql.execution.QueryExecution
27+
import org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand
28+
import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
29+
import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetDataSourceV2
30+
import org.apache.spark.sql.internal.SQLConf
31+
import org.apache.spark.sql.rapids.GpuFileSourceScanExec
32+
import org.apache.spark.sql.rapids.utils.RapidsSQLTestsTrait
33+
import org.apache.spark.sql.util.QueryExecutionListener
34+
35+
class RapidsFileDataSourceV2FallBackSuite
36+
extends FileDataSourceV2FallBackSuite with RapidsSQLTestsTrait {
37+
38+
testRapids("Fallback Parquet V2 to V1") {
39+
Seq("parquet", classOf[ParquetDataSourceV2].getCanonicalName).foreach { format =>
40+
withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> format) {
41+
val commands = ArrayBuffer.empty[(String, LogicalPlan)]
42+
val exceptions = ArrayBuffer.empty[(String, Exception)]
43+
val listener = new QueryExecutionListener {
44+
override def onFailure(
45+
funcName: String,
46+
qe: QueryExecution,
47+
exception: Exception): Unit = {
48+
exceptions += funcName -> exception
49+
}
50+
51+
override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = {
52+
commands += funcName -> qe.logical
53+
}
54+
}
55+
spark.listenerManager.register(listener)
56+
57+
try {
58+
withTempPath { path =>
59+
val inputData = spark.range(10)
60+
inputData.write.format(format).save(path.getCanonicalPath)
61+
sparkContext.listenerBus.waitUntilEmpty()
62+
assert(
63+
exceptions.isEmpty,
64+
s"Unexpected QueryExecutionListener failures: ${exceptions.mkString(", ")}")
65+
assert(commands.length == 1)
66+
assert(commands.head._1 == "command")
67+
assert(commands.head._2.isInstanceOf[InsertIntoHadoopFsRelationCommand])
68+
assert(commands.head._2.asInstanceOf[InsertIntoHadoopFsRelationCommand]
69+
.fileFormat.isInstanceOf[ParquetFileFormat])
70+
val df = spark.read.format(format).load(path.getCanonicalPath)
71+
checkAnswer(df, inputData.toDF())
72+
assert(df.queryExecution.executedPlan.exists(
73+
_.isInstanceOf[GpuFileSourceScanExec]))
74+
}
75+
} finally {
76+
spark.listenerManager.unregister(listener)
77+
}
78+
}
79+
}
80+
}
81+
}

‎tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsKeyGroupedPartitioningSuite.scala‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Copyright (c) 2026, NVIDIA CORPORATION.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
/*** spark-rapids-shim-json-lines
18+
{"spark": "330"}
19+
spark-rapids-shim-json-lines ***/
20+
package org.apache.spark.sql.rapids.suites
21+
22+
import org.apache.spark.sql.connector.KeyGroupedPartitioningSuite
23+
import org.apache.spark.sql.rapids.utils.RapidsSQLTestsTrait
24+
25+
class RapidsKeyGroupedPartitioningSuite
26+
extends KeyGroupedPartitioningSuite with RapidsSQLTestsTrait

‎tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsOrcSourceSuite.scala‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*
2+
* Copyright (c) 2026, NVIDIA CORPORATION.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
/*** spark-rapids-shim-json-lines
18+
{"spark": "330"}
19+
spark-rapids-shim-json-lines ***/
20+
package org.apache.spark.sql.rapids.suites
21+
22+
import org.apache.spark.sql.execution.datasources.orc.{OrcSourceV1Suite, OrcSourceV2Suite}
23+
import org.apache.spark.sql.rapids.utils.RapidsSQLTestsTrait
24+
25+
class RapidsOrcSourceV1Suite extends OrcSourceV1Suite with RapidsSQLTestsTrait
26+
27+
class RapidsOrcSourceV2Suite extends OrcSourceV2Suite with RapidsSQLTestsTrait

‎tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsV1ReadFallbackSuite.scala‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/*
2+
* Copyright (c) 2026, NVIDIA CORPORATION.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
/*** spark-rapids-shim-json-lines
18+
{"spark": "330"}
19+
spark-rapids-shim-json-lines ***/
20+
package org.apache.spark.sql.rapids.suites
21+
22+
import org.apache.spark.sql.connector.{
23+
V1ReadFallbackWithCatalogSuite,
24+
V1ReadFallbackWithDataFrameReaderSuite}
25+
import org.apache.spark.sql.rapids.utils.RapidsSQLTestsTrait
26+
27+
class RapidsV1ReadFallbackWithDataFrameReaderSuite
28+
extends V1ReadFallbackWithDataFrameReaderSuite with RapidsSQLTestsTrait
29+
30+
class RapidsV1ReadFallbackWithCatalogSuite
31+
extends V1ReadFallbackWithCatalogSuite with RapidsSQLTestsTrait

‎tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsWriteDistributionAndOrderingSuite.scala‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Copyright (c) 2026, NVIDIA CORPORATION.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
/*** spark-rapids-shim-json-lines
18+
{"spark": "330"}
19+
spark-rapids-shim-json-lines ***/
20+
package org.apache.spark.sql.rapids.suites
21+
22+
import org.apache.spark.sql.connector.WriteDistributionAndOrderingSuite
23+
import org.apache.spark.sql.rapids.utils.RapidsSQLTestsTrait
24+
25+
class RapidsWriteDistributionAndOrderingSuite
26+
extends WriteDistributionAndOrderingSuite with RapidsSQLTestsTrait

‎tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,66 @@ class RapidsTestSettings extends BackendTestSettings {
178178
enableSuite[RapidsDeprecatedAPISuite]
179179
enableSuite[RapidsDeprecatedDatasetAggregatorSuite]
180180
enableSuite[RapidsStatisticsCollectionSuite]
181+
enableSuite[RapidsWriteDistributionAndOrderingSuite]
182+
enableSuite[RapidsOrcSourceV1Suite]
183+
.exclude("Propagate Hadoop configs from orc options to underlying file system",
184+
KNOWN_ISSUE("https://github.com/NVIDIA/cudf-spark/issues/11602. " +
185+
"Recovery trigger: GPU ORC writes propagate data-source options; P1."))
186+
.exclude("Write Spark version into ORC file metadata",
187+
KNOWN_ISSUE("https://github.com/NVIDIA/cudf-spark/issues/15468. " +
188+
"Recovery trigger: GPU ORC files include Spark version metadata; P1."))
189+
.exclude("SPARK-31238: compatibility with Spark 2.4 in reading dates",
190+
KNOWN_ISSUE("https://github.com/NVIDIA/cudf-spark/issues/15470. " +
191+
"Recovery trigger: GPU ORC legacy date rebasing matches Spark CPU; P0."))
192+
.exclude("SPARK-31284: compatibility with Spark 2.4 in reading timestamps",
193+
KNOWN_ISSUE("https://github.com/NVIDIA/cudf-spark/issues/15471. " +
194+
"Recovery trigger: GPU ORC legacy timestamp reads match Spark CPU; P0."))
195+
.exclude("SPARK-31284, SPARK-31423: rebasing timestamps in write",
196+
KNOWN_ISSUE("https://github.com/NVIDIA/cudf-spark/issues/15473. " +
197+
"Recovery trigger: GPU ORC legacy timestamp round trips match Spark CPU; P0."))
198+
.exclude("Enforce direct encoding column-wise selectively",
199+
WONT_FIX_ISSUE("GPU ORC uses valid libcudf-selected encodings rather than the Spark ORC " +
200+
"selective direct-encoding option. See https://github.com/NVIDIA/cudf-spark/issues/15469. " +
201+
"Recovery trigger: the GPU writer supports this tuning contract or falls back; P2."))
202+
.exclude("SPARK-36663: OrcUtils.toCatalystSchema should correctly handle " +
203+
"a column name which consists of only numbers",
204+
KNOWN_ISSUE("https://github.com/NVIDIA/cudf-spark/issues/15472. " +
205+
"Recovery trigger: GPU ORC reader handles numeric-only field names; P0."))
206+
enableSuite[RapidsOrcSourceV2Suite]
207+
.exclude("Propagate Hadoop configs from orc options to underlying file system",
208+
KNOWN_ISSUE("https://github.com/NVIDIA/cudf-spark/issues/11602. " +
209+
"Recovery trigger: GPU ORC writes propagate data-source options; P1."))
210+
.exclude("Write Spark version into ORC file metadata",
211+
KNOWN_ISSUE("https://github.com/NVIDIA/cudf-spark/issues/15468. " +
212+
"Recovery trigger: GPU ORC files include Spark version metadata; P1."))
213+
.exclude("SPARK-31238: compatibility with Spark 2.4 in reading dates",
214+
KNOWN_ISSUE("https://github.com/NVIDIA/cudf-spark/issues/15470. " +
215+
"Recovery trigger: GPU ORC legacy date rebasing matches Spark CPU; P0."))
216+
.exclude("SPARK-31284: compatibility with Spark 2.4 in reading timestamps",
217+
KNOWN_ISSUE("https://github.com/NVIDIA/cudf-spark/issues/15471. " +
218+
"Recovery trigger: GPU ORC legacy timestamp reads match Spark CPU; P0."))
219+
.exclude("SPARK-31284, SPARK-31423: rebasing timestamps in write",
220+
KNOWN_ISSUE("https://github.com/NVIDIA/cudf-spark/issues/15473. " +
221+
"Recovery trigger: GPU ORC legacy timestamp round trips match Spark CPU; P0."))
222+
.exclude("Enforce direct encoding column-wise selectively",
223+
WONT_FIX_ISSUE("GPU ORC uses valid libcudf-selected encodings rather than the Spark ORC " +
224+
"selective direct-encoding option. See https://github.com/NVIDIA/cudf-spark/issues/15469. " +
225+
"Recovery trigger: the GPU writer supports this tuning contract or falls back; P2."))
226+
.exclude("SPARK-36663: OrcUtils.toCatalystSchema should correctly handle " +
227+
"a column name which consists of only numbers",
228+
KNOWN_ISSUE("https://github.com/NVIDIA/cudf-spark/issues/15472. " +
229+
"Recovery trigger: GPU ORC reader handles numeric-only field names; P0."))
230+
enableSuite[RapidsKeyGroupedPartitioningSuite]
231+
enableSuite[RapidsDataSourceV2DataFrameSuite]
232+
enableSuite[RapidsBucketedWriteWithoutHiveSupportSuite]
233+
enableSuite[RapidsCreateTableAsSelectSuite]
234+
enableSuite[RapidsFileDataSourceV2FallBackSuite]
235+
.exclude("Fallback Parquet V2 to V1",
236+
ADJUST_UT("Replaced by a testRapids version that preserves V1 fallback assertions and " +
237+
"checks GpuFileSourceScanExec. See https://github.com/NVIDIA/cudf-spark/issues/15465. " +
238+
"Recovery trigger: the Spark test accepts the GPU V1 scan node; P2."))
239+
enableSuite[RapidsV1ReadFallbackWithDataFrameReaderSuite]
240+
enableSuite[RapidsV1ReadFallbackWithCatalogSuite]
181241
enableSuite[RapidsFileSourceStrategySuite]
182242
.exclude("partitioned table - after scan filters", ADJUST_UT("Replaced by testRapids version that checks GpuFilterExec residual filters."))
183243
.exclude("[SPARK-16818] partition pruned file scans implement sameResult correctly", KNOWN_ISSUE("https://github.com/NVIDIA/cudf-spark/issues/15161"))

0 commit comments

Comments
 (0)