Merge branch 'main' into ctas-rtas-write-support

Author: nartal1

datagen/src/main/spark400/scala/org/apache/spark/sql/tests/datagen/DataGenExprShims.scala

@@ -20,11 +20,13 @@
 {"spark": "401"}
 {"spark": "402"}
 {"spark": "403"}
+{"spark": "404"}
 {"spark": "411"}
 {"spark": "412"}
 {"spark": "413"}
 {"spark": "420"}
 spark-rapids-shim-json-lines ***/
+
 package org.apache.spark.sql.tests.datagen
 
 import org.apache.spark.sql.Column

docs/design/rapids_shuffle_manager_v2_phase1_design.md

@@ -44,10 +44,10 @@ The plugin is designed to work on NVIDIA Volta, Turing, Ampere, Ada Lovelace, Ho
 		Apache Spark 3.3.0, 3.3.1, 3.3.2, 3.3.3, 3.3.4
 		Apache Spark 3.4.0, 3.4.1, 3.4.2, 3.4.3, 3.4.4
 		Apache Spark 3.5.0, 3.5.1, 3.5.2, 3.5.3, 3.5.4, 3.5.5, 3.5.6, 3.5.7, 3.5.8
-		Apache Spark 4.0.0, 4.0.1, 4.0.2, 4.0.3
+		Apache Spark 4.0.0, 4.0.1, 4.0.2, 4.0.3, 4.0.4
 		Apache Spark 4.1.1
 		Scala 2.12: Spark 3.3.0 through 3.5.8
-		Scala 2.13: Spark 3.5.0 through 3.5.8, and Spark 4.0.0, 4.0.1, 4.0.2, 4.0.3, and 4.1.1
+		Scala 2.13: Spark 3.5.0 through 3.5.8, and Spark 4.0.0, 4.0.1, 4.0.2, 4.0.3, 4.0.4, and 4.1.1
 
 	Supported Databricks runtime versions for Azure and AWS:
 		Databricks 13.3 ML LTS (GPU, Scala 2.12, Spark 3.4.1)

docs/download.md

@@ -18,6 +18,7 @@
 {"spark": "401"}
 {"spark": "402"}
 {"spark": "403"}
+{"spark": "404"}
 spark-rapids-shim-json-lines ***/
 
 package com.nvidia.spark.rapids.iceberg;

iceberg/iceberg-1-10-x/src/main/spark400/java/com/nvidia/spark/rapids/iceberg/GpuInternalRow.java

@@ -18,6 +18,7 @@
 {"spark": "401"}
 {"spark": "402"}
 {"spark": "403"}
+{"spark": "404"}
 spark-rapids-shim-json-lines ***/
 
 package com.nvidia.spark.rapids.iceberg;

iceberg/iceberg-1-11-x/src/main/spark400/java/com/nvidia/spark/rapids/iceberg/GpuInternalRow.java

@@ -16,6 +16,7 @@
 /*** spark-rapids-shim-json-lines
 {"spark": "402"}
 {"spark": "403"}
+{"spark": "404"}
 spark-rapids-shim-json-lines ***/
 
 package com.nvidia.spark.rapids.iceberg.iceberg111x

iceberg/iceberg-1-11-x/src/main/spark402/scala/com/nvidia/spark/rapids/iceberg/iceberg111x/IcebergProviderImpl.scala

@@ -17,7 +17,7 @@
 from asserts import assert_cpu_and_gpu_are_equal_collect_with_capture, assert_gpu_and_cpu_are_equal_collect
 from data_gen import *
 from marks import approximate_float, datagen_overrides, ignore_order, disable_ansi_mode
-from spark_session import with_cpu_session, is_spark_403, is_spark_412_or_later, \
+from spark_session import with_cpu_session, is_spark_403_or_404, is_spark_412_or_later, \
     is_spark_420_or_later
 import pyspark.sql.functions as f
 
@@ -61,9 +61,9 @@
 
 ast_boolean_descr = [(boolean_gen, True)]
 ast_double_descr = [(double_gen, True)]
-# AST is not expressive enough to support the ACOSH Spark emulation expression in Spark 4.0.3
-# and Spark 4.1.2+.
-ast_acosh_descr = [(double_gen, not (is_spark_403() or is_spark_412_or_later()))]
+# AST is not expressive enough to support the ACOSH Spark emulation expression in Spark 4.0.3,
+# Spark 4.0.4, and Spark 4.1.2+.
+ast_acosh_descr = [(double_gen, not (is_spark_403_or_404() or is_spark_412_or_later()))]
 
 _project_ast_enabled_conf = {"spark.rapids.sql.projectAstEnabled": "true"}
 

integration_tests/src/main/python/ast_test.py

@@ -267,6 +267,12 @@ def is_spark_401_or_later():
 def is_spark_403():
     return spark_version() == "4.0.3"
 
+def is_spark_404():
+    return spark_version() == "4.0.4"
+
+def is_spark_403_or_404():
+    return is_spark_403() or is_spark_404()
+
 def is_spark_411_or_later():
     return spark_version() >= "4.1.1"
 

integration_tests/src/main/python/spark_session.py

@@ -23,7 +23,7 @@
 import pyspark.sql.functions as f
 from spark_session import is_before_spark_320, is_databricks113_or_later, \
     is_databricks133_or_later, is_spark_350_or_later, spark_version, with_cpu_session, \
-    is_spark_340_or_later, is_spark_420_or_later
+    is_scala212, is_spark_340_or_later, is_spark_420_or_later
 import warnings
 
 # mark this test as ci_1 for mvn verify sanity check in pre-merge CI
@@ -2377,8 +2377,25 @@ def do_it(spark):
 @ignore_order(local=True)
 @allow_non_gpu(*non_utc_allow)
 def test_window_aggs_for_rows_collect_set():
+    data_gen = _gen_data_for_collect_set
+    if is_scala212():
+        # Scala 2.12 CPU window collect_set can retain both signed zeros, while the GPU and
+        # Scala 2.13 treat them as the same value. Exclude -0.0 from this Scala 2.12 test.
+        float_special_cases = [
+            FLOAT_MIN, FLOAT_MAX, 0.0, 1.0, -1.0,
+            float('inf'), float('-inf'), float('nan'), NEG_FLOAT_NAN_MAX_VALUE]
+        double_special_cases = [
+            DOUBLE_MIN, DOUBLE_MAX, 0.0, 1.0, -1.0,
+            float('inf'), float('-inf'), float('nan'), NEG_DOUBLE_NAN_MAX_VALUE]
+        collect_set_fp_gens = {
+            'c_float': RepeatSeqGen(FloatGen(special_cases=float_special_cases), length=15),
+            'c_double': RepeatSeqGen(DoubleGen(special_cases=double_special_cases), length=15)}
+        data_gen = [
+            (name, collect_set_fp_gens[name]) if name in collect_set_fp_gens else (name, gen)
+            for name, gen in data_gen]
+
     assert_gpu_and_cpu_are_equal_sql(
-        lambda spark: gen_df(spark, _gen_data_for_collect_set),
+        lambda spark: gen_df(spark, data_gen),
         "window_collect_table",
         '''
         select a, b,

integration_tests/src/main/python/window_function_test.py

@@ -20,12 +20,14 @@
 {"spark": "401"}
 {"spark": "402"}
 {"spark": "403"}
+{"spark": "404"}
 {"spark": "411"}
 {"spark": "412"}
 {"spark": "413"}
 {"spark": "420"}
 spark-rapids-shim-json-lines ***/
 
+
 package org.apache.spark.sql.rapids.shims
 
 import org.apache.spark.sql.classic.SparkSession