Merge branch 'main' into ray/15323

Author: liurenjie1024

AGENTS.md

@@ -113,7 +113,7 @@ spark-rapids-shim-json-lines ***/
 - When modifying a shim, update ALL related Spark version shims
   (the same logical change may need different adaptations per
   version — do not blindly copy-paste).
-- `db` suffix (e.g., `330db`, `341db`) = Databricks-specific shim.
+- `db` suffix (e.g., `330db`, `350db143`) = Databricks-specific shim.
 - **Scala 2.12 vs 2.13**: `sql-plugin` shims work identically for
   both Scala versions. After modifying any `pom.xml`, run
   `./build/make-scala-version-build-files.sh 2.13` to sync.

CONTRIBUTING.md

@@ -252,12 +252,12 @@ The following acronyms may appear in directory names:
 
 |Acronym|Definition  |Example|Example Explanation                           |
 |-------|------------|-------|----------------------------------------------|
-|db     |Databricks  |341db  |Databricks Spark based on Spark 3.4.1         |
+|db     |Databricks  |350db143|Databricks Spark based on Spark 3.5.0        |
 |cdh    |Cloudera CDH|(removed)|Cloudera CDH shims have been removed         |
 
 The version-specific directory names have one of the following forms / use cases:
 
-* `src/main/spark${buildver}`, example: `src/main/spark341db`
+* `src/main/spark${buildver}`, example: `src/main/spark350db143`
 * `src/test/spark${buildver}`, example: `src/test/spark340`
 
 with a special shim descriptor as a Scala/Java comment. See [shimplify.md][1]

datagen/src/main/spark330/scala/org/apache/spark/sql/tests/datagen/DataGenExprShims.scala

@@ -24,7 +24,6 @@
 {"spark": "334"}
 {"spark": "340"}
 {"spark": "341"}
-{"spark": "341db"}
 {"spark": "342"}
 {"spark": "343"}
 {"spark": "344"}

datagen/src/main/spark400/scala/org/apache/spark/sql/tests/datagen/DataGenExprShims.scala

@@ -20,11 +20,13 @@
 {"spark": "401"}
 {"spark": "402"}
 {"spark": "403"}
+{"spark": "404"}
 {"spark": "411"}
 {"spark": "412"}
 {"spark": "413"}
 {"spark": "420"}
 spark-rapids-shim-json-lines ***/
+
 package org.apache.spark.sql.tests.datagen
 
 import org.apache.spark.sql.Column

delta-lake/README.md

@@ -19,7 +19,6 @@ and directory contains the corresponding support code.
 | 3.3.x              | Spark 3.5.[3-]  | `delta-33x`        |
 | 4.0.x              | Spark 4.0.x     | `delta-40x`        |
 | 4.1.x              | Spark 4.1.x     | `delta-41x`        |
-| Databricks 13.3    | Databricks 13.3 | `delta-spark341db` |
 | Databricks 14.3    | Databricks 14.3 | `delta-spark350db143` |
 
 

delta-lake/common/src/main/databricks/scala/com/nvidia/spark/rapids/delta/DatabricksDeltaProviderBase.scala

@@ -116,8 +116,8 @@ trait DatabricksDeltaProviderBase extends DeltaProviderImplBase {
   }
 
   private def getWriteOptions(options: Any): Map[String, String] = {
-    // For Databricks 13.3 AtomicCreateTableAsSelectExec writeOptions is a Map[String, String]
-    // while in all the other versions it's a CaseInsensitiveMap
+    // Databricks 14.3 writeOptions is a Map[String, String], while newer supported
+    // Databricks runtimes use CaseInsensitiveStringMap.
     options match {
       case c: CaseInsensitiveStringMap => c.asCaseSensitiveMap().asScala.toMap
       case _ => options.asInstanceOf[Map[String, String]]