Enabled Databricks 17.3 tests and removed support for Databricks 12.2

NvTimLiu · NvTimLiu · commit 2bb8a8cc99df · 2026-04-01T13:23:06.000+08:00
Databricks 17.3 IT: Jenkins, POM cleanup, docs and build version updates

Signed-off-by: timl &lt;timl@nvidia.com&gt;
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -252,12 +252,12 @@ The following acronyms may appear in directory names:
 
 |Acronym|Definition  |Example|Example Explanation                           |
 |-------|------------|-------|----------------------------------------------|
-|db     |Databricks  |332db  |Databricks Spark based on Spark 3.3.2         |
+|db     |Databricks  |341db  |Databricks Spark based on Spark 3.4.1         |
 |cdh    |Cloudera CDH|(removed)|Cloudera CDH shims have been removed         |
 
 The version-specific directory names have one of the following forms / use cases:
 
-* `src/main/spark${buildver}`, example: `src/main/spark332db`
+* `src/main/spark${buildver}`, example: `src/main/spark341db`
 * `src/test/spark${buildver}`, example: `src/test/spark340`
 
 with a special shim descriptor as a Scala/Java comment. See [shimplify.md][1]
diff --git a/build/get_buildvers.py b/build/get_buildvers.py
@@ -57,7 +57,7 @@ def _get_buildvers(buildvers, pom_file, logger=None, ignore_excluded_shims=False
                     # exists. This is expected when pom.xml and scala2.13/pom.xml
                     # share the same exclusion list but have different profiles due
                     # to #if scala-2.12/#if scala-2.13 conditional comments
-                    # (e.g. 332db only in pom.xml, 400 only in scala2.13/pom.xml).
+                    # (e.g. profile split between pom.xml and scala2.13/pom.xml via #if scala comments).
                     if logger:
                         logger.debug(
                             "Shim %s listed in dyn.shim.excluded.releases not found in "
diff --git a/delta-lake/README.md b/delta-lake/README.md
@@ -18,9 +18,9 @@ and directory contains the corresponding support code.
 | 2.4.x              | Spark 3.4.x     | `delta-24x`        |
 | 3.3.x              | Spark 3.5.[3-]  | `delta-33x`        |
 | 4.0.x              | Spark 4.0.x     | `delta-40x`        |
-| Databricks 12.2    | Databricks 12.2 | `delta-spark332db` |
 | Databricks 13.3    | Databricks 13.3 | `delta-spark341db` |
 | Databricks 14.3    | Databricks 14.3 | `delta-spark350db143` |
+| Databricks 17.3    | Databricks 17.3 | `delta-spark400db173` |
 
 
 Delta Lake is not supported on all Spark versions, and for Spark versions where it is not
diff --git a/docs/dev/lore.md b/docs/dev/lore.md
@@ -121,7 +121,6 @@ restored data to console.
    `GpuShuffleExchangeExec`), leaf operator (e.g. `GpuFileSourceScanExec`) are not supported.
 3. **GpuDataWritingCommandExec is not supported for LORE dump on certain Spark versions**.
    The following versions are not supported due to compatibility issues with `GpuWriteFiles`:
-   - Spark 3.3.2 Databricks (`332db`)
    - Spark 3.4.0 (`340`)
    - Spark 3.4.1 (`341`)
    - Spark 3.4.1 Databricks (`341db`)
@@ -138,6 +137,7 @@ restored data to console.
    - Spark 3.5.6 (`356`)
    - Spark 3.5.7 (`357`)
    - Spark 4.0.0 (`400`)
+   - Spark 4.0.0 Databricks 17.3 (`400db173`)
    - Spark 4.0.1 (`401`)
 
    When attempting to dump a `GpuDataWritingCommandExec` on these versions, LORE will throw an
diff --git a/docs/download.md b/docs/download.md
@@ -44,9 +44,9 @@ The plugin is designed to work on NVIDIA Volta, Turing, Ampere, Ada Lovelace, Ho
         Scala 2.13: Spark 3.5.0+, 4.0.0, 4.0.1, 4.0.2, 4.1.1
 	
 	Supported Databricks runtime versions for Azure and AWS:
-		Databricks 12.2 ML LTS (GPU, Scala 2.12, Spark 3.3.2)
 		Databricks 13.3 ML LTS (GPU, Scala 2.12, Spark 3.4.1)
 		Databricks 14.3 ML LTS (GPU, Scala 2.12, Spark 3.5.0)
+		Databricks 17.3 ML LTS (GPU, Scala 2.13, Spark 4.0.0)
 	
 	Supported Dataproc versions (Debian/Ubuntu/Rocky):
 		GCP Dataproc 2.1
diff --git a/jenkins/Jenkinsfile-blossom.premerge-databricks b/jenkins/Jenkinsfile-blossom.premerge-databricks
@@ -93,7 +93,7 @@ pipeline {
                         // 'name' and 'value' only supprt literal string in the declarative Jenkins
                         // Refer to Jenkins issue https://issues.jenkins.io/browse/JENKINS-62127
                         name 'DB_RUNTIME'
-                        values '12.2', '13.3', '14.3', '17.3'
+                        values '13.3', '14.3', '17.3'
                     }
                 }
                 stages {
@@ -184,16 +184,13 @@ void databricksBuild() {
             }
         }
 
-        // TODO: Temporarily skip tests on Databricks 17.3 until the test failures are fixed
-        if (env.DB_RUNTIME != '17.3') {
-            stage("Test agaist $SPARK_MAJOR DB") {
-                dbStep = 'TEST'
-                withCredentials([file(credentialsId: 'SPARK_DATABRICKS_PRIVKEY', variable: 'DATABRICKS_PRIVKEY')]) {
-                    def TEST_PARAMS = DbUtils.getParameters(this, dbStep, "-c $CLUSTER_ID")
-                    // Get built tar from the path "$testArgs" on Databricks cluster
-                    // Refer to https://github.com/NVIDIA/spark-rapids/pull/11788/files#diff-db28879431d57d0e454a2c7ee89fdda9abdec463c61771333d6a6565bf96c062R52-R55
-                    sh "python3 ./jenkins/databricks/run-tests.py $TEST_PARAMS $testArgs"
-                }
+        stage("Test agaist $SPARK_MAJOR DB") {
+            dbStep = 'TEST'
+            withCredentials([file(credentialsId: 'SPARK_DATABRICKS_PRIVKEY', variable: 'DATABRICKS_PRIVKEY')]) {
+                def TEST_PARAMS = DbUtils.getParameters(this, dbStep, "-c $CLUSTER_ID")
+                // Get built tar from the path "$testArgs" on Databricks cluster
+                // Refer to https://github.com/NVIDIA/spark-rapids/pull/11788/files#diff-db28879431d57d0e454a2c7ee89fdda9abdec463c61771333d6a6565bf96c062R52-R55
+                sh "python3 ./jenkins/databricks/run-tests.py $TEST_PARAMS $testArgs"
             }
         }
     } finally {
diff --git a/pom.xml b/pom.xml
@@ -332,29 +332,6 @@
 
         <!-- Note Databricks requires 2 properties -Ddatabricks and -Dbuildver=3*db -->
         <!-- #if scala-2.12 -->
-        <profile>
-            <id>release332db</id>
-            <activation>
-                <property>
-                    <name>buildver</name>
-                    <value>332db</value>
-                </property>
-            </activation>
-            <properties>
-                <spark.version.classifier>spark332db</spark.version.classifier>
-                <spark.version>${spark332db.version}</spark.version>
-                <spark.test.version>${spark332db.version}</spark.test.version>
-                <hadoop.client.version>3.3.1</hadoop.client.version>
-                <rat.consoleOutput>true</rat.consoleOutput>
-                <parquet.hadoop.version>1.12.0</parquet.hadoop.version>
-                <rapids.delta.artifactId1>rapids-4-spark-delta-${spark.version.classifier}</rapids.delta.artifactId1>
-            </properties>
-            <modules>
-                <module>shim-deps/databricks</module>
-                <module>delta-lake/delta-spark332db</module>
-                <module>iceberg/iceberg-stub</module>
-            </modules>
-        </profile>
         <profile>
             <id>release341db</id>
             <activation>
@@ -907,7 +884,6 @@
         <spark342.version>3.4.2</spark342.version>
         <spark343.version>3.4.3</spark343.version>
         <spark344.version>3.4.4</spark344.version>
-        <spark332db.version>3.3.2-databricks</spark332db.version>
         <spark341db.version>3.4.1-databricks</spark341db.version>
         <spark350db143.version>3.5.0-databricks-143</spark350db143.version>
         <spark350.version>3.5.0</spark350.version>
diff --git a/scala2.13/pom.xml b/scala2.13/pom.xml
@@ -332,29 +332,6 @@
 
         <!-- Note Databricks requires 2 properties -Ddatabricks and -Dbuildver=3*db -->
         <!-- #if scala-2.12 --><!--
-        <profile>
-            <id>release332db</id>
-            <activation>
-                <property>
-                    <name>buildver</name>
-                    <value>332db</value>
-                </property>
-            </activation>
-            <properties>
-                <spark.version.classifier>spark332db</spark.version.classifier>
-                <spark.version>${spark332db.version}</spark.version>
-                <spark.test.version>${spark332db.version}</spark.test.version>
-                <hadoop.client.version>3.3.1</hadoop.client.version>
-                <rat.consoleOutput>true</rat.consoleOutput>
-                <parquet.hadoop.version>1.12.0</parquet.hadoop.version>
-                <rapids.delta.artifactId1>rapids-4-spark-delta-${spark.version.classifier}</rapids.delta.artifactId1>
-            </properties>
-            <modules>
-                <module>shim-deps/databricks</module>
-                <module>delta-lake/delta-spark332db</module>
-                <module>iceberg/iceberg-stub</module>
-            </modules>
-        </profile>
         <profile>
             <id>release341db</id>
             <activation>
@@ -907,7 +884,6 @@
         <spark342.version>3.4.2</spark342.version>
         <spark343.version>3.4.3</spark343.version>
         <spark344.version>3.4.4</spark344.version>
-        <spark332db.version>3.3.2-databricks</spark332db.version>
         <spark341db.version>3.4.1-databricks</spark341db.version>
         <spark350db143.version>3.5.0-databricks-143</spark350db143.version>
         <spark350.version>3.5.0</spark350.version>
diff --git a/scala2.13/shim-deps/pom.xml b/scala2.13/shim-deps/pom.xml
@@ -101,23 +101,6 @@
                 </dependency>
             </dependencies>
         </profile>
-        <profile>
-            <id>release332db</id>
-            <activation>
-                <property>
-                    <name>buildver</name>
-                    <value>332db</value>
-                </property>
-            </activation>
-            <dependencies>
-                <dependency>
-                    <groupId>org.apache.logging.log4j</groupId>
-                    <artifactId>log4j-core</artifactId>
-                    <version>${spark.version}</version>
-                    <scope>provided</scope>
-                </dependency>
-            </dependencies>
-        </profile>
         <profile>
             <id>release341db</id>
             <activation>
diff --git a/shim-deps/pom.xml b/shim-deps/pom.xml
@@ -101,23 +101,6 @@
                 </dependency>
             </dependencies>
         </profile>
-        <profile>
-            <id>release332db</id>
-            <activation>
-                <property>
-                    <name>buildver</name>
-                    <value>332db</value>
-                </property>
-            </activation>
-            <dependencies>
-                <dependency>
-                    <groupId>org.apache.logging.log4j</groupId>
-                    <artifactId>log4j-core</artifactId>
-                    <version>${spark.version}</version>
-                    <scope>provided</scope>
-                </dependency>
-            </dependencies>
-        </profile>
         <profile>
             <id>release341db</id>
             <activation>

Original file line number	Diff line number	Diff line change
`@@ -93,7 +93,7 @@ pipeline {`
`93`	`93`	`// 'name' and 'value' only supprt literal string in the declarative Jenkins`
`94`	`94`	`// Refer to Jenkins issue https://issues.jenkins.io/browse/JENKINS-62127`
`95`	`95`	`name 'DB_RUNTIME'`
`96`		`- values '12.2', '13.3', '14.3', '17.3'`
	`96`	`+ values '13.3', '14.3', '17.3'`
`97`	`97`	`}`
`98`	`98`	`}`
`99`	`99`	`stages {`
`@@ -184,16 +184,13 @@ void databricksBuild() {`
`184`	`184`	`}`
`185`	`185`	`}`
`186`	`186`
`187`		`- // TODO: Temporarily skip tests on Databricks 17.3 until the test failures are fixed`
`188`		`- if (env.DB_RUNTIME != '17.3') {`
`189`		`- stage("Test agaist $SPARK_MAJOR DB") {`
`190`		`- dbStep = 'TEST'`
`191`		`- withCredentials([file(credentialsId: 'SPARK_DATABRICKS_PRIVKEY', variable: 'DATABRICKS_PRIVKEY')]) {`
`192`		`- def TEST_PARAMS = DbUtils.getParameters(this, dbStep, "-c $CLUSTER_ID")`
`193`		`- // Get built tar from the path "$testArgs" on Databricks cluster`
`194`		`- // Refer to https://github.com/NVIDIA/spark-rapids/pull/11788/files#diff-db28879431d57d0e454a2c7ee89fdda9abdec463c61771333d6a6565bf96c062R52-R55`
`195`		`- sh "python3 ./jenkins/databricks/run-tests.py $TEST_PARAMS $testArgs"`
`196`		`- }`
	`187`	`+ stage("Test agaist $SPARK_MAJOR DB") {`
	`188`	`+ dbStep = 'TEST'`
	`189`	`+ withCredentials([file(credentialsId: 'SPARK_DATABRICKS_PRIVKEY', variable: 'DATABRICKS_PRIVKEY')]) {`
	`190`	`+ def TEST_PARAMS = DbUtils.getParameters(this, dbStep, "-c $CLUSTER_ID")`
	`191`	`+ // Get built tar from the path "$testArgs" on Databricks cluster`
	`192`	`+ // Refer to https://github.com/NVIDIA/spark-rapids/pull/11788/files#diff-db28879431d57d0e454a2c7ee89fdda9abdec463c61771333d6a6565bf96c062R52-R55`
	`193`	`+ sh "python3 ./jenkins/databricks/run-tests.py $TEST_PARAMS $testArgs"`
`197`	`194`	`}`
`198`	`195`	`}`
`199`	`196`	`} finally {`