Skip to content

Commit 3129c8c

Browse files
firestarmanfirestarman
authored
Fix named struct dataType null field names [databricks] (#15358)
Fixes #15316. ### Description - Fix `GpuCreateNamedStruct.dataType` to match Spark behavior for null field names, so Spark versions with SPARK-57736 no longer hit an NPE before type checking rejects the invalid input. - Add `CreateNamedStructShims` so legacy Spark shims keep the old `name.toString` behavior, while Spark 4.0.4, 4.1.3, and 4.2.0 use Spark's null-safe field-name behavior. - Add shim-specific Scala tests for both legacy and fixed behavior, covering null field names in `GpuCreateNamedStruct.dataType` and the expected input type-check failure. - Validated with focused Scala tests: `mvn -s /home/liangcail/.m2/settings_art.xml -f scala2.13/pom.xml -pl sql-plugin -Dbuildver=358 -Dcuda.version=cuda13 -DwildcardSuites=org.apache.spark.sql.rapids.GpuCreateNamedStructSuite test`, plus the same command for `buildver=403`, `404`, `412`, `413`, and `420`. ### Checklists Documentation - [ ] Updated for new or modified user-facing features or behaviors - [x] No user-facing change Testing - [x] Added or modified tests to cover new code paths - [ ] Covered by existing tests (Please provide the names of the existing tests in the PR description.) - [ ] Not required Performance - [ ] Tests ran and results are added in the PR description - [ ] Issue filed with a link in the PR description - [x] Not required --------- Signed-off-by: Firestarman <firestarmanllc@gmail.com>
1 parent 6dfa312 commit 3129c8c

5 files changed

Lines changed: 190 additions & 1 deletion

File tree

‎sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeCreator.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FUNC_ALIAS
2929
import org.apache.spark.sql.catalyst.expressions.{EmptyRow, Expression, NamedExpression}
3030
import org.apache.spark.sql.catalyst.util.TypeUtils
3131
import org.apache.spark.sql.internal.SQLConf
32+
import org.apache.spark.sql.rapids.shims.CreateNamedStructShims
3233
import org.apache.spark.sql.types.{ArrayType, DataType, MapType, Metadata, NullType, StringType, StructField, StructType}
3334
import org.apache.spark.sql.vectorized.ColumnarBatch
3435

@@ -205,7 +206,7 @@ case class GpuCreateNamedStruct(children: Seq[Expression]) extends GpuExpression
205206
case ne: NamedExpression => ne.metadata
206207
case _ => Metadata.empty
207208
}
208-
StructField(name.toString, expr.dataType, expr.nullable, metadata)
209+
StructField(CreateNamedStructShims.fieldName(name), expr.dataType, expr.nullable, metadata)
209210
}
210211
StructType(fields)
211212
}

‎sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/CreateNamedStructShims.scala‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* Copyright (c) 2026, NVIDIA CORPORATION.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
/*** spark-rapids-shim-json-lines
17+
{"spark": "330"}
18+
{"spark": "330db"}
19+
{"spark": "331"}
20+
{"spark": "332"}
21+
{"spark": "332db"}
22+
{"spark": "333"}
23+
{"spark": "334"}
24+
{"spark": "340"}
25+
{"spark": "341"}
26+
{"spark": "341db"}
27+
{"spark": "342"}
28+
{"spark": "343"}
29+
{"spark": "344"}
30+
{"spark": "350"}
31+
{"spark": "350db143"}
32+
{"spark": "351"}
33+
{"spark": "352"}
34+
{"spark": "353"}
35+
{"spark": "354"}
36+
{"spark": "355"}
37+
{"spark": "356"}
38+
{"spark": "357"}
39+
{"spark": "358"}
40+
{"spark": "400"}
41+
{"spark": "400db173"}
42+
{"spark": "401"}
43+
{"spark": "402"}
44+
{"spark": "403"}
45+
{"spark": "411"}
46+
{"spark": "412"}
47+
spark-rapids-shim-json-lines ***/
48+
49+
package org.apache.spark.sql.rapids.shims
50+
51+
object CreateNamedStructShims {
52+
def fieldName(name: Any): String = name.toString
53+
}

‎sql-plugin/src/main/spark404/scala/org/apache/spark/sql/rapids/shims/CreateNamedStructShims.scala‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Copyright (c) 2026, NVIDIA CORPORATION.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
/*** spark-rapids-shim-json-lines
17+
{"spark": "404"}
18+
{"spark": "413"}
19+
{"spark": "420"}
20+
spark-rapids-shim-json-lines ***/
21+
22+
package org.apache.spark.sql.rapids.shims
23+
24+
object CreateNamedStructShims {
25+
def fieldName(name: Any): String = if (name == null) null else name.toString
26+
}

‎sql-plugin/src/test/spark330/scala/org/apache/spark/sql/rapids/GpuCreateNamedStructSuite.scala‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/*
2+
* Copyright (c) 2026, NVIDIA CORPORATION.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
/*** spark-rapids-shim-json-lines
17+
{"spark": "330"}
18+
{"spark": "330db"}
19+
{"spark": "331"}
20+
{"spark": "332"}
21+
{"spark": "332db"}
22+
{"spark": "333"}
23+
{"spark": "334"}
24+
{"spark": "340"}
25+
{"spark": "341"}
26+
{"spark": "341db"}
27+
{"spark": "342"}
28+
{"spark": "343"}
29+
{"spark": "344"}
30+
{"spark": "350"}
31+
{"spark": "350db143"}
32+
{"spark": "351"}
33+
{"spark": "352"}
34+
{"spark": "353"}
35+
{"spark": "354"}
36+
{"spark": "355"}
37+
{"spark": "356"}
38+
{"spark": "357"}
39+
{"spark": "358"}
40+
{"spark": "400"}
41+
{"spark": "400db173"}
42+
{"spark": "401"}
43+
{"spark": "402"}
44+
{"spark": "403"}
45+
{"spark": "411"}
46+
{"spark": "412"}
47+
spark-rapids-shim-json-lines ***/
48+
49+
package org.apache.spark.sql.rapids
50+
51+
import com.nvidia.spark.rapids.{FQSuiteName, GpuLiteral}
52+
import org.scalatest.funsuite.AnyFunSuite
53+
54+
import org.apache.spark.sql.types.{IntegerType, StringType}
55+
56+
class GpuCreateNamedStructSuite extends AnyFunSuite with FQSuiteName {
57+
test("dataType follows legacy Spark behavior for a null field name") {
58+
val struct = GpuCreateNamedStruct(Seq(
59+
GpuLiteral.create(null, StringType),
60+
GpuLiteral.create(1, IntegerType)))
61+
62+
intercept[NullPointerException] {
63+
struct.dataType
64+
}
65+
assert(struct.checkInputDataTypes().isFailure)
66+
}
67+
}

‎sql-plugin/src/test/spark404/scala/org/apache/spark/sql/rapids/GpuCreateNamedStructSuite.scala‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* Copyright (c) 2026, NVIDIA CORPORATION.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
/*** spark-rapids-shim-json-lines
17+
{"spark": "404"}
18+
{"spark": "413"}
19+
{"spark": "420"}
20+
spark-rapids-shim-json-lines ***/
21+
22+
package org.apache.spark.sql.rapids
23+
24+
import com.nvidia.spark.rapids.{FQSuiteName, GpuLiteral}
25+
import org.scalatest.funsuite.AnyFunSuite
26+
27+
import org.apache.spark.sql.types.{IntegerType, StringType}
28+
29+
class GpuCreateNamedStructSuite extends AnyFunSuite with FQSuiteName {
30+
test("dataType is null-safe for a null field name") {
31+
val struct = GpuCreateNamedStruct(Seq(
32+
GpuLiteral.create(null, StringType),
33+
GpuLiteral.create(1, IntegerType)))
34+
val dataType = struct.dataType
35+
36+
assert(dataType.length === 1)
37+
assert(dataType.head.name === null)
38+
assert(dataType.head.dataType === IntegerType)
39+
assert(dataType.head.nullable === false)
40+
assert(struct.checkInputDataTypes().isFailure)
41+
}
42+
}

0 commit comments

Comments
 (0)