Skip to content

Commit 55944e5

Browse files
wjxiz1992claude
andcommitted
Fix GpuArrayRemove to fallback for unsupported element types (#14129)
Restrict ArrayRemove TypeSig to exclude BINARY, ARRAY, STRUCT, and MAP element types. cuDF's equalToNullAware does not support these types in binaryOp, causing CudfException at runtime. Unsupported types now correctly fall back to CPU execution. Closes #14129 Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Signed-off-by: Allen Xu <allxu@nvidia.com>
1 parent 5473dfa commit 55944e5

2 files changed

Lines changed: 3 additions & 7 deletions

File tree

sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3059,16 +3059,13 @@ object GpuOverrides extends Logging {
30593059
"Returns the array after removing all elements that equal to the input element (right) " +
30603060
"from the input array (left)",
30613061
ExprChecks.binaryProject(
3062-
TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL +
3063-
TypeSig.ARRAY + TypeSig.STRUCT + TypeSig.MAP),
3062+
TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL),
30643063
TypeSig.ARRAY.nested(TypeSig.all),
30653064
("array",
3066-
TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL +
3067-
TypeSig.ARRAY + TypeSig.STRUCT + TypeSig.MAP),
3065+
TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL),
30683066
TypeSig.all),
30693067
("element",
3070-
(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL +
3071-
TypeSig.ARRAY + TypeSig.STRUCT + TypeSig.MAP).nested(),
3068+
(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL),
30723069
TypeSig.all)),
30733070
(in, conf, p, r) => new BinaryExprMeta[ArrayRemove](in, conf, p, r) {
30743071
override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression =

tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@ class RapidsTestSettings extends BackendTestSettings {
5151
.exclude("Flatten", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/14127"))
5252
.exclude("MapFromEntries", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/14128"))
5353
.exclude("Array Intersect", ADJUST_UT("Replaced by testRapids version that doesn't check the order of the elements in the result array. See https://github.com/NVIDIA/spark-rapids/issues/13696 for more details."))
54-
.exclude("Array remove", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/14129"))
5554
.exclude("Shuffle", ADJUST_UT("Replaced by testRapids version that adjusts the expected results to match the running by --master local[2]."))
5655
enableSuite[RapidsColumnExpressionSuite]
5756
.exclude("input_file_name, input_file_block_start, input_file_block_length - HadoopRDD", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/14153"))

0 commit comments

Comments
 (0)