Skip to content

Commit 5aca224

Browse files
Chong Gaoclaude
andcommitted
[TEST] Iceberg: verify FillNull(map) executes for missing nested map
Extends the regression test "Missing nested optional map with required key does not throw" with a process()-level assertion. Action-plan construction succeeding is not the same guarantee as execute() succeeding for FillNull(map<...>); a future regression that builds the plan correctly but breaks execution would have slipped past the original assertion. Builds a ColumnarBatch matching the file schema, runs processor.process, and asserts the output info struct gains a second child (the new props map) that is all-null. Drops the BINARY 'name' field from the file schema to avoid an unrelated cudf-layout mismatch between FuzzerUtils's StringType column and the BINARY->STRING UpCast inserted for that field. The structural bug repro (missing MAP with required key inside a parent struct) is preserved. Verified: 15/15 GpuPostProcessorSuite tests pass under buildver=356. Signed-off-by: Chong Gao <res_life@163.com> Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
1 parent bf959c3 commit 5aca224

1 file changed

Lines changed: 40 additions & 14 deletions

File tree

tests/src/test/spark350/scala/com/nvidia/spark/rapids/iceberg/GpuPostProcessorSuite.scala

Lines changed: 40 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -943,37 +943,33 @@ class GpuPostProcessorSuite extends AnyFunSuite with BeforeAndAfterAll {
943943
// could emit FillNull.
944944
test("Missing nested optional map with required key does not throw") {
945945
val infoStructId = 1
946-
val nameFieldId = 2
947-
val scoreFieldId = 3
948-
val propsMapId = 4 // newly-added MAP, missing from file
949-
val propsKeyId = 5
950-
val propsValueId = 6
951-
952-
// Parquet file: info STRUCT<name: STRING, score: BIGINT> — no props
953-
val nameType =
954-
ShadedTypes.primitive(ShadedPrimitiveTypeName.BINARY, ShadedRepetition.OPTIONAL)
955-
.id(nameFieldId).named("name")
946+
val scoreFieldId = 2
947+
val propsMapId = 3 // newly-added MAP, missing from file
948+
val propsKeyId = 4
949+
val propsValueId = 5
950+
951+
// Parquet file: info STRUCT<score: BIGINT> — no props
956952
val scoreType =
957953
ShadedTypes.primitive(ShadedPrimitiveTypeName.INT64, ShadedRepetition.OPTIONAL)
958954
.id(scoreFieldId).named("score")
959-
val infoStruct = ShadedTypes.optionalGroup().addField(nameType).addField(scoreType)
955+
val infoStruct = ShadedTypes.optionalGroup().addField(scoreType)
960956
.id(infoStructId).named("info")
961957
val parquetSchema = new ShadedMessageType("test",
962958
Seq[ShadedType](infoStruct).asJava)
963959

964-
// Expected: info STRUCT<name, score, props: MAP<STRING, BIGINT>>
960+
// Expected: info STRUCT<score, props: MAP<STRING, BIGINT>>
965961
val expectedSchema = new Schema(
966962
Types.NestedField.optional(infoStructId, "info",
967963
Types.StructType.of(
968-
Types.NestedField.optional(nameFieldId, "name", Types.StringType.get()),
969964
Types.NestedField.optional(scoreFieldId, "score", Types.LongType.get()),
970965
Types.NestedField.optional(propsMapId, "props",
971966
Types.MapType.ofOptional(propsKeyId, propsValueId,
972967
Types.StringType.get(), Types.LongType.get()))
973968
))
974969
)
975970

976-
val (parquetInfo, shadedSchema) = createParquetInfo(parquetSchema)
971+
val rowCount = 3
972+
val (parquetInfo, shadedSchema) = createParquetInfo(parquetSchema, rowCount.toLong)
977973
val processor = new GpuParquetReaderPostProcessor(
978974
parquetInfo,
979975
new JHashMap[Integer, Any](),
@@ -986,6 +982,36 @@ class GpuPostProcessorSuite extends AnyFunSuite with BeforeAndAfterAll {
986982
// for an optional missing field). Children are discarded by the parent.
987983
assert(plan.contains("FillNull(map<"),
988984
s"expected FillNull for missing map in plan:\n$plan")
985+
986+
// Exercise process() too: action-tree construction succeeding is not the
987+
// same guarantee as execute() succeeding for FillNull(map<...>).
988+
import com.nvidia.spark.rapids.{FuzzerUtils, GpuColumnVector, SpillableColumnarBatch}
989+
import com.nvidia.spark.rapids.SpillPriorities
990+
import com.nvidia.spark.rapids.Arm.{closeOnExcept, withResource}
991+
import org.apache.spark.sql.types.{LongType, StructField, StructType => SparkStructType}
992+
993+
val inputSparkSchema = SparkStructType(Array(StructField(
994+
"info",
995+
SparkStructType(Seq(StructField("score", LongType, true))),
996+
true)))
997+
val inputBatch = FuzzerUtils.createColumnarBatch(inputSparkSchema, rowCount, seed = 42)
998+
val spillable = closeOnExcept(inputBatch) { batch =>
999+
SpillableColumnarBatch(batch, SpillPriorities.ACTIVE_ON_DECK_PRIORITY)
1000+
}
1001+
withResource(spillable) { _ =>
1002+
withResource(processor.process(spillable.getColumnarBatch())) { outputBatch =>
1003+
assert(outputBatch.numRows() == rowCount)
1004+
assert(outputBatch.numCols() == 1)
1005+
val infoCol = outputBatch.column(0).asInstanceOf[GpuColumnVector].getBase
1006+
// info struct gains props as a second child.
1007+
assert(infoCol.getNumChildren == 2,
1008+
s"expected info struct to have 2 children, got ${infoCol.getNumChildren}")
1009+
withResource(infoCol.getChildColumnView(1)) { propsCol =>
1010+
assert(propsCol.getNullCount == rowCount,
1011+
s"props should be all-null; nullCount=${propsCol.getNullCount} rowCount=$rowCount")
1012+
}
1013+
}
1014+
}
9891015
}
9901016

9911017
// Same defect, with LIST<required element> as the missing container.

0 commit comments

Comments
 (0)