Skip to content

Commit e52f576

Browse files
committed
skip test for databricks; fixup offset for synthetic block as well
1 parent 6ebc7e8 commit e52f576

2 files changed

Lines changed: 9 additions & 2 deletions

File tree

integration_tests/src/main/python/delta_lake_test.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -677,8 +677,10 @@ def read_table(spark):
677677
@ignore_order(local=True)
678678
@pytest.mark.parametrize("parquet_reader_type", ["PERFILE", "COALESCING", "MULTITHREADED"], ids=idfn)
679679
@pytest.mark.parametrize("footer_type", ["NATIVE", "JAVA"], ids=idfn)
680-
@pytest.mark.skipif(not supports_delta_lake_deletion_vectors(),
681-
reason="Delta Lake deletion vector support is required")
680+
@pytest.mark.skipif(is_before_spark_353(),
681+
reason="Spark-RAPIDS supports scan with deletion vectors starting in Spark 3.5.3")
682+
@pytest.mark.skipif(is_databricks_runtime(),
683+
reason="Deletion vector scan is not supported on Databricks")
682684
def test_delta_deletion_vector_native_footer_multi_row_group(spark_tmp_path, parquet_reader_type, footer_type):
683685
"""
684686
Tests deletion vector filtering on a Delta table whose single Parquet file has multiple

sql-plugin/src/main/scala/com/nvidia/spark/rapids/parquet/GpuParquetScan.scala

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -727,6 +727,11 @@ protected case class GpuParquetFileFilterHandler(
727727
val numRows = tableFooter.getNumRows
728728
val block = new BlockMetaData()
729729
block.setRowCount(numRows)
730+
// Fix up the row index offset
731+
val offsets = tableFooter.getRowIndexOffsets
732+
if (offsets.nonEmpty) {
733+
block.setRowIndexOffset(offsets(0))
734+
}
730735
val schema = new MessageType("root")
731736
return ParquetFileInfoWithBlockMeta(filePath, Seq(block), file.partitionValues,
732737
schema, readDataSchema, DateTimeRebaseLegacy, DateTimeRebaseLegacy,

0 commit comments

Comments
 (0)