Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion integration_tests/src/main/python/csv_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -719,4 +719,4 @@ def test_csv_read_gbk_encoded_data(std_input_path):
.option("header", "true")
.schema("name string, age int, city string, job string")
.csv(std_input_path + "/test_gbk.csv"),
conf={"spark.sql.legacy.javaCharsets": legacy_charset})
conf={"spark.sql.legacy.javaCharsets": legacy_charset})
22 changes: 22 additions & 0 deletions integration_tests/src/main/python/join_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -485,6 +485,28 @@ def do_join(spark):
return t0.crossJoin(t1).limit(21)
assert_gpu_and_cpu_are_equal_collect(do_join)

@allow_non_gpu('CollectLimitExec')
def test_empty_right_outer_side_with_limit(std_input_path):
built_csv_path = std_input_path + '/t1.csv'

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

NIT: Better add comment on why this file will be read as the built batch ?

stream_csv_path = std_input_path + '/t0.csv'

def create_views(spark):
spark.read.csv(built_csv_path, header=True, inferSchema=True).createOrReplaceTempView("built_table")
spark.read.csv(stream_csv_path, header=True, inferSchema=True).createOrReplaceTempView("stream_table")

# create views first on CPU
with_cpu_session(lambda spark: create_views(spark))

# limit to 10 rows to produce `LocalLimitExec` node

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

NIT: Better add comment on why this limit node is needed.

def do_join(spark):
return spark.sql("""
SELECT '1', CAST(CAST(stream_table.c0 AS int) as string)
FROM built_table
RIGHT OUTER JOIN stream_table
ON TRUE limit 10
""")
assert_gpu_and_cpu_are_equal_collect(do_join)

# local sort because of https://github.com/NVIDIA/spark-rapids/issues/84
# After 3.1.0 is the min spark version we can drop this
@ignore_order(local=True)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -643,7 +643,14 @@ abstract class GpuTextBasedPartitionReader[BUFF <: LineBufferer, FACT <: LineBuf
// val cols = (0 until table.getNumberOfColumns).map(i => table.getColumn(i))
// Some(new Table(cols: _*))
// }
Some(table)
if (table.getRowCount == 0) {
// CSV reader can return empty table, close it and return None
// E.g.: CSV file with only header and no data rows, empty table will be returned
table.close()

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

any concerns that this code can throw instead of returning None even if close fails?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The row count is zero, it means there is no GPU memory allocated although table has columns.
It is not likely to throw exceptions in practice.

None
} else {
Some(table)
}
}

override def next(): Boolean = {
Expand Down
Loading