Skip to content

Commit a1c47cb

Browse files
committed
Add variant extraction review coverage
Signed-off-by: Niranjan Artal <nartal@nvidia.com>
1 parent 4646e6a commit a1c47cb

3 files changed

Lines changed: 20 additions & 2 deletions

File tree

integration_tests/src/main/python/variant_test.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,7 @@ def write_data(spark):
198198
"try_variant_get(v, '$.smin', 'bigint') AS smin_as_bigint",
199199
"try_variant_get(v, '$.smax', 'smallint') AS smax",
200200
"try_variant_get(v, '$.imin', 'int') AS imin",
201+
"try_variant_get(v, '$.imin', 'bigint') AS imin_as_bigint",
201202
"try_variant_get(v, '$.imax', 'int') AS imax",
202203
"try_variant_get(v, '$.lmin', 'bigint') AS lmin",
203204
"try_variant_get(v, '$.lmax', 'bigint') AS lmax",
@@ -207,6 +208,23 @@ def write_data(spark):
207208
conf=_variant_parquet_conf)
208209

209210

211+
@incompat
212+
@pytest.mark.skipif(is_before_spark_400(), reason='VariantType is available in Spark 4.0+')
213+
def test_parquet_variant_pass_through_filter_project(spark_tmp_path):
214+
data_path = spark_tmp_path + '/VARIANT_PASS_THROUGH_PARQUET'
215+
with_cpu_session(lambda spark: _write_variant_parquet(spark, data_path))
216+
217+
assert_gpu_and_cpu_are_equal_collect(
218+
lambda spark: spark.read.parquet(data_path)
219+
.selectExpr("v", "try_variant_get(v, '$.x', 'int') AS x")
220+
.filter("x IS NULL OR x >= 7")
221+
.selectExpr(
222+
"try_variant_get(v, '$.n.inner', 'string') AS inner",
223+
"try_variant_get(v, '$.m', 'bigint') AS m")
224+
.orderBy("inner", "m"),
225+
conf=_variant_parquet_conf)
226+
227+
210228
@incompat
211229
@pytest.mark.skipif(is_before_spark_400(), reason='VariantType is available in Spark 4.0+')
212230
def test_parquet_variant_try_get_before_shuffle(spark_tmp_path):

sql-plugin/src/main/java/com/nvidia/spark/rapids/RapidsHostColumnVectorCore.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11

22
/*
3-
* Copyright (c) 2020-2021, NVIDIA CORPORATION.
3+
* Copyright (c) 2020-2026, NVIDIA CORPORATION.
44
*
55
* Licensed under the Apache License, Version 2.0 (the "License");
66
* you may not use this file except in compliance with the License.

sql-plugin/src/main/scala/com/nvidia/spark/rapids/literals.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2019-2025, NVIDIA CORPORATION.
2+
* Copyright (c) 2019-2026, NVIDIA CORPORATION.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.

0 commit comments

Comments
 (0)