@@ -198,6 +198,7 @@ def write_data(spark):
198198 "try_variant_get(v, '$.smin', 'bigint') AS smin_as_bigint" ,
199199 "try_variant_get(v, '$.smax', 'smallint') AS smax" ,
200200 "try_variant_get(v, '$.imin', 'int') AS imin" ,
201+ "try_variant_get(v, '$.imin', 'bigint') AS imin_as_bigint" ,
201202 "try_variant_get(v, '$.imax', 'int') AS imax" ,
202203 "try_variant_get(v, '$.lmin', 'bigint') AS lmin" ,
203204 "try_variant_get(v, '$.lmax', 'bigint') AS lmax" ,
@@ -207,6 +208,23 @@ def write_data(spark):
207208 conf = _variant_parquet_conf )
208209
209210
211+ @incompat
212+ @pytest .mark .skipif (is_before_spark_400 (), reason = 'VariantType is available in Spark 4.0+' )
213+ def test_parquet_variant_pass_through_filter_project (spark_tmp_path ):
214+ data_path = spark_tmp_path + '/VARIANT_PASS_THROUGH_PARQUET'
215+ with_cpu_session (lambda spark : _write_variant_parquet (spark , data_path ))
216+
217+ assert_gpu_and_cpu_are_equal_collect (
218+ lambda spark : spark .read .parquet (data_path )
219+ .selectExpr ("v" , "try_variant_get(v, '$.x', 'int') AS x" )
220+ .filter ("x IS NULL OR x >= 7" )
221+ .selectExpr (
222+ "try_variant_get(v, '$.n.inner', 'string') AS inner" ,
223+ "try_variant_get(v, '$.m', 'bigint') AS m" )
224+ .orderBy ("inner" , "m" ),
225+ conf = _variant_parquet_conf )
226+
227+
210228@incompat
211229@pytest .mark .skipif (is_before_spark_400 (), reason = 'VariantType is available in Spark 4.0+' )
212230def test_parquet_variant_try_get_before_shuffle (spark_tmp_path ):
0 commit comments