Skip to content

Commit 22e9741

Browse files
Matt711Matt711
authored
Validate PDS-DS Queries 53-57 (#21633)
All queries validated at SF1. Contributes to #19200 ```bash $ python python/cudf_polars/cudf_polars/experimental/benchmarks/pdsds.py --root ~/cudf/tpcds_decimal --scale 1.0 --executor streaming --no-print-results --validate --baseline duckdb --runtime rapidsmpf --native-parquet 53-57 ✅ Query 53 - Iteration 0 finished in 1.1211s ✅ Query 54 - Iteration 0 finished in 0.3374s ✅ Query 55 - Iteration 0 finished in 0.1473s ✅ Query 56 - Iteration 0 finished in 0.4598s ✅ Query 57 - Iteration 0 finished in 0.2935s ``` Authors: - Matthew Murray (https://github.com/Matt711) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: #21633
1 parent fefda52 commit 22e9741

6 files changed

Lines changed: 148 additions & 95 deletions

File tree

‎python/cudf_polars/cudf_polars/experimental/benchmarks/pdsds.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,14 @@ class PDSDSPolarsQueries(PDSDSQueries):
137137
pl.col("store_cumulative").cast(pl.Decimal(18, 2)),
138138
],
139139
52: [pl.col("ext_price").cast(pl.Decimal(18, 2))],
140+
53: [pl.col("sum_sales").cast(pl.Decimal(18, 2))],
141+
55: [pl.col("ext_price").cast(pl.Decimal(18, 2))],
142+
56: [pl.col("total_sales").cast(pl.Decimal(18, 2))],
143+
57: [
144+
pl.col("sum_sales").cast(pl.Decimal(18, 2)),
145+
pl.col("psum").cast(pl.Decimal(18, 2)),
146+
pl.col("nsum").cast(pl.Decimal(18, 2)),
147+
],
140148
63: [pl.col("sum_sales").cast(pl.Decimal(18, 2))],
141149
64: [
142150
pl.col("s1").cast(pl.Decimal(18, 2)),
@@ -195,6 +203,11 @@ class PDSDSPolarsQueries(PDSDSQueries):
195203
pl.col("91-120 days").cast(COUNT_DTYPE),
196204
pl.col(">120 days").cast(COUNT_DTYPE),
197205
],
206+
54: [
207+
pl.col("segment").cast(pl.Float64),
208+
pl.col("segment_base").cast(pl.Float64),
209+
pl.col("num_customers").cast(COUNT_DTYPE),
210+
],
198211
64: [
199212
pl.col("cnt").cast(COUNT_DTYPE),
200213
pl.col("cnt_1").cast(COUNT_DTYPE),

‎python/cudf_polars/cudf_polars/experimental/benchmarks/pdsds_queries/q53.py‎

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
import polars as pl
1111

1212
from cudf_polars.experimental.benchmarks.pdsds_parameters import load_parameters
13-
from cudf_polars.experimental.benchmarks.utils import get_data
13+
from cudf_polars.experimental.benchmarks.utils import QueryResult, get_data
1414

1515
if TYPE_CHECKING:
1616
from cudf_polars.experimental.benchmarks.utils import RunConfig
@@ -79,7 +79,7 @@ def duckdb_impl(run_config: RunConfig) -> str:
7979
"""
8080

8181

82-
def polars_impl(run_config: RunConfig) -> pl.LazyFrame:
82+
def polars_impl(run_config: RunConfig) -> QueryResult:
8383
"""Query 53."""
8484
params = load_parameters(
8585
int(run_config.scale_factor),
@@ -150,22 +150,24 @@ def polars_impl(run_config: RunConfig) -> pl.LazyFrame:
150150
"avg_quarterly_sales",
151151
]
152152
)
153-
return (
154-
inner_query.filter(
155-
# Percentage deviation > 10%
156-
pl.when(pl.col("avg_quarterly_sales") > 0)
157-
.then(
158-
(pl.col("sum_sales") - pl.col("avg_quarterly_sales")).abs()
159-
/ pl.col("avg_quarterly_sales")
153+
sort_by = {"avg_quarterly_sales": False, "sum_sales": False, "i_manufact_id": False}
154+
limit = 100
155+
return QueryResult(
156+
frame=(
157+
inner_query.filter(
158+
# Percentage deviation > 10%
159+
pl.when(pl.col("avg_quarterly_sales") > 0)
160+
.then(
161+
(pl.col("sum_sales") - pl.col("avg_quarterly_sales")).abs()
162+
/ pl.col("avg_quarterly_sales")
163+
)
164+
.otherwise(None)
165+
> 0.1
160166
)
161-
.otherwise(None)
162-
> 0.1
163-
)
164-
.select(["i_manufact_id", "sum_sales", "avg_quarterly_sales"])
165-
.sort(
166-
["avg_quarterly_sales", "sum_sales", "i_manufact_id"],
167-
nulls_last=True,
168-
descending=[False, False, False],
169-
)
170-
.limit(100)
167+
.select(["i_manufact_id", "sum_sales", "avg_quarterly_sales"])
168+
.sort(sort_by.keys(), nulls_last=True)
169+
.limit(limit)
170+
),
171+
sort_by=list(sort_by.items()),
172+
limit=limit,
171173
)

‎python/cudf_polars/cudf_polars/experimental/benchmarks/pdsds_queries/q54.py‎

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
import polars as pl
1111

1212
from cudf_polars.experimental.benchmarks.pdsds_parameters import load_parameters
13-
from cudf_polars.experimental.benchmarks.utils import get_data
13+
from cudf_polars.experimental.benchmarks.utils import QueryResult, get_data
1414

1515
if TYPE_CHECKING:
1616
from cudf_polars.experimental.benchmarks.utils import RunConfig
@@ -89,7 +89,7 @@ def duckdb_impl(run_config: RunConfig) -> str:
8989
"""
9090

9191

92-
def polars_impl(run_config: RunConfig) -> pl.LazyFrame:
92+
def polars_impl(run_config: RunConfig) -> QueryResult:
9393
"""Query 54."""
9494
params = load_parameters(
9595
int(run_config.scale_factor),
@@ -173,24 +173,26 @@ def polars_impl(run_config: RunConfig) -> pl.LazyFrame:
173173
)
174174

175175
segments = my_revenue.with_columns(
176-
(pl.col("revenue") / 50.0).cast(pl.Int32).alias("segment")
176+
(pl.col("revenue") / 50.0).alias("segment")
177177
).select("segment")
178178

179-
return (
180-
segments.group_by("segment")
181-
.agg([pl.len().alias("num_customers")])
182-
.with_columns((pl.col("segment") * 50).alias("segment_base"))
183-
.select(
184-
[
185-
"segment",
186-
pl.col("num_customers").cast(pl.Int64),
187-
"segment_base",
188-
]
189-
)
190-
.sort(
191-
["segment", "num_customers"],
192-
nulls_last=True,
193-
descending=[False, False],
194-
)
195-
.limit(100)
179+
sort_by = {"segment": False, "num_customers": False}
180+
limit = 100
181+
return QueryResult(
182+
frame=(
183+
segments.group_by("segment")
184+
.agg([pl.len().alias("num_customers")])
185+
.with_columns((pl.col("segment") * 50.0).alias("segment_base"))
186+
.select(
187+
[
188+
"segment",
189+
pl.col("num_customers"),
190+
"segment_base",
191+
]
192+
)
193+
.sort(sort_by.keys(), nulls_last=True)
194+
.limit(limit)
195+
),
196+
sort_by=list(sort_by.items()),
197+
limit=limit,
196198
)

‎python/cudf_polars/cudf_polars/experimental/benchmarks/pdsds_queries/q55.py‎

Lines changed: 31 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
import polars as pl
1111

1212
from cudf_polars.experimental.benchmarks.pdsds_parameters import load_parameters
13-
from cudf_polars.experimental.benchmarks.utils import get_data
13+
from cudf_polars.experimental.benchmarks.utils import QueryResult, get_data
1414

1515
if TYPE_CHECKING:
1616
from cudf_polars.experimental.benchmarks.utils import RunConfig
@@ -48,7 +48,7 @@ def duckdb_impl(run_config: RunConfig) -> str:
4848
"""
4949

5050

51-
def polars_impl(run_config: RunConfig) -> pl.LazyFrame:
51+
def polars_impl(run_config: RunConfig) -> QueryResult:
5252
"""Query 55."""
5353
params = load_parameters(
5454
int(run_config.scale_factor),
@@ -63,23 +63,33 @@ def polars_impl(run_config: RunConfig) -> pl.LazyFrame:
6363
date_dim = get_data(run_config.dataset_path, "date_dim", run_config.suffix)
6464
store_sales = get_data(run_config.dataset_path, "store_sales", run_config.suffix)
6565
item = get_data(run_config.dataset_path, "item", run_config.suffix)
66-
return (
67-
store_sales.join(date_dim, left_on="ss_sold_date_sk", right_on="d_date_sk")
68-
.join(item, left_on="ss_item_sk", right_on="i_item_sk")
69-
.filter(
70-
(pl.col("i_manager_id") == manager_id)
71-
& (pl.col("d_moy") == month)
72-
& (pl.col("d_year") == year)
73-
)
74-
.group_by(["i_brand", "i_brand_id"])
75-
.agg(pl.col("ss_ext_sales_price").sum().alias("ext_price"))
76-
.select(
77-
[
78-
pl.col("i_brand_id").alias("brand_id"),
79-
pl.col("i_brand").alias("brand"),
80-
pl.col("ext_price"),
81-
]
82-
)
83-
.sort(["ext_price", "brand_id"], descending=[True, False], nulls_last=True)
84-
.limit(100)
66+
sort_by = {"ext_price": True, "brand_id": False}
67+
limit = 100
68+
return QueryResult(
69+
frame=(
70+
store_sales.join(date_dim, left_on="ss_sold_date_sk", right_on="d_date_sk")
71+
.join(item, left_on="ss_item_sk", right_on="i_item_sk")
72+
.filter(
73+
(pl.col("i_manager_id") == manager_id)
74+
& (pl.col("d_moy") == month)
75+
& (pl.col("d_year") == year)
76+
)
77+
.group_by(["i_brand", "i_brand_id"])
78+
.agg(pl.col("ss_ext_sales_price").sum().alias("ext_price"))
79+
.select(
80+
[
81+
pl.col("i_brand_id").alias("brand_id"),
82+
pl.col("i_brand").alias("brand"),
83+
pl.col("ext_price"),
84+
]
85+
)
86+
.sort(
87+
list(sort_by.keys()),
88+
descending=list(sort_by.values()),
89+
nulls_last=True,
90+
)
91+
.limit(limit)
92+
),
93+
sort_by=list(sort_by.items()),
94+
limit=limit,
8595
)

‎python/cudf_polars/cudf_polars/experimental/benchmarks/pdsds_queries/q56.py‎

Lines changed: 32 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
import polars as pl
1111

1212
from cudf_polars.experimental.benchmarks.pdsds_parameters import load_parameters
13-
from cudf_polars.experimental.benchmarks.utils import get_data
13+
from cudf_polars.experimental.benchmarks.utils import QueryResult, get_data
1414

1515
if TYPE_CHECKING:
1616
from cudf_polars.experimental.benchmarks.utils import RunConfig
@@ -102,7 +102,7 @@ def duckdb_impl(run_config: RunConfig) -> str:
102102
"""
103103

104104

105-
def polars_impl(run_config: RunConfig) -> pl.LazyFrame:
105+
def polars_impl(run_config: RunConfig) -> QueryResult:
106106
"""Query 56."""
107107
params = load_parameters(
108108
int(run_config.scale_factor),
@@ -126,8 +126,8 @@ def polars_impl(run_config: RunConfig) -> pl.LazyFrame:
126126
)
127127
item = get_data(run_config.dataset_path, "item", run_config.suffix)
128128

129-
color_item_ids_lf = item.filter(pl.col("i_color").is_in(colors)).select(
130-
["i_item_id"]
129+
color_item_ids_lf = (
130+
item.filter(pl.col("i_color").is_in(colors)).select(["i_item_id"]).unique()
131131
)
132132

133133
channels = [
@@ -167,17 +167,37 @@ def polars_impl(run_config: RunConfig) -> pl.LazyFrame:
167167
& (pl.col("ca_gmt_offset") == gmt_offset)
168168
)
169169
.group_by("i_item_id")
170-
.agg(pl.col(str(ch["ext_col"])).sum().alias("total_sales"))
170+
.agg(
171+
# Polars sum() returns 0 for all-null groups; SQL returns NULL.
172+
# See https://github.com/rapidsai/cudf/issues/19560.
173+
pl.when(pl.col(str(ch["ext_col"])).count() > 0)
174+
.then(pl.col(str(ch["ext_col"])).sum())
175+
.otherwise(None)
176+
.alias("total_sales")
177+
)
171178
.select(["i_item_id", "total_sales"])
172179
)
173180
for ch in channels
174181
]
175182

176-
return (
177-
pl.concat(per_channel)
178-
.group_by("i_item_id")
179-
.agg(pl.col("total_sales").sum().alias("total_sales"))
180-
.select(["i_item_id", "total_sales"])
181-
.sort(["total_sales"], nulls_last=True, descending=[False])
182-
.limit(100)
183+
sort_by = {"total_sales": False}
184+
limit = 100
185+
return QueryResult(
186+
frame=(
187+
pl.concat(per_channel)
188+
.group_by("i_item_id")
189+
.agg(
190+
# Polars sum() returns 0 for all-null groups; SQL returns NULL.
191+
# See https://github.com/rapidsai/cudf/issues/19560.
192+
pl.when(pl.col("total_sales").count() > 0)
193+
.then(pl.col("total_sales").sum())
194+
.otherwise(None)
195+
.alias("total_sales")
196+
)
197+
.select(["i_item_id", "total_sales"])
198+
.sort(sort_by.keys(), nulls_last=True)
199+
.limit(limit)
200+
),
201+
sort_by=list(sort_by.items()),
202+
limit=limit,
183203
)

‎python/cudf_polars/cudf_polars/experimental/benchmarks/pdsds_queries/q57.py‎

Lines changed: 29 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
import polars as pl
1111

1212
from cudf_polars.experimental.benchmarks.pdsds_parameters import load_parameters
13-
from cudf_polars.experimental.benchmarks.utils import get_data
13+
from cudf_polars.experimental.benchmarks.utils import QueryResult, get_data
1414

1515
if TYPE_CHECKING:
1616
from cudf_polars.experimental.benchmarks.utils import RunConfig
@@ -95,7 +95,7 @@ def duckdb_impl(run_config: RunConfig) -> str:
9595
"""
9696

9797

98-
def polars_impl(run_config: RunConfig) -> pl.LazyFrame:
98+
def polars_impl(run_config: RunConfig) -> QueryResult:
9999
"""Query 57."""
100100
params = load_parameters(
101101
int(run_config.scale_factor),
@@ -185,27 +185,33 @@ def polars_impl(run_config: RunConfig) -> pl.LazyFrame:
185185
.select(["i_brand", "d_year", "avg_monthly_sales", "sum_sales", "psum", "nsum"])
186186
)
187187

188-
return (
189-
v2.filter(
190-
(pl.col("d_year") == year)
191-
& (pl.col("avg_monthly_sales") > 0)
192-
& (
193-
pl.when(pl.col("avg_monthly_sales") > 0)
194-
.then(
195-
(pl.col("sum_sales") - pl.col("avg_monthly_sales")).abs()
196-
/ pl.col("avg_monthly_sales")
188+
sort_by = {"avg_monthly_sales": False}
189+
limit = 100
190+
return QueryResult(
191+
frame=(
192+
v2.filter(
193+
(pl.col("d_year") == year)
194+
& (pl.col("avg_monthly_sales") > 0)
195+
& (
196+
pl.when(pl.col("avg_monthly_sales") > 0)
197+
.then(
198+
(pl.col("sum_sales") - pl.col("avg_monthly_sales")).abs()
199+
/ pl.col("avg_monthly_sales")
200+
)
201+
.otherwise(None)
202+
> 0.1
197203
)
198-
.otherwise(None)
199-
> 0.1
200204
)
201-
)
202-
.sort(
203-
by=[
204-
pl.col("sum_sales") - pl.col("avg_monthly_sales"),
205-
pl.col("avg_monthly_sales"),
206-
],
207-
descending=[False, False],
208-
nulls_last=True,
209-
)
210-
.limit(100)
205+
.sort(
206+
by=[
207+
pl.col("sum_sales") - pl.col("avg_monthly_sales"),
208+
pl.col("avg_monthly_sales"),
209+
],
210+
descending=[False, False],
211+
nulls_last=True,
212+
)
213+
.limit(limit)
214+
),
215+
sort_by=list(sort_by.items()),
216+
limit=limit,
211217
)

0 commit comments

Comments
 (0)