Skip to content

Commit 943816f

Browse files
before-Sunriserobd003
before-Sunrise
authored and
robd003
committed
[BugFix] fix spillable hash join probe crash (StarRocks#72397)
Signed-off-by: before-Sunrise <unclejyj@gmail.com>
1 parent 4c09812 commit 943816f

3 files changed

Lines changed: 156 additions & 1 deletion

File tree

‎be/src/exec/pipeline/hashjoin/spillable_hash_join_build_operator.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,8 @@ StatusOr<std::function<StatusOr<ChunkPtr>()>> SpillableHashJoinBuildOperator::_c
267267
_hash_table_iterate_idx++;
268268
for (; _hash_table_iterate_idx < _hash_tables.size(); _hash_table_iterate_idx++) {
269269
auto build_chunk = _hash_tables[_hash_table_iterate_idx]->get_build_chunk();
270-
if (build_chunk->num_rows() > 0) {
270+
// Every build chunk has a dummy row at index 0. Skip partitions that have no real build rows.
271+
if (build_chunk->num_rows() > kHashJoinKeyColumnOffset) {
271272
_hash_table_build_chunk_slice.reset(build_chunk);
272273
_hash_table_build_chunk_slice.skip(kHashJoinKeyColumnOffset);
273274
break;

‎test/sql/test_spill/R/test_spill_join_with_empty_partition‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
-- name: test_spill_join_with_empty_partition
2+
CREATE TABLE `hj_build` (
3+
`k` bigint(20) NULL COMMENT "",
4+
`c01` bigint(20) NULL COMMENT "",
5+
`c02` bigint(20) NULL COMMENT "",
6+
`c03` bigint(20) NULL COMMENT "",
7+
`c04` bigint(20) NULL COMMENT "",
8+
`c05` bigint(20) NULL COMMENT "",
9+
`c06` bigint(20) NULL COMMENT "",
10+
`c07` bigint(20) NULL COMMENT "",
11+
`c08` bigint(20) NULL COMMENT "",
12+
`c09` bigint(20) NULL COMMENT "",
13+
`c10` bigint(20) NULL COMMENT "",
14+
`c11` bigint(20) NULL COMMENT "",
15+
`c12` bigint(20) NULL COMMENT "",
16+
`c13` bigint(20) NULL COMMENT "",
17+
`c14` bigint(20) NULL COMMENT "",
18+
`c15` bigint(20) NULL COMMENT "",
19+
`c16` bigint(20) NULL COMMENT "",
20+
`c17` bigint(20) NULL COMMENT "",
21+
`c18` bigint(20) NULL COMMENT "",
22+
`c19` bigint(20) NULL COMMENT "",
23+
`c20` bigint(20) NULL COMMENT "",
24+
`c21` bigint(20) NULL COMMENT "",
25+
`c22` bigint(20) NULL COMMENT "",
26+
`c23` bigint(20) NULL COMMENT "",
27+
`c24` bigint(20) NULL COMMENT "",
28+
`c25` bigint(20) NULL COMMENT "",
29+
`c26` bigint(20) NULL COMMENT "",
30+
`c27` bigint(20) NULL COMMENT "",
31+
`c28` bigint(20) NULL COMMENT "",
32+
`c29` bigint(20) NULL COMMENT "",
33+
`c30` bigint(20) NULL COMMENT "",
34+
`c31` bigint(20) NULL COMMENT "",
35+
`c32` bigint(20) NULL COMMENT ""
36+
) ENGINE=OLAP
37+
DUPLICATE KEY(`k`)
38+
DISTRIBUTED BY HASH(`k`) BUCKETS 1
39+
PROPERTIES (
40+
"compression" = "LZ4",
41+
"fast_schema_evolution" = "true",
42+
"replicated_storage" = "true",
43+
"replication_num" = "1"
44+
);
45+
-- result:
46+
-- !result
47+
CREATE TABLE `hj_probe` (
48+
`k` bigint(20) NULL COMMENT ""
49+
) ENGINE=OLAP
50+
DUPLICATE KEY(`k`)
51+
DISTRIBUTED BY HASH(`k`) BUCKETS 1
52+
PROPERTIES (
53+
"compression" = "LZ4",
54+
"fast_schema_evolution" = "true",
55+
"replicated_storage" = "true",
56+
"replication_num" = "1"
57+
);
58+
-- result:
59+
-- !result
60+
insert into hj_probe values (1);
61+
-- result:
62+
-- !result
63+
insert into hj_build values ( 1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1 );
64+
-- result:
65+
-- !result
66+
set enable_spill = true;
67+
-- result:
68+
-- !result
69+
set spillable_operator_mask = 1;
70+
-- result:
71+
-- !result
72+
set spill_revocable_max_bytes = 1;
73+
-- result:
74+
-- !result
75+
set disable_join_reorder = true;
76+
-- result:
77+
-- !result
78+
set pipeline_dop = 1;
79+
-- result:
80+
-- !result
81+
select b.c01,b.c02,b.c03,b.c04,b.c05,b.c06,b.c07,b.c08, b.c09,b.c10,b.c11,b.c12,b.c13,b.c14,b.c15,b.c16, b.c17,b.c18,b.c19,b.c20,b.c21,b.c22,b.c23,b.c24, b.c25,b.c26,b.c27,b.c28,b.c29,b.c30,b.c31,b.c32 from hj_probe p right join hj_build b on p.k = b.k;
82+
-- result:
83+
1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
84+
-- !result

‎test/sql/test_spill/T/test_spill_join_with_empty_partition‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
-- name: test_spill_join_with_empty_partition
2+
CREATE TABLE `hj_build` (
3+
`k` bigint(20) NULL COMMENT "",
4+
`c01` bigint(20) NULL COMMENT "",
5+
`c02` bigint(20) NULL COMMENT "",
6+
`c03` bigint(20) NULL COMMENT "",
7+
`c04` bigint(20) NULL COMMENT "",
8+
`c05` bigint(20) NULL COMMENT "",
9+
`c06` bigint(20) NULL COMMENT "",
10+
`c07` bigint(20) NULL COMMENT "",
11+
`c08` bigint(20) NULL COMMENT "",
12+
`c09` bigint(20) NULL COMMENT "",
13+
`c10` bigint(20) NULL COMMENT "",
14+
`c11` bigint(20) NULL COMMENT "",
15+
`c12` bigint(20) NULL COMMENT "",
16+
`c13` bigint(20) NULL COMMENT "",
17+
`c14` bigint(20) NULL COMMENT "",
18+
`c15` bigint(20) NULL COMMENT "",
19+
`c16` bigint(20) NULL COMMENT "",
20+
`c17` bigint(20) NULL COMMENT "",
21+
`c18` bigint(20) NULL COMMENT "",
22+
`c19` bigint(20) NULL COMMENT "",
23+
`c20` bigint(20) NULL COMMENT "",
24+
`c21` bigint(20) NULL COMMENT "",
25+
`c22` bigint(20) NULL COMMENT "",
26+
`c23` bigint(20) NULL COMMENT "",
27+
`c24` bigint(20) NULL COMMENT "",
28+
`c25` bigint(20) NULL COMMENT "",
29+
`c26` bigint(20) NULL COMMENT "",
30+
`c27` bigint(20) NULL COMMENT "",
31+
`c28` bigint(20) NULL COMMENT "",
32+
`c29` bigint(20) NULL COMMENT "",
33+
`c30` bigint(20) NULL COMMENT "",
34+
`c31` bigint(20) NULL COMMENT "",
35+
`c32` bigint(20) NULL COMMENT ""
36+
) ENGINE=OLAP
37+
DUPLICATE KEY(`k`)
38+
DISTRIBUTED BY HASH(`k`) BUCKETS 1
39+
PROPERTIES (
40+
"compression" = "LZ4",
41+
"fast_schema_evolution" = "true",
42+
"replicated_storage" = "true",
43+
"replication_num" = "1"
44+
);
45+
46+
47+
CREATE TABLE `hj_probe` (
48+
`k` bigint(20) NULL COMMENT ""
49+
) ENGINE=OLAP
50+
DUPLICATE KEY(`k`)
51+
DISTRIBUTED BY HASH(`k`) BUCKETS 1
52+
PROPERTIES (
53+
"compression" = "LZ4",
54+
"fast_schema_evolution" = "true",
55+
"replicated_storage" = "true",
56+
"replication_num" = "1"
57+
);
58+
59+
insert into hj_probe values (1);
60+
insert into hj_build values ( 1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1 );
61+
62+
set enable_spill = true;
63+
set spillable_operator_mask = 1;
64+
set spill_revocable_max_bytes = 1;
65+
set disable_join_reorder = true;
66+
set pipeline_dop = 1;
67+
68+
69+
70+
select b.c01,b.c02,b.c03,b.c04,b.c05,b.c06,b.c07,b.c08, b.c09,b.c10,b.c11,b.c12,b.c13,b.c14,b.c15,b.c16, b.c17,b.c18,b.c19,b.c20,b.c21,b.c22,b.c23,b.c24, b.c25,b.c26,b.c27,b.c28,b.c29,b.c30,b.c31,b.c32 from hj_probe p right join hj_build b on p.k = b.k;

0 commit comments

Comments
 (0)