Skip to content

Commit 6c153f4

Browse files
fix resize batch
1 parent 64f3250 commit 6c153f4

1 file changed

Lines changed: 17 additions & 4 deletions

File tree

backends-velox/src/main/scala/org/apache/gluten/extension/AppendBatchResizeForShuffleInputAndOutput.scala

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import org.apache.gluten.execution.VeloxResizeBatchesExec
2222
import org.apache.spark.sql.catalyst.rules.Rule
2323
import org.apache.spark.sql.execution.{ColumnarShuffleExchangeExec, SparkPlan}
2424
import org.apache.spark.sql.execution.adaptive.{AQEShuffleReadExec, ShuffleQueryStageExec}
25+
import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
2526

2627
/**
2728
* Try to append [[VeloxResizeBatchesExec]] for shuffle input and output to make the batch sizes in
@@ -44,24 +45,36 @@ case class AppendBatchResizeForShuffleInputAndOutput() extends Rule[SparkPlan] {
4445
VeloxResizeBatchesExec(shuffle.child, range.min, range.max)
4546
shuffle.withNewChildren(Seq(appendBatches))
4647
case a @ AQEShuffleReadExec(
47-
ShuffleQueryStageExec(_, shuffle: ColumnarShuffleExchangeExec, _),
48+
ShuffleQueryStageExec(
49+
_,
50+
shuffle: ColumnarShuffleExchangeExec |
51+
ReusedExchangeExec(_, _: ColumnarShuffleExchangeExec),
52+
_),
4853
_)
4954
if resizeBatchesShuffleOutputEnabled &&
5055
shuffle.shuffleWriterType.requiresResizingShuffleOutput =>
5156
VeloxResizeBatchesExec(a, range.min, range.max)
52-
// Since it's transformed in a bottom to up order, so we may first encountered
57+
// Since it's transformed in a bottom to up order, so we may first encounter
5358
// ShuffeQueryStageExec, which is transformed to VeloxResizeBatchesExec(ShuffeQueryStageExec),
5459
// then we see AQEShuffleReadExec
5560
case a @ AQEShuffleReadExec(
5661
VeloxResizeBatchesExec(
57-
s @ ShuffleQueryStageExec(_, shuffle: ColumnarShuffleExchangeExec, _),
62+
s @ ShuffleQueryStageExec(
63+
_,
64+
shuffle: ColumnarShuffleExchangeExec |
65+
ReusedExchangeExec(_, _: ColumnarShuffleExchangeExec),
66+
_),
5867
_,
5968
_),
6069
_)
6170
if resizeBatchesShuffleOutputEnabled &&
6271
shuffle.shuffleWriterType.requiresResizingShuffleOutput =>
6372
VeloxResizeBatchesExec(a.copy(child = s), range.min, range.max)
64-
case s @ ShuffleQueryStageExec(_, shuffle: ColumnarShuffleExchangeExec, _)
73+
case s @ ShuffleQueryStageExec(
74+
_,
75+
shuffle: ColumnarShuffleExchangeExec |
76+
ReusedExchangeExec(_, _: ColumnarShuffleExchangeExec),
77+
_)
6578
if resizeBatchesShuffleOutputEnabled &&
6679
shuffle.shuffleWriterType.requiresResizingShuffleOutput =>
6780
VeloxResizeBatchesExec(s, range.min, range.max)

0 commit comments

Comments
 (0)