Skip to content

Commit 1e9c9f0

Browse files
authored
[auto-merge] release/26.04 to main [skip ci] [bot] (#14548)
auto-merge triggered by github actions on `release/26.04` to create a PR keeping `main` up-to-date. If this PR is unable to be merged due to conflicts, it will remain open until manually fix.
2 parents 997748c + 40100f3 commit 1e9c9f0

2 files changed

Lines changed: 14 additions & 2 deletions

File tree

sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuShuffleCoalesceExec.scala

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -481,6 +481,12 @@ case class CloseableTableSeqWithTargetSize[T <: AutoCloseable](
481481
override def length: Int = tables.length
482482
override def iterator: Iterator[T] = tables.iterator
483483
override def apply(idx: Int): T = tables.apply(idx)
484+
485+
// Keep retry-OOM reporting bounded. SeqLike.toString would stringify every table.
486+
override def toString: String = {
487+
s"CloseableTableSeqWithTargetSize(numTables=$length, " +
488+
s"targetSize=${targetSize.targetSize}, minSize=${targetSize.minSize})"
489+
}
484490
}
485491

486492

sql-plugin/src/main/scala/com/nvidia/spark/rapids/RmmRapidsRetryIterator.scala

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -487,8 +487,6 @@ object RmmRapidsRetryIterator extends Logging {
487487
}
488488
}
489489
val curAttempt = attemptStack.pop()
490-
// Get the info before running the split, since the attempt may be closed after splitting.
491-
val attemptAsString = closeOnExcept(curAttempt)(_.toString)
492490
val splitted = try {
493491
// splitPolicy must take ownership of the argument
494492
splitPolicy(curAttempt)
@@ -497,25 +495,33 @@ object RmmRapidsRetryIterator extends Logging {
497495
// same type to provide more context for the OOM.
498496
// This looks a little odd, because we can not change the type of root exception.
499497
// Otherwise, some unit tests will fail due to the wrong exception type returned.
498+
//
499+
// Stringify the attempt lazily (only on failure) to avoid O(n) cost on the
500+
// hot path when splits succeed. The attempt has not been closed yet at this
501+
// point because splitPolicy threw before taking ownership.
500502
case go: GpuRetryOOM =>
503+
val attemptAsString = curAttempt.toString
501504
throw new GpuRetryOOM(
502505
s"GPU OutOfMemory: " +
503506
s"Current threadCountBlockedUntilReady: ${threadCountBlockedUntilReady}. " +
504507
s"Could not split the current attempt: {$attemptAsString}"
505508
).initCause(go)
506509
case go: GpuSplitAndRetryOOM =>
510+
val attemptAsString = curAttempt.toString
507511
throw new GpuSplitAndRetryOOM(
508512
s"GPU OutOfMemory: " +
509513
s"Current threadCountBlockedUntilReady: ${threadCountBlockedUntilReady}. " +
510514
s"Could not split the current attempt: {$attemptAsString}"
511515
).initCause(go)
512516
case co: CpuRetryOOM =>
517+
val attemptAsString = curAttempt.toString
513518
throw new CpuRetryOOM(
514519
s"CPU OutOfMemory: " +
515520
s"Current threadCountBlockedUntilReady: ${threadCountBlockedUntilReady}. " +
516521
s"Could not split the current attempt: {$attemptAsString}"
517522
).initCause(co)
518523
case co: CpuSplitAndRetryOOM =>
524+
val attemptAsString = curAttempt.toString
519525
throw new CpuSplitAndRetryOOM(
520526
s"CPU OutOfMemory: " +
521527
s"Current threadCountBlockedUntilReady: ${threadCountBlockedUntilReady}. " +

0 commit comments

Comments
 (0)