Skip to content

Commit 0b11df1

Browse files
author
Chong Gao
committed
Address comments
Signed-off-by: Chong Gao <res_life@163.com>
1 parent ed611a9 commit 0b11df1

2 files changed

Lines changed: 14 additions & 9 deletions

File tree

sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuShuffleEnv.scala

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -108,20 +108,21 @@ object GpuShuffleEnv extends Logging {
108108
conf.getBoolean("spark.authenticate", false)
109109
}
110110

111+
// Returns true if row-based checksum is enabled, which is not supported
112+
// by the RAPIDS Shuffle Manager
113+
def isRowBasedChecksumEnabled: Boolean = {
114+
val conf = SparkEnv.get.conf
115+
// Row-based checksum feature was added in Spark 4.1.x (SPARK-51756).
116+
// Fully supporting this feature would require kernel development to compute
117+
// checksums on the GPU side.
118+
conf.getBoolean("spark.shuffle.checksum.enabled", false)
119+
}
120+
111121
//
112122
// The actual instantiation of the RAPIDS Shuffle Manager is lazy, and
113123
// this forces the initialization when we know we are ready in the driver and executor.
114124
//
115125
def initShuffleManager(): Unit = {
116-
val conf = SparkEnv.get.conf
117-
// Row-based checksum feature was added in Spark 4.1.x (SPARK-51756).
118-
// Fully supporting this feature would require kernel development to compute
119-
// checksums on the GPU side.
120-
if (conf.getBoolean("spark.shuffle.checksum.enabled", false)) {
121-
throw new IllegalStateException(
122-
"RAPIDS Shuffle Manager does not support spark.shuffle.checksum.enabled. " +
123-
"Please set spark.shuffle.checksum.enabled to false or disable the RAPIDS Shuffle Manager.")
124-
}
125126
val shuffleManager = SparkEnv.get.shuffleManager
126127
if (ShuffleManagerShimUtils.eagerlyInitialized) {
127128
// skip deferred init

sql-plugin/src/main/scala/org/apache/spark/sql/rapids/RapidsShuffleInternalManagerBase.scala

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1623,6 +1623,10 @@ class RapidsShuffleInternalManagerBase(conf: SparkConf, val isDriver: Boolean)
16231623
if (GpuShuffleEnv.isSparkAuthenticateEnabled) {
16241624
fallThroughReasons += "Spark authentication is enabled"
16251625
}
1626+
if (GpuShuffleEnv.isRowBasedChecksumEnabled) {
1627+
fallThroughReasons += "Detected spark.shuffle.checksum.enabled=true. " +
1628+
"This feature is supported in Spark 4.1+, but is not yet supported by Spark-Rapids."
1629+
}
16261630
}
16271631
if (rapidsConf.isSqlExplainOnlyEnabled) {
16281632
fallThroughReasons += "Plugin is in explain only mode"

0 commit comments

Comments
 (0)