Skip to content

Commit c5280f8

Browse files
zml1206zml1206
authored
[CORE] Move ClickHouse-only extended rule configs from GlutenConfig to CHConfig (#11897)
1 parent 3af2e33 commit c5280f8

11 files changed

Lines changed: 73 additions & 51 deletions

File tree

‎backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHConfig.scala‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,28 @@ object CHConfig extends ConfigRegistry {
102102
.doc("Enable local cache for CH backend.")
103103
.booleanConf
104104
.createWithDefault(false)
105+
106+
val EXTENDED_COLUMNAR_TRANSFORM_RULES =
107+
buildConf("spark.gluten.sql.columnar.extended.columnar.transform.rules")
108+
.internal()
109+
.withAlternative("spark.gluten.sql.columnar.extended.columnar.pre.rules")
110+
.doc("A comma-separated list of classes for the extended columnar transform rules.")
111+
.stringConf
112+
.createWithDefaultString("")
113+
114+
val EXTENDED_COLUMNAR_POST_RULES =
115+
buildConf("spark.gluten.sql.columnar.extended.columnar.post.rules")
116+
.internal()
117+
.doc("A comma-separated list of classes for the extended columnar post rules.")
118+
.stringConf
119+
.createWithDefaultString("")
120+
121+
val EXTENDED_EXPRESSION_TRAN_CONF =
122+
buildConf("spark.gluten.sql.columnar.extended.expressions.transformer")
123+
.internal()
124+
.doc("A class for the extended expressions transformer.")
125+
.stringConf
126+
.createWithDefaultString("")
105127
}
106128

107129
class CHConfig(conf: SQLConf) extends GlutenConfig(conf) {
@@ -128,6 +150,14 @@ class CHConfig(conf: SQLConf) extends GlutenConfig(conf) {
128150
getConf(ENABLE_CH_REWRITE_DATE_CONVERSION)
129151

130152
def enableGlutenLocalFileCache: Boolean = getConf(ENABLE_GLUTEN_LOCAL_FILE_CACHE)
153+
154+
// A comma-separated list of classes for the extended columnar pre rules
155+
def extendedColumnarTransformRules: String = getConf(EXTENDED_COLUMNAR_TRANSFORM_RULES)
156+
157+
// A comma-separated list of classes for the extended columnar post rules
158+
def extendedColumnarPostRules: String = getConf(EXTENDED_COLUMNAR_POST_RULES)
159+
160+
def extendedExpressionTransformer: String = getConf(EXTENDED_EXPRESSION_TRAN_CONF)
131161
}
132162

133163
object GlutenObjectStorageConfig {

‎backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ class CHListenerApi extends ListenerApi with Logging {
4949
initialize(pc.conf, isDriver = true)
5050

5151
val expressionExtensionTransformer = ExpressionUtil.extendedExpressionTransformer(
52-
pc.conf.get(GlutenConfig.EXTENDED_EXPRESSION_TRAN_CONF)
52+
pc.conf.get(CHConfig.EXTENDED_EXPRESSION_TRAN_CONF)
5353
)
5454
if (expressionExtensionTransformer != null) {
5555
ExpressionExtensionTrait.registerExpressionExtension(expressionExtensionTransformer)

‎backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHRuleApi.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ object CHRuleApi {
130130
c =>
131131
intercept(
132132
SparkPlanRules.extendedColumnarRule(
133-
new GlutenConfig(c.sqlConf).extendedColumnarTransformRules)(c.session)))
133+
new CHConfig(c.sqlConf).extendedColumnarTransformRules)(c.session)))
134134
injector.injectPostTransform(_ => CollectLimitTransformerRule())
135135
injector.injectPostTransform(_ => CollectTailTransformerRule())
136136
injector.injectPostTransform(c => InsertTransitions.create(c.outputsColumnar, CHBatchType))
@@ -153,8 +153,8 @@ object CHRuleApi {
153153
injector.injectPost(
154154
c =>
155155
intercept(
156-
SparkPlanRules.extendedColumnarRule(
157-
new GlutenConfig(c.sqlConf).extendedColumnarPostRules)(c.session)))
156+
SparkPlanRules
157+
.extendedColumnarRule(new CHConfig(c.sqlConf).extendedColumnarPostRules)(c.session)))
158158
injector.injectPost(c => GlutenNoopWriterRule.apply(c.session))
159159

160160
// Gluten columnar: Final rules.

‎backends-clickhouse/src/test/scala/org/apache/gluten/execution/extension/GlutenCustomAggExpressionSuite.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
*/
1717
package org.apache.gluten.execution.extension
1818

19-
import org.apache.gluten.config.GlutenConfig
19+
import org.apache.gluten.backendsapi.clickhouse.CHConfig
2020
import org.apache.gluten.execution._
2121
import org.apache.gluten.substrait.SubstraitContext
2222
import org.apache.gluten.utils.SubstraitPlanPrinterUtil
@@ -33,7 +33,7 @@ class GlutenCustomAggExpressionSuite extends ParquetSuite {
3333
super.sparkConf
3434
.set("spark.sql.adaptive.enabled", "false")
3535
.set(
36-
GlutenConfig.EXTENDED_EXPRESSION_TRAN_CONF.key,
36+
CHConfig.EXTENDED_EXPRESSION_TRAN_CONF.key,
3737
"org.apache.gluten.execution.extension.CustomAggExpressionTransformer")
3838
}
3939

‎backends-clickhouse/src/test/scala/org/apache/spark/sql/extension/GlutenClickhouseCustomerExpressionTransformerSuite.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
*/
1717
package org.apache.spark.sql.extension
1818

19-
import org.apache.gluten.config.GlutenConfig
19+
import org.apache.gluten.backendsapi.clickhouse.CHConfig
2020
import org.apache.gluten.execution.{GlutenClickHouseWholeStageTransformerSuite, ProjectExecTransformer}
2121
import org.apache.gluten.expression.ExpressionConverter
2222

@@ -88,7 +88,7 @@ class GlutenClickhouseCustomerExpressionTransformerSuite
8888
super.sparkConf
8989
.set("spark.sql.adaptive.enabled", "false")
9090
.set(
91-
GlutenConfig.EXTENDED_EXPRESSION_TRAN_CONF.key,
91+
CHConfig.EXTENDED_EXPRESSION_TRAN_CONF.key,
9292
"org.apache.spark.sql.extension.CustomerExpressionTransformer")
9393
}
9494

‎gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala‎

Lines changed: 0 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -286,14 +286,6 @@ class GlutenConfig(conf: SQLConf) extends GlutenCoreConfig(conf) {
286286

287287
def softAffinityLogLevel: String = getConf(SOFT_AFFINITY_LOG_LEVEL)
288288

289-
// A comma-separated list of classes for the extended columnar pre rules
290-
def extendedColumnarTransformRules: String = getConf(EXTENDED_COLUMNAR_TRANSFORM_RULES)
291-
292-
// A comma-separated list of classes for the extended columnar post rules
293-
def extendedColumnarPostRules: String = getConf(EXTENDED_COLUMNAR_POST_RULES)
294-
295-
def extendedExpressionTransformer: String = getConf(EXTENDED_EXPRESSION_TRAN_CONF)
296-
297289
def smallFileThreshold: Double = getConf(SMALL_FILE_THRESHOLD)
298290

299291
def expressionBlacklist: Set[String] = {
@@ -1314,31 +1306,6 @@ object GlutenConfig extends ConfigRegistry {
13141306
.booleanConf
13151307
.createWithDefault(true)
13161308

1317-
// FIXME: This only works with CH backend.
1318-
val EXTENDED_COLUMNAR_TRANSFORM_RULES =
1319-
buildConf("spark.gluten.sql.columnar.extended.columnar.transform.rules")
1320-
.internal()
1321-
.withAlternative("spark.gluten.sql.columnar.extended.columnar.pre.rules")
1322-
.doc("A comma-separated list of classes for the extended columnar transform rules.")
1323-
.stringConf
1324-
.createWithDefaultString("")
1325-
1326-
// FIXME: This only works with CH backend.
1327-
val EXTENDED_COLUMNAR_POST_RULES =
1328-
buildConf("spark.gluten.sql.columnar.extended.columnar.post.rules")
1329-
.internal()
1330-
.doc("A comma-separated list of classes for the extended columnar post rules.")
1331-
.stringConf
1332-
.createWithDefaultString("")
1333-
1334-
// FIXME: This only works with CH backend.
1335-
val EXTENDED_EXPRESSION_TRAN_CONF =
1336-
buildConf("spark.gluten.sql.columnar.extended.expressions.transformer")
1337-
.internal()
1338-
.doc("A class for the extended expressions transformer.")
1339-
.stringConf
1340-
.createWithDefaultString("")
1341-
13421309
val EXPRESSION_BLACK_LIST =
13431310
buildConf("spark.gluten.expression.blacklist")
13441311
.doc("A black list of expression to skip transform, multiple values separated by commas.")

‎gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,20 @@ import org.apache.spark.SparkConf
2222
import org.apache.spark.sql.GlutenSQLTestsTrait
2323

2424
class GlutenCustomerExtensionSuite extends GlutenSQLTestsTrait {
25+
// These configs only take effect on ClickHouse backend.
26+
private val ExtendedColumnarTransformRulesKey =
27+
"spark.gluten.sql.columnar.extended.columnar.transform.rules"
28+
private val ExtendedColumnarPostRulesKey =
29+
"spark.gluten.sql.columnar.extended.columnar.post.rules"
2530

2631
override def sparkConf: SparkConf = {
2732
super.sparkConf
2833
.set("spark.sql.adaptive.enabled", "false")
2934
.set(
30-
GlutenConfig.EXTENDED_COLUMNAR_TRANSFORM_RULES.key,
35+
ExtendedColumnarTransformRulesKey,
3136
"org.apache.spark.sql" +
3237
".extension.CustomerColumnarPreRules")
33-
.set(GlutenConfig.EXTENDED_COLUMNAR_POST_RULES.key, "")
38+
.set(ExtendedColumnarPostRulesKey, "")
3439
}
3540

3641
testGluten("test customer column rules") {

‎gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,20 @@ import org.apache.spark.SparkConf
2222
import org.apache.spark.sql.GlutenSQLTestsTrait
2323

2424
class GlutenCustomerExtensionSuite extends GlutenSQLTestsTrait {
25+
// These configs only take effect on ClickHouse backend.
26+
private val ExtendedColumnarTransformRulesKey =
27+
"spark.gluten.sql.columnar.extended.columnar.transform.rules"
28+
private val ExtendedColumnarPostRulesKey =
29+
"spark.gluten.sql.columnar.extended.columnar.post.rules"
2530

2631
override def sparkConf: SparkConf = {
2732
super.sparkConf
2833
.set("spark.sql.adaptive.enabled", "false")
2934
.set(
30-
GlutenConfig.EXTENDED_COLUMNAR_TRANSFORM_RULES.key,
35+
ExtendedColumnarTransformRulesKey,
3136
"org.apache.spark.sql" +
3237
".extension.CustomerColumnarPreRules")
33-
.set(GlutenConfig.EXTENDED_COLUMNAR_POST_RULES.key, "")
38+
.set(ExtendedColumnarPostRulesKey, "")
3439
}
3540

3641
testGluten("test customer column rules") {

‎gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,20 @@ import org.apache.spark.SparkConf
2222
import org.apache.spark.sql.GlutenSQLTestsTrait
2323

2424
class GlutenCustomerExtensionSuite extends GlutenSQLTestsTrait {
25+
// These configs only take effect on ClickHouse backend.
26+
private val ExtendedColumnarTransformRulesKey =
27+
"spark.gluten.sql.columnar.extended.columnar.transform.rules"
28+
private val ExtendedColumnarPostRulesKey =
29+
"spark.gluten.sql.columnar.extended.columnar.post.rules"
2530

2631
override def sparkConf: SparkConf = {
2732
super.sparkConf
2833
.set("spark.sql.adaptive.enabled", "false")
2934
.set(
30-
GlutenConfig.EXTENDED_COLUMNAR_TRANSFORM_RULES.key,
35+
ExtendedColumnarTransformRulesKey,
3136
"org.apache.spark.sql" +
3237
".extension.CustomerColumnarPreRules")
33-
.set(GlutenConfig.EXTENDED_COLUMNAR_POST_RULES.key, "")
38+
.set(ExtendedColumnarPostRulesKey, "")
3439
}
3540

3641
testGluten("test customer column rules") {

‎gluten-ut/spark40/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,20 @@ import org.apache.spark.SparkConf
2222
import org.apache.spark.sql.GlutenSQLTestsTrait
2323

2424
class GlutenCustomerExtensionSuite extends GlutenSQLTestsTrait {
25+
// These configs only take effect on ClickHouse backend.
26+
private val ExtendedColumnarTransformRulesKey =
27+
"spark.gluten.sql.columnar.extended.columnar.transform.rules"
28+
private val ExtendedColumnarPostRulesKey =
29+
"spark.gluten.sql.columnar.extended.columnar.post.rules"
2530

2631
override def sparkConf: SparkConf = {
2732
super.sparkConf
2833
.set("spark.sql.adaptive.enabled", "false")
2934
.set(
30-
GlutenConfig.EXTENDED_COLUMNAR_TRANSFORM_RULES.key,
35+
ExtendedColumnarTransformRulesKey,
3136
"org.apache.spark.sql" +
3237
".extension.CustomerColumnarPreRules")
33-
.set(GlutenConfig.EXTENDED_COLUMNAR_POST_RULES.key, "")
38+
.set(ExtendedColumnarPostRulesKey, "")
3439
}
3540

3641
testGluten("test customer column rules") {

0 commit comments

Comments
 (0)