Skip to content

Commit 83594fb

Browse files
committed
improve code
1 parent 43c9bf3 commit 83594fb

3 files changed

Lines changed: 5 additions & 9 deletions

File tree

backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ class VeloxConfig(conf: SQLConf) extends GlutenConfig(conf) {
9595
def cudfShuffleMaxPrefetchBytes: Long = getConf(CUDF_SHUFFLE_MAX_PREFETCH_BYTES)
9696

9797
def orcUseColumnNames: Boolean = getConf(ORC_USE_COLUMN_NAMES) &&
98-
!conf.getConfString(ORC_FORCE_POSITIONAL_EVOLUTION, "false").toBoolean
98+
!conf.getConfString(GlutenConfig.SPARK_ORC_FORCE_POSITIONAL_EVOLUTION, "false").toBoolean
9999

100100
def parquetUseColumnNames: Boolean = getConf(PARQUET_USE_COLUMN_NAMES)
101101

@@ -809,12 +809,6 @@ object VeloxConfig extends ConfigRegistry {
809809
.booleanConf
810810
.createWithDefault(true)
811811

812-
// The Hadoop/ORC config that forces position-based schema evolution. When set to true,
813-
// vanilla Spark reads ORC columns by position; Gluten/Velox must do the same, so it
814-
// overrides ORC_USE_COLUMN_NAMES to false. Kept as a plain conf key (not a Gluten
815-
// ConfigEntry) because it is a standard `spark.hadoop.*` Hadoop property.
816-
val ORC_FORCE_POSITIONAL_EVOLUTION = "spark.hadoop.orc.force.positional.evolution"
817-
818812
val PARQUET_USE_COLUMN_NAMES =
819813
buildConf("spark.gluten.sql.columnar.backend.velox.parquetUseColumnNames")
820814
.doc("Maps table field names to file field names using names, not indices for Parquet files.")

backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -354,7 +354,7 @@ class FallbackSuite extends VeloxWholeStageTransformerSuite with AdaptiveSparkPl
354354
forcePositional =>
355355
withSQLConf(
356356
VeloxConfig.ORC_USE_COLUMN_NAMES.key -> "true",
357-
VeloxConfig.ORC_FORCE_POSITIONAL_EVOLUTION -> forcePositional
357+
GlutenConfig.SPARK_ORC_FORCE_POSITIONAL_EVOLUTION -> forcePositional
358358
) {
359359
withTable("test") {
360360
spark

gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -433,6 +433,8 @@ object GlutenConfig extends ConfigRegistry {
433433
val SPARK_S3_ENDPOINT_REGION: String = HADOOP_PREFIX + S3_ENDPOINT_REGION
434434
val S3_AWS_IMDS_ENABLED = "fs.s3a.aws.imds.enabled"
435435
val SPARK_S3_AWS_IMDS_ENABLED: String = HADOOP_PREFIX + S3_AWS_IMDS_ENABLED
436+
val ORC_FORCE_POSITIONAL_EVOLUTION = "orc.force.positional.evolution"
437+
val SPARK_ORC_FORCE_POSITIONAL_EVOLUTION = HADOOP_PREFIX + ORC_FORCE_POSITIONAL_EVOLUTION
436438

437439
// ABFS config
438440
val ABFS_PREFIX = "fs.azure."
@@ -582,7 +584,7 @@ object GlutenConfig extends ConfigRegistry {
582584
// reads columns back as null/empty. Override the (Velox) orcUseColumnNames session conf
583585
// so native reads ORC by position too. Harmless for backends that ignore this key.
584586
// String literal is used because gluten-substrait cannot depend on backends-velox.
585-
if (conf.getOrElse("spark.hadoop.orc.force.positional.evolution", "false").toBoolean) {
587+
if (conf.getOrElse(SPARK_ORC_FORCE_POSITIONAL_EVOLUTION, "false").toBoolean) {
586588
nativeConfMap.put("spark.gluten.sql.columnar.backend.velox.orcUseColumnNames", "false")
587589
}
588590

0 commit comments

Comments
 (0)