File tree Expand file tree Collapse file tree
main/scala/org/apache/gluten/config
test/scala/org/apache/gluten/execution
gluten-substrait/src/main/scala/org/apache/gluten/config Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -95,7 +95,7 @@ class VeloxConfig(conf: SQLConf) extends GlutenConfig(conf) {
9595 def cudfShuffleMaxPrefetchBytes : Long = getConf(CUDF_SHUFFLE_MAX_PREFETCH_BYTES )
9696
9797 def orcUseColumnNames : Boolean = getConf(ORC_USE_COLUMN_NAMES ) &&
98- ! conf.getConfString(ORC_FORCE_POSITIONAL_EVOLUTION , " false" ).toBoolean
98+ ! conf.getConfString(GlutenConfig . SPARK_ORC_FORCE_POSITIONAL_EVOLUTION , " false" ).toBoolean
9999
100100 def parquetUseColumnNames : Boolean = getConf(PARQUET_USE_COLUMN_NAMES )
101101
@@ -809,12 +809,6 @@ object VeloxConfig extends ConfigRegistry {
809809 .booleanConf
810810 .createWithDefault(true )
811811
812- // The Hadoop/ORC config that forces position-based schema evolution. When set to true,
813- // vanilla Spark reads ORC columns by position; Gluten/Velox must do the same, so it
814- // overrides ORC_USE_COLUMN_NAMES to false. Kept as a plain conf key (not a Gluten
815- // ConfigEntry) because it is a standard `spark.hadoop.*` Hadoop property.
816- val ORC_FORCE_POSITIONAL_EVOLUTION = " spark.hadoop.orc.force.positional.evolution"
817-
818812 val PARQUET_USE_COLUMN_NAMES =
819813 buildConf(" spark.gluten.sql.columnar.backend.velox.parquetUseColumnNames" )
820814 .doc(" Maps table field names to file field names using names, not indices for Parquet files." )
Original file line number Diff line number Diff line change @@ -354,7 +354,7 @@ class FallbackSuite extends VeloxWholeStageTransformerSuite with AdaptiveSparkPl
354354 forcePositional =>
355355 withSQLConf(
356356 VeloxConfig .ORC_USE_COLUMN_NAMES .key -> " true" ,
357- VeloxConfig . ORC_FORCE_POSITIONAL_EVOLUTION -> forcePositional
357+ GlutenConfig . SPARK_ORC_FORCE_POSITIONAL_EVOLUTION -> forcePositional
358358 ) {
359359 withTable(" test" ) {
360360 spark
Original file line number Diff line number Diff line change @@ -433,6 +433,8 @@ object GlutenConfig extends ConfigRegistry {
433433 val SPARK_S3_ENDPOINT_REGION : String = HADOOP_PREFIX + S3_ENDPOINT_REGION
434434 val S3_AWS_IMDS_ENABLED = " fs.s3a.aws.imds.enabled"
435435 val SPARK_S3_AWS_IMDS_ENABLED : String = HADOOP_PREFIX + S3_AWS_IMDS_ENABLED
436+ val ORC_FORCE_POSITIONAL_EVOLUTION = " orc.force.positional.evolution"
437+ val SPARK_ORC_FORCE_POSITIONAL_EVOLUTION = HADOOP_PREFIX + ORC_FORCE_POSITIONAL_EVOLUTION
436438
437439 // ABFS config
438440 val ABFS_PREFIX = " fs.azure."
@@ -582,7 +584,7 @@ object GlutenConfig extends ConfigRegistry {
582584 // reads columns back as null/empty. Override the (Velox) orcUseColumnNames session conf
583585 // so native reads ORC by position too. Harmless for backends that ignore this key.
584586 // String literal is used because gluten-substrait cannot depend on backends-velox.
585- if (conf.getOrElse(" spark.hadoop.orc.force.positional.evolution " , " false" ).toBoolean) {
587+ if (conf.getOrElse(SPARK_ORC_FORCE_POSITIONAL_EVOLUTION , " false" ).toBoolean) {
586588 nativeConfMap.put(" spark.gluten.sql.columnar.backend.velox.orcUseColumnNames" , " false" )
587589 }
588590
You can’t perform that action at this time.
0 commit comments