Skip to content

Commit a7058c7

Browse files
malinjawiMohammad Linjawi
andauthored
[GLUTEN-11862][VL] Work around GMT session timezone validation failure on macOS (#11869)
Fixes #11862. This PR works around a macOS-specific native validation failure in the Velox backend when the effective Spark session timezone is GMT. Based on the discussion in #11862, the underlying behavior is likely related to upstream Folly/Velox timezone handling on macOS rather than a general Spark semantics problem. This patch keeps the workaround on the Gluten side so Spark session timezone values do not block native validation and execution in the Velox path. The changes in this PR are: normalize GMT and GMT+/-offset session timezone values to UTC-equivalent forms before passing them into native Velox query config use the runtime session timezone in SubstraitToVeloxPlanValidator instead of hardcoding GMT apply the same normalization when setting the Velox writer timezone config add a regression test covering execution with spark.sql.session.timeZone=GMT --------- Co-authored-by: Mohammad Linjawi <Mohammad.Linjawi@ibm.com>
1 parent 139c332 commit a7058c7

6 files changed

Lines changed: 28 additions & 3 deletions

File tree

backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1910,6 +1910,14 @@ class MiscOperatorSuite extends VeloxWholeStageTransformerSuite with AdaptiveSpa
19101910
assert(plan2.find(_.isInstanceOf[ProjectExecTransformer]).isDefined)
19111911
}
19121912

1913+
test("cast date to timestamp with GMT session timezone") {
1914+
withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT") {
1915+
runQueryAndCompare("SELECT cast(date'2023-01-02 01:01:01' as timestamp) as ts") {
1916+
checkGlutenPlan[ProjectExecTransformer]
1917+
}
1918+
}
1919+
}
1920+
19131921
test("cast timestamp to date") {
19141922
val query = "select cast(ts as date) from values (timestamp'2024-01-01 00:00:00') as tab(ts)"
19151923
runQueryAndCompare(query) {

cpp/core/config/GlutenConfig.cc

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,17 @@ parseConfMap(JNIEnv* env, const uint8_t* planData, const int32_t planDataLength)
4949
return sparkConfs;
5050
}
5151

52+
std::string normalizeSessionTimezone(std::string_view sessionTimezone) {
53+
if (sessionTimezone == "GMT") {
54+
return "UTC";
55+
}
56+
if (sessionTimezone.size() > 3 && sessionTimezone.substr(0, 3) == "GMT" &&
57+
(sessionTimezone[3] == '+' || sessionTimezone[3] == '-')) {
58+
return std::string("UTC").append(sessionTimezone.substr(3));
59+
}
60+
return std::string(sessionTimezone);
61+
}
62+
5263
std::string printConfig(const std::unordered_map<std::string, std::string>& conf) {
5364
std::ostringstream oss;
5465
oss << std::endl;

cpp/core/config/GlutenConfig.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include <jni.h>
2121
#include <cstdint>
2222
#include <string>
23+
#include <string_view>
2324
#include <unordered_map>
2425

2526
namespace gluten {
@@ -102,5 +103,7 @@ const std::string kDebugCudfDefault = "false";
102103
std::unordered_map<std::string, std::string>
103104
parseConfMap(JNIEnv* env, const uint8_t* planData, const int32_t planDataLength);
104105

106+
std::string normalizeSessionTimezone(std::string_view sessionTimezone);
107+
105108
std::string printConfig(const std::unordered_map<std::string, std::string>& conf);
106109
} // namespace gluten

cpp/velox/compute/WholeStageResultIterator.cc

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -575,7 +575,8 @@ std::unordered_map<std::string, std::string> WholeStageResultIterator::getQueryC
575575
std::to_string(veloxCfg_->get<uint64_t>(kVeloxPreferredBatchBytes, 10L << 20));
576576
try {
577577
configs[velox::core::QueryConfig::kSparkAnsiEnabled] = veloxCfg_->get<std::string>(kAnsiEnabled, "false");
578-
configs[velox::core::QueryConfig::kSessionTimezone] = veloxCfg_->get<std::string>(kSessionTimezone, "");
578+
configs[velox::core::QueryConfig::kSessionTimezone] =
579+
normalizeSessionTimezone(veloxCfg_->get<std::string>(kSessionTimezone, ""));
579580
// Adjust timestamp according to the above configured session timezone.
580581
configs[velox::core::QueryConfig::kAdjustTimestampToTimezone] = "true";
581582

cpp/velox/substrait/SubstraitToVeloxPlanValidator.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ class SubstraitToVeloxPlanValidator {
3131
public:
3232
SubstraitToVeloxPlanValidator(memory::MemoryPool* pool) {
3333
std::unordered_map<std::string, std::string> configs{
34-
{velox::core::QueryConfig::kSparkPartitionId, "0"}, {velox::core::QueryConfig::kSessionTimezone, "GMT"}};
34+
{velox::core::QueryConfig::kSparkPartitionId, "0"}, {velox::core::QueryConfig::kSessionTimezone, "UTC"}};
3535
veloxCfg_ = std::make_shared<facebook::velox::config::ConfigBase>(std::move(configs));
3636
planConverter_ = std::make_unique<SubstraitToVeloxPlanConverter>(
3737
pool, veloxCfg_.get(), std::vector<std::shared_ptr<ResultIterator>>{}, std::nullopt, std::nullopt, true);

cpp/velox/utils/VeloxWriterUtils.cc

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,9 @@ std::unique_ptr<WriterOptions> makeParquetWriteOption(const std::unordered_map<s
8989
writeOption->flushPolicyFactory = [maxRowGroupRows, maxRowGroupBytes]() {
9090
return std::make_unique<LambdaFlushPolicy>(maxRowGroupRows, maxRowGroupBytes, [&]() { return false; });
9191
};
92-
writeOption->parquetWriteTimestampTimeZone = getConfigValue(sparkConfs, kSessionTimezone, std::nullopt);
92+
if (auto it = sparkConfs.find(kSessionTimezone); it != sparkConfs.end()) {
93+
writeOption->parquetWriteTimestampTimeZone = normalizeSessionTimezone(it->second);
94+
}
9395
writeOption->arrowMemoryPool =
9496
getDefaultMemoryManager()->getOrCreateArrowMemoryPool("VeloxParquetWrite.ArrowMemoryPool");
9597
if (auto it = sparkConfs.find(kParquetDataPageSize); it != sparkConfs.end()) {

0 commit comments

Comments
 (0)