Skip to content

Commit 3738b82

Browse files
committed
ORC-2213: [C++] Add option to read timestamps with writer timezone
### What changes were proposed in this pull request? Add a RowReaderOptions flag that lets timestamp readers use the writer timezone recorded in each stripe footer instead of the configured reader timezone. This preserves the stored timestamp value for callers that need to avoid ORC timestamp timezone conversion. ### Why are the changes needed? In old days, ORC only supports orc::TIMESTAMP type. The semantics of orc::TIMESTAMP is equivalent to TIMESTAMP_NTZ but with a complex writer and reader timezone adjustment. So it is difficult to support both TIMESTAMP_NTZ and TIMESTAMP_LTZ types using a single orc::TIMESTAMP. In later versions, orc::TIMESTAMP_INSTANT has been added to support TIMESTAMP_LTZ type. However, users may not know the difference under the hood and still use legacy systems to write both TIMESTAMP_LTZ and TIMESTAMP_NTZ semantics to the old orc::TIMESTAMP type. When users use orc::TIMESTAMP as TIMESTAMP_LTZ values, we should set reader timezone to the writer timezone to avoid value conversion. However, users may not know the writer timezone in advance and ORC files may have different writer timezone (considering files are produced by teams in different time zones.) We need an approach to enforce reader to use writer timezone via explicit configuration. ### How was this patch tested? Added a new test case. ### Was this patch authored or co-authored using generative AI tooling? Generated-by: Codex GPT-5.6 Sol Closes #2699 from wgtmac/ORC-2213. Authored-by: Gang Wu <ustcwg@gmail.com> Signed-off-by: Gang Wu <ustcwg@gmail.com>
1 parent da33ff0 commit 3738b82

5 files changed

Lines changed: 66 additions & 2 deletions

File tree

c++/include/orc/Reader.hh

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -353,6 +353,16 @@ namespace orc {
353353
*/
354354
const std::string& getTimezoneName() const;
355355

356+
/**
357+
* Use the writer timezone when reading timestamp values.
358+
*/
359+
RowReaderOptions& setUseWriterTimezone(bool useWriterTimezone);
360+
361+
/**
362+
* Get whether to use the writer timezone when reading timestamp values.
363+
*/
364+
bool getUseWriterTimezone() const;
365+
356366
/**
357367
* Get the IdReadIntentMap map that was supplied by client.
358368
*/

c++/src/Options.hh

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ namespace orc {
151151
bool enableLazyDecoding;
152152
std::shared_ptr<SearchArgument> sargs;
153153
std::string readerTimezone;
154+
bool useWriterTimezone;
154155
RowReaderOptions::IdReadIntentMap idReadIntentMap;
155156
bool useTightNumericVector;
156157
std::shared_ptr<Type> readType;
@@ -167,6 +168,7 @@ namespace orc {
167168
forcedScaleOnHive11Decimal = 6;
168169
enableLazyDecoding = false;
169170
readerTimezone = "GMT";
171+
useWriterTimezone = false;
170172
useTightNumericVector = false;
171173
throwOnSchemaEvolutionOverflow = false;
172174
enableAsyncPrefetch = false;
@@ -318,13 +320,23 @@ namespace orc {
318320

319321
RowReaderOptions& RowReaderOptions::setTimezoneName(const std::string& zoneName) {
320322
privateBits_->readerTimezone = zoneName;
323+
privateBits_->useWriterTimezone = false;
321324
return *this;
322325
}
323326

324327
const std::string& RowReaderOptions::getTimezoneName() const {
325328
return privateBits_->readerTimezone;
326329
}
327330

331+
RowReaderOptions& RowReaderOptions::setUseWriterTimezone(bool useWriterTimezone) {
332+
privateBits_->useWriterTimezone = useWriterTimezone;
333+
return *this;
334+
}
335+
336+
bool RowReaderOptions::getUseWriterTimezone() const {
337+
return privateBits_->useWriterTimezone;
338+
}
339+
328340
const RowReaderOptions::IdReadIntentMap RowReaderOptions::getIdReadIntentMap() const {
329341
return privateBits_->idReadIntentMap;
330342
}

c++/src/Reader.cc

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -314,7 +314,9 @@ namespace orc {
314314
footer_(contents_->footer.get()),
315315
firstRowOfStripe_(*contents_->pool, 0),
316316
enableEncodedBlock_(opts.getEnableLazyDecoding()),
317-
readerTimezone_(getTimezoneByName(opts.getTimezoneName())),
317+
readerTimezone_(opts.getUseWriterTimezone() ? localTimezone_
318+
: getTimezoneByName(opts.getTimezoneName())),
319+
useWriterTimezone_(opts.getUseWriterTimezone()),
318320
schemaEvolution_(opts.getReadType(), contents_->schema.get()) {
319321
uint64_t numberOfStripes;
320322
numberOfStripes = static_cast<uint64_t>(footer_->stripes_size());
@@ -1366,7 +1368,8 @@ namespace orc {
13661368
: localTimezone_;
13671369
StripeStreamsImpl stripeStreams(*this, currentStripe_, currentStripeInfo_,
13681370
currentStripeFooter_, currentStripeInfo_.offset(),
1369-
*contents_->stream, writerTimezone, readerTimezone_);
1371+
*contents_->stream, writerTimezone,
1372+
useWriterTimezone_ ? writerTimezone : readerTimezone_);
13701373
reader_ = buildReader(*contents_->schema, stripeStreams, useTightNumericVector_,
13711374
throwOnSchemaEvolutionOverflow_, /*convertToReadType=*/true);
13721375

c++/src/Reader.hh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,7 @@ namespace orc {
189189

190190
// desired timezone to return data of timestamp types.
191191
const Timezone& readerTimezone_;
192+
const bool useWriterTimezone_;
192193

193194
// match read and file types
194195
SchemaEvolution schemaEvolution_;

c++/test/TestWriter.cc

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -840,6 +840,44 @@ namespace orc {
840840
"2014-06-06 12:34:56", IS_DST);
841841
}
842842

843+
TEST_P(WriterTest, readTimestampWithWriterTimezone) {
844+
MemoryOutputStream memStream(DEFAULT_MEM_STREAM_SIZE);
845+
MemoryPool* pool = getDefaultPool();
846+
std::unique_ptr<Type> type(Type::buildTypeFromString("struct<col1:timestamp>"));
847+
848+
std::unique_ptr<Writer> writer =
849+
createWriter(16 * 1024, 64, 1024, CompressionKind_ZLIB, *type, pool, &memStream,
850+
fileVersion, 0, "Asia/Shanghai");
851+
std::unique_ptr<ColumnVectorBatch> batch = writer->createRowBatch(1);
852+
auto* structBatch = dynamic_cast<StructVectorBatch*>(batch.get());
853+
auto* tsBatch = dynamic_cast<TimestampVectorBatch*>(structBatch->fields[0]);
854+
tsBatch->data[0] = 0;
855+
tsBatch->nanoseconds[0] = 123000000;
856+
structBatch->numElements = 1;
857+
tsBatch->numElements = 1;
858+
writer->add(*batch);
859+
writer->close();
860+
861+
auto readTimestamp = [&](bool useWriterTimezone) {
862+
auto inStream =
863+
std::make_unique<MemoryInputStream>(memStream.getData(), memStream.getLength());
864+
std::unique_ptr<Reader> reader = createReader(pool, std::move(inStream));
865+
RowReaderOptions rowReaderOptions;
866+
rowReaderOptions.setTimezoneName("GMT");
867+
rowReaderOptions.setUseWriterTimezone(useWriterTimezone);
868+
std::unique_ptr<RowReader> rowReader = reader->createRowReader(rowReaderOptions);
869+
std::unique_ptr<ColumnVectorBatch> readBatch = rowReader->createRowBatch(1);
870+
EXPECT_TRUE(rowReader->next(*readBatch));
871+
auto* readStructBatch = dynamic_cast<StructVectorBatch*>(readBatch.get());
872+
auto* readTsBatch = dynamic_cast<TimestampVectorBatch*>(readStructBatch->fields[0]);
873+
EXPECT_EQ(123000000, readTsBatch->nanoseconds[0]);
874+
return readTsBatch->data[0];
875+
};
876+
877+
EXPECT_EQ(8 * 60 * 60, readTimestamp(/*useWriterTimezone=*/false));
878+
EXPECT_EQ(0, readTimestamp(/*useWriterTimezone=*/true));
879+
}
880+
843881
// Test that the ORC-306 compensation (-1s for pre-1970 timestamps with
844882
// nanos > 999999) is applied BEFORE timezone conversion in the Reader.
845883
//

0 commit comments

Comments
 (0)