Preserve terminal ORC timezone offsets

Signed-off-by: Chong Gao <chongg@nvidia.com>

Author: Chong Gao

src/main/java/com/nvidia/spark/rapids/jni/GpuTimeZoneDB.java

@@ -532,17 +532,38 @@ private static int getOrc2015YearBaseOffsetMillis(String timezoneId, OrcTimezone
   }
 
   private static ColumnVector getTransitionsForUtilTZ(OrcTimezoneInfo info) {
-    try (HostColumnVector hcv = HostColumnVector.fromLongs(info.transitions)) {
+    long[] transitions = info.transitions;
+    if (needsTerminalOffsetSentinel(info)) {
+      transitions = Arrays.copyOf(transitions, transitions.length + 1);
+      transitions[transitions.length - 1] = Long.MAX_VALUE;
+    }
+    try (HostColumnVector hcv = HostColumnVector.fromLongs(transitions)) {
       return hcv.copyToDevice();
     }
   }
 
   private static ColumnVector getOffsetsForUtilTZ(OrcTimezoneInfo info) {
-    try (HostColumnVector hcv = HostColumnVector.fromInts(info.offsets)) {
+    int[] offsets = info.offsets;
+    if (needsTerminalOffsetSentinel(info)) {
+      offsets = Arrays.copyOf(offsets, offsets.length + 1);
+      offsets[offsets.length - 1] = offsets[offsets.length - 2];
+    }
+    try (HostColumnVector hcv = HostColumnVector.fromInts(offsets)) {
       return hcv.copyToDevice();
     }
   }
 
+  private static boolean needsTerminalOffsetSentinel(OrcTimezoneInfo info) {
+    // Native lookup normally falls back to rawOffset beyond the last historical
+    // transition. Some JDK TimeZone implementations instead retain the final
+    // wall offset indefinitely. Keep every timestamp_us lookup inside the table
+    // with a Long.MAX_VALUE-millisecond sentinel when those offsets differ.
+    return info.dstRule == null
+        && info.offsets != null
+        && info.offsets.length > 0
+        && info.offsets[info.offsets.length - 1] != info.rawOffset;
+  }
+
   private static Table getTableForUtilTZ(OrcTimezoneInfo info) {
     if (info.transitions == null) {
       return null;

src/main/java/com/nvidia/spark/rapids/jni/OrcDstRuleExtractor.java

@@ -135,7 +135,6 @@ static final class DstRule {
    *     {@code rules}
    * @param rules {@link ZoneRules} for the zone
    * @return the recurring DST rule, or {@code null} if the zone has no DST
-   *     ({@code rules.isFixedOffset()} or {@code !tz.useDaylightTime()})
    * @throws IllegalStateException if the zone reports DST but neither
    *     extraction path produces a usable rule — for example, an unsupported
    *     {@link ZoneRules#getTransitionRules()} count (not 0 and not 2), a
@@ -149,7 +148,7 @@ static DstRule extractDstRule(String timezoneId, TimeZone tz, ZoneRules rules) {
     // TimeZone.getTimeZone(zoneId) silently returns GMT for such ids on most
     // JVMs, which would leave `tz` describing a different zone than `rules`.
     // Mirrors the guard in OrcTimezoneInfo.buildRuntimeOrcTimezoneInfo.
-    if (rules.isFixedOffset() || !tz.useDaylightTime()) {
+    if (rules.isFixedOffset()) {
       return null;
     }
     // Sanity-check that tz and rules describe the same zone. Both Path A and
@@ -173,6 +172,12 @@ static DstRule extractDstRule(String timezoneId, TimeZone tz, ZoneRules rules) {
     if (rule != null) {
       return rule;
     }
+    // Some JDK timezone implementations report useDaylightTime() == false even
+    // though getOffset() still applies a recurring DST rule in future years.
+    // ORC conversion follows getOffset(), so only trust this flag after probing.
+    if (!tz.useDaylightTime()) {
+      return null;
+    }
     rule = extractDstRuleFromZoneRules(timezoneId, tz, rules);
     if (rule != null) {
       return rule;
@@ -351,7 +356,10 @@ private static DstTransitions findDstTransitionsByProbing(TimeZone tz, int refYe
   private static DstRule buildDstRuleFromProbedTransitions(TimeZone tz,
       DstTransitions transitions) {
     DstRule rule = new DstRule();
-    rule.dstSavings = tz.getDSTSavings();
+    rule.dstSavings = tz.useDaylightTime()
+        ? tz.getDSTSavings()
+        : tz.getOffset(transitions.dstOnTransition)
+            - tz.getOffset(transitions.dstOnTransition - 1);
 
     int[] startFields = decodeTransition(transitions.dstOnTransition, tz.getRawOffset());
     rule.startMonth = startFields[0];

src/test/java/com/nvidia/spark/rapids/jni/GpuTimeZoneDBTest.java

@@ -146,10 +146,14 @@ private static ColumnVector convertOrcFromUtcOnCPU(
   }
 
   private static Long[] getOrcFromUtcBoundaryMicros(String readerTzId) {
+    long minSupportedUs = LocalDateTime.of(1, 1, 1, 0, 0)
+        .toEpochSecond(ZoneOffset.UTC) * MICROS_PER_SECOND;
+    long maxSupportedUs = LocalDateTime.of(9999, 12, 31, 23, 59, 59)
+        .toEpochSecond(ZoneOffset.UTC) * MICROS_PER_SECOND + 999_999L;
     List<Long> values = new ArrayList<>(Arrays.asList(
         null,
-        Long.MIN_VALUE,
-        Long.MIN_VALUE + 1,
+        minSupportedUs,
+        minSupportedUs + 1,
         -3_649_379_812_521_628L,
         -2_957_649_381_472_612L,
         -1_501L,
@@ -161,8 +165,8 @@ private static Long[] getOrcFromUtcBoundaryMicros(String readerTzId) {
         999L,
         1_001L,
         514_952_012L,
-        Long.MAX_VALUE - 1,
-        Long.MAX_VALUE));
+        maxSupportedUs - 1,
+        maxSupportedUs));
 
     OrcTimezoneInfo readerInfo = OrcTimezoneInfo.get(readerTzId);
     if (readerInfo.transitions != null) {
@@ -282,6 +286,7 @@ void testConvertOrcFromUtcAllTimezones() {
     List<String> timezones = Arrays.asList(
         "UTC",
         "America/New_York",
+        "America/Vancouver",
         "America/Los_Angeles",
         "Europe/Paris",
         "Asia/Shanghai",
@@ -354,6 +359,7 @@ void testConvertOrcTimezones() {
 
     List<String> timezones = Arrays.asList(
         "America/Los_Angeles",
+        "America/Vancouver",
         "America/Cancun",
         "Asia/Shanghai",
         "Antarctica/DumontDUrville",

src/test/java/com/nvidia/spark/rapids/jni/OrcTimezoneInfoTest.java

@@ -31,9 +31,11 @@
 import java.time.zone.ZoneRules;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Calendar;
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
+import java.util.SimpleTimeZone;
 import java.util.TimeZone;
 
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
@@ -400,10 +402,50 @@ void testExtractDstRuleSouthernHemisphere() {
   @Test
   void testExtractDstRuleNoDstReturnsNull() {
     // Asia/Shanghai had DST historically (1940s, 1986-1991) but no current rule.
-    // tz.useDaylightTime() must be false → extractDstRule returns null.
+    // Probing finds no recurring transitions and useDaylightTime() is false.
     assertNull(extractDstRuleFor("Asia/Shanghai"));
   }
 
+  @Test
+  void testExtractDstRuleProbesBeforeUsingDaylightFlag() {
+    TimeZone tz = new SimpleTimeZone(
+        -8 * 3_600_000,
+        "Synthetic/FalseDaylightFlag",
+        Calendar.MARCH,
+        2,
+        Calendar.SUNDAY,
+        2 * 3_600_000,
+        Calendar.NOVEMBER,
+        1,
+        Calendar.SUNDAY,
+        2 * 3_600_000,
+        3_600_000) {
+      @Override
+      public boolean useDaylightTime() {
+        return false;
+      }
+    };
+    ZoneOffset baseOffset = ZoneOffset.ofHours(-8);
+    ZoneOffsetTransition historical = ZoneOffsetTransition.of(
+        LocalDateTime.of(1900, 1, 1, 0, 0),
+        ZoneOffset.ofHours(-9),
+        baseOffset);
+    ZoneRules rules = ZoneRules.of(
+        baseOffset,
+        baseOffset,
+        Collections.emptyList(),
+        Collections.singletonList(historical),
+        Collections.emptyList());
+
+    assertFalse(tz.useDaylightTime());
+    OrcDstRuleExtractor.DstRule rule = OrcDstRuleExtractor.extractDstRule(
+        tz.getID(), tz, rules);
+    assertNotNull(rule);
+    assertEquals(3_600_000, rule.dstSavings);
+    assertEquals(Calendar.MARCH, rule.startMonth);
+    assertEquals(Calendar.NOVEMBER, rule.endMonth);
+  }
+
   @Test
   void testExtractDstRuleFixedOffsetReturnsNull() {
     // Fixed-offset zones never observe DST.