Fix to_datetime nanosecond inference and DatetimeIndex tz wall-time interpretation (#23141)

- `to_datetime` infers nanosecond precision for strings with more than 6 fractional-second digits, matching pandas.
- `DatetimeIndex` with a timezone dtype treats timezone-naive strings/datetimes as wall time in the target timezone, while numeric data stays interpreted as UTC epoch values.
- Removes 13 now-passing entries from the pandas-testing plugin.
- Split out from #23138.

Authors:
  - GALI PREM SAGAR (https://github.com/galipremsagar)

Approvers:
  - Matthew Roeschke (https://github.com/mroeschke)

URL: #23141

Author: galipremsagar

python/cudf/cudf/core/index.py

@@ -3399,7 +3399,18 @@ def __init__(
             if dtype.kind != "M":
                 raise TypeError("dtype must be a datetime type")
             elif not isinstance(data.dtype, pd.DatetimeTZDtype):
-                data = data.astype(dtype)
+                if (
+                    isinstance(dtype, pd.DatetimeTZDtype)
+                    and data.dtype.kind not in "iuf"
+                ):
+                    # pandas interprets timezone-naive strings/datetimes as
+                    # wall time in the target timezone (numeric data stays
+                    # interpreted as UTC epoch values)
+                    data = data.astype(
+                        np.dtype(f"datetime64[{dtype.unit}]")
+                    ).tz_localize(str(dtype.tz))
+                else:
+                    data = data.astype(dtype)
         elif data.dtype.kind != "M":
             if is_dtype_obj_string(data.dtype):
                 # Pandas's array_to_datetime falls back to [s] when no

python/cudf/cudf/core/tools/datetimes.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION.
+# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
@@ -494,9 +494,16 @@ def _process_col(
                         element=col.element_indexing(0),
                         dayfirst=dayfirst,
                     )
+                target_unit = unit
+                if unit is None:
+                    # pandas infers nanosecond precision from strings with
+                    # more than 6 fractional-second digits
+                    subsecond = re.search(r"%(\d)f", format)
+                    if subsecond is not None and int(subsecond.group(1)) > 6:
+                        target_unit = "ns"
                 col = col.strptime(
                     dtype=np.dtype(
-                        _unit_dtype_map.get(unit, _unit_dtype_map["us"])  # type: ignore[arg-type]
+                        _unit_dtype_map.get(target_unit, _unit_dtype_map["us"])  # type: ignore[arg-type]
                     ),
                     format=format,
                 )

python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py

@@ -2320,14 +2320,8 @@ def pytest_unconfigure(config):
     "tests/indexes/datetimes/test_arithmetic.py::TestDatetimeIndexArithmetic::test_add_dti_day": "TypeError: unsupported operand type(s) for +: 'DatetimeArray' and 'DateOffset'",
     "tests/indexes/datetimes/test_constructors.py::TestDatetimeIndex::test_constructor_coverage": "TODO: Add a reason for failure",
     "tests/indexes/datetimes/test_constructors.py::TestDatetimeIndex::test_constructor_datetime64_tzformat[W-SUN]": "TODO: Add a reason for failure",
-    "tests/indexes/datetimes/test_constructors.py::TestDatetimeIndex::test_constructor_dtype": "AssertionError: Index are different",
     "tests/indexes/datetimes/test_constructors.py::TestDatetimeIndex::test_constructor_dtype_tz_mismatch_raises": "Failed: DID NOT RAISE <class 'ValueError'>",
-    "tests/indexes/datetimes/test_constructors.py::TestDatetimeIndex::test_constructor_start_end_with_tz[America/Los_Angeles]": "NotImplementedError: freq is not implemented yet",
-    "tests/indexes/datetimes/test_constructors.py::TestDatetimeIndex::test_constructor_start_end_with_tz[None]": "NotImplementedError: freq is not implemented yet",
-    "tests/indexes/datetimes/test_constructors.py::TestDatetimeIndex::test_constructor_start_end_with_tz[tz2]": "NotImplementedError: freq is not implemented yet",
-    "tests/indexes/datetimes/test_constructors.py::TestDatetimeIndex::test_constructor_start_end_with_tz[tz3]": "NotImplementedError: freq is not implemented yet",
     "tests/indexes/datetimes/test_constructors.py::TestDatetimeIndex::test_dti_from_tzaware_datetime[tz1]": "AssertionError: assert False",
-    "tests/indexes/datetimes/test_constructors.py::TestDatetimeIndex::test_integer_values_and_tz_interpreted_as_utc": "AssertionError: Index are different",
     "tests/indexes/datetimes/test_constructors.py::TestTimeSeries::test_constructor_int64_nocopy": "TODO: Add a reason for failure",
     "tests/indexes/datetimes/test_constructors.py::TestTimeSeries::test_dti_constructor_small_int[int16]": "AssertionError: Index are different",
     "tests/indexes/datetimes/test_constructors.py::TestTimeSeries::test_dti_constructor_small_int[int32]": "AssertionError: Index are different",
@@ -2402,8 +2396,6 @@ def pytest_unconfigure(config):
     "tests/indexes/datetimes/test_partial_slicing.py::TestSlicing::test_partial_slice_minutely": "TODO: Add a reason for failure",
     "tests/indexes/datetimes/test_partial_slicing.py::TestSlicing::test_partial_slicing_dataframe": "TODO: Add a reason for failure",
     "tests/indexes/datetimes/test_partial_slicing.py::TestSlicing::test_return_type_doesnt_depend_on_monotonicity": "TODO: Add a reason for failure",
-    "tests/indexes/datetimes/test_scalar_compat.py::TestDatetimeIndexOps::test_dti_time2[datetime64[ns, CET]]": "AssertionError: numpy array are different",
-    "tests/indexes/datetimes/test_scalar_compat.py::TestDatetimeIndexOps::test_dti_time2[datetime64[ns, EST]]": "AssertionError: numpy array are different",
     "tests/indexes/datetimes/test_setops.py::TestDatetimeIndexSetOps::test_difference_freq[False]": "TODO: Add a reason for failure",
     "tests/indexes/datetimes/test_setops.py::TestDatetimeIndexSetOps::test_difference_freq[None]": "TODO: Add a reason for failure",
     "tests/indexes/datetimes/test_setops.py::TestDatetimeIndexSetOps::test_intersection[False-Asia/Tokyo]": "TODO: Add a reason for failure",
@@ -2424,10 +2416,6 @@ def pytest_unconfigure(config):
     "tests/indexes/datetimes/test_timezones.py::TestDatetimeIndexTimezones::test_date_range_localize2[ns]": "assert nan == 3",
     "tests/indexes/datetimes/test_timezones.py::TestDatetimeIndexTimezones::test_date_range_localize2[s]": "assert nan == 3",
     "tests/indexes/datetimes/test_timezones.py::TestDatetimeIndexTimezones::test_date_range_localize2[us]": "assert nan == 3",
-    "tests/indexes/datetimes/test_timezones.py::TestDatetimeIndexTimezones::test_date_range_localize[ms]": "AssertionError: Index are different",
-    "tests/indexes/datetimes/test_timezones.py::TestDatetimeIndexTimezones::test_date_range_localize[ns]": "AssertionError: Index are different",
-    "tests/indexes/datetimes/test_timezones.py::TestDatetimeIndexTimezones::test_date_range_localize[s]": "AssertionError: Index are different",
-    "tests/indexes/datetimes/test_timezones.py::TestDatetimeIndexTimezones::test_date_range_localize[us]": "AssertionError: Index are different",
     "tests/indexes/datetimes/test_timezones.py::TestDatetimeIndexTimezones::test_dti_convert_tz_aware_datetime_datetime[tz0]": "AssertionError: assert zoneinfo.ZoneInfo(key='US/Eastern') is datetime.timezone.utc",
     "tests/indexes/datetimes/test_timezones.py::TestDatetimeIndexTimezones::test_dti_convert_tz_aware_datetime_datetime[tz1]": "AssertionError: assert False",
     "tests/indexes/interval/test_astype.py::TestDatetimelikeSubtype::test_subtype_integer[index0-int64]": "TODO: Add a reason for failure",
@@ -3418,7 +3406,6 @@ def pytest_unconfigure(config):
     "tests/reshape/concat/test_concat.py::test_concat_ignore_empty_object_float[object-datetime64[ns]]": "AssertionError: Attributes of DataFrame.iloc[:, 0] (column name='foo') are different",
     "tests/reshape/concat/test_concat.py::test_concat_ignore_empty_object_float[object-float64]": "AssertionError: Attributes of DataFrame.iloc[:, 0] (column name='foo') are different",
     "tests/reshape/concat/test_concat.py::test_concat_ignore_empty_object_float[object-int64]": "TODO: Add a reason for failure",
-    "tests/reshape/concat/test_datetimes.py::TestTimezoneConcat::test_concat_multiindex_with_tz": "AssertionError: DataFrame.index level [0] are different",
     "tests/reshape/concat/test_datetimes.py::test_concat_float_datetime64": "TODO: Add a reason for failure",
     "tests/reshape/concat/test_empty.py::TestEmptyConcat::test_concat_empty_dataframe_dtypes": "TODO: Add a reason for failure",
     "tests/reshape/concat/test_empty.py::TestEmptyConcat::test_concat_empty_df_object_dtype": "TODO: Add a reason for failure",
@@ -3704,14 +3691,6 @@ def pytest_unconfigure(config):
     "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_with_list_like_values[True-list]": "AssertionError: Attributes of DataFrame.iloc[:, 0] (column name='('baz', 'A')') are different",
     "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_with_list_like_values_nans[False-values0]": "TODO: Add a reason for failure",
     "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_with_list_like_values_nans[True-values0]": "TODO: Add a reason for failure",
-    "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_with_tz[ms-False]": "AssertionError: DataFrame.index are different",
-    "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_with_tz[ms-True]": "AssertionError: DataFrame.index are different",
-    "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_with_tz[ns-False]": "AssertionError: DataFrame.index are different",
-    "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_with_tz[ns-True]": "AssertionError: DataFrame.index are different",
-    "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_with_tz[s-False]": "AssertionError: DataFrame.index are different",
-    "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_with_tz[s-True]": "AssertionError: DataFrame.index are different",
-    "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_with_tz[us-False]": "AssertionError: DataFrame.index are different",
-    "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_with_tz[us-True]": "AssertionError: DataFrame.index are different",
     "tests/reshape/test_pivot_multilevel.py::test_pivot_list_like_columns[input_index1-input_columns1-values-expected_values1-expected_columns1-expected_index1]": "TODO: Add a reason for failure",
     "tests/reshape/test_pivot_multilevel.py::test_pivot_list_like_columns[input_index3-input_columns3-values-expected_values3-expected_columns3-expected_index3]": "TODO: Add a reason for failure",
     "tests/reshape/test_pivot_multilevel.py::test_pivot_list_like_columns[lev4-input_columns0-values-expected_values0-expected_columns0-expected_index0]": "TODO: Add a reason for failure",
@@ -4347,12 +4326,7 @@ def pytest_unconfigure(config):
     "tests/test_algos.py::TestFactorize::test_factorize[tuples-False]": "AssertionError: Index are different",
     "tests/test_algos.py::TestFactorize::test_factorize[tuples-True]": "AssertionError: Index are different",
     "tests/test_algos.py::TestFactorize::test_int_factorize_use_na_sentinel_false[data0-expected_codes0-expected_uniques0]": "TODO: Add a reason for failure",
-    "tests/test_algos.py::TestUnique::test_datetime64_dtype_array_returned": "TODO: Add a reason for failure",
     "tests/test_algos.py::TestUnique::test_factorize_multiindex_empty": "AssertionError: Index are different",
-    "tests/test_algos.py::TestUnique::test_order_of_appearance_dt64tz[ms]": "AssertionError: Index are different",
-    "tests/test_algos.py::TestUnique::test_order_of_appearance_dt64tz[ns]": "AssertionError: Index are different",
-    "tests/test_algos.py::TestUnique::test_order_of_appearance_dt64tz[s]": "AssertionError: Index are different",
-    "tests/test_algos.py::TestUnique::test_order_of_appearance_dt64tz[us]": "AssertionError: Index are different",
     "tests/test_algos.py::TestValueCounts::test_value_counts_dropna": "TODO: Add a reason for failure",
     "tests/test_algos.py::TestValueCounts::test_value_counts_stability": "AssertionError: Series.index are different",
     "tests/test_col.py::test_cached_property": "AssertionError: assert False",
@@ -4466,8 +4440,6 @@ def pytest_unconfigure(config):
     "tests/tools/test_to_datetime.py::TestOrigin::test_julian": "AssertionError: Attributes of Series are different",
     "tests/tools/test_to_datetime.py::TestOrigin::test_to_datetime_out_of_bounds_with_format_arg[%Y-%d-%m %H:%M:%S-None]": "TODO: Add a reason for failure",
     "tests/tools/test_to_datetime.py::TestOrigin::test_to_datetime_out_of_bounds_with_format_arg[%Y-%m-%d %H:%M:%S-None]": "TODO: Add a reason for failure",
-    "tests/tools/test_to_datetime.py::TestTimeConversionFormats::test_parse_nanoseconds_with_formula[False-2012-01-01 09:00:00.000000001]": "cudf strptime drops trailing nanosecond precision",
-    "tests/tools/test_to_datetime.py::TestTimeConversionFormats::test_parse_nanoseconds_with_formula[True-2012-01-01 09:00:00.000000001]": "cudf strptime drops trailing nanosecond precision",
     "tests/tools/test_to_datetime.py::TestTimeConversionFormats::test_to_datetime_format_weeks[False-2013020-%Y%U%w-2013-01-13]": "AssertionError: assert Timestamp('2013-01-19 00:00:00') == Timestamp('2013-01-13 00:00:00')",
     "tests/tools/test_to_datetime.py::TestTimeConversionFormats::test_to_datetime_format_weeks[True-2013020-%Y%U%w-2013-01-13]": "AssertionError: assert Timestamp('2013-01-19 00:00:00') == Timestamp('2013-01-13 00:00:00')",
     "tests/tools/test_to_datetime.py::TestToDatetime::test_mixed_offsets_with_native_datetime_utc_false_raises": "assert False",

python/cudf/cudf/tests/general_functions/test_to_datetime.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION.
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
 
@@ -407,3 +407,30 @@ def test_to_datetime_dataframe_with_ns_field_widens_to_ns():
     actual = cudf.to_datetime(cudf.from_pandas(df))
     assert actual.dtype == expected.dtype
     assert_eq(actual, expected)
+
+
+def test_to_datetime_nanosecond_precision_strings():
+    # pandas infers nanosecond precision from strings with more than 6
+    # fractional-second digits.
+    data = ["2015-01-03T00:00:00.000000000", "2015-01-01T00:00:00.000000000"]
+    assert_eq(pd.to_datetime(data), cudf.to_datetime(data))
+    data_us = ["2015-01-03T00:00:00.000000", "2015-01-01T00:00:00.000000"]
+    assert_eq(pd.to_datetime(data_us), cudf.to_datetime(data_us))
+
+
+@pytest.mark.parametrize(
+    "data",
+    [
+        ["2016-01-01 00:00:00"],
+        [1451606400, 1451610000],
+    ],
+)
+def test_datetime_index_tz_dtype_wall_time(data):
+    # Timezone-naive string data with a tz-aware dtype is interpreted as
+    # wall time in the target timezone, while integers are interpreted as
+    # epoch time (UTC), matching pandas.
+    result = cudf.DatetimeIndex(
+        data, dtype=pd.DatetimeTZDtype("s", "US/Eastern")
+    )
+    expected = pd.DatetimeIndex(data, dtype="datetime64[s, US/Eastern]")
+    assert_eq(result, expected)