Merge branch 'main' into fix-io-benchmarks-12739

Author: PointKernel

.gitignore

@@ -187,3 +187,6 @@ compile_commands.json
 # pytest artifacts
 rmm_log.txt
 python/cudf/cudf_pandas_tests/data/rmm_log.txt
+
+# Quent traces
+logs/*.ndjson

python/cudf/cudf/core/index.py

@@ -85,7 +85,7 @@
     from cudf.core.dataframe import DataFrame
     from cudf.core.multiindex import MultiIndex
     from cudf.core.series import Series
-    from cudf.core.tools.datetimes import DateOffset, MonthEnd, YearEnd
+    from cudf.core.tools.datetimes import DateOffset
 
 
 def ensure_index(index_like: Any, nan_as_null=no_default) -> Index:
@@ -2025,9 +2025,7 @@ def __repr__(self) -> str:
             and isinstance(self, DatetimeIndex)
             and self._freq is not None
         ):
-            keywords.append(
-                f"freq={self._freq._maybe_as_fast_pandas_offset().freqstr!r}"
-            )
+            keywords.append(f"freq={self._freq.freqstr!r}")
         joined_keywords = ", ".join(keywords)
         lines.append(f"{prior_to_dtype} {joined_keywords})")
         return "\n".join(lines)
@@ -3381,11 +3379,8 @@ def __init__(
         if yearfirst is not False:
             raise NotImplementedError("yearfirst == True is not yet supported")
 
-        if freq is None:
-            if isinstance(data, type(self)):
-                freq = data.freq
-            if was_pd_index and data.freq is not None:
-                freq = data.freq.freqstr
+        if freq is None and isinstance(data, (type(self), pd.DatetimeIndex)):
+            freq = data.freq
 
         name = _getdefault_name(data, name=name)
 
@@ -3439,22 +3434,32 @@ def __init__(
     @_performance_tracking
     def serialize(self):
         header, frames = super().serialize()
-        if self.freq is not None:
+        if self._freq is None:
+            header["freq"] = None
+        elif type(self._freq) is pd.DateOffset:
+            # generic offsets have no parseable freqstr; store the kwds
+            # plus n/normalize, which .kwds omits
             header["freq"] = {
-                "kwds": self.freq.kwds,
+                "kwds": self._freq.kwds,
+                "n": self._freq.n,
+                "normalize": self._freq.normalize,
             }
         else:
-            header["freq"] = None
+            header["freq"] = self._freq.freqstr
         return header, frames
 
     @classmethod
     @_performance_tracking
     def deserialize(cls, header, frames):
         obj = super().deserialize(header, frames)
-        if (header_payload := header.get("freq")) is not None:
-            freq = cudf.DateOffset(**header_payload["kwds"])
+        if isinstance(header_payload := header.get("freq"), dict):
+            freq = pd.DateOffset(
+                n=header_payload.get("n", 1),
+                normalize=header_payload.get("normalize", False),
+                **header_payload["kwds"],
+            )
         else:
-            freq = None
+            freq = header_payload
 
         obj._freq = _validate_freq(freq)
         return obj
@@ -3522,7 +3527,7 @@ def find_label_range(self, loc: slice) -> slice:
     @_performance_tracking
     def copy(self, name=None, deep=False):
         idx_copy = super().copy(name=name, deep=deep)
-        idx_copy._freq = _validate_freq(self._freq)
+        idx_copy._freq = self._freq
         return idx_copy
 
     def as_unit(self, unit: str, round_ok: bool = True) -> Self:
@@ -3578,8 +3583,8 @@ def asi8(self) -> cupy.ndarray:
         return self._column.astype(np.dtype(np.int64)).values
 
     @property
-    def inferred_freq(self) -> DateOffset | MonthEnd | YearEnd | None:
-        if self._freq:
+    def inferred_freq(self):
+        if self._freq is not None:
             return self._freq
 
         plc_col = self._column.plc_column
@@ -3635,7 +3640,12 @@ def inferred_freq(self) -> DateOffset | MonthEnd | YearEnd | None:
                 if (c := getattr(cmps, component)) != 0:
                     kwds[component] = c
 
-            return cudf.DateOffset(**kwds)
+            # not pd.DateOffset(**kwds): a generic pd.DateOffset never
+            # compares equal to the fast offsets pandas infers (e.g.
+            # pd.DateOffset(days=1) != pd.offsets.Day()) and its freqstr
+            # is not parseable, so single-unit offsets must be converted
+            # to their fast pandas equivalents
+            return cudf.DateOffset(**kwds)._maybe_as_fast_pandas_offset()
 
         # maximum unique count supported is months with 4 unique lengths
         # bail above that for now
@@ -3644,19 +3654,23 @@ def inferred_freq(self) -> DateOffset | MonthEnd | YearEnd | None:
             if all(x in self.YEARLY_PERIODS for x in uniques_host):
                 # Could be year end or could be an anchored year end
                 if self.is_year_end.all():
-                    return cudf.DateOffset._from_freqstr("YE-DEC")
+                    return pd.tseries.frequencies.to_offset("YE-DEC")
                 else:
                     raise NotImplementedError()
             elif all(x in self.MONTHLY_PERIODS for x in uniques_host):
                 if self.is_month_end.all():
-                    return cudf.DateOffset._from_freqstr("ME")
+                    return pd.tseries.frequencies.to_offset("ME")
             else:
                 raise NotImplementedError
         else:
             return None
         return None
 
     def _get_slice_frequency(self, slc=None):
+        if self._freq is None:
+            # match pandas: slicing an index without a cached freq
+            # produces a result with freq=None
+            return None
         if slc.step in (1, None):
             # no change in freq
             return self._freq
@@ -3665,12 +3679,7 @@ def _get_slice_frequency(self, slc=None):
         else:
             if slc:
                 # fastpath: dont introspect
-                # Multiply the pandas offset directly (pd.Timedelta(offset)
-                # fails for calendar-based offsets like Day in pandas 3).
-                new_freq = slc.step * self._freq._maybe_as_fast_pandas_offset()
-                return cudf.DateOffset._from_freqstr(
-                    pd.tseries.frequencies.to_offset(new_freq).freqstr
-                )
+                return slc.step * self._freq
             else:
                 return self.inferred_freq
 
@@ -3680,37 +3689,32 @@ def _validate_freq_against_data(self, freq) -> None:
         unique_vals = cudf.Series._from_column(
             self.to_series().diff()._column.unique()
         )
-        if freq == cudf.DateOffset(months=1):
+        if freq == pd.DateOffset(months=1):
             possible = pd.Series(list(self.MONTHLY_PERIODS | {pd.NaT}))
             if unique_vals.isin(possible).sum() != len(unique_vals):
                 raise ValueError(
                     f"Inferred frequency from passed values does not "
-                    f"conform to passed frequency "
-                    f"{freq._maybe_as_fast_pandas_offset().freqstr}"
+                    f"conform to passed frequency {freq.freqstr}"
                 )
-        elif freq == cudf.DateOffset(years=1):
+        elif freq == pd.DateOffset(years=1):
             possible = pd.Series(list(self.YEARLY_PERIODS | {pd.NaT}))
             if unique_vals.isin(possible).sum() != len(unique_vals):
                 raise ValueError(
                     f"Inferred frequency from passed values does not "
-                    f"conform to passed frequency "
-                    f"{freq._maybe_as_fast_pandas_offset().freqstr}"
+                    f"conform to passed frequency {freq.freqstr}"
                 )
         else:
             if len(unique_vals) > 2 or (
-                len(unique_vals) == 2
-                and unique_vals[1].value
-                != freq._maybe_as_fast_pandas_offset().nanos
+                len(unique_vals) == 2 and unique_vals[1].value != freq.nanos
             ):
                 raise ValueError(
                     f"Inferred frequency from passed values does not "
-                    f"conform to passed frequency "
-                    f"{freq._maybe_as_fast_pandas_offset().freqstr}"
+                    f"conform to passed frequency {freq.freqstr}"
                 )
 
     @property
-    def freq(self) -> DateOffset | None:
-        return self._freq  # type: ignore[return-value]  # (validated setter stores DateOffset-compatible value)
+    def freq(self) -> pd.tseries.offsets.BaseOffset | None:
+        return self._freq
 
     @freq.setter
     def freq(self, value) -> None:
@@ -4275,7 +4279,7 @@ def to_pandas(
             inferred = result.inferred_freq
             if inferred is None:
                 try:
-                    result.freq = self._freq._maybe_as_fast_pandas_offset()
+                    result.freq = self._freq
                 except ValueError:
                     pass
             else:
@@ -5779,11 +5783,28 @@ def _get_nearest_indexer(
     return indexer
 
 
-def _validate_freq(freq: Any) -> DateOffset | MonthEnd | YearEnd | None:
+def _validate_freq(freq: Any) -> pd.tseries.offsets.BaseOffset | None:
+    """Normalize ``freq`` to a canonical pandas offset.
+
+    Accepts freq strings, pandas offsets and cudf's own offset classes,
+    rejecting frequencies that cudf cannot represent.
+    """
+    from cudf.core.tools.datetimes import DateOffset, MonthEnd, YearEnd
+
+    if freq is None:
+        return None
+    if isinstance(freq, (DateOffset, MonthEnd, YearEnd)):
+        return freq._maybe_as_fast_pandas_offset()
     if isinstance(freq, str):
-        return cudf.DateOffset._from_freqstr(freq)
-    elif freq is None:
-        return freq
-    elif freq is not None and not isinstance(freq, cudf.DateOffset):
+        # cudf supports a subset of pandas frequency strings; parse with
+        # cudf's DateOffset (which raises for unsupported ones) and store
+        # the equivalent pandas offset.
+        return DateOffset._from_freqstr(freq)._maybe_as_fast_pandas_offset()
+    if not isinstance(freq, pd.tseries.offsets.BaseOffset):
         raise ValueError(f"Invalid frequency: {freq}")
-    return cast("cudf.DateOffset", freq)
+    # gate pandas offsets on the same supported subset
+    if type(freq) is pd.DateOffset:
+        DateOffset(**freq.kwds)
+    else:
+        DateOffset._from_freqstr(freq.freqstr)
+    return freq

python/cudf/cudf/pandas/_wrappers/pandas.py

@@ -2789,6 +2789,10 @@ def _unpickle_offset_obj(pickled_args):
 # same reducer/unpickler can be used for Timedelta:
 copyreg.dispatch_table[Timedelta] = _reduce_proxied_td_obj
 copyreg.dispatch_table[pd.Timedelta] = _reduce_obj
+# Period has no fast (cudf) representation; pickle the wrapped/real pandas
+# object directly (e.g. matplotlib figures store pandas.Period x-data).
+copyreg.dispatch_table[Period] = _reduce_proxied_td_obj
+copyreg.dispatch_table[pd.Period] = _reduce_obj
 
 # TODO: Need to find a way to unpickle cross-version(old) pickled objects.
 # Register custom reducer/unpickler functions for pandas objects
@@ -2820,6 +2824,18 @@ def _unpickle_offset_obj(pickled_args):
 )
 
 copyreg.dispatch_table[pd.DateOffset] = _reduce_offset_obj
+# Concrete pandas offsets (Day, Week, Minute, ...) are stored e.g. as the
+# ``freq`` of matplotlib's date converters. The module accelerator makes the
+# offset module attributes resolve to proxies, so pickle's class-identity
+# check fails; reduce each via its real ``__reduce__`` (with the accelerator
+# disabled). ``DateOffset`` keeps its dedicated reducer above.
+for _offset_cls in vars(pd.tseries.offsets).values():
+    if (
+        isinstance(_offset_cls, type)
+        and issubclass(_offset_cls, pd.tseries.offsets.BaseOffset)
+        and _offset_cls is not pd.DateOffset
+    ):
+        copyreg.dispatch_table.setdefault(_offset_cls, _reduce_obj)
 
 
 def _unpickle_NaT():

python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py

@@ -2296,19 +2296,11 @@ def pytest_unconfigure(config):
     "tests/indexes/datetimes/methods/test_shift.py::TestDatetimeIndexShift::test_dti_shift_across_dst[ns]": "AssertionError: Series.index are different",
     "tests/indexes/datetimes/methods/test_shift.py::TestDatetimeIndexShift::test_dti_shift_across_dst[s]": "AssertionError: Series.index are different",
     "tests/indexes/datetimes/methods/test_shift.py::TestDatetimeIndexShift::test_dti_shift_across_dst[us]": "AssertionError: Series.index are different",
-    "tests/indexes/datetimes/methods/test_shift.py::TestDatetimeIndexShift::test_dti_shift_int[ms]": "TypeError: unsupported operand type(s) for *: 'int' and 'DateOffset'",
-    "tests/indexes/datetimes/methods/test_shift.py::TestDatetimeIndexShift::test_dti_shift_int[ns]": "TypeError: unsupported operand type(s) for *: 'int' and 'DateOffset'",
-    "tests/indexes/datetimes/methods/test_shift.py::TestDatetimeIndexShift::test_dti_shift_int[s]": "TypeError: unsupported operand type(s) for *: 'int' and 'DateOffset'",
-    "tests/indexes/datetimes/methods/test_shift.py::TestDatetimeIndexShift::test_dti_shift_int[us]": "TypeError: unsupported operand type(s) for *: 'int' and 'DateOffset'",
-    "tests/indexes/datetimes/methods/test_to_period.py::TestToPeriod::test_to_period_tz[US/Eastern]": "TypeError: Argument 'freq' has incorrect type (expected str, got DateOffset)",
-    "tests/indexes/datetimes/methods/test_to_period.py::TestToPeriod::test_to_period_tz_utc_offset_consistency[Etc/GMT+1]": "TypeError: Argument 'freq' has incorrect type (expected str, got DateOffset)",
-    "tests/indexes/datetimes/methods/test_to_period.py::TestToPeriod::test_to_period_tz_utc_offset_consistency[Etc/GMT-1]": "TypeError: Argument 'freq' has incorrect type (expected str, got DateOffset)",
     "tests/indexes/datetimes/methods/test_tz_convert.py::TestTZConvert::test_dti_tz_convert_dst": "ValueError: Inferred frequency None from passed values does not conform to passed frequency h",
     "tests/indexes/datetimes/methods/test_tz_localize.py::TestTZLocalize::test_dti_tz_localize[]": "Failed: DID NOT RAISE <class 'ValueError'>",
     "tests/indexes/datetimes/methods/test_tz_localize.py::TestTZLocalize::test_dti_tz_localize_nonexistent_raise_coerce": "Failed: DID NOT RAISE <class 'ValueError'>",
     "tests/indexes/datetimes/methods/test_tz_localize.py::TestTZLocalize::test_dti_tz_localize_utc_conversion[tz2]": "Failed: DID NOT RAISE <class 'ValueError'>",
     "tests/indexes/datetimes/methods/test_unique.py::test_index_unique2": "TODO: Add a reason for failure",
-    "tests/indexes/datetimes/test_arithmetic.py::TestDatetimeIndexArithmetic::test_add_dti_day": "TypeError: unsupported operand type(s) for +: 'DatetimeArray' and 'DateOffset'",
     "tests/indexes/datetimes/test_constructors.py::TestDatetimeIndex::test_constructor_coverage": "TODO: Add a reason for failure",
     "tests/indexes/datetimes/test_constructors.py::TestDatetimeIndex::test_constructor_datetime64_tzformat[W-SUN]": "TODO: Add a reason for failure",
     "tests/indexes/datetimes/test_constructors.py::TestDatetimeIndex::test_constructor_dtype_tz_mismatch_raises": "Failed: DID NOT RAISE <class 'ValueError'>",
@@ -3220,20 +3212,6 @@ def pytest_unconfigure(config):
     "tests/plotting/test_datetimelike.py::TestTSPlot::test_from_resampling_area_line_mixed_high_to_low[area-line]": "ValueError: You must pass a freq argument as current index has none.",
     "tests/plotting/test_datetimelike.py::TestTSPlot::test_from_resampling_area_line_mixed_high_to_low[line-area]": "ValueError: You must pass a freq argument as current index has none.",
     "tests/plotting/test_datetimelike.py::TestTSPlot::test_from_weekly_resampling": "ValueError: You must pass a freq argument as current index has none.",
-    "tests/plotting/test_datetimelike.py::TestTSPlot::test_line_plot_datetime_frame[D]": "TODO: Add a reason for failure",
-    "tests/plotting/test_datetimelike.py::TestTSPlot::test_line_plot_datetime_frame[W]": "TODO: Add a reason for failure",
-    "tests/plotting/test_datetimelike.py::TestTSPlot::test_line_plot_datetime_frame[h]": "TODO: Add a reason for failure",
-    "tests/plotting/test_datetimelike.py::TestTSPlot::test_line_plot_datetime_frame[min]": "TODO: Add a reason for failure",
-    "tests/plotting/test_datetimelike.py::TestTSPlot::test_line_plot_datetime_frame[s]": "TODO: Add a reason for failure",
-    "tests/plotting/test_datetimelike.py::TestTSPlot::test_line_plot_inferred_freq[D]": "TODO: Add a reason for failure",
-    "tests/plotting/test_datetimelike.py::TestTSPlot::test_line_plot_inferred_freq[ME]": "TODO: Add a reason for failure",
-    "tests/plotting/test_datetimelike.py::TestTSPlot::test_line_plot_inferred_freq[YE]": "TODO: Add a reason for failure",
-    "tests/plotting/test_datetimelike.py::TestTSPlot::test_pickle_fig[DataFrame-idx0]": "_pickle.PicklingError: Can't pickle <class 'pandas.Period'>: it's not the same object as pandas.Period",
-    "tests/plotting/test_datetimelike.py::TestTSPlot::test_pickle_fig[DataFrame-idx1]": "_pickle.PicklingError: Can't pickle <class 'pandas.Period'>: it's not the same object as pandas.Period",
-    "tests/plotting/test_datetimelike.py::TestTSPlot::test_pickle_fig[DataFrame-idx3]": "_pickle.PicklingError: Can't pickle <class 'pandas.Period'>: it's not the same object as pandas.Period",
-    "tests/plotting/test_datetimelike.py::TestTSPlot::test_pickle_fig[Series-idx0]": "_pickle.PicklingError: Can't pickle <class 'pandas.Period'>: it's not the same object as pandas.Period",
-    "tests/plotting/test_datetimelike.py::TestTSPlot::test_pickle_fig[Series-idx1]": "_pickle.PicklingError: Can't pickle <class 'pandas.Period'>: it's not the same object as pandas.Period",
-    "tests/plotting/test_datetimelike.py::TestTSPlot::test_pickle_fig[Series-idx3]": "_pickle.PicklingError: Can't pickle <class 'pandas.Period'>: it's not the same object as pandas.Period",
     "tests/plotting/test_datetimelike.py::TestTSPlot::test_to_weekly_resampling": "AssertionError: assert <Week: weekday=4> == <DateOffset: weeks=1>",
     "tests/reductions/test_reductions.py::TestDatetime64SeriesReductions::test_minmax_nat_series[nat_ser0]": "TODO: Add a reason for failure",
     "tests/reductions/test_reductions.py::TestDatetime64SeriesReductions::test_minmax_nat_series[nat_ser1]": "TODO: Add a reason for failure",

python/cudf_polars/cudf_polars/dsl/expressions/boolean.py

@@ -99,6 +99,7 @@ def __init__(
             BooleanFunction.Name.All,
             BooleanFunction.Name.Any,
             BooleanFunction.Name.IsDuplicated,
+            BooleanFunction.Name.IsEmpty,
             BooleanFunction.Name.IsFirstDistinct,
             BooleanFunction.Name.IsLastDistinct,
             BooleanFunction.Name.IsSorted,
@@ -107,7 +108,6 @@ def __init__(
         if self.name in {
             BooleanFunction.Name.HasNulls,
             BooleanFunction.Name.IsClose,
-            BooleanFunction.Name.IsEmpty,
         }:
             raise NotImplementedError(
                 f"Boolean function {self.name}"
@@ -192,6 +192,19 @@ def do_evaluate(
         self, df: DataFrame, *, context: ExecutionContext = ExecutionContext.FRAME
     ) -> Column:
         """Evaluate this expression given a dataframe for context."""
+        if self.name is BooleanFunction.Name.IsEmpty:
+            (child,) = self.children
+            column = child.evaluate(df, context=context)
+            return Column(
+                plc.Column.from_scalar(
+                    plc.Scalar.from_py(
+                        column.size == 0, self.dtype.plc_type, stream=df.stream
+                    ),
+                    1,
+                    stream=df.stream,
+                ),
+                dtype=self.dtype,
+            )
         if self.name in (
             BooleanFunction.Name.IsFinite,
             BooleanFunction.Name.IsInfinite,