Fix LMEval ONNX evaluator device assignment with recent lm-eval

lm-eval's `LM` base class exposes `device` as a read-only property
(backed by `_device`). The ONNX evaluators (`LMEvalORTEvaluator`,
`LMEvalORTGenAIEvaluator`) assign `self.device` in `__init__`, which
raises `AttributeError: property 'device' ... has no setter` with recent
lm-eval versions (e.g. 0.4.12), breaking `get_model('ortgenai')` and
`get_model('ort')` based evaluation.

Add a `device` property with a setter to the shared `LMEvalOnnxBase` so
the existing `self.device = ...` assignments work regardless of lm-eval
version. Add a regression test for the settable property.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Signed-off-by: Justin Chu <11205048+justinchuby@users.noreply.github.com>

Author: justinchuby

olive/evaluator/lmeval_ort.py

@@ -38,6 +38,18 @@
 class LMEvalOnnxBase(TemplateLM):
     """Base class for ONNX model evaluation."""
 
+    @property
+    def device(self):
+        # lm-eval's LM base class exposes ``device`` as a read-only property
+        # (backed by ``_device``). The ONNX evaluators assign ``self.device``
+        # in ``__init__`` (e.g. for IOBinding placement), so provide a setter
+        # to keep that assignment working across lm-eval versions.
+        return getattr(self, "_device", None)
+
+    @device.setter
+    def device(self, value):
+        self._device = value
+
     @abstractmethod
     def prepare(self, requests: list[LogLikelihoodInputs]):
         pass

test/evaluator/test_lmeval_ort.py

@@ -0,0 +1,42 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+import pytest
+
+# The lmeval_ort module imports lm_eval at module load time.
+pytest.importorskip("lm_eval")
+
+from olive.evaluator.lmeval_ort import LMEvalOnnxBase
+
+
+def test_device_property_has_setter():
+    """LMEvalOnnxBase must override lm-eval's read-only ``device`` property.
+
+    lm-eval's ``LM`` base class exposes ``device`` as a read-only property,
+    but the ONNX evaluators assign ``self.device`` in ``__init__``. Without a
+    setter this raises ``AttributeError: property 'device' ... has no setter``.
+    This is a regression test for that incompatibility.
+    """
+    device_prop = LMEvalOnnxBase.__dict__.get("device")
+    assert isinstance(device_prop, property)
+    assert device_prop.fset is not None, "device property must define a setter"
+
+
+def test_device_property_round_trips():
+    """The setter stores and the getter returns the assigned value."""
+    device_prop = LMEvalOnnxBase.__dict__["device"]
+
+    # Use a bare holder to exercise the descriptor without instantiating the
+    # abstract base class (which has unrelated abstract methods).
+    class _Holder:
+        pass
+
+    holder = _Holder()
+    assert device_prop.fget(holder) is None  # unset -> None
+
+    device_prop.fset(holder, "cuda")
+    assert device_prop.fget(holder) == "cuda"
+
+    device_prop.fset(holder, "cpu")
+    assert device_prop.fget(holder) == "cpu"