Merge remote-tracking branch 'origin/main' into codex/tvm-skip-reporting

Author: strimo378

README.md

@@ -8,16 +8,6 @@ SPDX-License-Identifier: Apache-2.0
 
 This is a repository of the ONNX Backend Scoreboard which measures the compliance of ONNX backends with the standard.
 
-## 🛠 Maintainer Wanted
-
-We are currently **looking for a new maintainer** to help support and evolve the `backend-scoreboard` project.
-
-If you're passionate about the ONNX standard or contributing to the open source machine learning ecosystem, we'd love to hear from you! This is a great opportunity to contribute to a widely used project and collaborate with the ONNX community.
-
-**To express interest:**  
-Please open an issue or comment on [this thread](https://github.com/onnx/backend-scoreboard/issues) and let us know about your interest and background.
-
-
 ### :chart_with_upwards_trend: [Check out the scoreboard page](https://onnx.ai/backend-scoreboard/)
 
 - To add another backend to the scoreboard please follow [these instructions](ADD-BACKEND.md).

backends/emx_onnx_cgen/backend.py

@@ -1,13 +1,39 @@
 # SPDX-License-Identifier: Apache-2.0
 
-"""ONNX backend wrapper for Apache TVM (Relay frontend)."""
+"""ONNX backend wrapper for Apache TVM (Relay frontend, native ops only)."""
 
+import logging
 import multiprocessing as mp
+import sys
 
 import numpy as np
 from onnx.backend.base import Backend, BackendRep
 from onnx.backend.test.runner import BackendIsNotSupposedToImplementIt
 
+logging.basicConfig(level=logging.WARNING)
+logger = logging.getLogger(__name__)
+
+
+def _native_ops_only(model):
+    """Return ops in the model that have no native TVM Relay converter.
+
+    Uses TVM's internal converter map to detect ops that would silently fall
+    back to the ONNX reference runtime instead of being compiled natively.
+    Returns an empty list when the check cannot be performed.
+    """
+    try:
+        from tvm.relay.frontend.onnx import _get_convert_map
+
+        opset = max(
+            (x.version for x in model.opset_import if x.domain == ""),
+            default=1,
+        )
+        convert_map = _get_convert_map(opset)
+        unsupported = {n.op_type for n in model.graph.node if n.op_type not in convert_map}
+        return sorted(unsupported)
+    except (ImportError, AttributeError):
+        return []
+
 
 def _tvm_worker(model_bytes, inputs, input_names, output_count, result_queue):
     """Compile and run an ONNX model via TVM Relay in an isolated subprocess."""
@@ -18,6 +44,14 @@ def _tvm_worker(model_bytes, inputs, input_names, output_count, result_queue):
 
     model = onnx.ModelProto()
     model.ParseFromString(model_bytes)
+
+    unsupported = _native_ops_only(model)
+    if unsupported:
+        msg = f"no native TVM converter for: {unsupported}"
+        print(f"[tvm] SKIP {msg}", file=sys.stderr, flush=True)
+        result_queue.put(("error", msg))
+        return
+
     shape_dict = {
         name: np.asarray(inp).shape
         for name, inp in zip(input_names, inputs, strict=True)
@@ -33,7 +67,10 @@ def _tvm_worker(model_bytes, inputs, input_names, output_count, result_queue):
         outputs = [module.get_output(i).numpy() for i in range(output_count)]
         result_queue.put(("ok", outputs))
     except (tvm.TVMError, RuntimeError, ValueError, TypeError, OSError) as e:
-        result_queue.put(("error", str(e)))
+        ops = sorted({n.op_type for n in model.graph.node})
+        msg = f"ops={ops} error={type(e).__name__}: {e}"
+        print(f"[tvm] SKIP {msg}", file=sys.stderr, flush=True)
+        result_queue.put(("error", msg))
 
 
 class TVMBackendRep(BackendRep):
@@ -58,13 +95,16 @@ def run(self, inputs, **kwargs):
         if p.is_alive():
             p.terminate()
             p.join()
-            raise BackendIsNotSupposedToImplementIt("tvm process timed out")
+            msg = "tvm process timed out"
+            logger.warning(msg)
+            raise BackendIsNotSupposedToImplementIt(msg)
         if p.exitcode != 0:
-            raise BackendIsNotSupposedToImplementIt(
-                f"tvm process crashed (exit code {p.exitcode})"
-            )
+            msg = f"tvm process crashed (exit code {p.exitcode})"
+            logger.warning(msg)
+            raise BackendIsNotSupposedToImplementIt(msg)
         status, result = q.get_nowait()
         if status == "error":
+            logger.warning("tvm skip: %s", result)
             raise BackendIsNotSupposedToImplementIt(result)
         return result
 

backends/tvm/backend.py

@@ -1,40 +1,45 @@
-FROM ubuntu:22.04
+FROM ubuntu:24.04
 
 # Disable interactive installation mode
 ENV DEBIAN_FRONTEND=noninteractive
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+ENV CC=gcc-14
 
 # Install Python, gcc and locales dependencies
+# emx-onnx-cgen needs a compiler with C23 _BitInt(N) support. Ubuntu 24.04's
+# default gcc-13 does not provide it on this target, so install and prefer gcc-14.
 RUN apt-get update && apt-get install -y \
     python3 \
     python3-dev \
     python3-pip \
-    gcc \
+    gcc-14 \
+    g++-14 \
     locales && \
-    apt-get clean autoclean && apt-get autoremove -y && \
-    pip3 install --upgrade pip setuptools wheel
+    apt-get clean autoclean && apt-get autoremove -y
+
+RUN ln -sf /usr/bin/gcc-14 /usr/bin/gcc && \
+    ln -sf /usr/bin/g++-14 /usr/bin/g++
 
 # Copy local directories
 COPY ./test /root/test
 COPY ./setup /root/setup
-COPY ./backends/emx_onnx_cgen/backend.py /root/emx_backend.py
 
 # Install test report dependencies
 RUN pip3 install --no-cache-dir -r /root/setup/requirements_report.txt
 
 ############## ONNX Backend dependencies ###########
-ENV ONNX_BACKEND="emx_backend"
-ENV PYTHONPATH="/root"
+ENV ONNX_BACKEND="emx_onnx_cgen.onnx_backend"
 
 # Set locale which is required for StringNormalizer
 RUN locale-gen en_US.UTF-8 && update-locale LANG=en_US.UTF-8
 
-# Install emx-onnx-cgen and onnx
+# Install emx-onnx-cgen backend and onnx
 RUN pip3 install \
     onnx \
     emx-onnx-cgen
 ####################################################
 
-RUN useradd -m -u 1000 runner && chown -R runner:runner /root
+RUN useradd -m runner && chown -R runner:runner /root
 USER runner
 WORKDIR /root
 

runtimes/emx-onnx-cgen/stable/Dockerfile

@@ -15,12 +15,28 @@
 import json
 import os
 
+import pytest
+
 from datetime import datetime
 
 
 # Keys for values to save in report (matched with terminalreporter.stats)
 REPORT_KEYS = ["passed", "failed", "skipped"]
 
+# Ops seen across ALL collected test items (populated before -k deselection)
+_suite_ops: set = set()
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_collection_modifyitems(session, config, items):  # noqa: ARG001
+    """Collect all op names from onnx_coverage marks before -k deselection."""
+    for item in items:
+        for mark in item.iter_markers("onnx_coverage"):
+            proto = mark.args[0] if mark.args else None
+            if proto is not None and hasattr(proto, "graph"):
+                for node in proto.graph.node:
+                    _suite_ops.add(node.op_type)
+
 
 def pytest_addoption(parser):
     """Pytest hook function."""
@@ -116,6 +132,7 @@ def _prepare_summary(report, package_versions=None):
 
     summary = {"date": report.get("date", datetime.now().strftime("%m/%d/%Y %H:%M:%S"))}
     summary["versions"] = package_versions
+    summary["total_ops"] = len(_suite_ops)
     for key in report.keys():
         if isinstance(report.get(key), list):
             summary[key] = len(report.get(key))