dtdoctor: fix broken diagnosis paths and add test coverage

MAINTAINERS.yml

@@ -1276,6 +1276,21 @@ DSP subsystem:
   tests:
     - zdsp
 
+DT Doctor:
+  status: maintained
+  maintainers:
+    - kartben
+  files:
+    - cmake/sca/dtdoctor/
+    - doc/develop/sca/dtdoctor.rst
+    - scripts/dts/dtdoctor_analyzer.py
+    - scripts/dts/dtdoctor_sca_wrapper.py
+    - tests/misc/dtdoctor/
+  labels:
+    - "area: Devicetree"
+  tests:
+    - sca.dtdoctor
+
 Debug:
   status: maintained
   maintainers:

scripts/ci/undef_kconfig_allowlist.txt

@@ -68,6 +68,11 @@ CPU_V7  # U-Boot boards/luckfox/pico_ultra
 CRC  # Used in TI CC13x2 / CC26x2 SDK comment
 DEEP_SLEEP  # #defined by RV32M1 in ext/
 DESCRIPTION
+DTD_PLATFORM_IMPLY  # DT Doctor test fixture symbol
+DTD_PLATFORM_SELECT  # DT Doctor test fixture symbol
+DTD_SERIAL  # DT Doctor test fixture symbol
+DTD_SPECIAL_CORE  # DT Doctor test fixture symbol
+DTD_UART  # DT Doctor test fixture symbol
 DT_HAS_  # example from doc/build/dts/dt-vs-kconfig.rst
 EFI_LOADER  # U-Boot boards/luckfox/pico_ultra
 ERR

scripts/dts/dtdoctor_analyzer.py

@@ -55,9 +55,11 @@ def load_edt(path: str) -> edtlib.EDT:
         return pickle.load(f)
 
 
-def setup_kconfig() -> kconfiglib.Kconfig:
-    kconf = kconfiglib.Kconfig(os.path.join(os.environ.get("ZEPHYR_BASE"), "Kconfig"), warn=False)
-    return kconf
+def setup_kconfig() -> kconfiglib.Kconfig | None:
+    zephyr_base = os.environ.get("ZEPHYR_BASE")
+    if not zephyr_base:
+        return None
+    return kconfiglib.Kconfig(os.path.join(zephyr_base, "Kconfig"), warn=False)
 
 
 def format_node(node: edtlib.Node) -> str:
@@ -70,8 +72,16 @@ def find_kconfig_deps(kconf: kconfiglib.Kconfig, dt_has_symbol: str) -> set[str]
     """
     prefix = os.environ.get("CONFIG_", "CONFIG_")
     target = f"{prefix}{dt_has_symbol}"
+    # Word-boundary match so e.g. DT_HAS_FOO_ENABLED doesn't match DT_HAS_FOO_ENABLED_EXT
+    target_re = re.compile(rf"(?<!\w){re.escape(target)}(?!\w)")
     deps = set()
 
+    def expr_to_str(expr):
+        return kconfiglib.expr_str(
+            expr,
+            lambda sc: f"{prefix}{sc.name}" if hasattr(sc, 'name') and sc.name else str(sc),
+        )
+
     def collect_syms(expr):
         # Recursively collect all symbol names in the expression tree except the target
         for item in kconfiglib.expr_items(expr):
@@ -81,24 +91,19 @@ def collect_syms(expr):
             if sym_name != target:
                 deps.add(sym_name)
 
-    for sym in getattr(kconf, "unique_defined_syms", []):
+    for sym in kconf.unique_defined_syms:
         for node in sym.nodes:
             # Check dependencies
-            if node.dep is None:
-                continue
-            dep_str = kconfiglib.expr_str(
-                node.dep,
-                lambda sc: f"{prefix}{sc.name}" if hasattr(sc, 'name') and sc.name else str(sc),
-            )
-            if target in dep_str:
+            if node.dep is not None and target_re.search(expr_to_str(node.dep)):
                 collect_syms(node.dep)
 
-            # Check selects/implies
+            # A symbol whose select/imply is conditioned on the DT_HAS symbol is itself
+            # an option worth enabling
             for attr in ["orig_selects", "orig_implies"]:
-                for value, _ in getattr(node, attr, []) or []:
-                    value_str = kconfiglib.expr_str(value, str)
-                    if target in value_str:
-                        collect_syms(value)
+                for _, cond in getattr(node, attr, []) or []:
+                    if cond is not None and target_re.search(expr_to_str(cond)):
+                        deps.add(f"{prefix}{sym.name}")
+                        collect_syms(cond)
 
     return deps
 
@@ -111,8 +116,12 @@ def handle_enabled_node(node: edtlib.Node) -> list[str]:
     lines = [f"'{format_node(node)}' is enabled but no driver appears to be available for it.\n"]
 
     compats = list(getattr(node, "compats", []))
-    if compats:
-        kconf = setup_kconfig()
+    kconf = setup_kconfig() if compats else None
+    if not compats:
+        lines.append("Could not determine compatible; check driver Kconfig manually.")
+    elif not kconf:
+        lines.append("ZEPHYR_BASE is not set; check driver Kconfig manually.")
+    else:
         deps = set()
         for compat in compats:
             dt_has = f"DT_HAS_{edtlib.str_as_token(compat.upper())}_ENABLED"
@@ -121,8 +130,6 @@ def handle_enabled_node(node: edtlib.Node) -> list[str]:
         if deps:
             lines.append("Try enabling these Kconfig options:\n")
             lines.extend(f" - {dep}=y" for dep in sorted(deps))
-    else:
-        lines.append("Could not determine compatible; check driver Kconfig manually.")
 
     return lines
 
@@ -142,12 +149,8 @@ def handle_disabled_node(node: edtlib.Node) -> list[str]:
         lines.extend(f" - {u.path}" for u in users)
 
     # Show chosen/alias references
-    chosen_refs = [
-        name
-        for name, n in (getattr(edt, "chosen_nodes", {}) or getattr(edt, "chosen", {})).items()
-        if n is node
-    ]
-    alias_refs = [name for name, n in getattr(edt, "aliases", {}).items() if n is node]
+    chosen_refs = [name for name, n in edt.chosen_nodes.items() if n is node]
+    alias_refs = node.aliases
 
     if chosen_refs or alias_refs:
         lines.append("")

scripts/dts/dtdoctor_sca_wrapper.py

@@ -5,7 +5,7 @@
 
 """
 Compiler launcher wrapper that captures what appears to be Devicetree-related build errors, and
-diagnoses them using diagnose_build_error.py.
+diagnoses them using dtdoctor_analyzer.py.
 
 The tool is meant to be configured as a CMAKE_<LANG>_COMPILER_LAUNCHER or as a
 CMAKE_<LANG>_LINKER_LAUNCHER.
@@ -47,10 +47,11 @@ def main() -> int:
     # Extract __device_dts_ord_xxx symbols from errors and run diagnostics
     if proc.returncode != 0 and args.edt_pickle:
         patterns = [
-            r"(__device_dts_ord_\d+).*undeclared here",  # gcc
+            r"(__device_dts_ord_\d+).* undeclared",  # gcc (quote style depends on locale)
+            r"(__device_dts_ord_\d+).* was not declared",  # g++
             r"undefined reference to.*(__device_dts_ord_\d+)",  # ld
             r"use of undeclared identifier '(__device_dts_ord_\d+)'",  # LLVM/clang (ATfE)
-            r"undefined symbol: \(__device_dts_ord_(\d+)",  # LLVM/lld (ATfE)
+            r"undefined symbol: (__device_dts_ord_\d+)",  # LLVM/lld (ATfE)
         ]
         symbols = {m for p in patterns for m in re.findall(p, proc.stderr)}
 

tests/misc/dtdoctor/CMakeLists.txt

@@ -0,0 +1,38 @@
+# SPDX-FileCopyrightText: Copyright The Zephyr Project Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_minimum_required(VERSION 3.28.0)
+
+# The e2e suite replays the application's own compile commands through the
+# SCA wrapper, against deliberately-broken translation units
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
+
+project(dtdoctor_test)
+target_sources(app PRIVATE src/main.c src/main.cpp)
+
+enable_testing()
+include(CTest)
+
+# Unit tests for the dtdoctor scripts (fixture DTS and Kconfig trees)
+add_test(
+  NAME dtdoctor_unit
+  COMMAND ${PYTHON_EXECUTABLE} -m pytest ${CMAKE_CURRENT_SOURCE_DIR}/unit -v
+)
+
+add_test(
+  NAME dtdoctor_e2e
+  COMMAND ${PYTHON_EXECUTABLE} -m pytest
+    ${CMAKE_CURRENT_SOURCE_DIR}/e2e
+    --build-dir ${CMAKE_BINARY_DIR}
+    --cc ${CMAKE_C_COMPILER}
+    -v
+)
+
+# Reproduce the Kconfig environment the dtdoctor SCA launcher runs with, so the
+# analyzer can parse the full Zephyr Kconfig tree for the enabled-node diagnosis
+set_tests_properties(
+  dtdoctor_e2e
+  PROPERTIES ENVIRONMENT "${COMMON_KCONFIG_ENV_SETTINGS}"
+)

tests/misc/dtdoctor/README.rst

@@ -0,0 +1,36 @@
+DT Doctor tests
+###############
+
+Test suite for the DT Doctor static analysis tool
+(``scripts/dts/dtdoctor_sca_wrapper.py`` and ``scripts/dts/dtdoctor_analyzer.py``,
+documented in ``doc/develop/sca/dtdoctor.rst``).
+
+Twister builds this application like any other, with
+``ZEPHYR_SCA_VARIANT=dtdoctor``, and then runs two ctest entries on the host
+(``harness: ctest``) — the firmware itself never executes:
+
+* ``unit/`` is a unit-test pytest suite for the scripts. EDTs are built from
+  inline DTS snippets against the fixture bindings and Kconfig trees under
+  ``unit/fixture/``, so it needs no toolchain and no application build.
+
+* ``e2e/`` is an integration pytest suite run against this application's build
+  directory. It compiles deliberately-broken sources that use the real
+  devicetree macros on the fixture nodes from ``app.overlay``, replaying the
+  application's own compile commands (from ``compile_commands.json``) through
+  the real SCA wrapper, and checks the resulting diagnoses. This exercises the
+  whole chain — generated macros, ``<devicetree.h>`` expansion, the
+  toolchain's actual error messages, wrapper, analyzer — with the same C and
+  C++ compilers the application was built with.
+
+The application is only a build vehicle: ``app.overlay`` declares fake
+``vnd,dtdoctor-*`` devices that deliberately have no driver, and a C++ source
+file is included so a real C++ compile command is exported for the e2e suite.
+
+To run everything locally::
+
+   west twister -T tests/misc/dtdoctor
+
+or, against an existing build::
+
+   west build -b qemu_cortex_m3 tests/misc/dtdoctor -- -DZEPHYR_SCA_VARIANT=dtdoctor
+   ctest --test-dir build --output-on-failure

tests/misc/dtdoctor/app.overlay

@@ -0,0 +1,30 @@
+/*
+ * SPDX-FileCopyrightText: Copyright The Zephyr Project Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/*
+ * dtdoctor_disabled is the target of the deliberately-failing builds in
+ * e2e/test_dtdoctor.py; the chosen entry and alias below must show up in its
+ * diagnosis. dtdoctor_enabled exercises the "enabled but no driver" path
+ * (vnd,dtdoctor-device deliberately has no driver).
+ */
+/ {
+	chosen {
+		dtdoctor,dev = &dtdoctor_disabled;
+	};
+
+	aliases {
+		dtdoctor-dev = &dtdoctor_disabled;
+	};
+
+	dtdoctor_disabled: dtdoctor-disabled-device {
+		compatible = "vnd,dtdoctor-device";
+		status = "disabled";
+	};
+
+	dtdoctor_enabled: dtdoctor-enabled-device {
+		compatible = "vnd,dtdoctor-device";
+		status = "okay";
+	};
+};

tests/misc/dtdoctor/dts/bindings/vnd,dtdoctor-device.yaml

@@ -0,0 +1,6 @@
+# SPDX-FileCopyrightText: Copyright The Zephyr Project Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+description: Fake device without a driver, used by the DT Doctor integration test
+
+compatible: "vnd,dtdoctor-device"

tests/misc/dtdoctor/e2e/conftest.py

@@ -0,0 +1,96 @@
+# SPDX-FileCopyrightText: Copyright The Zephyr Project Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+"""Pytest configuration for the DT Doctor integration test."""
+
+import json
+import pickle
+import shlex
+import sys
+from pathlib import Path
+
+import pytest
+
+ZEPHYR_BASE = Path(__file__).parents[4]
+sys.path.insert(0, str(ZEPHYR_BASE / "scripts" / "dts" / "python-devicetree" / "src"))
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--build-dir",
+        action="store",
+        required=True,
+        help="Path to the build directory of the test application",
+    )
+    parser.addoption(
+        "--cc",
+        action="store",
+        required=True,
+        help="Path to the C compiler the application was built with",
+    )
+
+
+@pytest.fixture(scope="session")
+def build_dir(request):
+    return Path(request.config.getoption("--build-dir"))
+
+
+@pytest.fixture(scope="session")
+def cc(request):
+    return request.config.getoption("--cc")
+
+
+@pytest.fixture(scope="session")
+def edt_pickle(build_dir):
+    return build_dir / "zephyr" / "edt.pickle"
+
+
+@pytest.fixture(scope="session")
+def edt(edt_pickle):
+    from devicetree import edtlib  # noqa: F401 (needed to unpickle the EDT)
+
+    with open(edt_pickle, "rb") as f:
+        return pickle.load(f)
+
+
+@pytest.fixture(scope="session")
+def compile_cmd(build_dir):
+    """The application's real compile command for the source file with the given suffix.
+
+    Returns (argv, cwd, source_path), straight from the build's compile_commands.json.
+    """
+    with open(build_dir / "compile_commands.json", encoding="utf-8") as f:
+        entries = json.load(f)
+
+    def _get(suffix: str):
+        entry = next(e for e in entries if e["file"].endswith(suffix))
+        return shlex.split(entry["command"]), Path(entry["directory"]), entry["file"]
+
+    return _get
+
+
+def retarget(argv, source_file, new_source, new_obj):
+    """Point a compile command at another source file and output object.
+
+    Dependency-generation flags are dropped so the replay cannot touch the
+    application build's own .obj/.d files.
+    """
+    result = []
+    skip_next = False
+    for tok in argv:
+        if skip_next:
+            skip_next = False
+            continue
+        if tok in ("-MD", "-MMD"):
+            continue
+        if tok in ("-MT", "-MF", "-MQ"):
+            skip_next = True
+            continue
+        result.append(tok)
+
+    for i, tok in enumerate(result):
+        if tok == source_file:
+            result[i] = str(new_source)
+        elif tok == "-o":
+            result[i + 1] = str(new_obj)
+    return result