Merge pull request #846 from NVIDIA/ipod/cloudai-24/dse-report

DSE reporting

Author: podkidyshev

README.md

@@ -84,6 +84,13 @@ cloudai generate-report\
     --result-dir /path/to/result_directory
 ```
 
+Generated artifacts depend on the scenario contents:
+
+- A plain scenario status report is written as `<scenario>.html`.
+- If the scenario contains DSE test cases, an additional DSE-specific report is written as `<scenario>-dse-report.html`.
+- For DSE runs, the best discovered test configuration is also written as `<dse-case>/<iteration>/<dse-case>.toml`.
+- Custom reporters could generate additional artifacts.
+
 ### install
 This mode installs test prerequisites. For more details, please refer to the [installation guide](https://nvidia.github.io/cloudai/workloads_requirements_installation.html). It automatically runs as part of the `run` mode if prerequisites are not met.
 

pyproject.toml

@@ -145,8 +145,8 @@ root_package = "cloudai"
   [[tool.importlinter.contracts]]
   name = "Report generator is leaf dependency"
   type = "forbidden"
-  forbidden_modules = ["cloudai.systems", "cloudai.workloads", "cloudai.cli"]
-  allow_indirect_imports = true                                               # allow "from cloudai.core import ..."
+  forbidden_modules = ["cloudai.workloads", "cloudai.cli"]
+  allow_indirect_imports = true                            # allow "from cloudai.core import ..."
   source_modules = ["cloudai.report_generator"]
 
   [[tool.importlinter.contracts]]

src/cloudai/_core/registry.py

@@ -1,5 +1,5 @@
 # SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES
-# Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -228,7 +228,8 @@ def report_order(k: str) -> int:
             return {
                 "per_test": 0,  # first
                 "status": 2,
-                "tarball": 3,  # last
+                "dse": 3,
+                "tarball": 4,  # last
             }.get(k, 1)
 
         return sorted(self.scenario_reports.items(), key=lambda kv: report_order(kv[0]))

src/cloudai/registration.py

@@ -46,7 +46,7 @@ def register_all():
     )
     from cloudai.core import Registry
     from cloudai.models.scenario import ReportConfig
-    from cloudai.reporter import PerTestReporter, StatusReporter, TarballReporter
+    from cloudai.reporter import DSEReporter, PerTestReporter, StatusReporter, TarballReporter
 
     # Import systems
     from cloudai.systems.kubernetes import KubernetesInstaller, KubernetesRunner, KubernetesSystem
@@ -303,6 +303,7 @@ def register_all():
 
     Registry().add_scenario_report("per_test", PerTestReporter, ReportConfig(enable=True))
     Registry().add_scenario_report("status", StatusReporter, ReportConfig(enable=True))
+    Registry().add_scenario_report("dse", DSEReporter, ReportConfig(enable=True))
     Registry().add_scenario_report("tarball", TarballReporter, ReportConfig(enable=True))
     Registry().add_scenario_report(
         "nixl_bench_summary",