Add report generation

Author: avolkov-intel

sklbench/report/implementation.py

@@ -49,6 +49,10 @@
     ],
     "higher is better": [
         "throughput[samples/ms]",
+        # throughput mode
+        "total_iterations",
+        "total_throughput_iterations_per_sec",
+        "mean_throughput_per_instance",
         # classification
         "accuracy",
         "balanced accuracy",
@@ -76,6 +80,11 @@
         "1st-mean run ratio",
         "time CV",
         "cpu load[%]",
+        # throughput mode
+        "std_throughput_per_instance",
+        "min_iterations_per_instance",
+        "max_iterations_per_instance",
+        "measurement_wall_time_sec",
     ],
 }
 MEMORY_TYPES = ["RAM", "VRAM"]
@@ -274,13 +283,18 @@ def get_result_tables_as_df(
             for bench_case in results["bench_cases"]
         ]
     )
+    # Drop columns that contain non-scalar data (e.g. per-instance details)
+    for col in bench_cases.columns:
+        if bench_cases[col].apply(lambda x: isinstance(x, (list, dict))).any():
+            bench_cases.drop(columns=[col], inplace=True)
 
     if compatibility_mode:
         bench_cases = transform_results_to_compatible(bench_cases)
 
     for column in diffby_columns.copy():
-        if bench_cases[column].nunique() == 1:
-            bench_cases.drop(columns=[column], inplace=True)
+        if column not in bench_cases.columns or bench_cases[column].nunique() == 1:
+            if column in bench_cases.columns:
+                bench_cases.drop(columns=[column], inplace=True)
             diffby_columns.remove(column)
 
     return split_df_by_columns(bench_cases, splitby_columns)

sklbench/runner/implementation.py

@@ -128,8 +128,8 @@ def run_benchmarks(args: argparse.Namespace) -> int:
     with open(args.result_file, "w") as fp:
         json.dump(result, fp, indent=4)
 
-    # output as pandas dataframe (skip for throughput mode which has nested results)
-    if len(result["bench_cases"]) != 0 and not args.throughput_mode:
+    # output as pandas dataframe
+    if len(result["bench_cases"]) != 0:
         for key, df in get_result_tables_as_df(result).items():
             logger.info(f'{custom_format(key, bcolor="HEADER")}\n{df}')
 

sklbench/runner/throughput.py

@@ -254,7 +254,6 @@ def run_single_throughput_case(
     results = []
     estimator_name = get_bench_case_value(bench_case, "algorithm:estimator")
     library_name = get_bench_case_value(bench_case, "algorithm:library")
-    from .commands_helper import generate_benchmark_command
 
     from ..benchmarks.sklearn_estimator import estimator_to_task
 
@@ -264,6 +263,11 @@ def run_single_throughput_case(
     quality_metrics = instance_outputs[0].get("quality_metrics", {})
     final_estimator_params = instance_outputs[0].get("estimator_params", {})
 
+    # Dataset info from bench_case
+    from ..utils.bench_case import get_data_name
+
+    dataset_name = get_data_name(bench_case, shortened=True)
+
     for stage in stages:
         stage_result = aggregate_stage_results(
             instance_outputs, stage, measurement_duration, core_assignments
@@ -279,21 +283,26 @@ def run_single_throughput_case(
                 method = stage_data.get("method", "unknown")
                 break
 
+        # Flatten aggregate metrics to top-level for report compatibility
+        aggregate = stage_result.pop("aggregate")
+        instances_detail = stage_result.pop("instances")
+
         result_entry = {
             "mode": "throughput",
             "stage": stage,
             "method": method,
             "task": task,
             "estimator": estimator_name,
+            "dataset": dataset_name,
             "library": library_name,
             "device": get_bench_case_value(bench_case, "algorithm:device"),
             "num_instances": num_instances,
             "cores_per_instance": cores_per_instance,
             "measurement_duration_seconds": measurement_duration,
         }
+        result_entry.update(aggregate)
         result_entry.update(quality_metrics)
-        result_entry.update(final_estimator_params)
-        result_entry.update(stage_result)
+        result_entry["instances"] = instances_detail
         results.append(result_entry)
 
     return return_code, results