Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 44 additions & 14 deletions scripts/gpu_smoke_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from __future__ import annotations

import argparse
import json
import subprocess
import xml.etree.ElementTree as ET
from collections import OrderedDict
Expand Down Expand Up @@ -95,24 +96,23 @@ def parse_test_xml(path: str) -> list[dict[str, str]]:


def generate_report(results: list[dict[str, str]], target: str) -> str:
total_cases = len(results)
summary = summarize_results(results, target)
failed_cases = [result for result in results if result["outcome"] == "failed"]
passed_count = sum(1 for result in results if result["outcome"] == "passed")
skipped_count = sum(1 for result in results if result["outcome"] == "skipped")
failed_ops = list(
OrderedDict.fromkeys(result["op"] for result in failed_cases if result["op"])
)

correctness = f"{_PASS} Pass" if not failed_cases else f"{_FAIL} {len(failed_cases)} failed"
correctness = (
f"{_PASS} Pass" if summary["failed_count"] == 0 else f"{_FAIL} {summary['failed_count']} failed"
)
failures_ops_str = (
", ".join(_escape_markdown_cell(op) for op in failed_ops) if failed_ops else "-"
", ".join(_escape_markdown_cell(op) for op in summary["failed_ops"])
if summary["failed_ops"]
else "-"
)
health = _PASS if not failed_cases else _FAIL
health = _PASS if summary["failed_count"] == 0 else _FAIL

lines = [
f"# {health} TileOPs GPU Smoke Report",
"",
f"> `{_get_git_commit()}`",
f"> `{summary['git_commit']}`",
"",
"## Summary",
"",
Expand All @@ -121,10 +121,10 @@ def generate_report(results: list[dict[str, str]], target: str) -> str:
f"| **Correctness** | {correctness} |",
f"| **gpu-smoke target** | `{target}` |",
(
f"| **Gpu-smoke ops number** | {total_cases} "
f"({passed_count} passed, {skipped_count} skipped) |"
f"| **Gpu-smoke ops number** | {summary['total_cases']} "
f"({summary['passed_count']} passed, {summary['skipped_count']} skipped) |"
),
f"| **Gpu-smoke Failures** | {len(failed_cases)} |",
f"| **Gpu-smoke Failures** | {summary['failed_count']} |",
f"| **Failures ops** | {failures_ops_str} |",
"",
]
Expand All @@ -149,20 +149,50 @@ def generate_report(results: list[dict[str, str]], target: str) -> str:
return "\n".join(lines)


def summarize_results(results: list[dict[str, str]], target: str) -> dict[str, object]:
total_cases = len(results)
failed_cases = [result for result in results if result["outcome"] == "failed"]
passed_count = sum(1 for result in results if result["outcome"] == "passed")
skipped_count = sum(1 for result in results if result["outcome"] == "skipped")
failed_ops = list(
OrderedDict.fromkeys(result["op"] for result in failed_cases if result["op"])
)

return {
"target": target,
"git_commit": _get_git_commit(),
"total_cases": total_cases,
"passed_count": passed_count,
"skipped_count": skipped_count,
"failed_count": len(failed_cases),
"failed_ops": failed_ops,
}


def main() -> None:
parser = argparse.ArgumentParser(description="Generate TileOPs GPU smoke report")
parser.add_argument("--test-xml", required=True, help="Path to pytest JUnit XML")
parser.add_argument("--target", required=True, help="Displayed gpu-smoke target label")
parser.add_argument("--output", required=True, help="Output markdown report path")
parser.add_argument("--json-output", help="Optional JSON summary output path")
args = parser.parse_args()

results = parse_test_xml(args.test_xml)
report = generate_report(results, args.target)
output_path = Path(args.output)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(report)
output_path.write_text(report, encoding="utf-8")
print(f"Report written to {args.output}")

if args.json_output:
json_path = Path(args.json_output)
json_path.parent.mkdir(parents=True, exist_ok=True)
json_path.write_text(
json.dumps(summarize_results(results, args.target), indent=2, sort_keys=True),
encoding="utf-8",
)
print(f"JSON summary written to {args.json_output}")


if __name__ == "__main__":
main()
50 changes: 50 additions & 0 deletions tests/test_gpu_smoke_report_json.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import unittest
from pathlib import Path
import sys

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from scripts import gpu_smoke_report


class GpuSmokeReportJsonTest(unittest.TestCase):
def test_summarize_results_counts_outcomes(self):
original_get_git_commit = gpu_smoke_report._get_git_commit
gpu_smoke_report._get_git_commit = lambda: "abc123"
try:
results = [
{"outcome": "passed", "op": "softmax"},
{"outcome": "skipped", "op": "rms_norm"},
{"outcome": "failed", "op": "softmax"},
{"outcome": "failed", "op": "rms_norm"},
{"outcome": "failed", "op": "softmax"},
]

summary = gpu_smoke_report.summarize_results(results, "smoke")
finally:
gpu_smoke_report._get_git_commit = original_get_git_commit

self.assertEqual(
summary,
{
"target": "smoke",
"git_commit": "abc123",
"total_cases": 5,
"passed_count": 1,
"skipped_count": 1,
"failed_count": 3,
"failed_ops": ["softmax", "rms_norm"],
},
)

def test_generate_report_can_be_written_as_utf8(self):
report = gpu_smoke_report.generate_report(
[{"outcome": "failed", "op": "softmax", "name": "case", "nodeid": "node", "failure_reason": "bad"}],
"smoke",
)

self.assertIn("TileOPs GPU Smoke Report", report)
report.encode("utf-8")


if __name__ == "__main__":
unittest.main()
Loading