Add summary processing tool (#3032)

Author: tengqm

tools/psummary

@@ -0,0 +1,149 @@
+#!/usr/bin/env python
+
+import argparse
+import json
+import sys
+
+# ROOT = Path(__file__).parent.parent
+DATA = {}
+
+
+def load_data(data_path):
+    global DATA
+    try:
+        with open(data_path, "r") as f:
+            DATA = json.load(f)
+        return True
+    except Exception as ex:
+        print(f"Failed to load data: {ex}")
+    return False
+
+
+def format_record(fields, fmt):
+    if fmt == "csv":
+        return ",".join(fields)
+
+    if fmt == "markdown":
+        line = " | ".join(fields)
+        return f"| {line} |"
+
+    if fmt == "html":
+        line = "</rd><td>".join(fields)
+        return "<tr><td>" + line + "</td></tr>"
+
+    return ""
+
+
+def parse_lines(fmt):
+    """Write result to CSV.
+
+    The layout is as follows:
+    <id>,<astatus>,<pass/fail/skip>,<pstatus>,<fp16>,<fp32>,<bf16>,<cf64>,<i16>,<i32>,<i64>,<note>
+    """
+    data = DATA.get("result", {})
+    lines = []
+    for op, item in data.items():
+        fields = []
+        # print(f"{op} ....")
+        accu = item["accuracy"]["status"]
+        accud = "/".join(
+            [
+                str(item["accuracy"]["passed"]),
+                str(item["accuracy"]["failed"]),
+                str(item["accuracy"]["skipped"]),
+            ]
+        )
+        fields.extend([op, accu, accud])
+
+        perf_data = item["performance"]["data"]
+        perf_status = item["performance"]["status"]
+        if perf_status == "NotFound":
+            fields.append("NotFound")
+            fields.extend([""] * 8)
+
+        elif perf_status in ["Failed", "Skipped"]:
+            reason = item["performance"]["reason"]
+            reason = reason.split("\n")[0]
+            fields.append(perf_status)
+            fields.extend([""] * 7)
+            fields.append(f'"{reason}"')
+
+        else:
+            fields.append(perf_status)
+            for dtype in ["fp16", "fb32", "bf16", "cf64", "int16", "int32", "int64"]:
+                dobj = perf_data.get(dtype, {})
+                speedup = dobj.get("speedup", None)
+                if speedup:
+                    fields.append(f"{speedup:6.3f}")
+                else:
+                    fields.append("")
+
+        lines.append(format_record(fields, fmt))
+
+    return sorted(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("data", help="the data file to process")
+    parser.add_argument(
+        "-f",
+        "--format",
+        choices=["csv", "markdown", "html"],
+        default="csv",
+        help="the output format",
+    )
+    parser.add_argument(
+        "-o", "--output", default="<stdout>", help="path to the output file."
+    )
+
+    args = parser.parse_args()
+    if args.data is None:
+        parser.print_help()
+        return 0
+
+    if not load_data(args.data):
+        return 1
+
+    lines = parse_lines(args.format)
+    header_fields = [
+        "ID",
+        "AccRes",
+        "AccStat",
+        "PerfRes",
+        "FP16",
+        "FP32",
+        "BF16",
+        "CF64",
+        "I16",
+        "I32",
+        "I64",
+        "Note",
+    ]
+    if args.format == "csv":
+        lines.insert(0, ",".join(header_fields))
+    elif args.format == "markdown":
+        header = "|" + "----|" * 12
+        lines.insert(0, header)
+        header = "| " + " | ".join(header_fields) + " |"
+        lines.insert(0, header)
+    elif args.format == "html":
+        header = ["<table>", "<thead>", "<tr>"]
+        for field in header_fields:
+            header.append(f"<th>{field}</th>")
+        header.extend(["</tr>", "</thead>", "<tbody>"])
+        lines = header + lines
+        lines.append("</tbody></table")
+
+    if args.output == "<stdout>":
+        for ln in lines:
+            print(ln)
+    else:
+        with open(args.output, "w") as f:
+            f.writelines(lines)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())