generalize more script to new multipv format

diffmates.py

@@ -14,26 +14,29 @@ def open_file(filename):
     parser.add_argument("file1")
     parser.add_argument("file2")
     args = parser.parse_args()
-    p = re.compile(r"([0-9a-zA-Z/\- ]*) bm #([0-9\-]*);")
+    p = re.compile(r"^([1-8a-zA-Z/]+ [wb] [a-zA-Z\-]+ [a-h1-8\-]+)( bm #(-?\d+);)?")
 
     d = [{}, {}]
     for idx, filename in enumerate([args.file1, args.file2]):
         with open_file(filename) as f:
             for line in f:
+                if line.startswith("#"):  # ignore comments
+                    continue
                 m = p.match(line)
                 assert m, f"error for line '{line[:-1]}' in file {filename}"
-                fen, bm = m.group(1), int(m.group(2))
+                fen = m.group(1)
+                bm = int(m.group(3)) if m.group(2) is not None else None
                 assert fen not in d[idx], f'error: duplicate FEN "{fen}" in {filename}'
                 d[idx][fen] = bm, line
 
     for fen, (bm1, line1) in d[0].items():
         bm2, line2 = d[1].get(fen, (None, None))
-        if bm2 is None:
+        if line2 is None:
             print("> " + line1 + "<\n---")
         elif bm2 != bm1:
             print("> " + line1 + "< " + line2 + "---")
 
     for fen, (_, line2) in d[1].items():
-        bm1, _ = d[0].get(fen, (None, None))
-        if bm1 is None:
+        _, line1 = d[0].get(fen, (None, None))
+        if line1 is None:
             print(">\n< " + line2 + "---")

matestats.py

@@ -12,20 +12,24 @@ def open_file(filename):
 class data:
     def __init__(self, filename, debug=False):
         self.plies = Counter()
-        p = re.compile(r"([0-9a-zA-Z/\- ]*) bm #([0-9\-]*);")
+        p = re.compile(r"^([1-8a-zA-Z/]+ [wb] [a-zA-Z\-]+ [a-h1-8\-]+)( bm #(-?\d+);)?")
         loaded = set()
         self.bmplus = self.bmminus = 0
         with open_file(filename) as f:
             for line in f:
+                if line.startswith("#"):  # ignore comments
+                    continue
                 m = p.match(line)
                 if not m:
                     print("---------------------> IGNORING : ", line)
-                else:
-                    fen, bm = m.group(1), int(m.group(2))
-                    if fen in loaded:
-                        print(f"Warning: Found duplicate FEN {fen}.")
-                        continue
-                    loaded.add(fen)
+                    continue
+                fen = m.group(1)
+                bm = int(m.group(3)) if m.group(2) is not None else None
+                if fen in loaded:
+                    print(f"Warning: Found duplicate FEN {fen}.")
+                    continue
+                loaded.add(fen)
+                if bm:
                     plies = 2 * bm - 1 if bm > 0 else -2 * bm
                     self.plies[plies] += 1
                     if bm > 0:
@@ -35,13 +39,13 @@ def __init__(self, filename, debug=False):
         self.filename = filename[:-3] if filename.endswith(".gz") else filename
         self.bmmin = (min(self.plies.keys()) + 1) // 2
         self.bmmax = (max(self.plies.keys()) + 1) // 2
+        totalbm = self.bmplus + self.bmminus
         print(
-            f"Loaded {len(loaded)} unique EPDs with |bm| in [{self.bmmin}, {self.bmmax}]."
+            f"Loaded {len(loaded)} unique EPDs with {totalbm} bm values, with |bm| in [{self.bmmin}, {self.bmmax}]."
         )
         s = sum((key + 1) // 2 * count for key, count in self.plies.items())
-        l = sum(self.plies.values())
-        if l:
-            print(f"Average for |bm| is {s/l:.2f}.")
+        if totalbm:
+            print(f"Average for |bm| is {s/totalbm:.2f}.")
         if debug:
             print("bm frequencies:", end=" ")
             ply_count = sorted(self.plies.items(), key=lambda x: x[0])