Add new entries for '𰻝', '𰻞', and '𰻞𰻞麵' in BPMFBase.txt and phrase.occ

.github/workflows/continuous-build-data.yml

@@ -13,6 +13,9 @@ jobs:
   build:
     name: Build phrase database
     runs-on: ubuntu-latest
+    env:
+      LANG: en_US.UTF-8
+      LC_ALL: en_US.UTF-8
     steps:
       - uses: actions/checkout@v5
       - uses: actions/setup-python@v6

Source/Data/BPMFBase.txt

@@ -726,6 +726,8 @@
 臏 ㄅㄧㄣˋ bin4 1up4 big5
 髕 ㄅㄧㄣˋ bin4 1up4 big5
 鶣 ㄅㄧㄣˋ bin4 1up4 big5
+𰻞 ㄅㄧㄤˊ biang2 11u;6 utf8
+𰻝 ㄅㄧㄤˊ biang2 11u;6 utf8
 ㄉ ㄉ d 2 big5
 都 ㄉㄡ dou 2. big5
 兜 ㄉㄡ dou 2. big5

Source/Data/BPMFMappings.txt

@@ -144962,3 +144962,4 @@
 龜鑑 ㄍㄨㄟ ㄐㄧㄢˋ
 龜頭 ㄍㄨㄟ ㄊㄡˊ
 龜鱉目 ㄍㄨㄟ ㄅㄧㄝ ㄇㄨˋ
+𰻞𰻞麵 ㄅㄧㄤˊ ㄅㄧㄤˊ ㄇㄧㄢˋ

Source/Data/Makefile

@@ -63,10 +63,7 @@ sort:
 check: SHELL:=/bin/bash
 check: export LC_ALL=C
 check:
-	@awk 'length($$1)/3!=NF-1' BPMFMappings.txt
-	@awk '{print $$2,$$3,$$4,$$5,$$6,$$7}' BPMFMappings.txt | awk '/一/'
-	@diff -u <(awk '{print $$1}' BPMFMappings.txt|sort -u) \
-                 <(awk 'length($$1)>4{print $$1}' phrase.occ|sort -u)
+	@python3 scripts/check.py
 
 tidy:
 	@sed -i '' -e 's/ //g' phrase.occ

Source/Data/phrase.occ

@@ -152591,3 +152591,6 @@
 嗀 0
 𡻈 0
 𢻯 0
+𰻝 0
+𰻞 0
+𰻞𰻞麵 0

Source/Data/scripts/check.py

@@ -0,0 +1,63 @@
+def check_if_bpmf_mappings_matches():
+    issues = []
+    with open("BPMFMappings.txt", encoding="utf-8") as f:
+        for line in f:
+            components = line.strip().split(" ")
+            first = components[0]
+            if len(first) != len(components) - 1:
+                issues.append(f"Length mismatch: {line.strip()}")
+    return issues
+
+def check_if_readings_in_bpmf_mappings_contains_non_bpmf():
+    issues = []
+    bpmf = (
+        "ㄅㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄐㄑㄒㄓㄔㄕㄖㄗㄘㄙㄚㄛㄜㄝㄞㄟㄠㄡㄢㄣㄤㄥㄦㄧㄨㄩˊˇˋ˙"
+    )
+    with open("BPMFMappings.txt", encoding="utf-8") as f:
+        for line in f:
+            components = line.strip().split()[1:]
+            for c in components:
+                if any(char not in bpmf for char in c):
+                    issues.append(f"Non bpmf {c} in {line.strip()}")
+                    break
+    return issues
+
+def compare_bpmf_mappings_with_phrase_occ():
+    issues = []
+    bpmf_mappings_phrases = set()
+    phrase_occ_phrases = set()
+    with open("BPMFMappings.txt", encoding="utf-8") as f:
+        for line in f:
+            components = line.strip().split(" ")
+            first = components[0]
+            if len(first) > 1:
+                bpmf_mappings_phrases.add(first)
+    with open("phrase.occ", encoding="utf-8") as f:
+        for line in f:
+            components = line.strip().split(" ")
+            first = components[0]
+            if len(first) > 1:
+                phrase_occ_phrases.add(first)
+    missing_in_phrase_occ = bpmf_mappings_phrases - phrase_occ_phrases
+    missing_in_bpmf_mappings = phrase_occ_phrases - bpmf_mappings_phrases
+    if missing_in_phrase_occ:
+        issues.append(f"Missing in phrase.occ: {missing_in_phrase_occ}")
+    if missing_in_bpmf_mappings:
+        issues.append(f"Missing in BPMFMappings.txt: {missing_in_bpmf_mappings}")
+    return issues
+
+
+def main():
+    all_issues = []
+    all_issues.extend(check_if_bpmf_mappings_matches())
+    all_issues.extend(check_if_readings_in_bpmf_mappings_contains_non_bpmf())
+    all_issues.extend(compare_bpmf_mappings_with_phrase_occ())
+
+    for issue in all_issues:
+        print(issue)
+
+    if all_issues:
+        raise RuntimeError(f"Data check failed with {len(all_issues)} issue(s).")
+
+if __name__ == "__main__":
+    main()