Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion Source/Data/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@ all: data.txt data-plain-bpmf.txt associated-phrases-v2.txt
install: all

data-plain-bpmf.txt: curation/compilers/plain_bpmf_compiler.py BPMFBase.txt BPMFPunctuations.txt
$(PYTHON) -m curation.compilers.plain_bpmf_compiler BPMFBase.txt BPMFPunctuations.txt data-plain-bpmf.txt
$(PYTHON) -m curation.compilers.plain_bpmf_compiler \
--bpmf_base BPMFBase.txt \
--punctuations BPMFPunctuations.txt \
--output data-plain-bpmf.txt

data.txt: curation/compilers/main_compiler.py BPMFBase.txt BPMFMappings.txt BPMFPunctuations.txt \
PhraseFreq.txt phrase.occ Symbols.txt Macros.txt\
Expand Down
74 changes: 43 additions & 31 deletions Source/Data/curation/compilers/plain_bpmf_compiler.py
Original file line number Diff line number Diff line change
@@ -1,50 +1,62 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from .compiler_utils import HEADER, convert_vks_rows_to_sorted_kvs_rows
import argparse
import re
import sys

__author__ = "Lukhnos Liu and The McBopomofo Authors"
__copyright__ = "Copyright 2012 and onwards The McBopomofo Authors"
__license__ = "MIT"

skip = re.compile(r".\s+_punctuation.*_>")
# Punctuation fix for Plain Bopomofo mode
SKIP_RE = re.compile(r".\s+_punctuation.*_>")
PUNCTUATION_FIX = [".", '_punctuation_"', "0.0"]

insert = [".", "_punctuation_\"", "0.0"]


def main():
if len(sys.argv) < 4:
sys.exit('Usage: cook-plain-bpmf.py bpmf-base punctuation-list output')

bpmf_base = open(sys.argv[1], "r")
punctuation_list = open(sys.argv[2], "r")
def cook(bpmf_base_path, punctuations_path, output_path):
output = []

while True:
line = bpmf_base.readline()
if not line: break
kv = line.split(" ")
output.append((kv[0], kv[1], "0.0"))

while True:
line = punctuation_list.readline()
if not line:
break
if skip.search(line):
continue
row = line.rstrip().split(" ")
assert len(row) == 3
output.append(row)

output.append(insert)
with open(bpmf_base_path, "r") as bpmf_base:
Comment thread
lukhnos marked this conversation as resolved.
for line in bpmf_base:
line = line.strip()
if not line:
continue
elements = line.split(" ")
assert len(elements) >= 2
output.append((elements[0], elements[1], "0.0"))

with open(punctuations_path, "r") as punctuation_list:
Comment thread
lukhnos marked this conversation as resolved.
for line in punctuation_list:
if SKIP_RE.search(line):
continue
row = line.rstrip().split(" ")
assert len(row) == 3
output.append(row)

output.append(PUNCTUATION_FIX)

output = convert_vks_rows_to_sorted_kvs_rows(output)
with open(sys.argv[3], "w") as fout:
fout.write(HEADER)
with open(output_path, "w") as output_file:
Comment thread
lukhnos marked this conversation as resolved.
output_file.write(HEADER)
for row in output:
fout.write("%s %s %s\n" % tuple(row))
output_file.write("%s %s %s\n" % tuple(row))


if __name__ == '__main__':
def main():
parser = argparse.ArgumentParser(
description="compile plain Bopomofo phrases database"
)
parser.add_argument("--bpmf_base", required=True)
parser.add_argument("--punctuations", required=True)
parser.add_argument("--output", required=True)

args = parser.parse_args()
cook(
bpmf_base_path=args.bpmf_base,
punctuations_path=args.punctuations,
output_path=args.output,
)


if __name__ == "__main__":
main()