Skip to content

Commit 1f12635

Browse files
mgrafupre-commit-ci[bot]
mgrafu
and
pre-commit-ci[bot]
authored
change normalization of alphanum terms (#439)
* change normalization of alphanum terms Signed-off-by: Mariana Graterol Fuenmayor <marianag@nvidia.com> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * address review Signed-off-by: Mariana Graterol Fuenmayor <marianag@nvidia.com> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Mariana Graterol Fuenmayor <marianag@nvidia.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
1 parent c8ad73e commit 1f12635

4 files changed

Lines changed: 109 additions & 54 deletions

File tree

‎Jenkinsfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ pipeline {
1212
environment {
1313
AR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-11-26-0'
1414
DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-23-24-0'
15-
EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-26-3'
15+
EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-11-26-1'
1616
ES_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-25-24-0'
1717
ES_EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-30-24-0'
1818
HI_EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-26-4'

‎nemo_text_processing/text_normalization/en/taggers/serial.py‎

Lines changed: 105 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
from pynini.lib import pynutil
1919

2020
from nemo_text_processing.text_normalization.en.graph_utils import (
21+
MIN_NEG_WEIGHT,
22+
MIN_POS_WEIGHT,
2123
NEMO_ALPHA,
2224
NEMO_DIGIT,
2325
NEMO_NOT_SPACE,
@@ -28,16 +30,65 @@
2830
from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels
2931

3032

33+
def _leading_zero_graph(cardinal: GraphFst) -> "pynini.FstLike":
34+
return pynini.compose(pynini.accep("0") + pynini.closure(NEMO_DIGIT), cardinal.single_digits_graph).optimize()
35+
36+
37+
def _build_serial_graph(
38+
num_graph: "pynini.FstLike",
39+
delimiter: "pynini.FstLike",
40+
alphas: "pynini.FstLike",
41+
ordinal: GraphFst,
42+
) -> "pynini.FstLike":
43+
letter_num = alphas + delimiter + num_graph
44+
num_letter = pynini.closure(num_graph + delimiter, 1) + alphas
45+
next_alpha_or_num = pynini.closure(delimiter + (alphas | num_graph))
46+
next_alpha_or_num |= pynini.closure(
47+
delimiter
48+
+ num_graph
49+
+ plurals._priority_union(pynini.accep(" "), pynutil.insert(" "), NEMO_SIGMA).optimize()
50+
+ alphas
51+
)
52+
53+
serial_graph = letter_num + next_alpha_or_num
54+
serial_graph |= num_letter + next_alpha_or_num
55+
serial_graph |= num_graph + delimiter + num_graph + delimiter + num_graph + pynini.closure(delimiter + num_graph)
56+
57+
symbols = [x[0] for x in load_labels(get_abs_path("data/whitelist/symbol.tsv"))]
58+
symbols = pynini.union(*symbols)
59+
serial_graph |= pynini.compose(NEMO_SIGMA + symbols + NEMO_SIGMA, num_graph + delimiter + num_graph)
60+
61+
serial_graph = pynini.compose(
62+
pynini.difference(NEMO_SIGMA, pynini.project(ordinal.graph, "input")), serial_graph
63+
).optimize()
64+
65+
serial_graph = pynutil.add_weight(serial_graph, MIN_POS_WEIGHT)
66+
serial_graph |= (
67+
pynini.closure(NEMO_NOT_SPACE, 1) + (pynini.cross("^2", " squared") | pynini.cross("^3", " cubed")).optimize()
68+
)
69+
70+
serial_graph = (
71+
pynini.closure((serial_graph | num_graph | alphas) + delimiter)
72+
+ serial_graph
73+
+ pynini.closure(delimiter + (serial_graph | num_graph | alphas))
74+
)
75+
return serial_graph.optimize()
76+
77+
3178
class SerialFst(GraphFst):
3279
"""
33-
This class is a composite class of two other class instances
80+
Finite state transducer for classifying serial numbers without conventional delimiters.
81+
82+
Digit normalization within letter-digit tokens follows:
83+
1. 1-2 digits, or single digits followed by zeros -> cardinal
84+
2. 3 digits not ending in 00, or 4+ digits -> single-digit reading
85+
3. Digit-only tokens separated by ``/`` -> cardinal per segment (5+ digits stay single-digit)
3486
3587
Args:
36-
time: composed tagger and verbalizer
37-
date: composed tagger and verbalizer
38-
cardinal: tagger
88+
cardinal: cardinal tagger
89+
ordinal: ordinal tagger (used to exclude ordinal readings)
3990
deterministic: if True will provide a single transduction option,
40-
for False multiple transduction are generated (used for audio-based normalization)
91+
for False multiple transduction are generated (used for audio-based normalization)
4192
lm: whether to use for hybrid LM
4293
"""
4394

@@ -48,31 +99,56 @@ def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool =
4899
Finite state transducer for classifying serial (handles only cases without delimiters,
49100
values with delimiters are handled by default).
50101
The serial is a combination of digits, letters and dashes, e.g.:
51-
c325b -> tokens { cardinal { integer: "c three two five b" } }
102+
"H800" -> tokens { name: "H eight hundred" }
103+
"a320b" -> tokens { name: "a three two zero b" }
104+
"12/345/67890" -> tokens { name: "twelve/three hundred forty five/six seven eight nine zero" }
105+
52106
"""
53107
if deterministic:
54-
num_graph = pynini.compose(NEMO_DIGIT ** (6, ...), cardinal.single_digits_graph).optimize()
55-
num_graph |= pynini.compose(NEMO_DIGIT ** (1, 5), cardinal.graph).optimize()
56-
# to handle numbers starting with zero
57-
num_graph |= pynini.compose(
58-
pynini.accep("0") + pynini.closure(NEMO_DIGIT), cardinal.single_digits_graph
108+
num_graph_pure = (
109+
pynini.compose(NEMO_DIGIT ** (1, 3), cardinal.graph)
110+
| pynini.compose(NEMO_DIGIT ** (4, ...), cardinal.single_digits_graph)
111+
| _leading_zero_graph(cardinal)
112+
).optimize()
113+
114+
num_graph_alnum = (
115+
pynini.compose(NEMO_DIGIT, cardinal.graph)
116+
| pynini.compose(NEMO_DIGIT**2, cardinal.graph)
117+
| pynutil.add_weight(
118+
pynini.compose(NEMO_DIGIT + pynini.closure("0", 1), cardinal.graph), MIN_NEG_WEIGHT
119+
)
120+
| pynini.compose(
121+
pynini.difference(NEMO_DIGIT**3, NEMO_DIGIT + NEMO_DIGIT + "00"), cardinal.single_digits_graph
122+
)
123+
| pynini.compose(NEMO_DIGIT ** (4, ...), cardinal.single_digits_graph)
124+
| _leading_zero_graph(cardinal)
125+
).optimize()
126+
127+
num_graph_slash = (
128+
pynini.compose(NEMO_DIGIT ** (1, 4), cardinal.graph)
129+
| pynini.compose(NEMO_DIGIT ** (5, ...), cardinal.single_digits_graph)
130+
| _leading_zero_graph(cardinal)
59131
).optimize()
132+
60133
else:
61-
num_graph = cardinal.final_graph
134+
num_graph_pure = cardinal.final_graph
135+
num_graph_alnum = cardinal.final_graph
136+
num_graph_slash = cardinal.final_graph
62137

63138
# TODO: "#" doesn't work from the file
64139
symbols_graph = pynini.string_file(get_abs_path("data/whitelist/symbol.tsv")).optimize() | pynini.cross(
65140
"#", "hash"
66141
)
67-
num_graph |= symbols_graph
142+
num_graph_pure |= symbols_graph
143+
num_graph_alnum |= symbols_graph
68144

69145
if not self.deterministic and not lm:
70-
num_graph |= cardinal.single_digits_graph
71-
num_graph |= pynini.compose(num_graph, NEMO_SIGMA + pynutil.delete("hundred ") + NEMO_SIGMA)
72-
# also allow double digits to be pronounced as integer in serial number
73-
num_graph |= pynutil.add_weight(
74-
NEMO_DIGIT**2 @ cardinal.graph_hundred_component_at_least_one_none_zero_digit, weight=0.0001
146+
num_graph_pure |= cardinal.single_digits_graph
147+
num_graph_pure |= pynini.compose(num_graph_pure, NEMO_SIGMA + pynutil.delete("hundred ") + NEMO_SIGMA)
148+
num_graph_pure |= pynutil.add_weight(
149+
NEMO_DIGIT**2 @ cardinal.graph_hundred_component_at_least_one_none_zero_digit, weight=MIN_POS_WEIGHT
75150
)
151+
num_graph_alnum = num_graph_pure
76152

77153
# add space between letter and digit/symbol
78154
symbols = [x[0] for x in load_labels(get_abs_path("data/whitelist/symbol.tsv"))]
@@ -90,44 +166,21 @@ def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool =
90166
delimiter |= pynini.cross("-", " dash ") | pynini.cross("/", " slash ")
91167

92168
alphas = pynini.closure(NEMO_ALPHA, 1)
93-
letter_num = alphas + delimiter + num_graph
94-
num_letter = pynini.closure(num_graph + delimiter, 1) + alphas
95-
next_alpha_or_num = pynini.closure(delimiter + (alphas | num_graph))
96-
next_alpha_or_num |= pynini.closure(
97-
delimiter
98-
+ num_graph
99-
+ plurals._priority_union(pynini.accep(" "), pynutil.insert(" "), NEMO_SIGMA).optimize()
100-
+ alphas
101-
)
102-
103-
serial_graph = letter_num + next_alpha_or_num
104-
serial_graph |= num_letter + next_alpha_or_num
105-
# numbers only with 2+ delimiters
106-
serial_graph |= (
107-
num_graph + delimiter + num_graph + delimiter + num_graph + pynini.closure(delimiter + num_graph)
108-
)
109-
# 2+ symbols
110-
serial_graph |= pynini.compose(NEMO_SIGMA + symbols + NEMO_SIGMA, num_graph + delimiter + num_graph)
111-
112-
# exclude ordinal numbers from serial options
113-
serial_graph = pynini.compose(
114-
pynini.difference(NEMO_SIGMA, pynini.project(ordinal.graph, "input")), serial_graph
115-
).optimize()
116169

117-
serial_graph = pynutil.add_weight(serial_graph, 0.0001)
118-
serial_graph |= (
119-
pynini.closure(NEMO_NOT_SPACE, 1)
120-
+ (pynini.cross("^2", " squared") | pynini.cross("^3", " cubed")).optimize()
121-
)
170+
serial_graph = _build_serial_graph(num_graph_pure, delimiter, alphas, ordinal)
171+
serial_graph_alnum = _build_serial_graph(num_graph_alnum, delimiter, alphas, ordinal)
122172

123-
# at least one serial graph with alpha numeric value and optional additional serial/num/alpha values
124-
serial_graph = (
125-
pynini.closure((serial_graph | num_graph | alphas) + delimiter)
126-
+ serial_graph
127-
+ pynini.closure(delimiter + (serial_graph | num_graph | alphas))
173+
# Rule 3: tokens that contain only digits and slashes (e.g. 31/31/100, 123/261788/2021).
174+
slash_digit_token = (
175+
pynini.closure(NEMO_DIGIT, 1) + pynini.accep("/") + pynini.closure(NEMO_DIGIT | pynini.accep("/"), 0)
128176
)
177+
slash_serial = pynini.compose(
178+
slash_digit_token,
179+
pynini.closure(num_graph_slash + pynini.accep("/"), 1) + num_graph_slash,
180+
).optimize()
181+
serial_graph |= pynutil.add_weight(slash_serial, MIN_NEG_WEIGHT)
129182

130-
serial_graph |= pynini.compose(graph_with_space, serial_graph.optimize()).optimize()
183+
serial_graph |= pynini.compose(graph_with_space, serial_graph_alnum.optimize()).optimize()
131184
serial_graph = pynini.compose(pynini.closure(NEMO_NOT_SPACE, 2), serial_graph).optimize()
132185

133186
# this is not to verbolize "/" as "slash" in cases like "import/export"

‎tests/nemo_text_processing/en/data_text_normalization/test_cases_ordinal.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,4 @@
2424
21th~twenty one th
2525
121st~one hundred twenty first
2626
111th~one hundred eleventh
27-
111st~one hundred eleven st
27+
111st~one one one st

‎tests/nemo_text_processing/en/data_text_normalization/test_cases_serial.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,5 @@ a 4-kilogram bag~a four-kilogram bag
2929
100-car~one hundred-car
3030
123/261788/2021~one hundred twenty three/two six one seven eight eight/two thousand twenty one
3131
2*8~two asterisk eight
32+
my pnr is t2000~my pnr is t two thousand
33+
your otp is ab9453~your otp is ab nine four five three

0 commit comments

Comments
 (0)