Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ pip install nemo_text_processing

**_NOTE:_** This should work on any Linux OS with x86_64. Pip installation on MacOS and Windows are not supported due to the dependency [Pynini](https://www.openfst.org/twiki/bin/view/GRM/Pynini). On a platform other than Linux x86_64, installing from Pip tries to compile Pynini from scratch, and requires OpenFst headers and libraries to be in the expected place. So if it's working for you, it's because you happen to have installed OpenFst in the right way in the right place. So if you want to Pip install Pynini on MacOS, you have to have pre-compiled and pre-installed OpenFst. The Pynini README for that version should tell you which version it needs and what `--enable-foo` flags to use.
Instead, we recommend you to use conda-forge to install Pynini on MacOS or Windows:
`conda install -c conda-forge pynini=2.1.6.post1`.
`conda install -c conda-forge "pynini<=2.1.7"`.


### Pip from source
Expand Down
12 changes: 10 additions & 2 deletions nemo_text_processing/text_normalization/en/taggers/electronic.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

from nemo_text_processing.text_normalization.en.graph_utils import ( # common string literals
MIN_NEG_WEIGHT,
MIN_POS_WEIGHT,
NEMO_ALPHA,
NEMO_DIGIT,
NEMO_NOT_SPACE,
Expand Down Expand Up @@ -76,12 +77,19 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True):
)
dict_words_graph = dict_words_without_delimiter | dict_words

# Prefer complete dictionary words over partial matches such as "proGRAM".
spelled_letters = pynutil.add_weight(TO_UPPER, MIN_POS_WEIGHT) if deterministic else TO_UPPER

all_accepted_symbols_start = (
dict_words_graph | pynini.closure(TO_UPPER) | pynini.closure(NEMO_UPPER) | accepted_symbols
dict_words_graph | pynini.closure(spelled_letters) | pynini.closure(NEMO_UPPER) | accepted_symbols
).optimize()

all_accepted_symbols_end = (
dict_words_graph | numbers | pynini.closure(TO_UPPER) | pynini.closure(NEMO_UPPER) | accepted_symbols
dict_words_graph
| numbers
| pynini.closure(spelled_letters)
| pynini.closure(NEMO_UPPER)
| accepted_symbols
).optimize()

graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbol.tsv")).optimize()
Expand Down
6 changes: 5 additions & 1 deletion nemo_text_processing/text_normalization/en/taggers/serial.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,11 @@ def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool =
).optimize()
serial_graph |= pynutil.add_weight(slash_serial, MIN_NEG_WEIGHT)

serial_graph |= pynini.compose(graph_with_space, serial_graph_alnum.optimize()).optimize()
serial_graph_alnum = pynini.compose(graph_with_space, serial_graph_alnum.optimize()).optimize()
if deterministic:
# Prefer the delimited reading when both paths match, e.g. "133-ABC".
serial_graph_alnum = pynutil.add_weight(serial_graph_alnum, MIN_POS_WEIGHT)
serial_graph |= serial_graph_alnum
serial_graph = pynini.compose(pynini.closure(NEMO_NOT_SPACE, 2), serial_graph).optimize()

# this is not to verbolize "/" as "slash" in cases like "import/export"
Expand Down
2 changes: 1 addition & 1 deletion requirements/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ editdistance
inflect
joblib
pandas
pynini==2.1.6.post1
pynini<=2.1.7
regex
sacremoses>=0.0.43
setuptools>=65.5.1
Expand Down
2 changes: 1 addition & 1 deletion requirements/requirements_test.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ black==25.1.0
click>=8.0.2
isort[requirements]>5.1.0,<=6.0.1
parameterized
pynini==2.1.6.post1
pynini<=2.1.7
pytest
pytest-runner
ruamel.yaml
Expand Down
Loading