diff --git a/README.md b/README.md index 8bbb5c617..cf7574b46 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ pip install nemo_text_processing **_NOTE:_** This should work on any Linux OS with x86_64. Pip installation on MacOS and Windows are not supported due to the dependency [Pynini](https://www.openfst.org/twiki/bin/view/GRM/Pynini). On a platform other than Linux x86_64, installing from Pip tries to compile Pynini from scratch, and requires OpenFst headers and libraries to be in the expected place. So if it's working for you, it's because you happen to have installed OpenFst in the right way in the right place. So if you want to Pip install Pynini on MacOS, you have to have pre-compiled and pre-installed OpenFst. The Pynini README for that version should tell you which version it needs and what `--enable-foo` flags to use. Instead, we recommend you to use conda-forge to install Pynini on MacOS or Windows: -`conda install -c conda-forge pynini=2.1.6.post1`. +`conda install -c conda-forge "pynini<=2.1.7"`. ### Pip from source diff --git a/nemo_text_processing/text_normalization/en/taggers/electronic.py b/nemo_text_processing/text_normalization/en/taggers/electronic.py index 25c3c445a..6c046190a 100644 --- a/nemo_text_processing/text_normalization/en/taggers/electronic.py +++ b/nemo_text_processing/text_normalization/en/taggers/electronic.py @@ -18,6 +18,7 @@ from nemo_text_processing.text_normalization.en.graph_utils import ( # common string literals MIN_NEG_WEIGHT, + MIN_POS_WEIGHT, NEMO_ALPHA, NEMO_DIGIT, NEMO_NOT_SPACE, @@ -76,12 +77,19 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): ) dict_words_graph = dict_words_without_delimiter | dict_words + # Prefer complete dictionary words over partial matches such as "proGRAM". + spelled_letters = pynutil.add_weight(TO_UPPER, MIN_POS_WEIGHT) if deterministic else TO_UPPER + all_accepted_symbols_start = ( - dict_words_graph | pynini.closure(TO_UPPER) | pynini.closure(NEMO_UPPER) | accepted_symbols + dict_words_graph | pynini.closure(spelled_letters) | pynini.closure(NEMO_UPPER) | accepted_symbols ).optimize() all_accepted_symbols_end = ( - dict_words_graph | numbers | pynini.closure(TO_UPPER) | pynini.closure(NEMO_UPPER) | accepted_symbols + dict_words_graph + | numbers + | pynini.closure(spelled_letters) + | pynini.closure(NEMO_UPPER) + | accepted_symbols ).optimize() graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbol.tsv")).optimize() diff --git a/nemo_text_processing/text_normalization/en/taggers/serial.py b/nemo_text_processing/text_normalization/en/taggers/serial.py index 4a5d6bb9d..d42b22c99 100644 --- a/nemo_text_processing/text_normalization/en/taggers/serial.py +++ b/nemo_text_processing/text_normalization/en/taggers/serial.py @@ -180,7 +180,11 @@ def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = ).optimize() serial_graph |= pynutil.add_weight(slash_serial, MIN_NEG_WEIGHT) - serial_graph |= pynini.compose(graph_with_space, serial_graph_alnum.optimize()).optimize() + serial_graph_alnum = pynini.compose(graph_with_space, serial_graph_alnum.optimize()).optimize() + if deterministic: + # Prefer the delimited reading when both paths match, e.g. "133-ABC". + serial_graph_alnum = pynutil.add_weight(serial_graph_alnum, MIN_POS_WEIGHT) + serial_graph |= serial_graph_alnum serial_graph = pynini.compose(pynini.closure(NEMO_NOT_SPACE, 2), serial_graph).optimize() # this is not to verbolize "/" as "slash" in cases like "import/export" diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 6622e2d49..f2306a467 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -3,7 +3,7 @@ editdistance inflect joblib pandas -pynini==2.1.6.post1 +pynini<=2.1.7 regex sacremoses>=0.0.43 setuptools>=65.5.1 diff --git a/requirements/requirements_test.txt b/requirements/requirements_test.txt index aacfde319..25e2fc64e 100644 --- a/requirements/requirements_test.txt +++ b/requirements/requirements_test.txt @@ -2,7 +2,7 @@ black==25.1.0 click>=8.0.2 isort[requirements]>5.1.0,<=6.0.1 parameterized -pynini==2.1.6.post1 +pynini<=2.1.7 pytest pytest-runner ruamel.yaml