From 9c53e916240064e9cd9bd708802da712559a0a93 Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Wed, 21 Oct 2026 13:26:52 -0400 Subject: [PATCH] De ITN: decouple the money grammar from the TN grammar Signed-off-by: Adelina Dunina --- Jenkinsfile | 2 +- .../de/data/money/__init__.py | 13 ++ .../de/data/money/currency_major.tsv | 117 ++++++++++++++++ .../de/data/money/currency_minor.tsv | 34 +++++ .../de/data/money/currency_minor_to_major.tsv | 26 ++++ .../de/taggers/money.py | 128 +++++++++++------- .../de/taggers/tokenize_and_classify.py | 2 +- .../de/verbalizers/money.py | 23 ++-- .../test_cases_money.txt | 52 ++++--- 9 files changed, 315 insertions(+), 82 deletions(-) create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/money/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/money/currency_major.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/money/currency_minor.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/money/currency_minor_to_major.tsv diff --git a/Jenkinsfile b/Jenkinsfile index 078215712..c0597d1b4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,7 +11,7 @@ pipeline { } environment { AR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-11-26-0' - DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-28-26-0' + DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/11-10-26-0' EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-11-26-1' ES_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-25-24-0' ES_EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-30-24-0' diff --git a/nemo_text_processing/inverse_text_normalization/de/data/money/__init__.py b/nemo_text_processing/inverse_text_normalization/de/data/money/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/money/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/de/data/money/currency_major.tsv b/nemo_text_processing/inverse_text_normalization/de/data/money/currency_major.tsv new file mode 100644 index 000000000..1d9a1184d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/money/currency_major.tsv @@ -0,0 +1,117 @@ +euro € +dollar $ +u s dollar USD +us dollar USD +pfund pfund +pfund sterling £ +englische pfund £ +britische pfund £ +britischen pfund £ +yen ¥ +rupien ₹ +rupie ₹ +russische rubel ₽ +russischen rubel ₽ +rubel ₽ +australische dollar a$ +australischen dollar a$ +chinesische renminbi 元 +chinesische yuan 元 +yuan 元 +renminbi 元 +kanadische dollar CAD +kanadischen dollar CAD +zypern pfund CYP +indonesischen rupiah IDR +indonesische rupiah IDR +schwedische krone SEK +schwedische kronen SEK +schwedischen kronen SEK +norwegische kronen NOK +norwegischen kronen NOK +norwegische krone NOK +kronen kr +krone kr +lybische dinar LYD +baht ฿ +thai baht ฿ +belarus rubel BYN +frank CHF +franken CHF +ägyptische pfund EGP +ägyptischen pfund EGP +japanische yen ¥ +polnische zloty zł +zloty zł +złoty zł +dänische kronen DKK +dänische krone DKK +bolivianos BOB +algerischer dinar DZD +gulden ƒ +aruba gulden ƒ +ghanaischer cedi gh₵ +katar riyal QAR +kayman dollar ci$ +kaymaninseln dollar ci$ +lesothischer loti LSL +malaysischer ringgit MYR +mexikanischer peso MXN +peruanischer sol PEN +singapur dollar s$ +taiwanesischer dollar nt$ +burundi frank BIF +khmer riel ៛ +kambodschanische riel ៛ +südafrikanischer rand ZAR +israelische schekel ₪ +israelischen schekel ₪ +schekel ₪ +jordanischen dinar JOD +marokkanischen dirham MAD +marokkanische dirham MAD +kongolesische francs CDF +türkische lira ₺ +türkischen lira ₺ +lira ₺ +argentinische pesos ARS +argentinische peso ARS +neuseeländische dollar nz$ +nepalesische rupien NPR +reais r$ +brasilianischen reais r$ +brasilianische reais r$ +georgische lari ₾ +fidschi dollar FJD +zimbabwe dollar ZWL +chilenischen pesos CLP +guernsey pfund GPP +irakischen dinar IQD +isländischen kronen ISK +moldauischen leu MDL +ungarischen forint HUF +forint HUF +bahrain dinar BHD +hongkong dollar hk$ +schweizer franken CHF +indischen rupien ₹ +indische rupie ₹ +belarussische rubel BYN +belarus rubeln BYN +tschechische kronen CZK +tschechischen kronen CZK +tschechische krone CZK +ägyptisches pfund EG£ +eritreische nakfa ERN +algerische dinar DZD +algerischen dinar DZD +burundi franc BIF +mexikanische pesos MXN +südafrikanische rand ZAR +südafrikanischen rand ZAR +won ₩ +lempira HNL +naira ₦ +kuna kn +real r$ +reals r$ \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/money/currency_minor.tsv b/nemo_text_processing/inverse_text_normalization/de/data/money/currency_minor.tsv new file mode 100644 index 000000000..f1d85099c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/money/currency_minor.tsv @@ -0,0 +1,34 @@ +cent ¢ +cents ¢ +eurocent ¢ +eurocents ¢ +euro cent ¢ +euro cents ¢ +us cent ¢ +us cents ¢ +u s cent ¢ +u s cents ¢ +sen +sens +centime ¢ +centimes ¢ +centavo ¢ +centavos ¢ +penny p +pence p +rappen Rp. +piaster Pt. +paisa +paise +kopeken +kopeke +øre Ø +öre Ø +eyrir +grosz gr. +groszen gr. +heller +lipa +quindarka +jiao +kobo \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/money/currency_minor_to_major.tsv b/nemo_text_processing/inverse_text_normalization/de/data/money/currency_minor_to_major.tsv new file mode 100644 index 000000000..a5bc10285 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/money/currency_minor_to_major.tsv @@ -0,0 +1,26 @@ +eurocent euro +eurocents euro +euro cent euro +euro cents euro +us cent dollar +us cents dollar +u s cent dollar +u s cents dollar +penny pfund +pence pfund +sen yen +centime franken +centimes franken +rappen franken +paisa rupien +paise rupien +kopeke rubel +kopeken rubel +grosz złoty +groszen złoty +öre schwedische kronen +öre schwedische krone +øre kronen +øre krone +heller tschechische kronen +lipa kuna \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/money.py b/nemo_text_processing/inverse_text_normalization/de/taggers/money.py index c53b865ed..061ed21c5 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/money.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/money.py @@ -15,78 +15,108 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.de.taggers.money import maj_singular, min_plural, min_singular -from nemo_text_processing.text_normalization.en.graph_utils import ( +from nemo_text_processing.inverse_text_normalization.de.graph_utils import ( NEMO_DIGIT, - NEMO_SIGMA, GraphFst, - convert_space, delete_extra_space, delete_space, insert_space, ) +from nemo_text_processing.inverse_text_normalization.de.utils import get_abs_path class MoneyFst(GraphFst): """ - Finite state transducer for classifying money - e.g. elf euro und vier cent -> money { integer_part: "11" fractional_part: 04 currency: "€" } + Finite state transducer for classifying money. + The grammar assumes centesimal currency design, whereby the main unit of currency is divided into 100 subunits. + While this assumption will cover the vast majority of world currencies, there will be exceptions such as e.g. Mauritanian Ouguiya or Malagasy Ariary. + e.g. sieben euro -> money { integer_part: "7" currency: "€" } + e.g. eine million dollar -> money { integer_part: "1" quantity: "Million" currency: "$" } + e.g. sechshunderteinundzwanzig yen und achtzehn sen -> money { integer_part: "621" currency: "¥" fractional_part: "18" } + e.g. zehn pence -> money { integer_part: "0" fractional_part: "10" currency: "£" } Args: - itn_cardinal_tagger: ITN Cardinal Tagger - itn_decimal_tagger: ITN Decimal Tagger + cardinal: CardinalFst + decimal: DecimalFst """ - def __init__(self, itn_cardinal_tagger: GraphFst, itn_decimal_tagger: GraphFst, deterministic: bool = True): - super().__init__(name="money", kind="classify", deterministic=deterministic) - cardinal_graph = ( - pynini.cdrewrite(pynini.cross(pynini.union("ein", "eine"), "eins"), "[BOS]", "[EOS]", NEMO_SIGMA) - @ itn_cardinal_tagger.graph_no_exception + def __init__( + self, + cardinal: GraphFst, + decimal: GraphFst, + ): + super().__init__(name="money", kind="classify") + + graph_cardinal = cardinal.graph_no_exception + add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("0") + NEMO_DIGIT) + graph_centiles = (graph_cardinal @ add_leading_zero_to_double_digit).optimize() + graph_decimal_final = decimal.final_graph_wo_negative + currency_major = pynini.string_file(get_abs_path("data/money/currency_major.tsv")) + currency_minor = pynini.string_file(get_abs_path("data/money/currency_minor.tsv")) + currency_minor_to_major = pynini.string_file(get_abs_path("data/money/currency_minor_to_major.tsv")) + + graph_currency_major = pynutil.insert('currency: "') + currency_major + pynutil.insert('"') + graph_currency_min_to_maj = ( + pynutil.insert('currency: "') + (currency_minor_to_major @ currency_major) + pynutil.insert('"') ) - graph_decimal_final = itn_decimal_tagger.final_graph_wo_negative + cents = pynini.accep("cent") | pynini.accep("cents") - graph_unit = pynini.invert(maj_singular) - graph_unit = pynutil.insert("currency: \"") + convert_space(graph_unit) + pynutil.insert("\"") + # zehn Euro -> 10 € + graph_currency_major_only = ( + pynutil.insert('integer_part: "') + + graph_cardinal + + pynutil.insert('"') + + delete_extra_space + + graph_currency_major + ) - add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("0") + NEMO_DIGIT) - min_unit = pynini.project(min_singular | min_plural, "output") - # elf euro (und) vier cent, vier cent - cents_standalone = ( - pynutil.insert("fractional_part: \"") - + cardinal_graph @ add_leading_zero_to_double_digit + # zehn Cent -> 0,10 € + graph_currency_cents_only = ( + pynutil.insert('integer_part: "0"') + + insert_space + + pynutil.insert('fractional_part: "') + + graph_centiles + + pynutil.insert('"') + delete_space - + pynutil.delete(min_unit) - + pynutil.insert("\"") + + pynutil.delete(cents) + + insert_space + + pynutil.insert('currency: "€"') ) - optional_cents_standalone = pynini.closure( - delete_space - + pynini.closure(pynutil.delete("und") + delete_space, 0, 1) + # zehn Eurocent -> 0,10 € + # the subgraph maps the centile minor currencies to their major counterparts + # the abbreviations for minor currencies (e.g. Ø for the Danish øre) are not represented in this class + graph_currency_minor_to_major = ( + pynutil.insert('integer_part: "0"') + insert_space - + cents_standalone, - 0, - 1, + + pynutil.insert('fractional_part: "') + + graph_centiles + + pynutil.insert('"') + + delete_extra_space + + graph_currency_min_to_maj ) - # elf euro vierzig, only after integer - optional_cents_suffix = pynini.closure( - delete_extra_space - + pynutil.insert("fractional_part: \"") - + pynutil.add_weight(cardinal_graph @ add_leading_zero_to_double_digit, -0.7) - + pynutil.insert("\""), - 0, - 1, + + # zehn Euro und zehn Cent -> 10,10 € + cents_part = pynutil.insert('fractional_part: "') + graph_centiles + pynutil.insert('"') + minor_word = delete_space + pynutil.delete(pynini.project(currency_minor, "input")) + graph_currency_major_fractional = graph_currency_major_only + ( + (delete_space + pynutil.delete("und") + delete_extra_space + cents_part + minor_word) + | (delete_extra_space + cents_part + minor_word.ques) ) - graph_integer = ( - pynutil.insert("integer_part: \"") - + cardinal_graph - + pynutil.insert("\"") - + delete_extra_space - + graph_unit - + (optional_cents_standalone | optional_cents_suffix) + # zwei Millionen Euro -> 2 Millionen € + graph_currency_decimal = pynutil.add_weight( + graph_decimal_final + delete_extra_space + graph_currency_major, -0.1 ) - graph_decimal = graph_decimal_final + delete_extra_space + graph_unit - graph_decimal |= pynutil.insert("currency: \"€\" integer_part: \"0\" ") + cents_standalone - final_graph = graph_integer | graph_decimal - final_graph = self.add_tokens(final_graph) + + graph_all_currency = ( + graph_currency_major_only + | graph_currency_cents_only + | graph_currency_major_fractional + | graph_currency_minor_to_major + | graph_currency_decimal + ) + + currency_graph = graph_all_currency.optimize() + final_graph = self.add_tokens(currency_graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py index c45726cf0..41d77a979 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py @@ -108,7 +108,7 @@ def __init__( ).fst word_graph = WordFst().fst time_graph = TimeFst(tn_time_verbalizer=tn_time_verbalizer).fst - money_graph = MoneyFst(itn_cardinal_tagger=cardinal, itn_decimal_tagger=decimal).fst + money_graph = MoneyFst(cardinal=cardinal, decimal=decimal).fst whitelist_graph = WhiteListFst(tn_whitelist_tagger=tn_whitelist_tagger).fst punct_graph = PunctuationFst().fst electronic_graph = ElectronicFst( diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/money.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/money.py index f8bbc0793..11c263803 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/money.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/money.py @@ -14,28 +14,33 @@ import pynini from pynini.lib import pynutil - -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, delete_space +from nemo_text_processing.inverse_text_normalization.de.graph_utils import ( + NEMO_CHAR, + GraphFst, + delete_extra_space, + delete_space, +) class MoneyFst(GraphFst): """ - Finite state transducer for verbalizing money, e.g. - money { integer_part: "12" fractional_part: "05" currency: "$" } -> $12.05 + Finite state transducer for verbalizing money, + e.g. money { integer_part: "7" currency: "€" } -> 7 € + e.g. money { integer_part: "76.987" currency: "$" } -> 76.987 $ Args: - decimal: ITN Decimal verbalizer + decimal: DecimalFst """ - def __init__(self, decimal: GraphFst, deterministic: bool = True): - super().__init__(name="money", kind="verbalize", deterministic=deterministic) + def __init__(self, decimal: GraphFst): + super().__init__(name="money", kind="verbalize") unit = ( pynutil.delete("currency:") - + delete_space + + delete_extra_space + pynutil.delete("\"") + pynini.closure(NEMO_CHAR - " ", 1) + pynutil.delete("\"") ) - graph = unit + delete_space + decimal.numbers + graph = decimal.numbers + delete_space + unit delete_tokens = self.delete_tokens(graph) self.fst = delete_tokens.optimize() diff --git a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_money.txt index 4222ad22c..e8b83f758 100644 --- a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_money.txt +++ b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_money.txt @@ -1,23 +1,31 @@ -zwei dollar~$2 -ein dollar~$1 -eine million dollar~$1 Million -zwei komma null null null eins dollar~$2,0001 -zwei komma null eins dollar~$2,01 -zwei komma null null dollar~$2,00 -ein cent~€0,01 -zwei cent~€0,02 -zwanzig cent~€0,20 -zweiundzwanzig cent~€0,22 -einhundert cent~100 cent -zwei dollar zwanzig~$2,20 +zwei dollar~2 $ +ein dollar~1 $ +eine million dollar~1 Million $ +zwei komma null null null eins dollar~2,0001 $ +zwei komma null eins dollar~2,01 $ +zwei komma null null dollar~2,00 $ +ein cent~0,01 € +zwei cent~0,02 € +zwanzig cent~0,20 € +zweiundzwanzig cent~0,22 € +zwei dollar zwanzig~2,20 $ zweidollarzwanzig~zweidollarzwanzig -zwei dollar und zwanzig cent~$2,20 -zwei euro und zwanzig cent~€2,20 -zwei pfund und zwanzig pence~£2,20 -zwei euro zwanzig cent~€2,20 -zwei millionen euro~€2 Millionen -zwei komma zwei null null millionen euro~€2,200 Millionen -zwei komma zwei null eins millionen euro~€2,201 Millionen -zwei komma zwei eins millionen euro~€2,21 Millionen -zwei pfund und ein penny~£2,01 -zwei pfund und ein hundert penny~£2 und 100 penny \ No newline at end of file +zwei dollar und zwanzig cent~2,20 $ +zwei euro und zwanzig cent~2,20 € +zwei pfund und zwanzig pence~2,20 pfund +zwei euro zwanzig cent~2,20 € +zwei millionen euro~2 Millionen € +zwei komma zwei null null millionen euro~2,200 Millionen € +zwei komma zwei null eins millionen euro~2,201 Millionen € +zwei komma zwei eins millionen euro~2,21 Millionen € +zwei pfund und ein penny~2,01 pfund +zwei pfund und ein hundert penny~2 pfund und 100 penny +zwanzig euro cent~0,20 € +zwanzig eurocent~0,20 € +zweiundzwanzig us cent~0,22 $ +fünfzig yen und vierunddreißig sen~50,34 ¥ +fünfzig yen vierunddreißig~50,34 ¥ +sechsundsiebzig paise~0,76 ₹ +vierzig złoty zehn groszen~40,10 zł +vierzig złoty ein grosz~40,01 zł +dreizehn franken~13 CHF \ No newline at end of file