Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ pipeline {
}
environment {
AR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-11-26-0'
DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-28-26-0'
DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/11-10-26-0'
EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-11-26-1'
ES_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-25-24-0'
ES_EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-30-24-0'
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
euro €
dollar $
u s dollar USD
us dollar USD
pfund pfund
pfund sterling £
englische pfund £
britische pfund £
britischen pfund £
yen ¥
rupien ₹
rupie ₹
russische rubel ₽
russischen rubel ₽
rubel ₽
australische dollar a$
australischen dollar a$
chinesische renminbi 元
chinesische yuan 元
yuan 元
renminbi 元
kanadische dollar CAD
kanadischen dollar CAD
zypern pfund CYP
indonesischen rupiah IDR
indonesische rupiah IDR
schwedische krone SEK
schwedische kronen SEK
schwedischen kronen SEK
norwegische kronen NOK
norwegischen kronen NOK
norwegische krone NOK
kronen kr
krone kr
lybische dinar LYD
baht ฿
thai baht ฿
belarus rubel BYN
frank CHF
franken CHF
ägyptische pfund EGP
ägyptischen pfund EGP
japanische yen ¥
polnische zloty zł
zloty zł
złoty zł
dänische kronen DKK
dänische krone DKK
bolivianos BOB
algerischer dinar DZD
gulden ƒ
aruba gulden ƒ
ghanaischer cedi gh₵
katar riyal QAR
kayman dollar ci$
kaymaninseln dollar ci$
lesothischer loti LSL
malaysischer ringgit MYR
mexikanischer peso MXN
peruanischer sol PEN
singapur dollar s$
taiwanesischer dollar nt$
burundi frank BIF
khmer riel ៛
kambodschanische riel ៛
südafrikanischer rand ZAR
israelische schekel ₪
israelischen schekel ₪
schekel ₪
jordanischen dinar JOD
marokkanischen dirham MAD
marokkanische dirham MAD
kongolesische francs CDF
türkische lira ₺
türkischen lira ₺
lira ₺
argentinische pesos ARS
argentinische peso ARS
neuseeländische dollar nz$
nepalesische rupien NPR
reais r$
brasilianischen reais r$
brasilianische reais r$
georgische lari ₾
fidschi dollar FJD
zimbabwe dollar ZWL
chilenischen pesos CLP
guernsey pfund GPP
irakischen dinar IQD
isländischen kronen ISK
moldauischen leu MDL
ungarischen forint HUF
forint HUF
bahrain dinar BHD
hongkong dollar hk$
schweizer franken CHF
indischen rupien ₹
indische rupie ₹
belarussische rubel BYN
belarus rubeln BYN
tschechische kronen CZK
tschechischen kronen CZK
tschechische krone CZK
ägyptisches pfund EG£
eritreische nakfa ERN
algerische dinar DZD
algerischen dinar DZD
burundi franc BIF
mexikanische pesos MXN
südafrikanische rand ZAR
südafrikanischen rand ZAR
won ₩
lempira HNL
naira ₦
kuna kn
real r$
reals r$
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
cent ¢
cents ¢
eurocent ¢
eurocents ¢
euro cent ¢
euro cents ¢
us cent ¢
us cents ¢
u s cent ¢
u s cents ¢
sen
sens
centime ¢
centimes ¢
centavo ¢
centavos ¢
penny p
pence p
rappen Rp.
piaster Pt.
paisa
paise
kopeken
kopeke
øre Ø
öre Ø
eyrir
grosz gr.
groszen gr.
heller
lipa
quindarka
jiao
kobo
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
eurocent euro
eurocents euro
euro cent euro
euro cents euro
us cent dollar
us cents dollar
u s cent dollar
u s cents dollar
penny pfund
pence pfund
sen yen
centime franken
centimes franken
rappen franken
paisa rupien
paise rupien
kopeke rubel
kopeken rubel
grosz złoty
groszen złoty
öre schwedische kronen
öre schwedische krone
øre kronen
øre krone
heller tschechische kronen
lipa kuna
128 changes: 79 additions & 49 deletions nemo_text_processing/inverse_text_normalization/de/taggers/money.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,78 +15,108 @@
import pynini
from pynini.lib import pynutil

from nemo_text_processing.text_normalization.de.taggers.money import maj_singular, min_plural, min_singular
from nemo_text_processing.text_normalization.en.graph_utils import (
from nemo_text_processing.inverse_text_normalization.de.graph_utils import (
NEMO_DIGIT,
NEMO_SIGMA,
GraphFst,
convert_space,
delete_extra_space,
delete_space,
insert_space,
)
from nemo_text_processing.inverse_text_normalization.de.utils import get_abs_path


class MoneyFst(GraphFst):
"""
Finite state transducer for classifying money
e.g. elf euro und vier cent -> money { integer_part: "11" fractional_part: 04 currency: "€" }
Finite state transducer for classifying money.
The grammar assumes centesimal currency design, whereby the main unit of currency is divided into 100 subunits.
While this assumption will cover the vast majority of world currencies, there will be exceptions such as e.g. Mauritanian Ouguiya or Malagasy Ariary.
e.g. sieben euro -> money { integer_part: "7" currency: "€" }
e.g. eine million dollar -> money { integer_part: "1" quantity: "Million" currency: "$" }
e.g. sechshunderteinundzwanzig yen und achtzehn sen -> money { integer_part: "621" currency: "¥" fractional_part: "18" }
e.g. zehn pence -> money { integer_part: "0" fractional_part: "10" currency: "£" }

Args:
itn_cardinal_tagger: ITN Cardinal Tagger
itn_decimal_tagger: ITN Decimal Tagger
cardinal: CardinalFst
decimal: DecimalFst
"""

def __init__(self, itn_cardinal_tagger: GraphFst, itn_decimal_tagger: GraphFst, deterministic: bool = True):
super().__init__(name="money", kind="classify", deterministic=deterministic)
cardinal_graph = (
pynini.cdrewrite(pynini.cross(pynini.union("ein", "eine"), "eins"), "[BOS]", "[EOS]", NEMO_SIGMA)
@ itn_cardinal_tagger.graph_no_exception
def __init__(
self,
cardinal: GraphFst,
decimal: GraphFst,
):
super().__init__(name="money", kind="classify")

graph_cardinal = cardinal.graph_no_exception
add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("0") + NEMO_DIGIT)
graph_centiles = (graph_cardinal @ add_leading_zero_to_double_digit).optimize()
graph_decimal_final = decimal.final_graph_wo_negative
currency_major = pynini.string_file(get_abs_path("data/money/currency_major.tsv"))
currency_minor = pynini.string_file(get_abs_path("data/money/currency_minor.tsv"))
currency_minor_to_major = pynini.string_file(get_abs_path("data/money/currency_minor_to_major.tsv"))

graph_currency_major = pynutil.insert('currency: "') + currency_major + pynutil.insert('"')
graph_currency_min_to_maj = (
pynutil.insert('currency: "') + (currency_minor_to_major @ currency_major) + pynutil.insert('"')
)
graph_decimal_final = itn_decimal_tagger.final_graph_wo_negative
cents = pynini.accep("cent") | pynini.accep("cents")

graph_unit = pynini.invert(maj_singular)
graph_unit = pynutil.insert("currency: \"") + convert_space(graph_unit) + pynutil.insert("\"")
# zehn Euro -> 10 €
graph_currency_major_only = (
pynutil.insert('integer_part: "')
+ graph_cardinal
+ pynutil.insert('"')
+ delete_extra_space
+ graph_currency_major
)

add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("0") + NEMO_DIGIT)
min_unit = pynini.project(min_singular | min_plural, "output")
# elf euro (und) vier cent, vier cent
cents_standalone = (
pynutil.insert("fractional_part: \"")
+ cardinal_graph @ add_leading_zero_to_double_digit
# zehn Cent -> 0,10 €
graph_currency_cents_only = (
pynutil.insert('integer_part: "0"')
+ insert_space
+ pynutil.insert('fractional_part: "')
+ graph_centiles
+ pynutil.insert('"')
+ delete_space
+ pynutil.delete(min_unit)
+ pynutil.insert("\"")
+ pynutil.delete(cents)
+ insert_space
+ pynutil.insert('currency: "€"')
)

optional_cents_standalone = pynini.closure(
delete_space
+ pynini.closure(pynutil.delete("und") + delete_space, 0, 1)
# zehn Eurocent -> 0,10 €
# the subgraph maps the centile minor currencies to their major counterparts
# the abbreviations for minor currencies (e.g. Ø for the Danish øre) are not represented in this class
graph_currency_minor_to_major = (
pynutil.insert('integer_part: "0"')
+ insert_space
+ cents_standalone,
0,
1,
+ pynutil.insert('fractional_part: "')
+ graph_centiles
+ pynutil.insert('"')
+ delete_extra_space
+ graph_currency_min_to_maj
)
# elf euro vierzig, only after integer
optional_cents_suffix = pynini.closure(
delete_extra_space
+ pynutil.insert("fractional_part: \"")
+ pynutil.add_weight(cardinal_graph @ add_leading_zero_to_double_digit, -0.7)
+ pynutil.insert("\""),
0,
1,

# zehn Euro und zehn Cent -> 10,10 €
cents_part = pynutil.insert('fractional_part: "') + graph_centiles + pynutil.insert('"')
minor_word = delete_space + pynutil.delete(pynini.project(currency_minor, "input"))
graph_currency_major_fractional = graph_currency_major_only + (
(delete_space + pynutil.delete("und") + delete_extra_space + cents_part + minor_word)
| (delete_extra_space + cents_part + minor_word.ques)
)

graph_integer = (
pynutil.insert("integer_part: \"")
+ cardinal_graph
+ pynutil.insert("\"")
+ delete_extra_space
+ graph_unit
+ (optional_cents_standalone | optional_cents_suffix)
# zwei Millionen Euro -> 2 Millionen €
graph_currency_decimal = pynutil.add_weight(
graph_decimal_final + delete_extra_space + graph_currency_major, -0.1
)
graph_decimal = graph_decimal_final + delete_extra_space + graph_unit
graph_decimal |= pynutil.insert("currency: \"€\" integer_part: \"0\" ") + cents_standalone
final_graph = graph_integer | graph_decimal
final_graph = self.add_tokens(final_graph)

graph_all_currency = (
graph_currency_major_only
| graph_currency_cents_only
| graph_currency_major_fractional
| graph_currency_minor_to_major
| graph_currency_decimal
)

currency_graph = graph_all_currency.optimize()
final_graph = self.add_tokens(currency_graph)
self.fst = final_graph.optimize()
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ def __init__(
).fst
word_graph = WordFst().fst
time_graph = TimeFst(tn_time_verbalizer=tn_time_verbalizer).fst
money_graph = MoneyFst(itn_cardinal_tagger=cardinal, itn_decimal_tagger=decimal).fst
money_graph = MoneyFst(cardinal=cardinal, decimal=decimal).fst
whitelist_graph = WhiteListFst(tn_whitelist_tagger=tn_whitelist_tagger).fst
punct_graph = PunctuationFst().fst
electronic_graph = ElectronicFst(
Expand Down
Loading