From 2477d0b1a01723f2f67166c856f5cf785b94016c Mon Sep 17 00:00:00 2001 From: Cal3bdev Date: Sun, 25 Oct 2026 10:56:45 +0300 Subject: [PATCH 1/9] Add shared Bantu ITN kernel with a test-only language profile Adds nemo_text_processing/inverse_text_normalization/bantu/, a shared inverse text normalization kernel that Bantu languages configure with a language profile. It supports cardinal and money, resolved in a sentence-level classifier; the verbalizers only format. The kernel is not listed in the language codes. Tests build the classifier and verbalizer directly with a made-up test language (currency code XTS), run through NeMo's inverse_normalize runtime, and check that every test input has exactly one best reading. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Cal3bdev --- .../bantu/__init__.py | 13 ++ .../bantu/profile.py | 189 +++++++++++++++++ .../bantu/taggers/__init__.py | 13 ++ .../bantu/taggers/cardinal.py | 85 ++++++++ .../bantu/taggers/money.py | 38 ++++ .../bantu/taggers/scale.py | 143 +++++++++++++ .../bantu/taggers/sentence.py | 196 ++++++++++++++++++ .../bantu/taggers/tokenize_and_classify.py | 56 +++++ .../bantu/verbalizers/__init__.py | 13 ++ .../bantu/verbalizers/cardinal.py | 44 ++++ .../bantu/verbalizers/money.py | 50 +++++ .../bantu/verbalizers/verbalize.py | 35 ++++ .../bantu/verbalizers/verbalize_final.py | 46 ++++ .../bantu/verbalizers/word.py | 30 +++ tests/nemo_text_processing/bantu/__init__.py | 13 ++ .../bantu/test_cardinal.py | 104 ++++++++++ .../nemo_text_processing/bantu/test_money.py | 142 +++++++++++++ .../bantu/test_profile.py | 91 ++++++++ .../bantu/test_segmentation.py | 69 ++++++ .../bantu/test_verbalizer.py | 55 +++++ .../nemo_text_processing/bantu/toy_profile.py | 136 ++++++++++++ tests/nemo_text_processing/bantu/utils.py | 42 ++++ 22 files changed, 1603 insertions(+) create mode 100644 nemo_text_processing/inverse_text_normalization/bantu/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/bantu/profile.py create mode 100644 nemo_text_processing/inverse_text_normalization/bantu/taggers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/bantu/taggers/money.py create mode 100644 nemo_text_processing/inverse_text_normalization/bantu/taggers/scale.py create mode 100644 nemo_text_processing/inverse_text_normalization/bantu/taggers/sentence.py create mode 100644 nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/inverse_text_normalization/bantu/verbalizers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/bantu/verbalizers/money.py create mode 100644 nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize.py create mode 100644 nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize_final.py create mode 100644 nemo_text_processing/inverse_text_normalization/bantu/verbalizers/word.py create mode 100644 tests/nemo_text_processing/bantu/__init__.py create mode 100644 tests/nemo_text_processing/bantu/test_cardinal.py create mode 100644 tests/nemo_text_processing/bantu/test_money.py create mode 100644 tests/nemo_text_processing/bantu/test_profile.py create mode 100644 tests/nemo_text_processing/bantu/test_segmentation.py create mode 100644 tests/nemo_text_processing/bantu/test_verbalizer.py create mode 100644 tests/nemo_text_processing/bantu/toy_profile.py create mode 100644 tests/nemo_text_processing/bantu/utils.py diff --git a/nemo_text_processing/inverse_text_normalization/bantu/__init__.py b/nemo_text_processing/inverse_text_normalization/bantu/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/bantu/profile.py b/nemo_text_processing/inverse_text_normalization/bantu/profile.py new file mode 100644 index 000000000..6b6bc59e7 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/profile.py @@ -0,0 +1,189 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Small explicit resource contract, not a universal Bantu grammar schema.""" +from dataclasses import dataclass, replace + +Forms = tuple[tuple[str, str], ...] +VOWELS = "aeiou" + + +@dataclass(frozen=True) +class NounContext: + class_id: str + noun: str + atomic_units: Forms + final_units: Forms + compounds: bool = True + teen_variants: Forms = () + + +@dataclass(frozen=True) +class Place: + # Each form outputs its nonzero coefficient, not the complete place value. + power: int + forms: Forms + remainder_join: str + + +@dataclass(frozen=True) +class CurrencyContext: + # A spoken currency phrase sets the amount's agreement, like a counted noun. + phrase: str + code: str + agreement_class: str + roles: tuple[str, ...] = () # extra readings licensed only after this phrase + + +@dataclass(frozen=True) +class CountedScale: + """A scale noun followed by its count, e.g. Luganda mitwalo abiri = 20 x 10,000.""" + + singular: str # head counted once, e.g. mutwalo + singular_count: str # its agreeing "one", e.g. gumu + plural: str # e.g. mitwalo + power: int # 4 for ten-thousands + count_class: str # agreement class of unit words inside the count + tens: Forms # tens words inside a count, coefficient 1-9 + remainder_class: str # agreement class of the amount after the scale + contracted_teen_joins: tuple[str, ...] = () # teen join before a vowel, e.g. n' + implied_currency: str = "" # payment policy: a bare amount means this currency + count_ones: Forms = () # a "one" ending a count, e.g. kkumi na kamu (11) + count_hundreds: Forms = () # hundreds inside a count, e.g. obukadde kikumi (100) + # Heads with the augment (Luganda e-/a-/o-) count the noun before them; the bare + # heads above are money. Decided with the native speaker 2026-10-15/18. + counting_plural: str = "" # e.g. emitwalo, obukadde + counting_ones: tuple[str, ...] = () # e.g. akakadde kamu + money_only_ones: tuple[str, ...] = () # one, as money only: akakadde (not ebitabo akakadde) + implied_anywhere: bool = False # bare heads are money wherever they appear + counting_money_at_start: bool = False # augmented heads are money at the start of input + counts_any_word: bool = False # word + augmented head counts that word, listed or not + after_counted_nouns: bool = False # augmented heads count listed nouns, with agreement + + +@dataclass(frozen=True) +class MoneyCues: + """Words that signal a payment just before an amount (e.g. Luganda money verbs).""" + + stems: tuple[str, ...] = () # match anywhere inside a word + words: tuple[str, ...] = () # match only the whole word + not_cues: tuple[str, ...] = () # whole words containing a stem that are not cues + + +@dataclass(frozen=True) +class CardinalProfile: + language: str + contexts: tuple[NounContext, ...] + ten: str + teen_join: str + places: tuple[Place, ...] + exact_boundaries: Forms + roles: tuple[tuple[str, Forms], ...] + max_value: int + currencies: tuple[CurrencyContext, ...] = () + counted_scales: tuple[CountedScale, ...] = () + money_cues: MoneyCues = MoneyCues() + # Words that continue a number but are not handled yet (e.g. "and a half"). + # They join the number's run, so the whole phrase stays unchanged. + held_back_words: tuple[str, ...] = () + + def join_words(self) -> frozenset[str]: + """Words that join number parts; they may also be ordinary words (e.g. locative mu).""" + joins = [self.teen_join] + [place.remainder_join for place in self.places] + return frozenset(word for join in joins for word in join.split()) + + def number_words(self) -> frozenset[str]: + """Lexical run boundaries, including joins and role-only candidates.""" + surfaces = [self.ten, self.teen_join] + for place in self.places: + surfaces.append(place.remainder_join) + surfaces.extend(form for form, _ in place.forms) + for context in self.contexts: + surfaces.extend(form for form, _ in context.atomic_units + context.final_units + context.teen_variants) + surfaces.extend(form for form, _ in self.exact_boundaries) + for _, forms in self.roles: + surfaces.extend(form for form, _ in forms) + for scale in self.counted_scales: + surfaces.extend( + (scale.singular, scale.singular_count, scale.plural) + scale.counting_ones + scale.money_only_ones + ) + if scale.counting_plural: + surfaces.append(scale.counting_plural) + surfaces.extend(form for form, _ in scale.tens + scale.count_ones + scale.count_hundreds) + surfaces.extend(scale_teen_words(self, scale)) + surfaces.extend(self.held_back_words) + return frozenset(word for form in surfaces for word in form.split()) + + +def scale_teen_words(profile: CardinalProfile, scale: CountedScale) -> list[str]: + """Contracted teen words such as n'ebiri / nebiri (one word each).""" + context = agreement_context(profile, scale.count_class) + return [ + prefix + form + for form, _ in context.final_units + scale.count_ones + if form[0] in VOWELS + for prefix in scale.contracted_teen_joins + ] + + +def agreement_context(profile: CardinalProfile, class_id: str) -> NounContext: + context = next((c for c in profile.contexts if c.class_id == class_id and c.compounds), None) + if context is None: + raise ValueError(f"no compound agreement context for class {class_id}") + return context + + +def invariant_context(profile: CardinalProfile) -> NounContext: + """Unit forms that are the same in every class (Luganda 6-9), for unknown nouns. + + A number after a noun the profile does not list may only end in these, so + nothing that must agree with that noun is guessed. + """ + contexts = [c for c in profile.contexts if c.compounds] + shared = set(contexts[0].final_units) + for context in contexts[1:]: + shared &= set(context.final_units) + units = tuple(sorted(shared, key=lambda form: form[1])) + return NounContext("invariant", "", units + ((profile.ten, "10"),), units) + + +def with_spelling_variants(profile: CardinalProfile, variants: dict[str, tuple[str, ...]]) -> CardinalProfile: + """Accept extra spellings of whole words (e.g. abiiri for abiri) in every form.""" + + def spell(forms: Forms) -> Forms: + extra = [] + for form, value in forms: + words = form.split() + for index, word in enumerate(words): + for variant in variants.get(word, ()): + extra.append((" ".join(words[:index] + [variant] + words[index + 1 :]), value)) + return forms + tuple(extra) + + return replace( + profile, + contexts=tuple( + replace( + c, + atomic_units=spell(c.atomic_units), + final_units=spell(c.final_units), + teen_variants=spell(c.teen_variants), + ) + for c in profile.contexts + ), + places=tuple(replace(p, forms=spell(p.forms)) for p in profile.places), + roles=tuple((name, spell(forms)) for name, forms in profile.roles), + counted_scales=tuple( + replace(s, tens=spell(s.tens), count_hundreds=spell(s.count_hundreds)) for s in profile.counted_scales + ), + ) diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/__init__.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py new file mode 100644 index 000000000..2e409eebb --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py @@ -0,0 +1,85 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Bounded decimal place-value FSTs, built from a profile. No runtime arithmetic.""" +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile, Forms, NounContext + +DIGIT = pynini.union(*"0123456789").optimize() +NONZERO = pynini.union(*"123456789").optimize() + + +def mapping(forms: Forms) -> pynini.Fst: + return pynini.string_map(forms).optimize() if forms else pynini.Fst() + + +def integer_domain(max_value: int) -> pynini.Fst: + """Canonical positive decimal strings <= bound, without integer enumeration.""" + if isinstance(max_value, bool) or not isinstance(max_value, int) or max_value < 1: + raise ValueError("max_value must be a positive integer") + bound = str(max_value) + branches = [pynini.accep(bound)] + if len(bound) > 1: + branches.append(NONZERO + pynini.closure(DIGIT, 0, len(bound) - 2)) + for index, limit in enumerate(bound): + minimum = 1 if index == 0 else 0 + if int(limit) > minimum: + lower = pynini.union(*(str(n) for n in range(minimum, int(limit)))) + branches.append( + pynini.accep(bound[:index]) + + lower + + pynini.closure(DIGIT, len(bound) - index - 1, len(bound) - index - 1) + ) + return pynini.union(*branches).optimize() + + +def build_cardinal(profile: CardinalProfile, context: NounContext) -> pynini.Fst: + """Compose fixed-width places, supply skipped zeros, then canonicalize once.""" + atomic = mapping(context.atomic_units) + if not context.compounds: + return (atomic @ integer_domain(profile.max_value)).optimize() + units = mapping(context.final_units) + lower = units + expected_power = 1 + for place in profile.places: + if place.power != expected_power: + raise ValueError("places must be consecutive ascending decimal powers") + if any(value not in "123456789" or len(value) != 1 for _, value in place.forms): + raise ValueError("place coefficients must be single nonzero digits") + # A present high place either consumes a smaller remainder or emits zeros. + high = mapping(place.forms) + ( + pynutil.delete(place.remainder_join) + lower | pynutil.insert("0" * place.power) + ) + if place.power == 1: + high |= pynini.cross(profile.ten, "10") + high |= pynini.cross(profile.ten, "1") + pynutil.delete(profile.teen_join) + units + high |= mapping(context.teen_variants) + lower = (pynutil.insert("0") + lower | high).optimize() + expected_power += 1 + canonical = pynutil.delete(pynini.closure("0")) + NONZERO + pynini.closure(DIGIT) + compounds = lower @ canonical + # A final-one form need not be licensed as a standalone plural count. + compounds @= NONZERO + pynini.closure(DIGIT, 1) + result = atomic | compounds | mapping(profile.exact_boundaries) + return (result @ integer_domain(profile.max_value)).optimize() + + +def build_role(profile: CardinalProfile, role: str) -> pynini.Fst: + """Explicit role probe; not a standalone sentence or currency grammar.""" + for name, forms in profile.roles: + if name == role: + return mapping(forms) + raise KeyError(role) diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/money.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/money.py new file mode 100644 index 000000000..11bdbd947 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/money.py @@ -0,0 +1,38 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Currency-context amounts: reuse cardinal composition, then add licensed roles.""" +import pynini + +from nemo_text_processing.inverse_text_normalization.bantu.profile import ( + CardinalProfile, + CurrencyContext, + agreement_context, +) +from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import ( + build_cardinal, + build_role, + integer_domain, +) +from nemo_text_processing.inverse_text_normalization.bantu.taggers.scale import build_scale_amount + + +def build_amount(profile: CardinalProfile, currency: CurrencyContext) -> pynini.Fst: + """Digits for an amount after this currency phrase; no Python arithmetic.""" + amount = build_cardinal(profile, agreement_context(profile, currency.agreement_class)) + for role in currency.roles: + amount |= build_role(profile, role) @ integer_domain(profile.max_value) + for scale in profile.counted_scales: + amount |= build_scale_amount(profile, scale) + return amount.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/scale.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/scale.py new file mode 100644 index 000000000..13d19d892 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/scale.py @@ -0,0 +1,143 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Counted scales: a scale noun, its count (1-999), then an optional smaller remainder. + +The remainder may itself be a smaller counted scale: +obukadde bubiri mu mitwalo ataano = 2 x 1,000,000 + 50 x 10,000. + +Example (Luganda): mitwalo abiri mu etaano mu nkumi ttaano = 25 x 10,000 + 5,000. +Unit words in the count agree with the scale noun; the remainder agrees with what +is counted. A remainder must be at least 10, so a single unit after the join always +belongs to the count (6-9 have the same form in every class). +""" +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.bantu.profile import ( + VOWELS, + CardinalProfile, + CountedScale, + NounContext, + agreement_context, +) +from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import ( + DIGIT, + NONZERO, + build_cardinal, + integer_domain, + mapping, +) + + +def build_count(profile: CardinalProfile, scale: CountedScale) -> tuple[pynini.Fst, pynini.Fst]: + """Digits for a plural count: 02-99, or 100-999 when the scale has count_hundreds. + + Returns (count, round_hundreds): round_hundreds is the part of count that is + hundreds alone (obukadde kikumi), which a remainder below 100 may not follow. + """ + context = agreement_context(profile, scale.count_class) + final_units = context.final_units + scale.count_ones + units = mapping(final_units) # 1-9 + plural_units = mapping(tuple((f, v) for f, v in context.atomic_units if v != "10")) + tens = mapping(scale.tens) + higher_tens = mapping(tuple((f, v) for f, v in scale.tens if v != "1")) + join = profile.places[0].remainder_join + teen_units = [] + for form, value in final_units: + if form[0] in VOWELS: + spoken = [" " + prefix + form for prefix in scale.contracted_teen_joins] + else: + spoken = [profile.teen_join + form] + teen_units.extend(pynini.cross(s, value) for s in spoken) + teens = pynini.cross(profile.ten, "1") + pynini.union(*teen_units) + two_digits = ( + pynutil.insert("0") + plural_units + | tens + pynutil.insert("0") + | teens + | higher_tens + pynutil.delete(join) + units + ).optimize() + if not scale.count_hundreds: + return two_digits, pynini.Fst() + # obukadde kikumi mu ataano = 150: hundreds, then an optional mu + 01-99. + below_hundred = two_digits | pynutil.insert("0") + units + hundreds = mapping(scale.count_hundreds) + count = two_digits | hundreds + pynutil.delete(join) + below_hundred + return count.optimize(), (hundreds + pynutil.insert("00")).optimize() + + +HEADS = ("bare", "counting", "counting_money", "all") + + +def build_scale_amount( + profile: CardinalProfile, scale: CountedScale, heads: str = "all", remainder_context: NounContext | None = None +) -> pynini.Fst: + """Digits for a counted-scale amount, bounded by the profile maximum. + + heads picks the spoken heads: "bare" (mitwalo, kakadde kamu), "counting" (with + the augment: emitwalo, akakadde kamu), "counting_money" (counting + money-only + ones such as akakadde), or "all". Empty if the scale has no such heads. + remainder_context sets the agreement after the join (default: remainder_class), + e.g. the counted noun in "ebitabo obukadde bubiri". + """ + if heads not in HEADS: + raise ValueError(f"unknown heads {heads!r}") + join = profile.places[0].remainder_join + count, round_hundreds = build_count(profile, scale) + ones, plurals = [], [] + if heads in ("bare", "all"): + ones.append(f"{scale.singular} {scale.singular_count}") + plurals.append(scale.plural) + if heads != "bare": + ones.extend(scale.counting_ones) + plurals.extend([scale.counting_plural] if scale.counting_plural else []) + if heads in ("counting_money", "all"): + ones.extend(scale.money_only_ones) + if not ones and not plurals: + return pynini.Fst() + plural = pynini.union(*(pynutil.delete(p + " ") for p in plurals)) if plurals else pynini.Fst() + head = plural + count + if ones: + head |= pynini.union(*(pynini.cross(one, "01") for one in ones)) + width = scale.power + + def padded(smallest_digits): + return pynini.union( + *( + pynutil.insert("0" * (width - k)) + NONZERO + pynini.closure(DIGIT, k - 1, k - 1) + for k in range(smallest_digits, width + 1) + ) + ) + + context = remainder_context or agreement_context(profile, scale.remainder_class) + remainder = build_cardinal(profile, context) + for smaller in profile.counted_scales: + if smaller.power < scale.power: + remainder |= build_scale_amount(profile, smaller, "all", context) + amount = head + (pynutil.insert("0" * width) | pynutil.delete(join) + (remainder @ padded(2))) + # After round hundreds, tens/units words continue the count (decision 2026-10-16): + # obukadde lwenda mu kyenda mu mwenda = 999 million, never 900 million + 99. + amount |= plural + round_hundreds + (pynutil.insert("0" * width) | pynutil.delete(join) + (remainder @ padded(3))) + canonical = pynutil.delete(pynini.closure("0")) + NONZERO + pynini.closure(DIGIT) + return (amount @ canonical @ integer_domain(profile.max_value)).optimize() + + +def build_noun_scale_amount(profile: CardinalProfile, context: NounContext) -> pynini.Fst: + """Scale amounts that may count a noun (ebitabo obukadde bubiri), agreeing with it.""" + amounts = [ + build_scale_amount(profile, scale, "counting", context) + for scale in profile.counted_scales + if scale.after_counted_nouns and context.compounds + ] + return pynini.union(*amounts).optimize() if amounts else pynini.Fst() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/sentence.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/sentence.py new file mode 100644 index 000000000..74db7ffea --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/sentence.py @@ -0,0 +1,196 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sentence-level tagging: numbers are resolved in context, over maximal runs of known number words. + +Segmentation is internal to the compiled classifier, not a Python runtime pass. +Weights only prefer a normalization over leaving words unchanged. All numeric +readings have equal weight, so structure alone must make the reading unique; +the kernel tests check that every test input has exactly one best reading. +""" +import string + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile, invariant_context +from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import build_cardinal +from nemo_text_processing.inverse_text_normalization.bantu.taggers.money import build_amount +from nemo_text_processing.inverse_text_normalization.bantu.taggers.scale import ( + build_noun_scale_amount, + build_scale_amount, +) + +# Input words: lowercase ASCII letters (an apostrophe is allowed inside a word, +# e.g. Luganda n'ebiri), one space between words. Cased input and punctuation are +# handled by each language's own classifier, not by the kernel. +WORD = pynini.closure(pynini.union(*string.ascii_lowercase, "'"), 1).optimize() +# Internal markers on implied-currency amounts; removed before output. +IMPLIED = "#" # allowed at the start of the input or right after a money cue +CUE_ONLY = "%" # allowed only right after a money cue + + +def word_token(graph: pynini.Fst) -> pynini.Fst: + return pynutil.insert('tokens { name: "') + graph + pynutil.insert('" }') + + +def cardinal_token(graph: pynini.Fst) -> pynini.Fst: + return pynutil.insert('tokens { cardinal { integer: "') + graph + pynutil.insert('" } }') + + +def join_aware_runs(number_words, join_words): + """Marks maximal runs of number words; a join word followed by an ordinary word ends the run. + + Joins such as Luganda mu also have nonnumeric uses ("in"). In + "ente bbiri mu nnimiro" the run is "bbiri"; "mu nnimiro" stays as words. + A run may still end on a join at the end of the input, where it is malformed + and falls back whole (a cut-off amount is not guessed). A join between two + number words still extends the run, so malformed internals fall back whole. + """ + numeric_word = pynini.union(*sorted(number_words)).optimize() + join_word = pynini.union(*sorted(join_words)).optimize() + other_word = pynini.difference(WORD, numeric_word).optimize() + plain_number = pynini.difference(numeric_word, join_word).optimize() + any_word = (other_word | join_word).optimize() + # Between or after runs: words and joins, containing at least one ordinary word. + others = (pynini.closure(any_word + " ") + other_word + pynini.closure(" " + any_word)).optimize() + numbers = numeric_word + pynini.closure(" " + numeric_word) + before_others = pynini.closure(numeric_word + " ") + plain_number + + def marked(graph): + return pynutil.insert("<") + graph + pynutil.insert(">") + + runs = pynini.closure(marked(before_others) + " " + others + " ") + ( + marked(before_others) + " " + others | marked(numbers) + ) + segmenter = (others | pynini.closure(others + " ", 0, 1) + runs).optimize() + return numeric_word, any_word, segmenter + + +def money_token(phrase: str, amount: pynini.Fst, code: str) -> pynini.Fst: + return ( + pynutil.delete(phrase + " <") + + pynutil.insert('tokens { money { integer_part: "') + + amount + + pynutil.insert(f'" currency: "{code}" }} }}') + + pynutil.delete(">") + ) + + +def implied_money_token(amount: pynini.Fst, code: str, marker: str) -> pynini.Fst: + """A bare amount whose currency is implied by policy (e.g. mitwalo -> UGX).""" + return ( + pynutil.delete("<") + + pynutil.insert(marker + 'tokens { money { integer_part: "') + + amount + + pynutil.insert(f'" currency: "{code}" }} }}') + + pynutil.delete(">") + ) + + +def money_cue_word(profile: CardinalProfile) -> pynini.Fst: + cues = profile.money_cues + letters = pynini.closure(pynini.union(*string.ascii_lowercase, "'")) + options = [letters + pynini.union(*cues.stems) + letters] if cues.stems else [] + options += [pynini.union(*cues.words)] if cues.words else [] + if not options: + return pynini.Fst() + cue = pynini.union(*options).optimize() + if cues.not_cues: + cue = pynini.difference(cue, pynini.union(*cues.not_cues).optimize()) + return cue.optimize() + + +def only_with_money_signal(profile: CardinalProfile) -> pynini.Fst: + """Keep implied money only where something signals a payment, then drop markers. + + "nsasula mitwalo ebiri" (I pay ...) is money; "embuzi mitwalo ebiri" (goats) + is left alone, because embuzi is not a money cue. + """ + sigma = pynini.union(*(pynini.escape(chr(code)) for code in range(32, 127))).optimize() + plain = pynini.difference(sigma, pynini.union(IMPLIED, CUE_ONLY)).optimize() + after_cue = 'tokens { name: "' + money_cue_word(profile) + '" } ' + pynini.union(IMPLIED, CUE_ONLY) + allowed = (pynini.closure(pynini.accep(IMPLIED), 0, 1) + pynini.closure(plain | after_cue)).optimize() + strip = pynini.closure(plain | pynutil.delete(IMPLIED) | pynutil.delete(CUE_ONLY)) + return (allowed @ strip.optimize()).optimize() + + +def build_tagger(profile: CardinalProfile) -> pynini.Fst: + numeric_word, other_word, segmenter = join_aware_runs(profile.number_words(), profile.join_words()) + fallback_run = ( + pynutil.delete("<") + + word_token(numeric_word) + + pynini.closure(" " + word_token(numeric_word)) + + pynutil.delete(">") + ) + recognized = [] + for context in profile.contexts: + recognized.append( + word_token(pynini.accep(context.noun)) + + " " + + pynutil.delete("<") + + cardinal_token(build_cardinal(profile, context) | build_noun_scale_amount(profile, context)) + + pynutil.delete(">") + ) + for currency in profile.currencies: + recognized.append(money_token(currency.phrase, build_amount(profile, currency), currency.code)) + implied = [] + for scale in profile.counted_scales: + if not scale.implied_currency: + continue + # Bare heads (mitwalo): money anywhere, or only with a signal. + implied.append( + implied_money_token( + build_scale_amount(profile, scale, "bare"), + scale.implied_currency, + "" if scale.implied_anywhere else IMPLIED, + ) + ) + # Augmented heads (emitwalo, akakadde): money only after a cue (or at the start). + if scale.counting_plural or scale.counting_ones or scale.money_only_ones: + implied.append( + implied_money_token( + build_scale_amount(profile, scale, "counting_money"), + scale.implied_currency, + IMPLIED if scale.counting_money_at_start else CUE_ONLY, + ) + ) + recognized.extend(implied) + # Any other word + an augmented head counts that word, whether or not it is listed + # (embuzi emitwalo ebiri = 20,000 goats). Money cues, currency words and listed + # nouns are excluded: after a cue or a currency it is money, and listed nouns + # already agree fully above. + listed = [context.noun for context in profile.contexts] + listed += [currency.phrase.split()[-1] for currency in profile.currencies] + if profile.money_cues.stems or profile.money_cues.words: + listed_or_cue = pynini.union(*listed, money_cue_word(profile)) + else: + listed_or_cue = pynini.union(*listed) + any_noun = pynini.difference(other_word, listed_or_cue.optimize()).optimize() + for scale in profile.counted_scales: + if scale.counts_any_word: + amount = build_scale_amount(profile, scale, "counting", invariant_context(profile)) + recognized.append( + word_token(any_noun) + " " + pynutil.delete("<") + cardinal_token(amount) + pynutil.delete(">") + ) + chunks = pynini.union( + *recognized, + pynutil.add_weight(word_token(other_word), 1), + pynutil.add_weight(fallback_run, 1), + ) + sentence = chunks + pynini.closure(" " + chunks) + tagger = segmenter @ sentence + if implied: + tagger @= only_with_money_signal(profile) + return tagger.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..c636d55cb --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py @@ -0,0 +1,56 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini + +from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile +from nemo_text_processing.inverse_text_normalization.bantu.taggers.sentence import build_tagger +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, generator_main + + +class ClassifyFst(GraphFst): + """ + Sentence-level classifier for a Bantu language, configured by a language profile. + + Unlike most languages, the whole sentence is read at once: number words are + resolved in their context (the counted noun, a currency phrase, a money verb, + or a join word followed by an ordinary word), and the value is fully resolved + here. The verbalizer only formats it. Output is the standard token format, e.g. + "ente bbiri" -> tokens { name: "ente" } tokens { cardinal { integer: "2" } } + + Args: + profile: the language profile (word lists, agreement, scales, currencies) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, profile: CardinalProfile, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="tokenize_and_classify", kind="classify") + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"{profile.language}_itn.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logging.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logging.info(f"Creating ClassifyFst grammars for {profile.language}.") + self.fst = build_tagger(profile) + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py new file mode 100644 index 000000000..374c2918a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py @@ -0,0 +1,44 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import DIGIT, integer_domain +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_space + + +def format_integer(max_value: int, thousands_separator: str) -> "pynini.FstLike": + """Groups digits in threes, e.g. 25000 -> 25,000. Rejects values outside 1..max_value.""" + first = pynini.closure(DIGIT, 1, 3) + triple = pynini.closure(DIGIT, 3, 3) + grouped = first + pynini.closure(pynutil.insert(thousands_separator) + triple) + return (integer_domain(max_value) @ grouped).optimize() + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing a resolved cardinal, e.g. + cardinal { integer: "25000" } -> 25,000 + + Args: + max_value: largest value the language profile allows + thousands_separator: written separator between digit groups + """ + + def __init__(self, max_value: int, thousands_separator: str): + super().__init__(name="cardinal", kind="verbalize") + number = format_integer(max_value, thousands_separator) + graph = pynutil.delete("integer:") + delete_space + pynutil.delete('"') + number + pynutil.delete('"') + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/money.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/money.py new file mode 100644 index 000000000..7e651d3b7 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/money.py @@ -0,0 +1,50 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.cardinal import format_integer +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_UPPER, GraphFst, delete_space, insert_space + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing a resolved amount as " ", e.g. + money { integer_part: "20000" currency: "UGX" } -> 20,000 UGX + + Args: + max_value: largest value the language profile allows + thousands_separator: written separator between digit groups + """ + + def __init__(self, max_value: int, thousands_separator: str): + super().__init__(name="money", kind="verbalize") + number = format_integer(max_value, thousands_separator) + code = pynini.closure(NEMO_UPPER, 3, 3) + graph = ( + pynutil.delete("integer_part:") + + delete_space + + pynutil.delete('"') + + number + + pynutil.delete('"') + + delete_space + + pynutil.delete("currency:") + + delete_space + + pynutil.delete('"') + + insert_space + + code + + pynutil.delete('"') + ) + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize.py new file mode 100644 index 000000000..e68c1674b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize.py @@ -0,0 +1,35 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile +from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.money import MoneyFst +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + + +class VerbalizeFst(GraphFst): + """ + Composes the verbalizer grammars. Values arrive fully resolved from the + classifier, so verbalizing is formatting only. + + Args: + profile: the language profile (sets the largest value) + thousands_separator: written separator between digit groups + """ + + def __init__(self, profile: CardinalProfile, thousands_separator: str = ","): + super().__init__(name="verbalize", kind="verbalize") + cardinal = CardinalFst(profile.max_value, thousands_separator).fst + money = MoneyFst(profile.max_value, thousands_separator).fst + self.fst = (cardinal | money).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize_final.py new file mode 100644 index 000000000..ec288affc --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize_final.py @@ -0,0 +1,46 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile +from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.word import WordFst +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence. + + Args: + profile: the language profile (sets the largest value) + thousands_separator: written separator between digit groups + """ + + def __init__(self, profile: CardinalProfile, thousands_separator: str = ","): + super().__init__(name="verbalize_final", kind="verbalize") + types = VerbalizeFst(profile, thousands_separator).fst | WordFst().fst + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/word.py new file mode 100644 index 000000000..34fef5049 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/word.py @@ -0,0 +1,30 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.bantu.taggers.sentence import WORD +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_space + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing a plain word, e.g. + tokens { name: "ente" } -> ente + """ + + def __init__(self): + super().__init__(name="word", kind="verbalize") + graph = pynutil.delete("name:") + delete_space + pynutil.delete('"') + WORD + pynutil.delete('"') + self.fst = graph.optimize() diff --git a/tests/nemo_text_processing/bantu/__init__.py b/tests/nemo_text_processing/bantu/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/tests/nemo_text_processing/bantu/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/bantu/test_cardinal.py b/tests/nemo_text_processing/bantu/test_cardinal.py new file mode 100644 index 000000000..2fd7e7442 --- /dev/null +++ b/tests/nemo_text_processing/bantu/test_cardinal.py @@ -0,0 +1,104 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Kernel cardinals with the made-up test profile: agreement, composition, malformed input.""" + +import pytest +from parameterized import parameterized + +from .toy_profile import TOY_PROFILE +from .utils import best_tags, build_normalizer + +NORMALIZER = build_normalizer(TOY_PROFILE) + + +def check(spoken, expected): + # Exactly one best reading: the kernel never leaves a tie for NeMo to pick from. + assert len(best_tags(NORMALIZER, spoken)) == 1 + assert NORMALIZER.inverse_normalize(spoken, verbose=False) == expected + + +class TestBantuKernelCardinal: + # Number words agree with the counted noun's class; a wrong class stays unchanged. + @parameterized.expand( + [ + ("bakopa bapi", "bakopa 2"), + ("zikopa zipi", "zikopa 2"), + ("makopa mapi", "makopa 2"), + ("kopa ima", "kopa 1"), + ("bakopa zipi", "bakopa zipi"), + ("bakopa ima", "bakopa ima"), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_agreement(self, spoken, expected): + check(spoken, expected) + + # 6-9 and ten have one form in every class. + @parameterized.expand( + [ + ("bakopa sei", "bakopa 6"), + ("zikopa nona", "zikopa 9"), + ("makopa deka", "makopa 10"), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_invariant_units(self, spoken, expected): + check(spoken, expected) + + # Teens join with ke; tens, hundreds and thousands join with vo; a trailing one agrees. + @parameterized.expand( + [ + ("zikopa deka ke zipi", "zikopa 12"), + ("bakopa pideka vo baima", "bakopa 21"), + ("zikopa hanto pi vo kodeka vo ziko", "zikopa 233"), + ("bakopa milo", "bakopa 1,000"), + ("bakopa milo vo hanto", "bakopa 1,100"), + ("zikopa milo nona vo hanto nona vo nonadeka vo nona", "zikopa 9,999"), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_composition(self, spoken, expected): + check(spoken, expected) + + # Profile extras: a fixed phrase, a special teen form and a spelling variant. + @parameterized.expand( + [ + ("bakopa tausi", "bakopa 1,000"), + ("makopa dekafu", "makopa 15"), + ("bakopa piideka", "bakopa 20"), + ("bakopa piideka vo baima", "bakopa 21"), + ("koin piideka", "20 XTS"), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_profile_extras(self, spoken, expected): + check(spoken, expected) + + # A malformed run of number words is left unchanged as a whole, never partly converted. + @parameterized.expand( + [ + ("bakopa hanto vo milo", "bakopa hanto vo milo"), + ("bakopa deka ke", "bakopa deka ke"), + ("bakopa bapi bako", "bakopa bapi bako"), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_malformed(self, spoken, expected): + check(spoken, expected) diff --git a/tests/nemo_text_processing/bantu/test_money.py b/tests/nemo_text_processing/bantu/test_money.py new file mode 100644 index 000000000..824945bc5 --- /dev/null +++ b/tests/nemo_text_processing/bantu/test_money.py @@ -0,0 +1,142 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Kernel money with the made-up test profile: currency context, counted scales, implied currency.""" + +import pytest +from parameterized import parameterized + +from .toy_profile import TOY_PROFILE +from .utils import best_tags, build_normalizer + +NORMALIZER = build_normalizer(TOY_PROFILE) + + +def check(spoken, expected): + assert len(best_tags(NORMALIZER, spoken)) == 1 + assert NORMALIZER.inverse_normalize(spoken, verbose=False) == expected + + +class TestBantuKernelMoney: + # A currency phrase sets the agreement and licenses extra readings: mapi is 20 only after koin. + @parameterized.expand( + [ + ("koin zipi", "2 XTS"), + ("koin mapi", "20 XTS"), + ("makopa mapi", "makopa 2"), + ("koin milo vo hanto", "1,100 XTS"), + ("koin", "koin"), + ("koin kasa", "koin kasa"), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_currency_context(self, spoken, expected): + check(spoken, expected) + + # Ten-thousands: mina + count (agreeing with mina's class) + optional vo remainder. + @parameterized.expand( + [ + ("muna ema", "10,000 XTS"), + ("mina epi", "20,000 XTS"), + ("mina pideka vo efu vo milo fu", "255,000 XTS"), + ("koin mina epi", "20,000 XTS"), + ("mina zipi", "mina zipi"), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_counted_scale(self, spoken, expected): + check(spoken, expected) + + # A teen join before a vowel contracts (k'epi or kepi); the full form is not accepted. + @parameterized.expand( + [ + ("mina deka k'epi", "120,000 XTS"), + ("mina deka kepi", "120,000 XTS"), + ("mina deka ke epi", "mina deka ke epi"), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_contracted_teens(self, spoken, expected): + check(spoken, expected) + + # Millions: one forms, teens with a "one", hundreds in the count, and a smaller + # scale as the remainder. Tens/units after round hundreds continue the count. + @parameterized.expand( + [ + ("kamega kaima", "1,000,000 XTS"), + ("bumega bapi", "2,000,000 XTS"), + ("bumega deka ke kaima", "11,000,000 XTS"), + ("bumega hanto vo fudeka", "150,000,000 XTS"), + ("bumega hanto nona vo nonadeka vo nona", "999,000,000 XTS"), + ("bumega hanto vo milo fu", "100,005,000 XTS"), + ("bumega bapi vo mina fudeka", "2,500,000 XTS"), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_larger_scale(self, spoken, expected): + check(spoken, expected) + + # Implied currency: bare mina is always money; word + emina counts that word, + # unless the word is a money cue; emina with nothing before it stays words. + @parameterized.expand( + [ + ("kasa mina epi", "kasa 20,000 XTS"), + ("kasa emina epi", "kasa 20,000"), + ("lipoka emina epi", "lipoka 20,000 XTS"), + ("dalo emina epi", "dalo 20,000 XTS"), + ("olipon emina epi", "olipon 20,000"), + ("emina epi", "emina epi"), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_implied_currency(self, spoken, expected): + check(spoken, expected) + + # Millions: augmented forms are also money at the start of input, count a listed + # or unlisted noun before them, and need "one" (kaima) to count a single million. + @parameterized.expand( + [ + ("obumega bapi", "2,000,000 XTS"), + ("akamega", "1,000,000 XTS"), + ("akamega kaima", "1,000,000 XTS"), + ("kasa bumega bapi", "kasa 2,000,000 XTS"), + ("kasa obumega bapi", "kasa 2,000,000"), + ("kasa akamega kaima", "kasa 1,000,000"), + ("kasa akamega", "kasa akamega"), + ("bakopa obumega bapi", "bakopa 2,000,000"), + ("bakopa akamega kaima", "bakopa 1,000,000"), + ("lipoka obumega bapi", "lipoka 2,000,000 XTS"), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_augmented_larger_scale(self, spoken, expected): + check(spoken, expected) + + # A held-back word continuing the number keeps the whole phrase unchanged. + @parameterized.expand( + [ + ("mina epi halfu", "mina epi halfu"), + ("lipoka mina epi halfu", "lipoka mina epi halfu"), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_held_back(self, spoken, expected): + check(spoken, expected) diff --git a/tests/nemo_text_processing/bantu/test_profile.py b/tests/nemo_text_processing/bantu/test_profile.py new file mode 100644 index 000000000..240402044 --- /dev/null +++ b/tests/nemo_text_processing/bantu/test_profile.py @@ -0,0 +1,91 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Kernel behaviour that depends on the profile itself: validation, optional features, caching.""" + +from dataclasses import replace + +import pytest + +from nemo_text_processing.inverse_text_normalization.bantu.profile import MoneyCues, Place +from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import build_role, integer_domain +from nemo_text_processing.inverse_text_normalization.bantu.taggers.scale import build_scale_amount +from nemo_text_processing.inverse_text_normalization.bantu.taggers.tokenize_and_classify import ClassifyFst + +from .toy_profile import TOY_PROFILE +from .utils import build_normalizer + + +class TestBantuKernelProfile: + # A broken profile is rejected when the grammar is built, with a clear error. + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_invalid_profiles_are_rejected(self): + with pytest.raises(ValueError, match="no compound agreement context"): + ClassifyFst(replace(TOY_PROFILE, currencies=(replace(TOY_PROFILE.currencies[0], agreement_class="99"),))) + with pytest.raises(ValueError, match="consecutive ascending"): + ClassifyFst(replace(TOY_PROFILE, places=TOY_PROFILE.places[1:])) + with pytest.raises(ValueError, match="single nonzero digits"): + ClassifyFst(replace(TOY_PROFILE, places=(Place(1, (("pideka", "20"),), " vo "),) + TOY_PROFILE.places[1:])) + with pytest.raises(ValueError, match="positive integer"): + integer_domain(0) + with pytest.raises(KeyError): + build_role(TOY_PROFILE, "no-such-role") + with pytest.raises(ValueError, match="unknown heads"): + build_scale_amount(TOY_PROFILE, TOY_PROFILE.counted_scales[0], "no-such-heads") + + # Optional features can be left out: no money cues, and scales with no implied currency. + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_profile_without_money_cues_or_implied_currency(self): + plain = replace( + TOY_PROFILE, + money_cues=MoneyCues(), + counted_scales=tuple(replace(s, implied_currency="") for s in TOY_PROFILE.counted_scales), + ) + normalizer = build_normalizer(plain) + assert normalizer.inverse_normalize("mina epi", verbose=False) == "mina epi" + assert normalizer.inverse_normalize("koin mina epi", verbose=False) == "20,000 XTS" + assert normalizer.inverse_normalize("kasa emina epi", verbose=False) == "kasa 20,000" + assert normalizer.inverse_normalize("lipoka emina epi", verbose=False) == "lipoka 20,000" + + # Implied currency without any money cues: only the start-of-input and bare-form rules apply. + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_implied_currency_without_money_cues(self): + normalizer = build_normalizer(replace(TOY_PROFILE, money_cues=MoneyCues())) + assert normalizer.inverse_normalize("obumega bapi", verbose=False) == "2,000,000 XTS" + assert normalizer.inverse_normalize("kasa mina epi", verbose=False) == "kasa 20,000 XTS" + assert normalizer.inverse_normalize("lipoka emina epi", verbose=False) == "lipoka 20,000" + + # A scale without counting heads simply has no counting reading. + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_scale_without_counting_heads(self): + bare_only = replace(TOY_PROFILE.counted_scales[0], counting_plural="", counting_ones=()) + assert build_scale_amount(TOY_PROFILE, bare_only, "counting").num_states() == 0 + + # The compiled classifier is saved to and restored from a FAR file with the same results. + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_far_cache_roundtrip(self, tmp_path): + built = ClassifyFst(TOY_PROFILE, cache_dir=str(tmp_path), overwrite_cache=True) + assert (tmp_path / "toy_itn.far").exists() + restored = ClassifyFst(TOY_PROFILE, cache_dir=str(tmp_path)) + normalizer = build_normalizer(TOY_PROFILE) + for spoken in ("bakopa pideka vo baima", "kasa emina epi", "lipoka obumega bapi", "bakopa bapi vo kasa"): + normalizer.tagger = built + expected = normalizer.inverse_normalize(spoken, verbose=False) + normalizer.tagger = restored + assert normalizer.inverse_normalize(spoken, verbose=False) == expected diff --git a/tests/nemo_text_processing/bantu/test_segmentation.py b/tests/nemo_text_processing/bantu/test_segmentation.py new file mode 100644 index 000000000..c4e1550c7 --- /dev/null +++ b/tests/nemo_text_processing/bantu/test_segmentation.py @@ -0,0 +1,69 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Kernel sentence segmentation with the made-up test profile: numbers inside sentences.""" + +import pytest +from parameterized import parameterized + +from .toy_profile import TOY_PROFILE +from .utils import best_tags, build_normalizer + +NORMALIZER = build_normalizer(TOY_PROFILE) + + +def check(spoken, expected): + assert len(best_tags(NORMALIZER, spoken)) == 1 + assert NORMALIZER.inverse_normalize(spoken, verbose=False) == expected + + +class TestBantuKernelSegmentation: + # The join word vo is also an ordinary word ("in"): followed by an ordinary word, + # it ends the number instead of joining a remainder. + @parameterized.expand( + [ + ("bakopa bapi vo kasa", "bakopa 2 vo kasa"), + ("zikopa pideka vo kasa", "zikopa 20 vo kasa"), + ("rana bakopa bako vo kasa lena", "rana bakopa 3 vo kasa lena"), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_join_word_as_ordinary_word(self, spoken, expected): + check(spoken, expected) + + # Several numbers and plain words in one sentence. + @parameterized.expand( + [ + ("rana bakopa bapi lena zikopa ziko", "rana bakopa 2 lena zikopa 3"), + ("bakopa bapi zikopa ziko", "bakopa 2 zikopa 3"), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_numbers_in_sentences(self, spoken, expected): + check(spoken, expected) + + # No counted noun, no currency: number words stay as words. + @parameterized.expand( + [ + ("bapi", "bapi"), + ("rana bapi lena", "rana bapi lena"), + ("kasa vo rana", "kasa vo rana"), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_unchanged(self, spoken, expected): + check(spoken, expected) diff --git a/tests/nemo_text_processing/bantu/test_verbalizer.py b/tests/nemo_text_processing/bantu/test_verbalizer.py new file mode 100644 index 000000000..1e1d0c4d4 --- /dev/null +++ b/tests/nemo_text_processing/bantu/test_verbalizer.py @@ -0,0 +1,55 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Kernel verbalizers on their own: formatting only, values are resolved by the classifier.""" + +import pytest +from parameterized import parameterized +from pynini.lib import rewrite + +from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.verbalize_final import VerbalizeFinalFst + +from .toy_profile import TOY_PROFILE + +VERBALIZER = VerbalizeFinalFst(TOY_PROFILE).fst + + +class TestBantuKernelVerbalizer: + @parameterized.expand( + [ + ('tokens { cardinal { integer: "2" } }', "2"), + ('tokens { cardinal { integer: "999999999" } }', "999,999,999"), + ('tokens { money { integer_part: "25000" currency: "XTS" } }', "25,000 XTS"), + ('tokens { name: "kasa" } tokens { cardinal { integer: "1000" } }', "kasa 1,000"), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_formatting(self, tokens, expected): + assert rewrite.top_rewrite(tokens, VERBALIZER) == expected + + # Values outside 1..max_value, non-canonical digits and lowercase codes are rejected. + @parameterized.expand( + [ + ('tokens { cardinal { integer: "0" } }',), + ('tokens { cardinal { integer: "02" } }',), + ('tokens { cardinal { integer: "1000000000" } }',), + ('tokens { money { integer_part: "20" currency: "xts" } }',), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_rejects_invalid(self, tokens): + with pytest.raises(rewrite.Error): + rewrite.top_rewrite(tokens, VERBALIZER) diff --git a/tests/nemo_text_processing/bantu/toy_profile.py b/tests/nemo_text_processing/bantu/toy_profile.py new file mode 100644 index 000000000..602c9b6ea --- /dev/null +++ b/tests/nemo_text_processing/bantu/toy_profile.py @@ -0,0 +1,136 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A small made-up language for testing the shared Bantu kernel on its own. + +Every word here is invented. The language is shaped like a Bantu language so that +each kernel feature is exercised (Luganda equivalents in brackets): +- noun classes with agreeing number words: bakopa bapi / zikopa zipi = 2 [abantu babiri / emiti ebiri] +- 6-9 and ten do not agree: sei, siti, oto, nona, deka [mukaaga ... kkumi] +- teens join with "ke", larger parts with "vo" [na, mu] +- "vo" is also an ordinary word, "in" [mu] +- a currency phrase that licenses a special reading: koin mapi = 20 XTS, but makopa mapi = 2 [abiri] +- a counted scale of ten-thousands: mina + count [mitwalo] +- implied currency: bare "mina" is always money; word + "emina" counts that word, + except after a money verb [mitwalo / emitwalo] +- a held-back word that continues a number but is not handled: halfu [n'ekitundu] +- a count class with vowel-initial forms, so ke + epi contracts to k'epi [n'ebiri] +- a larger scale of millions: bumega + count, with augmented forms that count a noun, + "one" forms, hundreds inside the count, and money at the start [obukadde / akakadde] +- a fixed phrase (tausi = 1,000), a special teen form (dekafu = 15) and a spelling + variant (piideka = pideka) [akakadde, kkumi nataano, abiiri] + +A failing test with this profile points at the kernel, not at real language data. +XTS is the ISO 4217 code reserved for testing. +""" +from nemo_text_processing.inverse_text_normalization.bantu.profile import ( + CardinalProfile, + CountedScale, + CurrencyContext, + MoneyCues, + NounContext, + Place, + with_spelling_variants, +) + +INVARIANT = (("sei", "6"), ("siti", "7"), ("oto", "8"), ("nona", "9")) +STEMS = (("pi", "2"), ("ko", "3"), ("de", "4"), ("fu", "5")) +TENS = ( + ("pideka", "2"), + ("kodeka", "3"), + ("dedeka", "4"), + ("fudeka", "5"), + ("seideka", "6"), + ("sitideka", "7"), + ("otodeka", "8"), + ("nonadeka", "9"), +) + + +def plural(class_id, noun, prefix, final_one=None, teen_variants=()): + units = tuple((prefix + stem, value) for stem, value in STEMS) + INVARIANT + final = units + (((final_one, "1"),) if final_one else ()) + return NounContext(class_id, noun, units + (("deka", "10"),), final, teen_variants=teen_variants) + + +def multiples(scale): + words = (("pi", "2"), ("ko", "3"), ("de", "4"), ("fu", "5")) + INVARIANT + return ((scale, "1"),) + tuple((f"{scale} {word}", value) for word, value in words) + + +TOY_PROFILE = with_spelling_variants( + CardinalProfile( + language="toy", + contexts=( + NounContext("1", "kopa", (("ima", "1"),), (), compounds=False), + plural("2", "bakopa", "ba", final_one="baima"), + plural("4", "zikopa", "zi", final_one="zima"), + plural("6", "makopa", "ma", teen_variants=(("dekafu", "15"),)), + plural("8", "ekopa", "e", final_one="ema"), + ), + ten="deka", + teen_join=" ke ", + places=( + Place(1, TENS, " vo "), + Place(2, multiples("hanto"), " vo "), + Place(3, multiples("milo"), " vo "), + ), + exact_boundaries=(("tausi", "1000"),), + roles=(("short-tens", (("mapi", "20"),)),), + max_value=999_999_999, + currencies=(CurrencyContext("koin", "XTS", agreement_class="4", roles=("short-tens",)),), + counted_scales=( + # Ten-thousands [mitwalo]: bare mina is money anywhere; emina counts a word. + CountedScale( + singular="muna", + singular_count="ema", + plural="mina", + power=4, + count_class="8", + tens=(("deka", "1"),) + TENS, + remainder_class="4", + contracted_teen_joins=("k'", "k"), + implied_currency="XTS", + counting_plural="emina", + implied_anywhere=True, + counts_any_word=True, + ), + # Millions [obukadde]: augmented forms are also money at the start of input, + # count listed nouns with agreement, and need "one" to count (akamega kaima). + CountedScale( + singular="kamega", + singular_count="kaima", + plural="bumega", + power=6, + count_class="2", + tens=(("deka", "1"),) + TENS, + remainder_class="4", + count_ones=(("kaima", "1"),), + count_hundreds=multiples("hanto"), + implied_currency="XTS", + counting_plural="obumega", + counting_ones=("akamega kaima",), + money_only_ones=("akamega",), + implied_anywhere=True, + counting_money_at_start=True, + counts_any_word=True, + after_counted_nouns=True, + ), + ), + money_cues=MoneyCues(stems=("lipo",), words=("dalo",), not_cues=("olipon",)), + held_back_words=("halfu",), + ), + {"pideka": ("piideka",)}, +) diff --git a/tests/nemo_text_processing/bantu/utils.py b/tests/nemo_text_processing/bantu/utils.py new file mode 100644 index 000000000..987c7220b --- /dev/null +++ b/tests/nemo_text_processing/bantu/utils.py @@ -0,0 +1,42 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib import rewrite + +from nemo_text_processing.inverse_text_normalization.bantu.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.verbalize_final import VerbalizeFinalFst +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.token_parser import TokenParser + + +def build_normalizer(profile) -> InverseNormalizer: + """ + NeMo's own inverse-normalization runtime with the kernel's tagger and verbalizer + built directly from a profile. The kernel is not a selectable language, so it is + not constructed through InverseNormalizer(lang=...); the attributes set here + mirror the end of InverseNormalizer.__init__. + """ + normalizer = InverseNormalizer.__new__(InverseNormalizer) + normalizer.lang = profile.language + normalizer.tagger = ClassifyFst(profile) + normalizer.verbalizer = VerbalizeFinalFst(profile) + normalizer.parser = TokenParser() + normalizer.max_number_of_permutations_per_split = 729 # InverseNormalizer default + normalizer.post_processor = None + return normalizer + + +def best_tags(normalizer: InverseNormalizer, text: str) -> set: + """All equally best tagger outputs. The kernel must give exactly one for any input.""" + return set(rewrite.optimal_rewrites(text, normalizer.tagger.fst)) From 2fe906ae4864c8485968e091c544d1e279d7a558 Mon Sep 17 00:00:00 2001 From: Cal3bdev Date: Sun, 25 Oct 2026 15:04:01 +0300 Subject: [PATCH 2/9] Bantu ITN kernel: align with NeMo conventions - Pass any other word (capitals, digits, punctuation, any script) through unchanged and collapse repeated spaces, instead of failing on such input. - Split the taggers into CardinalFst, MoneyFst and WordFst classes, with the place-value builders in number.py. - Move test cases into test_cases_*.txt files read by parse_test_case_file. - Add a Jenkinsfile stage for the kernel tests. - Use typing.Optional/Tuple, NeMo's logger and the {lang}_itn_lower_cased.far cache name; move the vowel list into the profile. - Fix the money-signal docstring and remove project-internal notes. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Cal3bdev --- Jenkinsfile | 5 + .../bantu/profile.py | 48 +++--- .../bantu/taggers/cardinal.py | 104 +++++------- .../bantu/taggers/money.py | 58 ++++++- .../bantu/taggers/number.py | 88 ++++++++++ .../bantu/taggers/scale.py | 13 +- .../bantu/taggers/sentence.py | 159 +++++------------- .../bantu/taggers/tokenize_and_classify.py | 41 +++-- .../bantu/taggers/word.py | 41 +++++ .../bantu/verbalizers/cardinal.py | 2 +- .../bantu/verbalizers/word.py | 12 +- .../test_cases_cardinal.txt | 32 ++++ .../test_cases_money.txt | 39 +++++ .../test_cases_word.txt | 9 + .../bantu/test_cardinal.py | 88 +--------- .../nemo_text_processing/bantu/test_money.py | 126 +------------- .../bantu/test_profile.py | 4 +- .../bantu/test_segmentation.py | 69 -------- tests/nemo_text_processing/bantu/test_word.py | 34 ++++ 19 files changed, 478 insertions(+), 494 deletions(-) create mode 100644 nemo_text_processing/inverse_text_normalization/bantu/taggers/number.py create mode 100644 nemo_text_processing/inverse_text_normalization/bantu/taggers/word.py create mode 100644 tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_money.txt create mode 100644 tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_word.txt delete mode 100644 tests/nemo_text_processing/bantu/test_segmentation.py create mode 100644 tests/nemo_text_processing/bantu/test_word.py diff --git a/Jenkinsfile b/Jenkinsfile index f5d114f84..30bfb3213 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -483,6 +483,11 @@ pipeline { sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/ko/ -m "not pleasefixme" --cpu --tn_cache_dir ${KO_TN_CACHE}' } } + stage('L1: Run Bantu kernel ITN tests') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/bantu/ -m "not pleasefixme" --cpu' + } + } } } diff --git a/nemo_text_processing/inverse_text_normalization/bantu/profile.py b/nemo_text_processing/inverse_text_normalization/bantu/profile.py index 6b6bc59e7..bf15161b4 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/profile.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/profile.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Small explicit resource contract, not a universal Bantu grammar schema.""" +"""The settings a language supplies to the shared Bantu kernel: its number words, agreement and scales.""" from dataclasses import dataclass, replace +from typing import Dict, FrozenSet, List, Tuple -Forms = tuple[tuple[str, str], ...] -VOWELS = "aeiou" +Forms = Tuple[Tuple[str, str], ...] @dataclass(frozen=True) @@ -43,7 +43,7 @@ class CurrencyContext: phrase: str code: str agreement_class: str - roles: tuple[str, ...] = () # extra readings licensed only after this phrase + roles: Tuple[str, ...] = () # extra readings licensed only after this phrase @dataclass(frozen=True) @@ -57,15 +57,15 @@ class CountedScale: count_class: str # agreement class of unit words inside the count tens: Forms # tens words inside a count, coefficient 1-9 remainder_class: str # agreement class of the amount after the scale - contracted_teen_joins: tuple[str, ...] = () # teen join before a vowel, e.g. n' + contracted_teen_joins: Tuple[str, ...] = () # teen join before a vowel, e.g. n' implied_currency: str = "" # payment policy: a bare amount means this currency count_ones: Forms = () # a "one" ending a count, e.g. kkumi na kamu (11) count_hundreds: Forms = () # hundreds inside a count, e.g. obukadde kikumi (100) # Heads with the augment (Luganda e-/a-/o-) count the noun before them; the bare - # heads above are money. Decided with the native speaker 2026-10-15/18. + # heads above are money. counting_plural: str = "" # e.g. emitwalo, obukadde - counting_ones: tuple[str, ...] = () # e.g. akakadde kamu - money_only_ones: tuple[str, ...] = () # one, as money only: akakadde (not ebitabo akakadde) + counting_ones: Tuple[str, ...] = () # e.g. akakadde kamu + money_only_ones: Tuple[str, ...] = () # one, as money only: akakadde (not ebitabo akakadde) implied_anywhere: bool = False # bare heads are money wherever they appear counting_money_at_start: bool = False # augmented heads are money at the start of input counts_any_word: bool = False # word + augmented head counts that word, listed or not @@ -76,35 +76,37 @@ class CountedScale: class MoneyCues: """Words that signal a payment just before an amount (e.g. Luganda money verbs).""" - stems: tuple[str, ...] = () # match anywhere inside a word - words: tuple[str, ...] = () # match only the whole word - not_cues: tuple[str, ...] = () # whole words containing a stem that are not cues + stems: Tuple[str, ...] = () # match anywhere inside a word + words: Tuple[str, ...] = () # match only the whole word + not_cues: Tuple[str, ...] = () # whole words containing a stem that are not cues @dataclass(frozen=True) class CardinalProfile: language: str - contexts: tuple[NounContext, ...] + contexts: Tuple[NounContext, ...] ten: str teen_join: str - places: tuple[Place, ...] + places: Tuple[Place, ...] exact_boundaries: Forms - roles: tuple[tuple[str, Forms], ...] + roles: Tuple[Tuple[str, Forms], ...] max_value: int - currencies: tuple[CurrencyContext, ...] = () - counted_scales: tuple[CountedScale, ...] = () + currencies: Tuple[CurrencyContext, ...] = () + counted_scales: Tuple[CountedScale, ...] = () money_cues: MoneyCues = MoneyCues() # Words that continue a number but are not handled yet (e.g. "and a half"). # They join the number's run, so the whole phrase stays unchanged. - held_back_words: tuple[str, ...] = () + held_back_words: Tuple[str, ...] = () + # Vowels, for contracting a join before a vowel-initial word (Luganda na + ebiri -> n'ebiri). + vowels: str = "aeiou" - def join_words(self) -> frozenset[str]: + def join_words(self) -> FrozenSet[str]: """Words that join number parts; they may also be ordinary words (e.g. locative mu).""" joins = [self.teen_join] + [place.remainder_join for place in self.places] return frozenset(word for join in joins for word in join.split()) - def number_words(self) -> frozenset[str]: - """Lexical run boundaries, including joins and role-only candidates.""" + def number_words(self) -> FrozenSet[str]: + """Every word that can be part of a number, including joins and role-only words.""" surfaces = [self.ten, self.teen_join] for place in self.places: surfaces.append(place.remainder_join) @@ -126,13 +128,13 @@ def number_words(self) -> frozenset[str]: return frozenset(word for form in surfaces for word in form.split()) -def scale_teen_words(profile: CardinalProfile, scale: CountedScale) -> list[str]: +def scale_teen_words(profile: CardinalProfile, scale: CountedScale) -> List[str]: """Contracted teen words such as n'ebiri / nebiri (one word each).""" context = agreement_context(profile, scale.count_class) return [ prefix + form for form, _ in context.final_units + scale.count_ones - if form[0] in VOWELS + if form[0] in profile.vowels for prefix in scale.contracted_teen_joins ] @@ -158,7 +160,7 @@ def invariant_context(profile: CardinalProfile) -> NounContext: return NounContext("invariant", "", units + ((profile.ten, "10"),), units) -def with_spelling_variants(profile: CardinalProfile, variants: dict[str, tuple[str, ...]]) -> CardinalProfile: +def with_spelling_variants(profile: CardinalProfile, variants: Dict[str, Tuple[str, ...]]) -> CardinalProfile: """Accept extra spellings of whole words (e.g. abiiri for abiri) in every form.""" def spell(forms: Forms) -> Forms: diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py index 2e409eebb..c4b109d06 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py @@ -12,74 +12,56 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Bounded decimal place-value FSTs, built from a profile. No runtime arithmetic.""" import pynini -from pynini.lib import pynutil -from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile, Forms, NounContext +from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile, invariant_context +from nemo_text_processing.inverse_text_normalization.bantu.taggers.number import build_cardinal +from nemo_text_processing.inverse_text_normalization.bantu.taggers.scale import ( + build_noun_scale_amount, + build_scale_amount, +) +from nemo_text_processing.inverse_text_normalization.bantu.taggers.sentence import ( + cardinal_token, + money_cue_word, + run, + word_token, +) +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst -DIGIT = pynini.union(*"0123456789").optimize() -NONZERO = pynini.union(*"123456789").optimize() +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying a counted number: a noun followed by a + run of number words that agree with it. The noun stays a word; the number is + fully resolved here, e.g. + bakopa pideka vo baima -> tokens { name: "bakopa" } tokens { cardinal { integer: "21" } } -def mapping(forms: Forms) -> pynini.Fst: - return pynini.string_map(forms).optimize() if forms else pynini.Fst() + Any other word followed by a counting scale (Luganda embuzi emitwalo ebiri, + 20,000 goats) is also a counted number. Here the count may only end in forms + that are the same in every class, so no agreement with an unlisted noun is guessed. + Args: + profile: the language profile + ordinary_word: any word that is not a number word + """ -def integer_domain(max_value: int) -> pynini.Fst: - """Canonical positive decimal strings <= bound, without integer enumeration.""" - if isinstance(max_value, bool) or not isinstance(max_value, int) or max_value < 1: - raise ValueError("max_value must be a positive integer") - bound = str(max_value) - branches = [pynini.accep(bound)] - if len(bound) > 1: - branches.append(NONZERO + pynini.closure(DIGIT, 0, len(bound) - 2)) - for index, limit in enumerate(bound): - minimum = 1 if index == 0 else 0 - if int(limit) > minimum: - lower = pynini.union(*(str(n) for n in range(minimum, int(limit)))) - branches.append( - pynini.accep(bound[:index]) - + lower - + pynini.closure(DIGIT, len(bound) - index - 1, len(bound) - index - 1) - ) - return pynini.union(*branches).optimize() + def __init__(self, profile: CardinalProfile, ordinary_word: pynini.Fst): + super().__init__(name="cardinal", kind="classify") + graphs = [] + for context in profile.contexts: + number = build_cardinal(profile, context) | build_noun_scale_amount(profile, context) + graphs.append(word_token(pynini.accep(context.noun)) + " " + run(cardinal_token(number))) -def build_cardinal(profile: CardinalProfile, context: NounContext) -> pynini.Fst: - """Compose fixed-width places, supply skipped zeros, then canonicalize once.""" - atomic = mapping(context.atomic_units) - if not context.compounds: - return (atomic @ integer_domain(profile.max_value)).optimize() - units = mapping(context.final_units) - lower = units - expected_power = 1 - for place in profile.places: - if place.power != expected_power: - raise ValueError("places must be consecutive ascending decimal powers") - if any(value not in "123456789" or len(value) != 1 for _, value in place.forms): - raise ValueError("place coefficients must be single nonzero digits") - # A present high place either consumes a smaller remainder or emits zeros. - high = mapping(place.forms) + ( - pynutil.delete(place.remainder_join) + lower | pynutil.insert("0" * place.power) - ) - if place.power == 1: - high |= pynini.cross(profile.ten, "10") - high |= pynini.cross(profile.ten, "1") + pynutil.delete(profile.teen_join) + units - high |= mapping(context.teen_variants) - lower = (pynutil.insert("0") + lower | high).optimize() - expected_power += 1 - canonical = pynutil.delete(pynini.closure("0")) + NONZERO + pynini.closure(DIGIT) - compounds = lower @ canonical - # A final-one form need not be licensed as a standalone plural count. - compounds @= NONZERO + pynini.closure(DIGIT, 1) - result = atomic | compounds | mapping(profile.exact_boundaries) - return (result @ integer_domain(profile.max_value)).optimize() + # Listed nouns already agree fully above. After a money cue or a currency word + # the amount is money, handled by MoneyFst. + excluded = [context.noun for context in profile.contexts] + excluded += [currency.phrase.split()[-1] for currency in profile.currencies] + excluded = pynini.union(*excluded, money_cue_word(profile)).optimize() + any_noun = pynini.difference(ordinary_word, excluded).optimize() + for scale in profile.counted_scales: + if scale.counts_any_word: + amount = build_scale_amount(profile, scale, "counting", invariant_context(profile)) + graphs.append(word_token(any_noun) + " " + run(cardinal_token(amount))) - -def build_role(profile: CardinalProfile, role: str) -> pynini.Fst: - """Explicit role probe; not a standalone sentence or currency grammar.""" - for name, forms in profile.roles: - if name == role: - return mapping(forms) - raise KeyError(role) + self.fst = pynini.union(*graphs).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/money.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/money.py index 11bdbd947..759c17be7 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/money.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/money.py @@ -12,27 +12,79 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Currency-context amounts: reuse cardinal composition, then add licensed roles.""" import pynini +from pynini.lib import pynutil from nemo_text_processing.inverse_text_normalization.bantu.profile import ( CardinalProfile, CurrencyContext, agreement_context, ) -from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import ( +from nemo_text_processing.inverse_text_normalization.bantu.taggers.number import ( build_cardinal, build_role, integer_domain, ) from nemo_text_processing.inverse_text_normalization.bantu.taggers.scale import build_scale_amount +from nemo_text_processing.inverse_text_normalization.bantu.taggers.sentence import CUE_ONLY, IMPLIED, run +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst def build_amount(profile: CardinalProfile, currency: CurrencyContext) -> pynini.Fst: - """Digits for an amount after this currency phrase; no Python arithmetic.""" + """Digits for an amount after this currency phrase: agreeing numbers, licensed roles and counted scales.""" amount = build_cardinal(profile, agreement_context(profile, currency.agreement_class)) for role in currency.roles: amount |= build_role(profile, role) @ integer_domain(profile.max_value) for scale in profile.counted_scales: amount |= build_scale_amount(profile, scale) return amount.optimize() + + +def money_token(amount: pynini.Fst, code: str, marker: str = "") -> pynini.Fst: + return run( + pynutil.insert(marker + 'tokens { money { integer_part: "') + + amount + + pynutil.insert(f'" currency: "{code}" }} }}') + ) + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money, e.g. + koin mina epi -> tokens { money { integer_part: "20000" currency: "XTS" } } + + An amount after a spoken currency phrase is always money. A counted scale may + also imply a currency without one (Luganda mitwalo ebiri, 20,000 UGX): + - bare scale heads (mitwalo) are money anywhere, or only after a money cue + when the scale's implied_anywhere is not set; + - heads with the augment (emitwalo, akakadde) are money only after a money cue, + or also at the start of the input when counting_money_at_start is set. + Implied amounts carry a marker, checked and removed by only_with_money_signal + once the whole sentence is tagged. + + Args: + profile: the language profile + """ + + def __init__(self, profile: CardinalProfile): + super().__init__(name="money", kind="classify") + + graphs = [] + for currency in profile.currencies: + graphs.append( + pynutil.delete(currency.phrase + " ") + money_token(build_amount(profile, currency), currency.code) + ) + + self.has_implied_currency = False + for scale in profile.counted_scales: + if not scale.implied_currency: + continue + self.has_implied_currency = True + bare = build_scale_amount(profile, scale, "bare") + graphs.append(money_token(bare, scale.implied_currency, "" if scale.implied_anywhere else IMPLIED)) + if scale.counting_plural or scale.counting_ones or scale.money_only_ones: + augmented = build_scale_amount(profile, scale, "counting_money") + marker = IMPLIED if scale.counting_money_at_start else CUE_ONLY + graphs.append(money_token(augmented, scale.implied_currency, marker)) + + self.fst = pynini.union(*graphs).optimize() if graphs else pynini.Fst() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/number.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/number.py new file mode 100644 index 000000000..6606cc524 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/number.py @@ -0,0 +1,88 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Place-value number FSTs built from a profile, shared by the cardinal, money and scale taggers. + +Values are composed digit by digit inside the FST; there is no runtime arithmetic. +""" +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile, Forms, NounContext + +DIGIT = pynini.union(*"0123456789").optimize() +NONZERO = pynini.union(*"123456789").optimize() + + +def mapping(forms: Forms) -> pynini.Fst: + return pynini.string_map(forms).optimize() if forms else pynini.Fst() + + +def integer_domain(max_value: int) -> pynini.Fst: + """Canonical positive decimal strings <= bound, without integer enumeration.""" + if isinstance(max_value, bool) or not isinstance(max_value, int) or max_value < 1: + raise ValueError("max_value must be a positive integer") + bound = str(max_value) + branches = [pynini.accep(bound)] + if len(bound) > 1: + branches.append(NONZERO + pynini.closure(DIGIT, 0, len(bound) - 2)) + for index, limit in enumerate(bound): + minimum = 1 if index == 0 else 0 + if int(limit) > minimum: + lower = pynini.union(*(str(n) for n in range(minimum, int(limit)))) + branches.append( + pynini.accep(bound[:index]) + + lower + + pynini.closure(DIGIT, len(bound) - index - 1, len(bound) - index - 1) + ) + return pynini.union(*branches).optimize() + + +def build_cardinal(profile: CardinalProfile, context: NounContext) -> pynini.Fst: + """Compose fixed-width places, supply skipped zeros, then canonicalize once.""" + atomic = mapping(context.atomic_units) + if not context.compounds: + return (atomic @ integer_domain(profile.max_value)).optimize() + units = mapping(context.final_units) + lower = units + expected_power = 1 + for place in profile.places: + if place.power != expected_power: + raise ValueError("places must be consecutive ascending decimal powers") + if any(value not in "123456789" or len(value) != 1 for _, value in place.forms): + raise ValueError("place coefficients must be single nonzero digits") + # A present high place either consumes a smaller remainder or emits zeros. + high = mapping(place.forms) + ( + pynutil.delete(place.remainder_join) + lower | pynutil.insert("0" * place.power) + ) + if place.power == 1: + high |= pynini.cross(profile.ten, "10") + high |= pynini.cross(profile.ten, "1") + pynutil.delete(profile.teen_join) + units + high |= mapping(context.teen_variants) + lower = (pynutil.insert("0") + lower | high).optimize() + expected_power += 1 + canonical = pynutil.delete(pynini.closure("0")) + NONZERO + pynini.closure(DIGIT) + compounds = lower @ canonical + # A final-one form need not be licensed as a standalone plural count. + compounds @= NONZERO + pynini.closure(DIGIT, 1) + result = atomic | compounds | mapping(profile.exact_boundaries) + return (result @ integer_domain(profile.max_value)).optimize() + + +def build_role(profile: CardinalProfile, role: str) -> pynini.Fst: + """Explicit role probe; not a standalone sentence or currency grammar.""" + for name, forms in profile.roles: + if name == role: + return mapping(forms) + raise KeyError(role) diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/scale.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/scale.py index 13d19d892..2fb64b580 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/scale.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/scale.py @@ -22,17 +22,18 @@ is counted. A remainder must be at least 10, so a single unit after the join always belongs to the count (6-9 have the same form in every class). """ +from typing import Optional, Tuple + import pynini from pynini.lib import pynutil from nemo_text_processing.inverse_text_normalization.bantu.profile import ( - VOWELS, CardinalProfile, CountedScale, NounContext, agreement_context, ) -from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import ( +from nemo_text_processing.inverse_text_normalization.bantu.taggers.number import ( DIGIT, NONZERO, build_cardinal, @@ -41,7 +42,7 @@ ) -def build_count(profile: CardinalProfile, scale: CountedScale) -> tuple[pynini.Fst, pynini.Fst]: +def build_count(profile: CardinalProfile, scale: CountedScale) -> Tuple[pynini.Fst, pynini.Fst]: """Digits for a plural count: 02-99, or 100-999 when the scale has count_hundreds. Returns (count, round_hundreds): round_hundreds is the part of count that is @@ -56,7 +57,7 @@ def build_count(profile: CardinalProfile, scale: CountedScale) -> tuple[pynini.F join = profile.places[0].remainder_join teen_units = [] for form, value in final_units: - if form[0] in VOWELS: + if form[0] in profile.vowels: spoken = [" " + prefix + form for prefix in scale.contracted_teen_joins] else: spoken = [profile.teen_join + form] @@ -81,7 +82,7 @@ def build_count(profile: CardinalProfile, scale: CountedScale) -> tuple[pynini.F def build_scale_amount( - profile: CardinalProfile, scale: CountedScale, heads: str = "all", remainder_context: NounContext | None = None + profile: CardinalProfile, scale: CountedScale, heads: str = "all", remainder_context: Optional[NounContext] = None ) -> pynini.Fst: """Digits for a counted-scale amount, bounded by the profile maximum. @@ -126,7 +127,7 @@ def padded(smallest_digits): if smaller.power < scale.power: remainder |= build_scale_amount(profile, smaller, "all", context) amount = head + (pynutil.insert("0" * width) | pynutil.delete(join) + (remainder @ padded(2))) - # After round hundreds, tens/units words continue the count (decision 2026-10-16): + # After round hundreds, tens/units words continue the count: # obukadde lwenda mu kyenda mu mwenda = 999 million, never 900 million + 99. amount |= plural + round_hundreds + (pynutil.insert("0" * width) | pynutil.delete(join) + (remainder @ padded(3))) canonical = pynutil.delete(pynini.closure("0")) + NONZERO + pynini.closure(DIGIT) diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/sentence.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/sentence.py index 74db7ffea..00452d61c 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/sentence.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/sentence.py @@ -12,33 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Sentence-level tagging: numbers are resolved in context, over maximal runs of known number words. - -Segmentation is internal to the compiled classifier, not a Python runtime pass. -Weights only prefer a normalization over leaving words unchanged. All numeric -readings have equal weight, so structure alone must make the reading unique; -the kernel tests check that every test input has exactly one best reading. +"""Sentence-level helpers shared by the taggers: number-word runs, token formats and the money-signal filter. + +The classifier reads the whole sentence at once. Maximal runs of number words are +marked first, so each run is read as a single number in the context of the word +before it (a counted noun, a currency phrase or a money cue). Weights only prefer a +normalization over leaving words unchanged. All numeric readings have equal weight, +so structure alone must make the reading unique; the kernel tests check that every +test input has exactly one best reading. """ -import string - import pynini from pynini.lib import pynutil -from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile, invariant_context -from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import build_cardinal -from nemo_text_processing.inverse_text_normalization.bantu.taggers.money import build_amount -from nemo_text_processing.inverse_text_normalization.bantu.taggers.scale import ( - build_noun_scale_amount, - build_scale_amount, -) +from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_NOT_SPACE, NEMO_WHITE_SPACE -# Input words: lowercase ASCII letters (an apostrophe is allowed inside a word, -# e.g. Luganda n'ebiri), one space between words. Cased input and punctuation are -# handled by each language's own classifier, not by the kernel. -WORD = pynini.closure(pynini.union(*string.ascii_lowercase, "'"), 1).optimize() -# Internal markers on implied-currency amounts; removed before output. -IMPLIED = "#" # allowed at the start of the input or right after a money cue -CUE_ONLY = "%" # allowed only right after a money cue +# Internal markers, removed before output. Control characters, so they never clash with input text. +RUN_START = "\x01" # start of a run of number words +RUN_END = "\x02" # end of a run of number words +IMPLIED = "\x03" # implied-currency amount: allowed at the start of the input or right after a money cue +CUE_ONLY = "\x04" # implied-currency amount: allowed only right after a money cue +MARKERS = pynini.union(RUN_START, RUN_END, IMPLIED, CUE_ONLY).optimize() + +# Any input word: every character except whitespace. An apostrophe may be part of a +# number word (Luganda n'ebiri). Words with capitals, digits or punctuation are +# ordinary words and pass through unchanged. +WORD_CHAR = pynini.difference(NEMO_NOT_SPACE, MARKERS).optimize() +WORD = pynini.closure(WORD_CHAR, 1).optimize() +# Runs of whitespace between words become a single space. +SINGLE_SPACES = pynini.closure(WORD_CHAR | pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ")).optimize() def word_token(graph: pynini.Fst) -> pynini.Fst: @@ -49,6 +51,11 @@ def cardinal_token(graph: pynini.Fst) -> pynini.Fst: return pynutil.insert('tokens { cardinal { integer: "') + graph + pynutil.insert('" } }') +def run(graph: pynini.Fst) -> pynini.Fst: + """Reads a whole marked run of number words with graph.""" + return pynutil.delete(RUN_START) + graph + pynutil.delete(RUN_END) + + def join_aware_runs(number_words, join_words): """Marks maximal runs of number words; a join word followed by an ordinary word ends the run. @@ -57,6 +64,8 @@ def join_aware_runs(number_words, join_words): A run may still end on a join at the end of the input, where it is malformed and falls back whole (a cut-off amount is not guessed). A join between two number words still extends the run, so malformed internals fall back whole. + + Returns (number word, ordinary word or join, segmenter). """ numeric_word = pynini.union(*sorted(number_words)).optimize() join_word = pynini.union(*sorted(join_words)).optimize() @@ -69,7 +78,7 @@ def join_aware_runs(number_words, join_words): before_others = pynini.closure(numeric_word + " ") + plain_number def marked(graph): - return pynutil.insert("<") + graph + pynutil.insert(">") + return pynutil.insert(RUN_START) + graph + pynutil.insert(RUN_END) runs = pynini.closure(marked(before_others) + " " + others + " ") + ( marked(before_others) + " " + others | marked(numbers) @@ -78,30 +87,10 @@ def marked(graph): return numeric_word, any_word, segmenter -def money_token(phrase: str, amount: pynini.Fst, code: str) -> pynini.Fst: - return ( - pynutil.delete(phrase + " <") - + pynutil.insert('tokens { money { integer_part: "') - + amount - + pynutil.insert(f'" currency: "{code}" }} }}') - + pynutil.delete(">") - ) - - -def implied_money_token(amount: pynini.Fst, code: str, marker: str) -> pynini.Fst: - """A bare amount whose currency is implied by policy (e.g. mitwalo -> UGX).""" - return ( - pynutil.delete("<") - + pynutil.insert(marker + 'tokens { money { integer_part: "') - + amount - + pynutil.insert(f'" currency: "{code}" }} }}') - + pynutil.delete(">") - ) - - def money_cue_word(profile: CardinalProfile) -> pynini.Fst: + """Words that signal a payment (e.g. Luganda money verbs); empty if the profile has none.""" cues = profile.money_cues - letters = pynini.closure(pynini.union(*string.ascii_lowercase, "'")) + letters = pynini.closure(WORD_CHAR) options = [letters + pynini.union(*cues.stems) + letters] if cues.stems else [] options += [pynini.union(*cues.words)] if cues.words else [] if not options: @@ -113,84 +102,14 @@ def money_cue_word(profile: CardinalProfile) -> pynini.Fst: def only_with_money_signal(profile: CardinalProfile) -> pynini.Fst: - """Keep implied money only where something signals a payment, then drop markers. + """Keeps marked implied-currency amounts only where they are allowed, then drops the markers. - "nsasula mitwalo ebiri" (I pay ...) is money; "embuzi mitwalo ebiri" (goats) - is left alone, because embuzi is not a money cue. + An amount marked IMPLIED is money at the start of the input or right after a money + cue; one marked CUE_ONLY only right after a money cue. Otherwise the tagging is + rejected and another reading (the words unchanged, or a counted noun) is used. """ - sigma = pynini.union(*(pynini.escape(chr(code)) for code in range(32, 127))).optimize() - plain = pynini.difference(sigma, pynini.union(IMPLIED, CUE_ONLY)).optimize() + plain = pynini.difference(NEMO_CHAR, pynini.union(IMPLIED, CUE_ONLY)).optimize() after_cue = 'tokens { name: "' + money_cue_word(profile) + '" } ' + pynini.union(IMPLIED, CUE_ONLY) allowed = (pynini.closure(pynini.accep(IMPLIED), 0, 1) + pynini.closure(plain | after_cue)).optimize() strip = pynini.closure(plain | pynutil.delete(IMPLIED) | pynutil.delete(CUE_ONLY)) return (allowed @ strip.optimize()).optimize() - - -def build_tagger(profile: CardinalProfile) -> pynini.Fst: - numeric_word, other_word, segmenter = join_aware_runs(profile.number_words(), profile.join_words()) - fallback_run = ( - pynutil.delete("<") - + word_token(numeric_word) - + pynini.closure(" " + word_token(numeric_word)) - + pynutil.delete(">") - ) - recognized = [] - for context in profile.contexts: - recognized.append( - word_token(pynini.accep(context.noun)) - + " " - + pynutil.delete("<") - + cardinal_token(build_cardinal(profile, context) | build_noun_scale_amount(profile, context)) - + pynutil.delete(">") - ) - for currency in profile.currencies: - recognized.append(money_token(currency.phrase, build_amount(profile, currency), currency.code)) - implied = [] - for scale in profile.counted_scales: - if not scale.implied_currency: - continue - # Bare heads (mitwalo): money anywhere, or only with a signal. - implied.append( - implied_money_token( - build_scale_amount(profile, scale, "bare"), - scale.implied_currency, - "" if scale.implied_anywhere else IMPLIED, - ) - ) - # Augmented heads (emitwalo, akakadde): money only after a cue (or at the start). - if scale.counting_plural or scale.counting_ones or scale.money_only_ones: - implied.append( - implied_money_token( - build_scale_amount(profile, scale, "counting_money"), - scale.implied_currency, - IMPLIED if scale.counting_money_at_start else CUE_ONLY, - ) - ) - recognized.extend(implied) - # Any other word + an augmented head counts that word, whether or not it is listed - # (embuzi emitwalo ebiri = 20,000 goats). Money cues, currency words and listed - # nouns are excluded: after a cue or a currency it is money, and listed nouns - # already agree fully above. - listed = [context.noun for context in profile.contexts] - listed += [currency.phrase.split()[-1] for currency in profile.currencies] - if profile.money_cues.stems or profile.money_cues.words: - listed_or_cue = pynini.union(*listed, money_cue_word(profile)) - else: - listed_or_cue = pynini.union(*listed) - any_noun = pynini.difference(other_word, listed_or_cue.optimize()).optimize() - for scale in profile.counted_scales: - if scale.counts_any_word: - amount = build_scale_amount(profile, scale, "counting", invariant_context(profile)) - recognized.append( - word_token(any_noun) + " " + pynutil.delete("<") + cardinal_token(amount) + pynutil.delete(">") - ) - chunks = pynini.union( - *recognized, - pynutil.add_weight(word_token(other_word), 1), - pynutil.add_weight(fallback_run, 1), - ) - sentence = chunks + pynini.closure(" " + chunks) - tagger = segmenter @ sentence - if implied: - tagger @= only_with_money_signal(profile) - return tagger.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py index c636d55cb..f0f67606a 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py @@ -12,14 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile -from nemo_text_processing.inverse_text_normalization.bantu.taggers.sentence import build_tagger -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, generator_main +from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.bantu.taggers.money import MoneyFst +from nemo_text_processing.inverse_text_normalization.bantu.taggers.sentence import ( + SINGLE_SPACES, + join_aware_runs, + only_with_money_signal, +) +from nemo_text_processing.inverse_text_normalization.bantu.taggers.word import WordFst +from nemo_text_processing.text_normalization.en.graph_utils import INPUT_LOWER_CASED, GraphFst, generator_main +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -27,10 +34,13 @@ class ClassifyFst(GraphFst): Sentence-level classifier for a Bantu language, configured by a language profile. Unlike most languages, the whole sentence is read at once: number words are - resolved in their context (the counted noun, a currency phrase, a money verb, + resolved in their context (the counted noun, a currency phrase, a money cue, or a join word followed by an ordinary word), and the value is fully resolved here. The verbalizer only formats it. Output is the standard token format, e.g. - "ente bbiri" -> tokens { name: "ente" } tokens { cardinal { integer: "2" } } + bakopa bapi -> tokens { name: "bakopa" } tokens { cardinal { integer: "2" } } + + The kernel reads lower-cased input. Other words, including words with capitals, + digits or punctuation, pass through unchanged. Args: profile: the language profile (word lists, agreement, scales, currencies) @@ -44,13 +54,24 @@ def __init__(self, profile: CardinalProfile, cache_dir: str = None, overwrite_ca far_file = None if cache_dir is not None and cache_dir != "None": os.makedirs(cache_dir, exist_ok=True) - far_file = os.path.join(cache_dir, f"{profile.language}_itn.far") + far_file = os.path.join(cache_dir, f"{profile.language}_itn_{INPUT_LOWER_CASED}.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f"ClassifyFst.fst was restored from {far_file}.") + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars for {profile.language}.") - self.fst = build_tagger(profile) + logger.info(f"Creating ClassifyFst grammars for {profile.language}.") + number_word, ordinary_word, segmenter = join_aware_runs(profile.number_words(), profile.join_words()) + cardinal = CardinalFst(profile, ordinary_word) + money = MoneyFst(profile) + word = WordFst(number_word, ordinary_word) + + token = pynini.union(cardinal.fst, money.fst, word.fst) + sentence = token + pynini.closure(" " + token) + graph = SINGLE_SPACES @ segmenter @ sentence + if money.has_implied_currency: + graph @= only_with_money_signal(profile) + self.fst = graph.optimize() + if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") + logger.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/word.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/word.py new file mode 100644 index 000000000..f53de9c72 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/word.py @@ -0,0 +1,41 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.bantu.taggers.sentence import run, word_token +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying words left unchanged, e.g. + kasa -> tokens { name: "kasa" } + + A run of number words that cannot be read as a number in its context also stays + unchanged as a whole, word by word; it is never partly converted. Both carry a + weight, so any valid number reading is preferred. + + Args: + number_word: any word that can be part of a number + ordinary_word: any other word, including a join word used as an ordinary word + """ + + def __init__(self, number_word: pynini.Fst, ordinary_word: pynini.Fst): + super().__init__(name="word", kind="classify") + + unread_run = run(word_token(number_word) + pynini.closure(" " + word_token(number_word))) + graph = word_token(ordinary_word) | unread_run + self.fst = pynutil.add_weight(graph, 1).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py index 374c2918a..6840620e7 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py @@ -15,7 +15,7 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import DIGIT, integer_domain +from nemo_text_processing.inverse_text_normalization.bantu.taggers.number import DIGIT, integer_domain from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_space diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/word.py index 34fef5049..091b0058d 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/word.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/word.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pynini from pynini.lib import pynutil -from nemo_text_processing.inverse_text_normalization.bantu.taggers.sentence import WORD -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_space +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst, delete_space class WordFst(GraphFst): @@ -26,5 +26,11 @@ class WordFst(GraphFst): def __init__(self): super().__init__(name="word", kind="verbalize") - graph = pynutil.delete("name:") + delete_space + pynutil.delete('"') + WORD + pynutil.delete('"') + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_SPACE, 1) + + pynutil.delete('"') + ) self.fst = graph.optimize() diff --git a/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..22e260907 --- /dev/null +++ b/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,32 @@ +bakopa bapi~bakopa 2 +zikopa zipi~zikopa 2 +makopa mapi~makopa 2 +kopa ima~kopa 1 +bakopa zipi~bakopa zipi +bakopa ima~bakopa ima +bakopa sei~bakopa 6 +zikopa nona~zikopa 9 +makopa deka~makopa 10 +zikopa deka ke zipi~zikopa 12 +bakopa pideka vo baima~bakopa 21 +zikopa hanto pi vo kodeka vo ziko~zikopa 233 +bakopa milo~bakopa 1,000 +bakopa milo vo hanto~bakopa 1,100 +zikopa milo nona vo hanto nona vo nonadeka vo nona~zikopa 9,999 +bakopa tausi~bakopa 1,000 +makopa dekafu~makopa 15 +bakopa piideka~bakopa 20 +bakopa piideka vo baima~bakopa 21 +koin piideka~20 XTS +bakopa hanto vo milo~bakopa hanto vo milo +bakopa deka ke~bakopa deka ke +bakopa bapi bako~bakopa bapi bako +bakopa bapi vo kasa~bakopa 2 vo kasa +zikopa pideka vo kasa~zikopa 20 vo kasa +rana bakopa bako vo kasa lena~rana bakopa 3 vo kasa lena +rana bakopa bapi lena zikopa ziko~rana bakopa 2 lena zikopa 3 +bakopa bapi zikopa ziko~bakopa 2 zikopa 3 +bakopa bapi~bakopa 2 +bakopa bapi !~bakopa 2 ! +zikopa ziko, bakopa bapi~zikopa ziko, bakopa 2 +ŋoma bakopa bapi~ŋoma bakopa 2 diff --git a/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_money.txt new file mode 100644 index 000000000..8435001d6 --- /dev/null +++ b/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_money.txt @@ -0,0 +1,39 @@ +koin zipi~2 XTS +koin mapi~20 XTS +makopa mapi~makopa 2 +koin milo vo hanto~1,100 XTS +koin~koin +koin kasa~koin kasa +muna ema~10,000 XTS +mina epi~20,000 XTS +mina pideka vo efu vo milo fu~255,000 XTS +koin mina epi~20,000 XTS +mina zipi~mina zipi +mina deka k'epi~120,000 XTS +mina deka kepi~120,000 XTS +mina deka ke epi~mina deka ke epi +kamega kaima~1,000,000 XTS +bumega bapi~2,000,000 XTS +bumega deka ke kaima~11,000,000 XTS +bumega hanto vo fudeka~150,000,000 XTS +bumega hanto nona vo nonadeka vo nona~999,000,000 XTS +bumega hanto vo milo fu~100,005,000 XTS +bumega bapi vo mina fudeka~2,500,000 XTS +kasa mina epi~kasa 20,000 XTS +kasa emina epi~kasa 20,000 +lipoka emina epi~lipoka 20,000 XTS +dalo emina epi~dalo 20,000 XTS +olipon emina epi~olipon 20,000 +emina epi~emina epi +obumega bapi~2,000,000 XTS +akamega~1,000,000 XTS +akamega kaima~1,000,000 XTS +kasa bumega bapi~kasa 2,000,000 XTS +kasa obumega bapi~kasa 2,000,000 +kasa akamega kaima~kasa 1,000,000 +kasa akamega~kasa akamega +bakopa obumega bapi~bakopa 2,000,000 +bakopa akamega kaima~bakopa 1,000,000 +lipoka obumega bapi~lipoka 2,000,000 XTS +mina epi halfu~mina epi halfu +lipoka mina epi halfu~lipoka mina epi halfu diff --git a/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_word.txt new file mode 100644 index 000000000..0e4cbaaff --- /dev/null +++ b/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_word.txt @@ -0,0 +1,9 @@ +bapi~bapi +rana bapi lena~rana bapi lena +kasa vo rana~kasa vo rana +Bakopa bapi~Bakopa bapi +bakopa bapi.~bakopa bapi. +bakopa 2~bakopa 2 +bakopa-bapi~bakopa-bapi +kasa é~kasa é +50% #1 ~50% #1 diff --git a/tests/nemo_text_processing/bantu/test_cardinal.py b/tests/nemo_text_processing/bantu/test_cardinal.py index 2fd7e7442..0e1a6b003 100644 --- a/tests/nemo_text_processing/bantu/test_cardinal.py +++ b/tests/nemo_text_processing/bantu/test_cardinal.py @@ -12,93 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Kernel cardinals with the made-up test profile: agreement, composition, malformed input.""" - import pytest from parameterized import parameterized +from ..utils import parse_test_case_file from .toy_profile import TOY_PROFILE from .utils import best_tags, build_normalizer -NORMALIZER = build_normalizer(TOY_PROFILE) - - -def check(spoken, expected): - # Exactly one best reading: the kernel never leaves a tie for NeMo to pick from. - assert len(best_tags(NORMALIZER, spoken)) == 1 - assert NORMALIZER.inverse_normalize(spoken, verbose=False) == expected - - -class TestBantuKernelCardinal: - # Number words agree with the counted noun's class; a wrong class stays unchanged. - @parameterized.expand( - [ - ("bakopa bapi", "bakopa 2"), - ("zikopa zipi", "zikopa 2"), - ("makopa mapi", "makopa 2"), - ("kopa ima", "kopa 1"), - ("bakopa zipi", "bakopa zipi"), - ("bakopa ima", "bakopa ima"), - ] - ) - @pytest.mark.run_only_on('CPU') - @pytest.mark.unit - def test_agreement(self, spoken, expected): - check(spoken, expected) - - # 6-9 and ten have one form in every class. - @parameterized.expand( - [ - ("bakopa sei", "bakopa 6"), - ("zikopa nona", "zikopa 9"), - ("makopa deka", "makopa 10"), - ] - ) - @pytest.mark.run_only_on('CPU') - @pytest.mark.unit - def test_invariant_units(self, spoken, expected): - check(spoken, expected) - # Teens join with ke; tens, hundreds and thousands join with vo; a trailing one agrees. - @parameterized.expand( - [ - ("zikopa deka ke zipi", "zikopa 12"), - ("bakopa pideka vo baima", "bakopa 21"), - ("zikopa hanto pi vo kodeka vo ziko", "zikopa 233"), - ("bakopa milo", "bakopa 1,000"), - ("bakopa milo vo hanto", "bakopa 1,100"), - ("zikopa milo nona vo hanto nona vo nonadeka vo nona", "zikopa 9,999"), - ] - ) - @pytest.mark.run_only_on('CPU') - @pytest.mark.unit - def test_composition(self, spoken, expected): - check(spoken, expected) +class TestCardinal: - # Profile extras: a fixed phrase, a special teen form and a spelling variant. - @parameterized.expand( - [ - ("bakopa tausi", "bakopa 1,000"), - ("makopa dekafu", "makopa 15"), - ("bakopa piideka", "bakopa 20"), - ("bakopa piideka vo baima", "bakopa 21"), - ("koin piideka", "20 XTS"), - ] - ) - @pytest.mark.run_only_on('CPU') - @pytest.mark.unit - def test_profile_extras(self, spoken, expected): - check(spoken, expected) + inverse_normalizer = build_normalizer(TOY_PROFILE) - # A malformed run of number words is left unchanged as a whole, never partly converted. - @parameterized.expand( - [ - ("bakopa hanto vo milo", "bakopa hanto vo milo"), - ("bakopa deka ke", "bakopa deka ke"), - ("bakopa bapi bako", "bakopa bapi bako"), - ] - ) + @parameterized.expand(parse_test_case_file('bantu/data_inverse_text_normalization/test_cases_cardinal.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit - def test_malformed(self, spoken, expected): - check(spoken, expected) + def test_denorm(self, test_input, expected): + # Exactly one best reading: the kernel never leaves a tie for NeMo's runtime to pick from. + assert len(best_tags(self.inverse_normalizer, test_input)) == 1 + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/bantu/test_money.py b/tests/nemo_text_processing/bantu/test_money.py index 824945bc5..d72efb81e 100644 --- a/tests/nemo_text_processing/bantu/test_money.py +++ b/tests/nemo_text_processing/bantu/test_money.py @@ -12,131 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Kernel money with the made-up test profile: currency context, counted scales, implied currency.""" - import pytest from parameterized import parameterized +from ..utils import parse_test_case_file from .toy_profile import TOY_PROFILE from .utils import best_tags, build_normalizer -NORMALIZER = build_normalizer(TOY_PROFILE) - - -def check(spoken, expected): - assert len(best_tags(NORMALIZER, spoken)) == 1 - assert NORMALIZER.inverse_normalize(spoken, verbose=False) == expected - - -class TestBantuKernelMoney: - # A currency phrase sets the agreement and licenses extra readings: mapi is 20 only after koin. - @parameterized.expand( - [ - ("koin zipi", "2 XTS"), - ("koin mapi", "20 XTS"), - ("makopa mapi", "makopa 2"), - ("koin milo vo hanto", "1,100 XTS"), - ("koin", "koin"), - ("koin kasa", "koin kasa"), - ] - ) - @pytest.mark.run_only_on('CPU') - @pytest.mark.unit - def test_currency_context(self, spoken, expected): - check(spoken, expected) - - # Ten-thousands: mina + count (agreeing with mina's class) + optional vo remainder. - @parameterized.expand( - [ - ("muna ema", "10,000 XTS"), - ("mina epi", "20,000 XTS"), - ("mina pideka vo efu vo milo fu", "255,000 XTS"), - ("koin mina epi", "20,000 XTS"), - ("mina zipi", "mina zipi"), - ] - ) - @pytest.mark.run_only_on('CPU') - @pytest.mark.unit - def test_counted_scale(self, spoken, expected): - check(spoken, expected) - - # A teen join before a vowel contracts (k'epi or kepi); the full form is not accepted. - @parameterized.expand( - [ - ("mina deka k'epi", "120,000 XTS"), - ("mina deka kepi", "120,000 XTS"), - ("mina deka ke epi", "mina deka ke epi"), - ] - ) - @pytest.mark.run_only_on('CPU') - @pytest.mark.unit - def test_contracted_teens(self, spoken, expected): - check(spoken, expected) - - # Millions: one forms, teens with a "one", hundreds in the count, and a smaller - # scale as the remainder. Tens/units after round hundreds continue the count. - @parameterized.expand( - [ - ("kamega kaima", "1,000,000 XTS"), - ("bumega bapi", "2,000,000 XTS"), - ("bumega deka ke kaima", "11,000,000 XTS"), - ("bumega hanto vo fudeka", "150,000,000 XTS"), - ("bumega hanto nona vo nonadeka vo nona", "999,000,000 XTS"), - ("bumega hanto vo milo fu", "100,005,000 XTS"), - ("bumega bapi vo mina fudeka", "2,500,000 XTS"), - ] - ) - @pytest.mark.run_only_on('CPU') - @pytest.mark.unit - def test_larger_scale(self, spoken, expected): - check(spoken, expected) - # Implied currency: bare mina is always money; word + emina counts that word, - # unless the word is a money cue; emina with nothing before it stays words. - @parameterized.expand( - [ - ("kasa mina epi", "kasa 20,000 XTS"), - ("kasa emina epi", "kasa 20,000"), - ("lipoka emina epi", "lipoka 20,000 XTS"), - ("dalo emina epi", "dalo 20,000 XTS"), - ("olipon emina epi", "olipon 20,000"), - ("emina epi", "emina epi"), - ] - ) - @pytest.mark.run_only_on('CPU') - @pytest.mark.unit - def test_implied_currency(self, spoken, expected): - check(spoken, expected) +class TestMoney: - # Millions: augmented forms are also money at the start of input, count a listed - # or unlisted noun before them, and need "one" (kaima) to count a single million. - @parameterized.expand( - [ - ("obumega bapi", "2,000,000 XTS"), - ("akamega", "1,000,000 XTS"), - ("akamega kaima", "1,000,000 XTS"), - ("kasa bumega bapi", "kasa 2,000,000 XTS"), - ("kasa obumega bapi", "kasa 2,000,000"), - ("kasa akamega kaima", "kasa 1,000,000"), - ("kasa akamega", "kasa akamega"), - ("bakopa obumega bapi", "bakopa 2,000,000"), - ("bakopa akamega kaima", "bakopa 1,000,000"), - ("lipoka obumega bapi", "lipoka 2,000,000 XTS"), - ] - ) - @pytest.mark.run_only_on('CPU') - @pytest.mark.unit - def test_augmented_larger_scale(self, spoken, expected): - check(spoken, expected) + inverse_normalizer = build_normalizer(TOY_PROFILE) - # A held-back word continuing the number keeps the whole phrase unchanged. - @parameterized.expand( - [ - ("mina epi halfu", "mina epi halfu"), - ("lipoka mina epi halfu", "lipoka mina epi halfu"), - ] - ) + @parameterized.expand(parse_test_case_file('bantu/data_inverse_text_normalization/test_cases_money.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit - def test_held_back(self, spoken, expected): - check(spoken, expected) + def test_denorm(self, test_input, expected): + # Exactly one best reading: the kernel never leaves a tie for NeMo's runtime to pick from. + assert len(best_tags(self.inverse_normalizer, test_input)) == 1 + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/bantu/test_profile.py b/tests/nemo_text_processing/bantu/test_profile.py index 240402044..1ebc6daec 100644 --- a/tests/nemo_text_processing/bantu/test_profile.py +++ b/tests/nemo_text_processing/bantu/test_profile.py @@ -19,7 +19,7 @@ import pytest from nemo_text_processing.inverse_text_normalization.bantu.profile import MoneyCues, Place -from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import build_role, integer_domain +from nemo_text_processing.inverse_text_normalization.bantu.taggers.number import build_role, integer_domain from nemo_text_processing.inverse_text_normalization.bantu.taggers.scale import build_scale_amount from nemo_text_processing.inverse_text_normalization.bantu.taggers.tokenize_and_classify import ClassifyFst @@ -81,7 +81,7 @@ def test_scale_without_counting_heads(self): @pytest.mark.unit def test_far_cache_roundtrip(self, tmp_path): built = ClassifyFst(TOY_PROFILE, cache_dir=str(tmp_path), overwrite_cache=True) - assert (tmp_path / "toy_itn.far").exists() + assert (tmp_path / "toy_itn_lower_cased.far").exists() restored = ClassifyFst(TOY_PROFILE, cache_dir=str(tmp_path)) normalizer = build_normalizer(TOY_PROFILE) for spoken in ("bakopa pideka vo baima", "kasa emina epi", "lipoka obumega bapi", "bakopa bapi vo kasa"): diff --git a/tests/nemo_text_processing/bantu/test_segmentation.py b/tests/nemo_text_processing/bantu/test_segmentation.py deleted file mode 100644 index c4e1550c7..000000000 --- a/tests/nemo_text_processing/bantu/test_segmentation.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Kernel sentence segmentation with the made-up test profile: numbers inside sentences.""" - -import pytest -from parameterized import parameterized - -from .toy_profile import TOY_PROFILE -from .utils import best_tags, build_normalizer - -NORMALIZER = build_normalizer(TOY_PROFILE) - - -def check(spoken, expected): - assert len(best_tags(NORMALIZER, spoken)) == 1 - assert NORMALIZER.inverse_normalize(spoken, verbose=False) == expected - - -class TestBantuKernelSegmentation: - # The join word vo is also an ordinary word ("in"): followed by an ordinary word, - # it ends the number instead of joining a remainder. - @parameterized.expand( - [ - ("bakopa bapi vo kasa", "bakopa 2 vo kasa"), - ("zikopa pideka vo kasa", "zikopa 20 vo kasa"), - ("rana bakopa bako vo kasa lena", "rana bakopa 3 vo kasa lena"), - ] - ) - @pytest.mark.run_only_on('CPU') - @pytest.mark.unit - def test_join_word_as_ordinary_word(self, spoken, expected): - check(spoken, expected) - - # Several numbers and plain words in one sentence. - @parameterized.expand( - [ - ("rana bakopa bapi lena zikopa ziko", "rana bakopa 2 lena zikopa 3"), - ("bakopa bapi zikopa ziko", "bakopa 2 zikopa 3"), - ] - ) - @pytest.mark.run_only_on('CPU') - @pytest.mark.unit - def test_numbers_in_sentences(self, spoken, expected): - check(spoken, expected) - - # No counted noun, no currency: number words stay as words. - @parameterized.expand( - [ - ("bapi", "bapi"), - ("rana bapi lena", "rana bapi lena"), - ("kasa vo rana", "kasa vo rana"), - ] - ) - @pytest.mark.run_only_on('CPU') - @pytest.mark.unit - def test_unchanged(self, spoken, expected): - check(spoken, expected) diff --git a/tests/nemo_text_processing/bantu/test_word.py b/tests/nemo_text_processing/bantu/test_word.py new file mode 100644 index 000000000..ef9ddbca5 --- /dev/null +++ b/tests/nemo_text_processing/bantu/test_word.py @@ -0,0 +1,34 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from ..utils import parse_test_case_file +from .toy_profile import TOY_PROFILE +from .utils import best_tags, build_normalizer + + +class TestWord: + + inverse_normalizer = build_normalizer(TOY_PROFILE) + + @parameterized.expand(parse_test_case_file('bantu/data_inverse_text_normalization/test_cases_word.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + # Exactly one best reading: the kernel never leaves a tie for NeMo's runtime to pick from. + assert len(best_tags(self.inverse_normalizer, test_input)) == 1 + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected From 4fb7a86e74b53488283c67bf9c9c3334ea7a0be5 Mon Sep 17 00:00:00 2001 From: Cal3bdev Date: Sun, 25 Oct 2026 19:04:05 +0300 Subject: [PATCH 3/9] Bantu ITN kernel: count listed nouns with any number of ten-thousands The test language now has a ten-thousands number place and counts a listed noun with its ten-thousands scale, as for millions, so a count of 10 or more after a listed noun is read like after any other word. Corrects the comment on how listed nouns are counted and adds test cases. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Cal3bdev --- .../bantu/taggers/cardinal.py | 6 ++++-- .../test_cases_cardinal.txt | 9 +++++++++ tests/nemo_text_processing/bantu/toy_profile.py | 10 +++++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py index c4b109d06..fda8f6271 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py @@ -53,8 +53,10 @@ def __init__(self, profile: CardinalProfile, ordinary_word: pynini.Fst): number = build_cardinal(profile, context) | build_noun_scale_amount(profile, context) graphs.append(word_token(pynini.accep(context.noun)) + " " + run(cardinal_token(number))) - # Listed nouns already agree fully above. After a money cue or a currency word - # the amount is money, handled by MoneyFst. + # Listed nouns are counted by their own cardinal above: a profile covers large + # amounts after them with its places (Luganda abantu emitwalo ebiri) or with + # after_counted_nouns. After a money cue or a currency word the amount is money, + # handled by MoneyFst. excluded = [context.noun for context in profile.contexts] excluded += [currency.phrase.split()[-1] for currency in profile.currencies] excluded = pynini.union(*excluded, money_cue_word(profile)).optimize() diff --git a/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_cardinal.txt index 22e260907..4ccfde562 100644 --- a/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_cardinal.txt @@ -30,3 +30,12 @@ bakopa bapi~bakopa 2 bakopa bapi !~bakopa 2 ! zikopa ziko, bakopa bapi~zikopa ziko, bakopa 2 ŋoma bakopa bapi~ŋoma bakopa 2 +bakopa omuna~bakopa 10,000 +bakopa emina epi~bakopa 20,000 +zikopa emina nona vo milo fu vo hanto~zikopa 95,100 +ekopa emina eko vo kasa~ekopa 30,000 vo kasa +bakopa emina pideka~bakopa 200,000 +bakopa emina deka k'epi~bakopa 120,000 +bakopa emina pideka vo ema~bakopa 210,000 +bakopa emina pideka vo milo fu~bakopa 205,000 +bakopa emina pideka vo baima~bakopa emina pideka vo baima diff --git a/tests/nemo_text_processing/bantu/toy_profile.py b/tests/nemo_text_processing/bantu/toy_profile.py index 602c9b6ea..daa54920f 100644 --- a/tests/nemo_text_processing/bantu/toy_profile.py +++ b/tests/nemo_text_processing/bantu/toy_profile.py @@ -22,7 +22,9 @@ - teens join with "ke", larger parts with "vo" [na, mu] - "vo" is also an ordinary word, "in" [mu] - a currency phrase that licenses a special reading: koin mapi = 20 XTS, but makopa mapi = 2 [abiri] -- a counted scale of ten-thousands: mina + count [mitwalo] +- a counted scale of ten-thousands: mina + count [mitwalo]; after a listed noun, + emina counts that noun, also as an ordinary number place for 1-9: + bakopa emina epi = 20,000, bakopa emina pideka = 200,000 [abantu emitwalo ebiri / abiri] - implied currency: bare "mina" is always money; word + "emina" counts that word, except after a money verb [mitwalo / emitwalo] - a held-back word that continues a number but is not handled: halfu [n'ekitundu] @@ -59,6 +61,10 @@ ) +# Count forms agreeing with class 8 (ekopa, and the ten-thousands noun mina): 2-9. +EIGHT_COUNTS = tuple(("e" + stem, value) for stem, value in STEMS) + INVARIANT + + def plural(class_id, noun, prefix, final_one=None, teen_variants=()): units = tuple((prefix + stem, value) for stem, value in STEMS) + INVARIANT final = units + (((final_one, "1"),) if final_one else ()) @@ -86,6 +92,7 @@ def multiples(scale): Place(1, TENS, " vo "), Place(2, multiples("hanto"), " vo "), Place(3, multiples("milo"), " vo "), + Place(4, (("omuna", "1"),) + tuple((f"emina {word}", value) for word, value in EIGHT_COUNTS), " vo "), ), exact_boundaries=(("tausi", "1000"),), roles=(("short-tens", (("mapi", "20"),)),), @@ -106,6 +113,7 @@ def multiples(scale): counting_plural="emina", implied_anywhere=True, counts_any_word=True, + after_counted_nouns=True, ), # Millions [obukadde]: augmented forms are also money at the start of input, # count listed nouns with agreement, and need "one" to count (akamega kaima). From d2752d1e04f8091966f27ec6a6f87ecb2de52813 Mon Sep 17 00:00:00 2001 From: Cal3bdev Date: Thu, 29 Oct 2026 13:22:51 +0300 Subject: [PATCH 4/9] Bantu ITN kernel: address review comments - add Crane Labs to the licence headers - move taggers/sentence.py to bantu/graph_utils.py - use NEMO_SPACE, string.digits and GraphFst.add_tokens - move the thousands separator into the profile - cardinal cases: enumerate 1-20, add million-range samples - money cases: base cases for USD, GBP, EUR and JPY Co-Authored-By: Claude Opus 5.5 Signed-off-by: Cal3bdev --- .../bantu/__init__.py | 1 + .../{taggers/sentence.py => graph_utils.py} | 31 +++++++------ .../bantu/profile.py | 3 ++ .../bantu/taggers/__init__.py | 1 + .../bantu/taggers/cardinal.py | 18 ++++---- .../bantu/taggers/money.py | 20 ++++----- .../bantu/taggers/number.py | 10 +++-- .../bantu/taggers/scale.py | 6 ++- .../bantu/taggers/tokenize_and_classify.py | 18 +++++--- .../bantu/taggers/word.py | 7 +-- .../bantu/verbalizers/__init__.py | 1 + .../bantu/verbalizers/cardinal.py | 1 + .../bantu/verbalizers/money.py | 1 + .../bantu/verbalizers/verbalize.py | 10 ++--- .../bantu/verbalizers/verbalize_final.py | 8 ++-- .../bantu/verbalizers/word.py | 1 + tests/nemo_text_processing/bantu/__init__.py | 1 + .../test_cases_cardinal.txt | 43 +++++++++++++------ .../test_cases_money.txt | 13 ++++++ .../bantu/test_cardinal.py | 1 + .../nemo_text_processing/bantu/test_money.py | 1 + .../bantu/test_profile.py | 1 + .../bantu/test_verbalizer.py | 1 + tests/nemo_text_processing/bantu/test_word.py | 1 + .../nemo_text_processing/bantu/toy_profile.py | 12 +++++- tests/nemo_text_processing/bantu/utils.py | 1 + 26 files changed, 144 insertions(+), 68 deletions(-) rename nemo_text_processing/inverse_text_normalization/bantu/{taggers/sentence.py => graph_utils.py} (82%) diff --git a/nemo_text_processing/inverse_text_normalization/bantu/__init__.py b/nemo_text_processing/inverse_text_normalization/bantu/__init__.py index 9e3fb699d..eb60428b9 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/__init__.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/__init__.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/sentence.py b/nemo_text_processing/inverse_text_normalization/bantu/graph_utils.py similarity index 82% rename from nemo_text_processing/inverse_text_normalization/bantu/taggers/sentence.py rename to nemo_text_processing/inverse_text_normalization/bantu/graph_utils.py index 00452d61c..58aea3884 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/sentence.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/graph_utils.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Sentence-level helpers shared by the taggers: number-word runs, token formats and the money-signal filter. +"""Helpers shared by the Bantu taggers: number-word runs, token formats and the money-signal filter. The classifier reads the whole sentence at once. Maximal runs of number words are marked first, so each run is read as a single number in the context of the word @@ -25,7 +26,12 @@ from pynini.lib import pynutil from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_NOT_SPACE, NEMO_WHITE_SPACE +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_NOT_SPACE, + NEMO_SPACE, + NEMO_WHITE_SPACE, +) # Internal markers, removed before output. Control characters, so they never clash with input text. RUN_START = "\x01" # start of a run of number words @@ -40,15 +46,16 @@ WORD_CHAR = pynini.difference(NEMO_NOT_SPACE, MARKERS).optimize() WORD = pynini.closure(WORD_CHAR, 1).optimize() # Runs of whitespace between words become a single space. -SINGLE_SPACES = pynini.closure(WORD_CHAR | pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ")).optimize() +SINGLE_SPACES = pynini.closure(WORD_CHAR | pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), NEMO_SPACE)).optimize() def word_token(graph: pynini.Fst) -> pynini.Fst: return pynutil.insert('tokens { name: "') + graph + pynutil.insert('" }') -def cardinal_token(graph: pynini.Fst) -> pynini.Fst: - return pynutil.insert('tokens { cardinal { integer: "') + graph + pynutil.insert('" } }') +def token(graph: pynini.Fst) -> pynini.Fst: + """Wraps a class token, e.g. cardinal { ... }, as one item of the sentence.""" + return pynutil.insert("tokens { ") + graph + pynutil.insert(" }") def run(graph: pynini.Fst) -> pynini.Fst: @@ -73,17 +80,17 @@ def join_aware_runs(number_words, join_words): plain_number = pynini.difference(numeric_word, join_word).optimize() any_word = (other_word | join_word).optimize() # Between or after runs: words and joins, containing at least one ordinary word. - others = (pynini.closure(any_word + " ") + other_word + pynini.closure(" " + any_word)).optimize() - numbers = numeric_word + pynini.closure(" " + numeric_word) - before_others = pynini.closure(numeric_word + " ") + plain_number + others = (pynini.closure(any_word + NEMO_SPACE) + other_word + pynini.closure(NEMO_SPACE + any_word)).optimize() + numbers = numeric_word + pynini.closure(NEMO_SPACE + numeric_word) + before_others = pynini.closure(numeric_word + NEMO_SPACE) + plain_number def marked(graph): return pynutil.insert(RUN_START) + graph + pynutil.insert(RUN_END) - runs = pynini.closure(marked(before_others) + " " + others + " ") + ( - marked(before_others) + " " + others | marked(numbers) + runs = pynini.closure(marked(before_others) + NEMO_SPACE + others + NEMO_SPACE) + ( + marked(before_others) + NEMO_SPACE + others | marked(numbers) ) - segmenter = (others | pynini.closure(others + " ", 0, 1) + runs).optimize() + segmenter = (others | pynini.closure(others + NEMO_SPACE, 0, 1) + runs).optimize() return numeric_word, any_word, segmenter @@ -109,7 +116,7 @@ def only_with_money_signal(profile: CardinalProfile) -> pynini.Fst: rejected and another reading (the words unchanged, or a counted noun) is used. """ plain = pynini.difference(NEMO_CHAR, pynini.union(IMPLIED, CUE_ONLY)).optimize() - after_cue = 'tokens { name: "' + money_cue_word(profile) + '" } ' + pynini.union(IMPLIED, CUE_ONLY) + after_cue = 'tokens { name: "' + money_cue_word(profile) + '" }' + NEMO_SPACE + pynini.union(IMPLIED, CUE_ONLY) allowed = (pynini.closure(pynini.accep(IMPLIED), 0, 1) + pynini.closure(plain | after_cue)).optimize() strip = pynini.closure(plain | pynutil.delete(IMPLIED) | pynutil.delete(CUE_ONLY)) return (allowed @ strip.optimize()).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/profile.py b/nemo_text_processing/inverse_text_normalization/bantu/profile.py index bf15161b4..fd3efe85e 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/profile.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/profile.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -99,6 +100,8 @@ class CardinalProfile: held_back_words: Tuple[str, ...] = () # Vowels, for contracting a join before a vowel-initial word (Luganda na + ebiri -> n'ebiri). vowels: str = "aeiou" + # Written separator between digit groups in the output (25,000). + thousands_separator: str = "," def join_words(self) -> FrozenSet[str]: """Words that join number parts; they may also be ordinary words (e.g. locative mu).""" diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/__init__.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/__init__.py index 9e3fb699d..eb60428b9 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/__init__.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/__init__.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py index fda8f6271..a1f5be6e4 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,20 +14,16 @@ # limitations under the License. import pynini +from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import money_cue_word, run, token, word_token from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile, invariant_context from nemo_text_processing.inverse_text_normalization.bantu.taggers.number import build_cardinal from nemo_text_processing.inverse_text_normalization.bantu.taggers.scale import ( build_noun_scale_amount, build_scale_amount, ) -from nemo_text_processing.inverse_text_normalization.bantu.taggers.sentence import ( - cardinal_token, - money_cue_word, - run, - word_token, -) -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE, GraphFst class CardinalFst(GraphFst): @@ -48,10 +45,13 @@ class CardinalFst(GraphFst): def __init__(self, profile: CardinalProfile, ordinary_word: pynini.Fst): super().__init__(name="cardinal", kind="classify") + def cardinal_token(number): + return token(self.add_tokens(pynutil.insert('integer: "') + number + pynutil.insert('"'))) + graphs = [] for context in profile.contexts: number = build_cardinal(profile, context) | build_noun_scale_amount(profile, context) - graphs.append(word_token(pynini.accep(context.noun)) + " " + run(cardinal_token(number))) + graphs.append(word_token(pynini.accep(context.noun)) + NEMO_SPACE + run(cardinal_token(number))) # Listed nouns are counted by their own cardinal above: a profile covers large # amounts after them with its places (Luganda abantu emitwalo ebiri) or with @@ -64,6 +64,6 @@ def __init__(self, profile: CardinalProfile, ordinary_word: pynini.Fst): for scale in profile.counted_scales: if scale.counts_any_word: amount = build_scale_amount(profile, scale, "counting", invariant_context(profile)) - graphs.append(word_token(any_noun) + " " + run(cardinal_token(amount))) + graphs.append(word_token(any_noun) + NEMO_SPACE + run(cardinal_token(amount))) self.fst = pynini.union(*graphs).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/money.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/money.py index 759c17be7..d435aca76 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/money.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/money.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +16,7 @@ import pynini from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import CUE_ONLY, IMPLIED, run, token from nemo_text_processing.inverse_text_normalization.bantu.profile import ( CardinalProfile, CurrencyContext, @@ -26,8 +28,7 @@ integer_domain, ) from nemo_text_processing.inverse_text_normalization.bantu.taggers.scale import build_scale_amount -from nemo_text_processing.inverse_text_normalization.bantu.taggers.sentence import CUE_ONLY, IMPLIED, run -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE, GraphFst def build_amount(profile: CardinalProfile, currency: CurrencyContext) -> pynini.Fst: @@ -40,14 +41,6 @@ def build_amount(profile: CardinalProfile, currency: CurrencyContext) -> pynini. return amount.optimize() -def money_token(amount: pynini.Fst, code: str, marker: str = "") -> pynini.Fst: - return run( - pynutil.insert(marker + 'tokens { money { integer_part: "') - + amount - + pynutil.insert(f'" currency: "{code}" }} }}') - ) - - class MoneyFst(GraphFst): """ Finite state transducer for classifying money, e.g. @@ -69,10 +62,15 @@ class MoneyFst(GraphFst): def __init__(self, profile: CardinalProfile): super().__init__(name="money", kind="classify") + def money_token(amount, code, marker=""): + fields = pynutil.insert('integer_part: "') + amount + pynutil.insert(f'" currency: "{code}"') + return run(pynutil.insert(marker) + token(self.add_tokens(fields))) + graphs = [] for currency in profile.currencies: graphs.append( - pynutil.delete(currency.phrase + " ") + money_token(build_amount(profile, currency), currency.code) + pynutil.delete(currency.phrase + NEMO_SPACE) + + money_token(build_amount(profile, currency), currency.code) ) self.has_implied_currency = False diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/number.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/number.py index 6606cc524..f4ea3ca6c 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/number.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/number.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,13 +17,16 @@ Values are composed digit by digit inside the FST; there is no runtime arithmetic. """ +import string + import pynini from pynini.lib import pynutil from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile, Forms, NounContext -DIGIT = pynini.union(*"0123456789").optimize() -NONZERO = pynini.union(*"123456789").optimize() +NONZERO_DIGITS = string.digits[1:] +DIGIT = pynini.union(*string.digits).optimize() +NONZERO = pynini.union(*NONZERO_DIGITS).optimize() def mapping(forms: Forms) -> pynini.Fst: @@ -60,7 +64,7 @@ def build_cardinal(profile: CardinalProfile, context: NounContext) -> pynini.Fst for place in profile.places: if place.power != expected_power: raise ValueError("places must be consecutive ascending decimal powers") - if any(value not in "123456789" or len(value) != 1 for _, value in place.forms): + if any(value not in NONZERO_DIGITS or len(value) != 1 for _, value in place.forms): raise ValueError("place coefficients must be single nonzero digits") # A present high place either consumes a smaller remainder or emits zeros. high = mapping(place.forms) + ( diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/scale.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/scale.py index 2fb64b580..e08c33e1c 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/scale.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/scale.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,6 +41,7 @@ integer_domain, mapping, ) +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE def build_count(profile: CardinalProfile, scale: CountedScale) -> Tuple[pynini.Fst, pynini.Fst]: @@ -58,7 +60,7 @@ def build_count(profile: CardinalProfile, scale: CountedScale) -> Tuple[pynini.F teen_units = [] for form, value in final_units: if form[0] in profile.vowels: - spoken = [" " + prefix + form for prefix in scale.contracted_teen_joins] + spoken = [NEMO_SPACE + prefix + form for prefix in scale.contracted_teen_joins] else: spoken = [profile.teen_join + form] teen_units.extend(pynini.cross(s, value) for s in spoken) @@ -107,7 +109,7 @@ def build_scale_amount( ones.extend(scale.money_only_ones) if not ones and not plurals: return pynini.Fst() - plural = pynini.union(*(pynutil.delete(p + " ") for p in plurals)) if plurals else pynini.Fst() + plural = pynini.union(*(pynutil.delete(p + NEMO_SPACE) for p in plurals)) if plurals else pynini.Fst() head = plural + count if ones: head |= pynini.union(*(pynini.cross(one, "01") for one in ones)) diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py index f0f67606a..95e82375e 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,16 +17,21 @@ import pynini -from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile -from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import CardinalFst -from nemo_text_processing.inverse_text_normalization.bantu.taggers.money import MoneyFst -from nemo_text_processing.inverse_text_normalization.bantu.taggers.sentence import ( +from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import ( SINGLE_SPACES, join_aware_runs, only_with_money_signal, ) +from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile +from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.bantu.taggers.money import MoneyFst from nemo_text_processing.inverse_text_normalization.bantu.taggers.word import WordFst -from nemo_text_processing.text_normalization.en.graph_utils import INPUT_LOWER_CASED, GraphFst, generator_main +from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_LOWER_CASED, + NEMO_SPACE, + GraphFst, + generator_main, +) from nemo_text_processing.utils.logging import logger @@ -66,7 +72,7 @@ def __init__(self, profile: CardinalProfile, cache_dir: str = None, overwrite_ca word = WordFst(number_word, ordinary_word) token = pynini.union(cardinal.fst, money.fst, word.fst) - sentence = token + pynini.closure(" " + token) + sentence = token + pynini.closure(NEMO_SPACE + token) graph = SINGLE_SPACES @ segmenter @ sentence if money.has_implied_currency: graph @= only_with_money_signal(profile) diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/word.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/word.py index f53de9c72..d5e5f7291 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/word.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/word.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,8 +16,8 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.inverse_text_normalization.bantu.taggers.sentence import run, word_token -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import run, word_token +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE, GraphFst class WordFst(GraphFst): @@ -36,6 +37,6 @@ class WordFst(GraphFst): def __init__(self, number_word: pynini.Fst, ordinary_word: pynini.Fst): super().__init__(name="word", kind="classify") - unread_run = run(word_token(number_word) + pynini.closure(" " + word_token(number_word))) + unread_run = run(word_token(number_word) + pynini.closure(NEMO_SPACE + word_token(number_word))) graph = word_token(ordinary_word) | unread_run self.fst = pynutil.add_weight(graph, 1).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/__init__.py index 9e3fb699d..eb60428b9 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/__init__.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/__init__.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py index 6840620e7..66dd51bba 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/money.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/money.py index 7e651d3b7..439991465 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/money.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/money.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize.py index e68c1674b..cf923ea07 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,12 +25,11 @@ class VerbalizeFst(GraphFst): classifier, so verbalizing is formatting only. Args: - profile: the language profile (sets the largest value) - thousands_separator: written separator between digit groups + profile: the language profile (sets the largest value and the thousands separator) """ - def __init__(self, profile: CardinalProfile, thousands_separator: str = ","): + def __init__(self, profile: CardinalProfile): super().__init__(name="verbalize", kind="verbalize") - cardinal = CardinalFst(profile.max_value, thousands_separator).fst - money = MoneyFst(profile.max_value, thousands_separator).fst + cardinal = CardinalFst(profile.max_value, profile.thousands_separator).fst + money = MoneyFst(profile.max_value, profile.thousands_separator).fst self.fst = (cardinal | money).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize_final.py index ec288affc..23f173e4a 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize_final.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize_final.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,13 +27,12 @@ class VerbalizeFinalFst(GraphFst): Finite state transducer that verbalizes an entire sentence. Args: - profile: the language profile (sets the largest value) - thousands_separator: written separator between digit groups + profile: the language profile (sets the largest value and the thousands separator) """ - def __init__(self, profile: CardinalProfile, thousands_separator: str = ","): + def __init__(self, profile: CardinalProfile): super().__init__(name="verbalize_final", kind="verbalize") - types = VerbalizeFst(profile, thousands_separator).fst | WordFst().fst + types = VerbalizeFst(profile).fst | WordFst().fst graph = ( pynutil.delete("tokens") + delete_space diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/word.py index 091b0058d..c2382e72b 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/word.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/word.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/nemo_text_processing/bantu/__init__.py b/tests/nemo_text_processing/bantu/__init__.py index 9e3fb699d..eb60428b9 100644 --- a/tests/nemo_text_processing/bantu/__init__.py +++ b/tests/nemo_text_processing/bantu/__init__.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_cardinal.txt index 4ccfde562..be1e7318e 100644 --- a/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_cardinal.txt @@ -1,13 +1,29 @@ -bakopa bapi~bakopa 2 +kopa ima~kopa 1 zikopa zipi~zikopa 2 +zikopa ziko~zikopa 3 +zikopa zide~zikopa 4 +zikopa zifu~zikopa 5 +zikopa sei~zikopa 6 +zikopa siti~zikopa 7 +zikopa oto~zikopa 8 +zikopa nona~zikopa 9 +zikopa deka~zikopa 10 +zikopa deka ke zima~zikopa 11 +zikopa deka ke zipi~zikopa 12 +zikopa deka ke ziko~zikopa 13 +zikopa deka ke zide~zikopa 14 +zikopa deka ke zifu~zikopa 15 +zikopa deka ke sei~zikopa 16 +zikopa deka ke siti~zikopa 17 +zikopa deka ke oto~zikopa 18 +zikopa deka ke nona~zikopa 19 +zikopa pideka~zikopa 20 +bakopa bapi~bakopa 2 makopa mapi~makopa 2 -kopa ima~kopa 1 bakopa zipi~bakopa zipi bakopa ima~bakopa ima bakopa sei~bakopa 6 -zikopa nona~zikopa 9 makopa deka~makopa 10 -zikopa deka ke zipi~zikopa 12 bakopa pideka vo baima~bakopa 21 zikopa hanto pi vo kodeka vo ziko~zikopa 233 bakopa milo~bakopa 1,000 @@ -20,16 +36,16 @@ bakopa piideka vo baima~bakopa 21 koin piideka~20 XTS bakopa hanto vo milo~bakopa hanto vo milo bakopa deka ke~bakopa deka ke -bakopa bapi bako~bakopa bapi bako -bakopa bapi vo kasa~bakopa 2 vo kasa +bakopa bade bako~bakopa bade bako +bakopa bafu vo kasa~bakopa 5 vo kasa zikopa pideka vo kasa~zikopa 20 vo kasa rana bakopa bako vo kasa lena~rana bakopa 3 vo kasa lena -rana bakopa bapi lena zikopa ziko~rana bakopa 2 lena zikopa 3 -bakopa bapi zikopa ziko~bakopa 2 zikopa 3 -bakopa bapi~bakopa 2 -bakopa bapi !~bakopa 2 ! -zikopa ziko, bakopa bapi~zikopa ziko, bakopa 2 -ŋoma bakopa bapi~ŋoma bakopa 2 +rana bakopa bade lena zikopa ziko~rana bakopa 4 lena zikopa 3 +bakopa bako zikopa zifu~bakopa 3 zikopa 5 +bakopa bafu~bakopa 5 +bakopa sei !~bakopa 6 ! +zikopa ziko, bakopa bade~zikopa ziko, bakopa 4 +ŋoma bakopa siti~ŋoma bakopa 7 bakopa omuna~bakopa 10,000 bakopa emina epi~bakopa 20,000 zikopa emina nona vo milo fu vo hanto~zikopa 95,100 @@ -39,3 +55,6 @@ bakopa emina deka k'epi~bakopa 120,000 bakopa emina pideka vo ema~bakopa 210,000 bakopa emina pideka vo milo fu~bakopa 205,000 bakopa emina pideka vo baima~bakopa emina pideka vo baima +bakopa obumega bako vo emina dedeka vo efu vo milo siti vo hanto sei vo pideka vo baima~bakopa 3,457,621 +bakopa akamega kaima vo milo fu vo kodeka~bakopa 1,005,030 +zikopa obumega bafu vo emina eko vo kodeka~zikopa 5,030,030 diff --git a/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_money.txt index 8435001d6..c4e0a2990 100644 --- a/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_money.txt +++ b/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_money.txt @@ -37,3 +37,16 @@ bakopa akamega kaima~bakopa 1,000,000 lipoka obumega bapi~lipoka 2,000,000 XTS mina epi halfu~mina epi halfu lipoka mina epi halfu~lipoka mina epi halfu +dolari zide~4 USD +dolari pideka vo zima~21 USD +dolari mina epi~20,000 USD +paundi zifu~5 GBP +paundi milo vo hanto~1,100 GBP +yuro mafu~5 EUR +yuro deka ke mapi~12 EUR +yeni efu~5 JPY +yeni bumega bako~3,000,000 JPY +lipoka dolari ziko~lipoka 3 USD +dolari mapi~dolari mapi +yuro kasa~yuro kasa +yeni~yeni diff --git a/tests/nemo_text_processing/bantu/test_cardinal.py b/tests/nemo_text_processing/bantu/test_cardinal.py index 0e1a6b003..2c46c403f 100644 --- a/tests/nemo_text_processing/bantu/test_cardinal.py +++ b/tests/nemo_text_processing/bantu/test_cardinal.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/nemo_text_processing/bantu/test_money.py b/tests/nemo_text_processing/bantu/test_money.py index d72efb81e..b92ec49ec 100644 --- a/tests/nemo_text_processing/bantu/test_money.py +++ b/tests/nemo_text_processing/bantu/test_money.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/nemo_text_processing/bantu/test_profile.py b/tests/nemo_text_processing/bantu/test_profile.py index 1ebc6daec..6787c95dc 100644 --- a/tests/nemo_text_processing/bantu/test_profile.py +++ b/tests/nemo_text_processing/bantu/test_profile.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/nemo_text_processing/bantu/test_verbalizer.py b/tests/nemo_text_processing/bantu/test_verbalizer.py index 1e1d0c4d4..fb130e839 100644 --- a/tests/nemo_text_processing/bantu/test_verbalizer.py +++ b/tests/nemo_text_processing/bantu/test_verbalizer.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/nemo_text_processing/bantu/test_word.py b/tests/nemo_text_processing/bantu/test_word.py index ef9ddbca5..a4eccd707 100644 --- a/tests/nemo_text_processing/bantu/test_word.py +++ b/tests/nemo_text_processing/bantu/test_word.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/nemo_text_processing/bantu/toy_profile.py b/tests/nemo_text_processing/bantu/toy_profile.py index daa54920f..c1d0e8bc5 100644 --- a/tests/nemo_text_processing/bantu/toy_profile.py +++ b/tests/nemo_text_processing/bantu/toy_profile.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,6 +23,8 @@ - teens join with "ke", larger parts with "vo" [na, mu] - "vo" is also an ordinary word, "in" [mu] - a currency phrase that licenses a special reading: koin mapi = 20 XTS, but makopa mapi = 2 [abiri] +- other currency phrases, each agreeing with its own class: dolari zipi = 2 USD, + yuro mafu = 5 EUR, yeni efu = 5 JPY - a counted scale of ten-thousands: mina + count [mitwalo]; after a listed noun, emina counts that noun, also as an ordinary number place for 1-9: bakopa emina epi = 20,000, bakopa emina pideka = 200,000 [abantu emitwalo ebiri / abiri] @@ -97,7 +100,14 @@ def multiples(scale): exact_boundaries=(("tausi", "1000"),), roles=(("short-tens", (("mapi", "20"),)),), max_value=999_999_999, - currencies=(CurrencyContext("koin", "XTS", agreement_class="4", roles=("short-tens",)),), + currencies=( + CurrencyContext("koin", "XTS", agreement_class="4", roles=("short-tens",)), + # Base cases for common currencies; the words are made up like the rest. + CurrencyContext("dolari", "USD", agreement_class="4"), + CurrencyContext("paundi", "GBP", agreement_class="4"), + CurrencyContext("yuro", "EUR", agreement_class="6"), + CurrencyContext("yeni", "JPY", agreement_class="8"), + ), counted_scales=( # Ten-thousands [mitwalo]: bare mina is money anywhere; emina counts a word. CountedScale( diff --git a/tests/nemo_text_processing/bantu/utils.py b/tests/nemo_text_processing/bantu/utils.py index 987c7220b..44079d4d8 100644 --- a/tests/nemo_text_processing/bantu/utils.py +++ b/tests/nemo_text_processing/bantu/utils.py @@ -1,4 +1,5 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 0c617e8eb7ad774a0c83d038333e4daf175bef29 Mon Sep 17 00:00:00 2001 From: Cal3bdev Date: Thu, 5 Nov 2026 13:30:28 +0300 Subject: [PATCH 5/9] Bantu ITN kernel: fold number builders into CardinalFst; MoneyFst takes the cardinal Digit helpers (DIGIT, NONZERO, mapping, integer_domain) move to bantu/graph_utils.py; build_cardinal and build_role move into taggers/cardinal.py. CardinalFst keeps the agreeing graph per noun class and the role graphs, and MoneyFst builds its amounts from them. Output is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Cal3bdev --- .../bantu/graph_utils.py | 36 +++++++- .../bantu/taggers/cardinal.py | 83 +++++++++++++++-- .../bantu/taggers/money.py | 28 +++--- .../bantu/taggers/number.py | 92 ------------------- .../bantu/taggers/scale.py | 9 +- .../bantu/taggers/tokenize_and_classify.py | 2 +- .../bantu/verbalizers/cardinal.py | 2 +- .../bantu/test_profile.py | 20 +++- 8 files changed, 144 insertions(+), 128 deletions(-) delete mode 100644 nemo_text_processing/inverse_text_normalization/bantu/taggers/number.py diff --git a/nemo_text_processing/inverse_text_normalization/bantu/graph_utils.py b/nemo_text_processing/inverse_text_normalization/bantu/graph_utils.py index 58aea3884..80d381471 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/graph_utils.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/graph_utils.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Helpers shared by the Bantu taggers: number-word runs, token formats and the money-signal filter. +"""Helpers shared by the Bantu taggers: digit strings, number-word runs, token formats and the money-signal filter. The classifier reads the whole sentence at once. Maximal runs of number words are marked first, so each run is read as a single number in the context of the word @@ -22,10 +22,12 @@ so structure alone must make the reading unique; the kernel tests check that every test input has exactly one best reading. """ +import string + import pynini from pynini.lib import pynutil -from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile +from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile, Forms from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_CHAR, NEMO_NOT_SPACE, @@ -33,6 +35,36 @@ NEMO_WHITE_SPACE, ) +# Values are composed digit by digit inside the FSTs; there is no runtime arithmetic. +NONZERO_DIGITS = string.digits[1:] +DIGIT = pynini.union(*string.digits).optimize() +NONZERO = pynini.union(*NONZERO_DIGITS).optimize() + + +def mapping(forms: Forms) -> pynini.Fst: + return pynini.string_map(forms).optimize() if forms else pynini.Fst() + + +def integer_domain(max_value: int) -> pynini.Fst: + """Canonical positive decimal strings <= bound, without integer enumeration.""" + if isinstance(max_value, bool) or not isinstance(max_value, int) or max_value < 1: + raise ValueError("max_value must be a positive integer") + bound = str(max_value) + branches = [pynini.accep(bound)] + if len(bound) > 1: + branches.append(NONZERO + pynini.closure(DIGIT, 0, len(bound) - 2)) + for index, limit in enumerate(bound): + minimum = 1 if index == 0 else 0 + if int(limit) > minimum: + lower = pynini.union(*(str(n) for n in range(minimum, int(limit)))) + branches.append( + pynini.accep(bound[:index]) + + lower + + pynini.closure(DIGIT, len(bound) - index - 1, len(bound) - index - 1) + ) + return pynini.union(*branches).optimize() + + # Internal markers, removed before output. Control characters, so they never clash with input text. RUN_START = "\x01" # start of a run of number words RUN_END = "\x02" # end of a run of number words diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py index a1f5be6e4..ae462929c 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py @@ -16,16 +16,64 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import money_cue_word, run, token, word_token -from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile, invariant_context -from nemo_text_processing.inverse_text_normalization.bantu.taggers.number import build_cardinal -from nemo_text_processing.inverse_text_normalization.bantu.taggers.scale import ( - build_noun_scale_amount, - build_scale_amount, +from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import ( + DIGIT, + NONZERO, + NONZERO_DIGITS, + integer_domain, + mapping, + money_cue_word, + run, + token, + word_token, +) +from nemo_text_processing.inverse_text_normalization.bantu.profile import ( + CardinalProfile, + NounContext, + invariant_context, ) from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE, GraphFst +def build_cardinal(profile: CardinalProfile, context: NounContext) -> pynini.Fst: + """Compose fixed-width places, supply skipped zeros, then canonicalize once.""" + atomic = mapping(context.atomic_units) + if not context.compounds: + return (atomic @ integer_domain(profile.max_value)).optimize() + units = mapping(context.final_units) + lower = units + expected_power = 1 + for place in profile.places: + if place.power != expected_power: + raise ValueError("places must be consecutive ascending decimal powers") + if any(value not in NONZERO_DIGITS or len(value) != 1 for _, value in place.forms): + raise ValueError("place coefficients must be single nonzero digits") + # A present high place either consumes a smaller remainder or emits zeros. + high = mapping(place.forms) + ( + pynutil.delete(place.remainder_join) + lower | pynutil.insert("0" * place.power) + ) + if place.power == 1: + high |= pynini.cross(profile.ten, "10") + high |= pynini.cross(profile.ten, "1") + pynutil.delete(profile.teen_join) + units + high |= mapping(context.teen_variants) + lower = (pynutil.insert("0") + lower | high).optimize() + expected_power += 1 + canonical = pynutil.delete(pynini.closure("0")) + NONZERO + pynini.closure(DIGIT) + compounds = lower @ canonical + # A final-one form need not be licensed as a standalone plural count. + compounds @= NONZERO + pynini.closure(DIGIT, 1) + result = atomic | compounds | mapping(profile.exact_boundaries) + return (result @ integer_domain(profile.max_value)).optimize() + + +def build_role(profile: CardinalProfile, role: str) -> pynini.Fst: + """Explicit role probe; not a standalone sentence or currency grammar.""" + for name, forms in profile.roles: + if name == role: + return mapping(forms) + raise KeyError(role) + + class CardinalFst(GraphFst): """ Finite state transducer for classifying a counted number: a noun followed by a @@ -37,6 +85,10 @@ class CardinalFst(GraphFst): 20,000 goats) is also a counted number. Here the count may only end in forms that are the same in every class, so no agreement with an unlisted noun is guessed. + The graphs other classes reuse are kept as attributes: + by_class: the agreeing number graph for each noun class that compounds + roles: each role's graph (e.g. a currency-only reading), bounded by the profile maximum + Args: profile: the language profile ordinary_word: any word that is not a number word @@ -44,13 +96,28 @@ class CardinalFst(GraphFst): def __init__(self, profile: CardinalProfile, ordinary_word: pynini.Fst): super().__init__(name="cardinal", kind="classify") + # Imported here: the scale builders use build_cardinal from this module. + from nemo_text_processing.inverse_text_normalization.bantu.taggers.scale import ( + build_noun_scale_amount, + build_scale_amount, + ) + + self.profile = profile + numbers = [build_cardinal(profile, context) for context in profile.contexts] + self.by_class = {} + for context, number in zip(profile.contexts, numbers): + if context.compounds: + self.by_class.setdefault(context.class_id, number) + self.roles = {} + for name, _ in profile.roles: + self.roles.setdefault(name, (build_role(profile, name) @ integer_domain(profile.max_value)).optimize()) def cardinal_token(number): return token(self.add_tokens(pynutil.insert('integer: "') + number + pynutil.insert('"'))) graphs = [] - for context in profile.contexts: - number = build_cardinal(profile, context) | build_noun_scale_amount(profile, context) + for context, number in zip(profile.contexts, numbers): + number = number | build_noun_scale_amount(profile, context) graphs.append(word_token(pynini.accep(context.noun)) + NEMO_SPACE + run(cardinal_token(number))) # Listed nouns are counted by their own cardinal above: a profile covers large diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/money.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/money.py index d435aca76..78ad943e0 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/money.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/money.py @@ -17,25 +17,20 @@ from pynini.lib import pynutil from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import CUE_ONLY, IMPLIED, run, token -from nemo_text_processing.inverse_text_normalization.bantu.profile import ( - CardinalProfile, - CurrencyContext, - agreement_context, -) -from nemo_text_processing.inverse_text_normalization.bantu.taggers.number import ( - build_cardinal, - build_role, - integer_domain, -) +from nemo_text_processing.inverse_text_normalization.bantu.profile import CurrencyContext, agreement_context +from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import CardinalFst from nemo_text_processing.inverse_text_normalization.bantu.taggers.scale import build_scale_amount from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE, GraphFst -def build_amount(profile: CardinalProfile, currency: CurrencyContext) -> pynini.Fst: +def build_amount(cardinal: CardinalFst, currency: CurrencyContext) -> pynini.Fst: """Digits for an amount after this currency phrase: agreeing numbers, licensed roles and counted scales.""" - amount = build_cardinal(profile, agreement_context(profile, currency.agreement_class)) + profile = cardinal.profile + if currency.agreement_class not in cardinal.by_class: + agreement_context(profile, currency.agreement_class) # raises a clear error + amount = cardinal.by_class[currency.agreement_class].copy() # |= below edits in place for role in currency.roles: - amount |= build_role(profile, role) @ integer_domain(profile.max_value) + amount |= cardinal.roles[role] for scale in profile.counted_scales: amount |= build_scale_amount(profile, scale) return amount.optimize() @@ -56,11 +51,12 @@ class MoneyFst(GraphFst): once the whole sentence is tagged. Args: - profile: the language profile + cardinal: the CardinalFst, whose number graphs the amounts reuse """ - def __init__(self, profile: CardinalProfile): + def __init__(self, cardinal: CardinalFst): super().__init__(name="money", kind="classify") + profile = cardinal.profile def money_token(amount, code, marker=""): fields = pynutil.insert('integer_part: "') + amount + pynutil.insert(f'" currency: "{code}"') @@ -70,7 +66,7 @@ def money_token(amount, code, marker=""): for currency in profile.currencies: graphs.append( pynutil.delete(currency.phrase + NEMO_SPACE) - + money_token(build_amount(profile, currency), currency.code) + + money_token(build_amount(cardinal, currency), currency.code) ) self.has_implied_currency = False diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/number.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/number.py deleted file mode 100644 index f4ea3ca6c..000000000 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/number.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# Copyright (c) 2026, Crane Labs. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Place-value number FSTs built from a profile, shared by the cardinal, money and scale taggers. - -Values are composed digit by digit inside the FST; there is no runtime arithmetic. -""" -import string - -import pynini -from pynini.lib import pynutil - -from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile, Forms, NounContext - -NONZERO_DIGITS = string.digits[1:] -DIGIT = pynini.union(*string.digits).optimize() -NONZERO = pynini.union(*NONZERO_DIGITS).optimize() - - -def mapping(forms: Forms) -> pynini.Fst: - return pynini.string_map(forms).optimize() if forms else pynini.Fst() - - -def integer_domain(max_value: int) -> pynini.Fst: - """Canonical positive decimal strings <= bound, without integer enumeration.""" - if isinstance(max_value, bool) or not isinstance(max_value, int) or max_value < 1: - raise ValueError("max_value must be a positive integer") - bound = str(max_value) - branches = [pynini.accep(bound)] - if len(bound) > 1: - branches.append(NONZERO + pynini.closure(DIGIT, 0, len(bound) - 2)) - for index, limit in enumerate(bound): - minimum = 1 if index == 0 else 0 - if int(limit) > minimum: - lower = pynini.union(*(str(n) for n in range(minimum, int(limit)))) - branches.append( - pynini.accep(bound[:index]) - + lower - + pynini.closure(DIGIT, len(bound) - index - 1, len(bound) - index - 1) - ) - return pynini.union(*branches).optimize() - - -def build_cardinal(profile: CardinalProfile, context: NounContext) -> pynini.Fst: - """Compose fixed-width places, supply skipped zeros, then canonicalize once.""" - atomic = mapping(context.atomic_units) - if not context.compounds: - return (atomic @ integer_domain(profile.max_value)).optimize() - units = mapping(context.final_units) - lower = units - expected_power = 1 - for place in profile.places: - if place.power != expected_power: - raise ValueError("places must be consecutive ascending decimal powers") - if any(value not in NONZERO_DIGITS or len(value) != 1 for _, value in place.forms): - raise ValueError("place coefficients must be single nonzero digits") - # A present high place either consumes a smaller remainder or emits zeros. - high = mapping(place.forms) + ( - pynutil.delete(place.remainder_join) + lower | pynutil.insert("0" * place.power) - ) - if place.power == 1: - high |= pynini.cross(profile.ten, "10") - high |= pynini.cross(profile.ten, "1") + pynutil.delete(profile.teen_join) + units - high |= mapping(context.teen_variants) - lower = (pynutil.insert("0") + lower | high).optimize() - expected_power += 1 - canonical = pynutil.delete(pynini.closure("0")) + NONZERO + pynini.closure(DIGIT) - compounds = lower @ canonical - # A final-one form need not be licensed as a standalone plural count. - compounds @= NONZERO + pynini.closure(DIGIT, 1) - result = atomic | compounds | mapping(profile.exact_boundaries) - return (result @ integer_domain(profile.max_value)).optimize() - - -def build_role(profile: CardinalProfile, role: str) -> pynini.Fst: - """Explicit role probe; not a standalone sentence or currency grammar.""" - for name, forms in profile.roles: - if name == role: - return mapping(forms) - raise KeyError(role) diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/scale.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/scale.py index e08c33e1c..aaa269163 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/scale.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/scale.py @@ -28,19 +28,14 @@ import pynini from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import DIGIT, NONZERO, integer_domain, mapping from nemo_text_processing.inverse_text_normalization.bantu.profile import ( CardinalProfile, CountedScale, NounContext, agreement_context, ) -from nemo_text_processing.inverse_text_normalization.bantu.taggers.number import ( - DIGIT, - NONZERO, - build_cardinal, - integer_domain, - mapping, -) +from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import build_cardinal from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py index 95e82375e..86a42bb52 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py @@ -68,7 +68,7 @@ def __init__(self, profile: CardinalProfile, cache_dir: str = None, overwrite_ca logger.info(f"Creating ClassifyFst grammars for {profile.language}.") number_word, ordinary_word, segmenter = join_aware_runs(profile.number_words(), profile.join_words()) cardinal = CardinalFst(profile, ordinary_word) - money = MoneyFst(profile) + money = MoneyFst(cardinal) word = WordFst(number_word, ordinary_word) token = pynini.union(cardinal.fst, money.fst, word.fst) diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py index 66dd51bba..f19570445 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py @@ -16,7 +16,7 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.inverse_text_normalization.bantu.taggers.number import DIGIT, integer_domain +from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import DIGIT, integer_domain from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_space diff --git a/tests/nemo_text_processing/bantu/test_profile.py b/tests/nemo_text_processing/bantu/test_profile.py index 6787c95dc..12826deaa 100644 --- a/tests/nemo_text_processing/bantu/test_profile.py +++ b/tests/nemo_text_processing/bantu/test_profile.py @@ -19,8 +19,9 @@ import pytest +from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import integer_domain from nemo_text_processing.inverse_text_normalization.bantu.profile import MoneyCues, Place -from nemo_text_processing.inverse_text_normalization.bantu.taggers.number import build_role, integer_domain +from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import build_role from nemo_text_processing.inverse_text_normalization.bantu.taggers.scale import build_scale_amount from nemo_text_processing.inverse_text_normalization.bantu.taggers.tokenize_and_classify import ClassifyFst @@ -90,3 +91,20 @@ def test_far_cache_roundtrip(self, tmp_path): expected = normalizer.inverse_normalize(spoken, verbose=False) normalizer.tagger = restored assert normalizer.inverse_normalize(spoken, verbose=False) == expected + + # Money reuses the cardinal's graphs, as in other NeMo languages. + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_money_takes_the_cardinal(self): + from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import join_aware_runs + from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import CardinalFst + from nemo_text_processing.inverse_text_normalization.bantu.taggers.money import MoneyFst + + _, ordinary_word, _ = join_aware_runs(TOY_PROFILE.number_words(), TOY_PROFILE.join_words()) + cardinal = CardinalFst(TOY_PROFILE, ordinary_word) + assert cardinal.profile is TOY_PROFILE + assert set(cardinal.by_class) == {"2", "4", "6", "8"} + assert set(cardinal.roles) == {"short-tens"} + money = MoneyFst(cardinal) + assert money.has_implied_currency + assert money.fst.num_states() > 0 From e134e962d0e9ce29da06395969b3c4cbabea8ffc Mon Sep 17 00:00:00 2001 From: Cal3bdev Date: Thu, 5 Nov 2026 13:31:45 +0300 Subject: [PATCH 6/9] Bantu ITN kernel: plain builder functions as entry points build_classifier(profile) and build_final_verbalizer(max_value, thousands_separator) replace the kernel's ClassifyFst and VerbalizeFinalFst classes, so each language keeps NeMo's standard constructors and calls these. format_integer moves to bantu/graph_utils.py. Output is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Cal3bdev --- .../bantu/graph_utils.py | 8 +++ .../bantu/taggers/tokenize_and_classify.py | 56 +++++-------------- .../bantu/verbalizers/cardinal.py | 11 +--- .../bantu/verbalizers/money.py | 2 +- .../bantu/verbalizers/verbalize.py | 18 +++--- .../bantu/verbalizers/verbalize_final.py | 39 ++++++------- .../bantu/test_profile.py | 26 +++------ .../bantu/test_verbalizer.py | 4 +- tests/nemo_text_processing/bantu/utils.py | 10 ++-- 9 files changed, 67 insertions(+), 107 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/bantu/graph_utils.py b/nemo_text_processing/inverse_text_normalization/bantu/graph_utils.py index 80d381471..15e852e54 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/graph_utils.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/graph_utils.py @@ -65,6 +65,14 @@ def integer_domain(max_value: int) -> pynini.Fst: return pynini.union(*branches).optimize() +def format_integer(max_value: int, thousands_separator: str) -> "pynini.FstLike": + """Groups digits in threes, e.g. 25000 -> 25,000. Rejects values outside 1..max_value.""" + first = pynini.closure(DIGIT, 1, 3) + triple = pynini.closure(DIGIT, 3, 3) + grouped = first + pynini.closure(pynutil.insert(thousands_separator) + triple) + return (integer_domain(max_value) @ grouped).optimize() + + # Internal markers, removed before output. Control characters, so they never clash with input text. RUN_START = "\x01" # start of a run of number words RUN_END = "\x02" # end of a run of number words diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py index 86a42bb52..ea0551555 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - import pynini from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import ( @@ -26,18 +24,13 @@ from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import CardinalFst from nemo_text_processing.inverse_text_normalization.bantu.taggers.money import MoneyFst from nemo_text_processing.inverse_text_normalization.bantu.taggers.word import WordFst -from nemo_text_processing.text_normalization.en.graph_utils import ( - INPUT_LOWER_CASED, - NEMO_SPACE, - GraphFst, - generator_main, -) -from nemo_text_processing.utils.logging import logger +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE -class ClassifyFst(GraphFst): +def build_classifier(profile: CardinalProfile) -> pynini.Fst: """ - Sentence-level classifier for a Bantu language, configured by a language profile. + Sentence-level classifier for a Bantu language, built from its profile. Each + language's ClassifyFst calls this and handles caching. Unlike most languages, the whole sentence is read at once: number words are resolved in their context (the counted noun, a currency phrase, a money cue, @@ -50,34 +43,15 @@ class ClassifyFst(GraphFst): Args: profile: the language profile (word lists, agreement, scales, currencies) - cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. - overwrite_cache: set to True to overwrite .far files """ - - def __init__(self, profile: CardinalProfile, cache_dir: str = None, overwrite_cache: bool = False): - super().__init__(name="tokenize_and_classify", kind="classify") - - far_file = None - if cache_dir is not None and cache_dir != "None": - os.makedirs(cache_dir, exist_ok=True) - far_file = os.path.join(cache_dir, f"{profile.language}_itn_{INPUT_LOWER_CASED}.far") - if not overwrite_cache and far_file and os.path.exists(far_file): - self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logger.info(f"ClassifyFst.fst was restored from {far_file}.") - else: - logger.info(f"Creating ClassifyFst grammars for {profile.language}.") - number_word, ordinary_word, segmenter = join_aware_runs(profile.number_words(), profile.join_words()) - cardinal = CardinalFst(profile, ordinary_word) - money = MoneyFst(cardinal) - word = WordFst(number_word, ordinary_word) - - token = pynini.union(cardinal.fst, money.fst, word.fst) - sentence = token + pynini.closure(NEMO_SPACE + token) - graph = SINGLE_SPACES @ segmenter @ sentence - if money.has_implied_currency: - graph @= only_with_money_signal(profile) - self.fst = graph.optimize() - - if far_file: - generator_main(far_file, {"tokenize_and_classify": self.fst}) - logger.info(f"ClassifyFst grammars are saved to {far_file}.") + number_word, ordinary_word, segmenter = join_aware_runs(profile.number_words(), profile.join_words()) + cardinal = CardinalFst(profile, ordinary_word) + money = MoneyFst(cardinal) + word = WordFst(number_word, ordinary_word) + + token = pynini.union(cardinal.fst, money.fst, word.fst) + sentence = token + pynini.closure(NEMO_SPACE + token) + graph = SINGLE_SPACES @ segmenter @ sentence + if money.has_implied_currency: + graph @= only_with_money_signal(profile) + return graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py index f19570445..386ff23c9 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py @@ -13,21 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pynini from pynini.lib import pynutil -from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import DIGIT, integer_domain +from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import format_integer from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_space -def format_integer(max_value: int, thousands_separator: str) -> "pynini.FstLike": - """Groups digits in threes, e.g. 25000 -> 25,000. Rejects values outside 1..max_value.""" - first = pynini.closure(DIGIT, 1, 3) - triple = pynini.closure(DIGIT, 3, 3) - grouped = first + pynini.closure(pynutil.insert(thousands_separator) + triple) - return (integer_domain(max_value) @ grouped).optimize() - - class CardinalFst(GraphFst): """ Finite state transducer for verbalizing a resolved cardinal, e.g. diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/money.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/money.py index 439991465..6edc64835 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/money.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/money.py @@ -16,7 +16,7 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.cardinal import format_integer +from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import format_integer from nemo_text_processing.text_normalization.en.graph_utils import NEMO_UPPER, GraphFst, delete_space, insert_space diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize.py index cf923ea07..35d162710 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize.py @@ -13,23 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile +import pynini + from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.cardinal import CardinalFst from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.money import MoneyFst -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst -class VerbalizeFst(GraphFst): +def build_verbalizer(max_value: int, thousands_separator: str) -> pynini.Fst: """ Composes the verbalizer grammars. Values arrive fully resolved from the classifier, so verbalizing is formatting only. Args: - profile: the language profile (sets the largest value and the thousands separator) + max_value: largest value the language allows + thousands_separator: written separator between digit groups """ - - def __init__(self, profile: CardinalProfile): - super().__init__(name="verbalize", kind="verbalize") - cardinal = CardinalFst(profile.max_value, profile.thousands_separator).fst - money = MoneyFst(profile.max_value, profile.thousands_separator).fst - self.fst = (cardinal | money).optimize() + cardinal = CardinalFst(max_value, thousands_separator).fst + money = MoneyFst(max_value, thousands_separator).fst + return (cardinal | money).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize_final.py index 23f173e4a..61f8c6cbe 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize_final.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize_final.py @@ -16,31 +16,28 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile -from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.verbalize import build_verbalizer from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.word import WordFst -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space +from nemo_text_processing.text_normalization.en.graph_utils import delete_extra_space, delete_space -class VerbalizeFinalFst(GraphFst): +def build_final_verbalizer(max_value: int, thousands_separator: str) -> pynini.Fst: """ - Finite state transducer that verbalizes an entire sentence. + Verbalizes an entire sentence. Each language's VerbalizeFinalFst calls this. Args: - profile: the language profile (sets the largest value and the thousands separator) + max_value: largest value the language allows + thousands_separator: written separator between digit groups """ - - def __init__(self, profile: CardinalProfile): - super().__init__(name="verbalize_final", kind="verbalize") - types = VerbalizeFst(profile).fst | WordFst().fst - graph = ( - pynutil.delete("tokens") - + delete_space - + pynutil.delete("{") - + delete_space - + types - + delete_space - + pynutil.delete("}") - ) - graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space - self.fst = graph.optimize() + types = build_verbalizer(max_value, thousands_separator) | WordFst().fst + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + return graph.optimize() diff --git a/tests/nemo_text_processing/bantu/test_profile.py b/tests/nemo_text_processing/bantu/test_profile.py index 12826deaa..386540b93 100644 --- a/tests/nemo_text_processing/bantu/test_profile.py +++ b/tests/nemo_text_processing/bantu/test_profile.py @@ -23,7 +23,7 @@ from nemo_text_processing.inverse_text_normalization.bantu.profile import MoneyCues, Place from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import build_role from nemo_text_processing.inverse_text_normalization.bantu.taggers.scale import build_scale_amount -from nemo_text_processing.inverse_text_normalization.bantu.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.bantu.taggers.tokenize_and_classify import build_classifier from .toy_profile import TOY_PROFILE from .utils import build_normalizer @@ -35,11 +35,15 @@ class TestBantuKernelProfile: @pytest.mark.unit def test_invalid_profiles_are_rejected(self): with pytest.raises(ValueError, match="no compound agreement context"): - ClassifyFst(replace(TOY_PROFILE, currencies=(replace(TOY_PROFILE.currencies[0], agreement_class="99"),))) + build_classifier( + replace(TOY_PROFILE, currencies=(replace(TOY_PROFILE.currencies[0], agreement_class="99"),)) + ) with pytest.raises(ValueError, match="consecutive ascending"): - ClassifyFst(replace(TOY_PROFILE, places=TOY_PROFILE.places[1:])) + build_classifier(replace(TOY_PROFILE, places=TOY_PROFILE.places[1:])) with pytest.raises(ValueError, match="single nonzero digits"): - ClassifyFst(replace(TOY_PROFILE, places=(Place(1, (("pideka", "20"),), " vo "),) + TOY_PROFILE.places[1:])) + build_classifier( + replace(TOY_PROFILE, places=(Place(1, (("pideka", "20"),), " vo "),) + TOY_PROFILE.places[1:]) + ) with pytest.raises(ValueError, match="positive integer"): integer_domain(0) with pytest.raises(KeyError): @@ -78,20 +82,6 @@ def test_scale_without_counting_heads(self): bare_only = replace(TOY_PROFILE.counted_scales[0], counting_plural="", counting_ones=()) assert build_scale_amount(TOY_PROFILE, bare_only, "counting").num_states() == 0 - # The compiled classifier is saved to and restored from a FAR file with the same results. - @pytest.mark.run_only_on('CPU') - @pytest.mark.unit - def test_far_cache_roundtrip(self, tmp_path): - built = ClassifyFst(TOY_PROFILE, cache_dir=str(tmp_path), overwrite_cache=True) - assert (tmp_path / "toy_itn_lower_cased.far").exists() - restored = ClassifyFst(TOY_PROFILE, cache_dir=str(tmp_path)) - normalizer = build_normalizer(TOY_PROFILE) - for spoken in ("bakopa pideka vo baima", "kasa emina epi", "lipoka obumega bapi", "bakopa bapi vo kasa"): - normalizer.tagger = built - expected = normalizer.inverse_normalize(spoken, verbose=False) - normalizer.tagger = restored - assert normalizer.inverse_normalize(spoken, verbose=False) == expected - # Money reuses the cardinal's graphs, as in other NeMo languages. @pytest.mark.run_only_on('CPU') @pytest.mark.unit diff --git a/tests/nemo_text_processing/bantu/test_verbalizer.py b/tests/nemo_text_processing/bantu/test_verbalizer.py index fb130e839..2f0ed0618 100644 --- a/tests/nemo_text_processing/bantu/test_verbalizer.py +++ b/tests/nemo_text_processing/bantu/test_verbalizer.py @@ -19,11 +19,11 @@ from parameterized import parameterized from pynini.lib import rewrite -from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.verbalize_final import VerbalizeFinalFst +from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.verbalize_final import build_final_verbalizer from .toy_profile import TOY_PROFILE -VERBALIZER = VerbalizeFinalFst(TOY_PROFILE).fst +VERBALIZER = build_final_verbalizer(TOY_PROFILE.max_value, TOY_PROFILE.thousands_separator) class TestBantuKernelVerbalizer: diff --git a/tests/nemo_text_processing/bantu/utils.py b/tests/nemo_text_processing/bantu/utils.py index 44079d4d8..c6f65ebc5 100644 --- a/tests/nemo_text_processing/bantu/utils.py +++ b/tests/nemo_text_processing/bantu/utils.py @@ -13,10 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from types import SimpleNamespace + from pynini.lib import rewrite -from nemo_text_processing.inverse_text_normalization.bantu.taggers.tokenize_and_classify import ClassifyFst -from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.verbalize_final import VerbalizeFinalFst +from nemo_text_processing.inverse_text_normalization.bantu.taggers.tokenize_and_classify import build_classifier +from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.verbalize_final import build_final_verbalizer from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.token_parser import TokenParser @@ -30,8 +32,8 @@ def build_normalizer(profile) -> InverseNormalizer: """ normalizer = InverseNormalizer.__new__(InverseNormalizer) normalizer.lang = profile.language - normalizer.tagger = ClassifyFst(profile) - normalizer.verbalizer = VerbalizeFinalFst(profile) + normalizer.tagger = SimpleNamespace(fst=build_classifier(profile)) + normalizer.verbalizer = SimpleNamespace(fst=build_final_verbalizer(profile.max_value, profile.thousands_separator)) normalizer.parser = TokenParser() normalizer.max_number_of_permutations_per_split = 729 # InverseNormalizer default normalizer.post_processor = None From 886a5890b87cc046f79600e5e15d6fbed3de0455 Mon Sep 17 00:00:00 2001 From: Cal3bdev Date: Thu, 5 Nov 2026 13:34:05 +0300 Subject: [PATCH 7/9] Bantu ITN: shared TSV loader; test language data as TSVs bantu/data_loader.py fills a language's profile skeleton (the shape of its grammar, written in settings.py) with the word lists in its data/*.tsv files. The test language's word lists move from tests/.../toy_profile.py to inverse_text_normalization/toy/data/, and its skeleton to toy/settings.py; the loaded profile equals the old one field by field. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Cal3bdev --- .../bantu/data_loader.py | 150 ++++++++++++++ .../toy/__init__.py | 14 ++ .../toy/data/__init__.py | 14 ++ .../toy/data/money/__init__.py | 14 ++ .../toy/data/money/cues.tsv | 3 + .../toy/data/money/currency.tsv | 5 + .../toy/data/numbers/__init__.py | 14 ++ .../toy/data/numbers/exact.tsv | 1 + .../toy/data/numbers/held_back.tsv | 1 + .../toy/data/numbers/places.tsv | 35 ++++ .../toy/data/numbers/roles.tsv | 1 + .../toy/data/numbers/scale_words.tsv | 28 +++ .../toy/data/numbers/spelling_variants.tsv | 1 + .../toy/data/numbers/units.tsv | 41 ++++ .../toy/settings.py | 115 +++++++++++ .../bantu/test_data_loader.py | 185 ++++++++++++++++++ 16 files changed, 622 insertions(+) create mode 100644 nemo_text_processing/inverse_text_normalization/bantu/data_loader.py create mode 100644 nemo_text_processing/inverse_text_normalization/toy/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/toy/data/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/toy/data/money/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/toy/data/money/cues.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/toy/data/money/currency.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/toy/data/numbers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/toy/data/numbers/exact.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/toy/data/numbers/held_back.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/toy/data/numbers/places.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/toy/data/numbers/roles.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/toy/data/numbers/scale_words.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/toy/data/numbers/spelling_variants.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/toy/data/numbers/units.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/toy/settings.py create mode 100644 tests/nemo_text_processing/bantu/test_data_loader.py diff --git a/nemo_text_processing/inverse_text_normalization/bantu/data_loader.py b/nemo_text_processing/inverse_text_normalization/bantu/data_loader.py new file mode 100644 index 000000000..961d0dc63 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/data_loader.py @@ -0,0 +1,150 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fills a language's profile skeleton with the word lists in its data folder. + +Each language writes the shape of its grammar in settings.py: a CardinalProfile whose +word lists are empty (noun classes, places, counted scales with their head words and +switches). Its word lists live in tab-separated files under data/; every file is +optional, and a missing file means an empty list: + + numbers/units.tsv form, value, class, where (any, atomic, final, teen) + numbers/places.tsv form, coefficient, power + numbers/exact.tsv form, value + numbers/roles.tsv form, value, role + numbers/scale_words.tsv scale plural, part (tens, ones, hundreds), form, value + numbers/held_back.tsv word + numbers/spelling_variants.tsv word, variant + money/currency.tsv phrase, code, class, roles (comma-separated, may be empty) + money/cues.tsv word, kind (stem, word, not) + +A "where" of any puts a unit form both inside compounds (atomic) and at the end of a +number (final); teen forms are whole teen words such as a contracted 15. +""" +import csv +import os +from collections import OrderedDict +from dataclasses import replace +from typing import Dict, List, Tuple + +from nemo_text_processing.inverse_text_normalization.bantu.profile import ( + CardinalProfile, + CurrencyContext, + MoneyCues, + with_spelling_variants, +) + +WHERE = { + "any": ("atomic_units", "final_units"), + "atomic": ("atomic_units",), + "final": ("final_units",), + "teen": ("teen_variants",), +} +SCALE_PARTS = {"tens": "tens", "ones": "count_ones", "hundreds": "count_hundreds"} +CUE_KINDS = {"stem": "stems", "word": "words", "not": "not_cues"} + + +def read_rows(data_dir: str, name: str, columns: int) -> List[Tuple[str, List[str]]]: + """Non-blank rows of one TSV as (location, row); location is "path:line" for error messages.""" + path = os.path.join(data_dir, name) + if not os.path.exists(path): + return [] + rows = [] + with open(path, encoding="utf-8", newline="") as tsv: + for line, row in enumerate(csv.reader(tsv, delimiter="\t"), start=1): + if not row: + continue + if len(row) != columns: + raise ValueError(f"{path}:{line}: expected {columns} columns, found {len(row)}") + rows.append((f"{path}:{line}", row)) + return rows + + +def lookup(table: Dict[str, str], key: str, location: str, what: str) -> str: + if key not in table: + raise ValueError(f"{location}: unknown {what} {key!r}") + return table[key] + + +def load_profile(data_dir: str, skeleton: CardinalProfile) -> CardinalProfile: + """ + Returns the skeleton with its word lists filled from data_dir. + + Args: + data_dir: the language's data folder + skeleton: the language's profile with empty word lists (from its settings.py) + """ + contexts = { + context.class_id: {"atomic_units": [], "final_units": [], "teen_variants": []} for context in skeleton.contexts + } + for location, (form, value, class_id, where) in read_rows(data_dir, "numbers/units.tsv", 4): + lists = contexts.get(class_id) + if lists is None: + raise ValueError(f"{location}: unknown class {class_id!r}") + for field in lookup(WHERE, where, location, "where"): + lists[field].append((form, value)) + + places = {str(place.power): [] for place in skeleton.places} + for location, (form, value, power) in read_rows(data_dir, "numbers/places.tsv", 3): + if power not in places: + raise ValueError(f"{location}: unknown power {power!r}") + places[power].append((form, value)) + + scales = {scale.plural: {field: [] for field in SCALE_PARTS.values()} for scale in skeleton.counted_scales} + for location, (plural, part, form, value) in read_rows(data_dir, "numbers/scale_words.tsv", 4): + if plural not in scales: + raise ValueError(f"{location}: unknown scale {plural!r}") + scales[plural][lookup(SCALE_PARTS, part, location, "part")].append((form, value)) + + roles = OrderedDict() + for _, (form, value, role) in read_rows(data_dir, "numbers/roles.tsv", 3): + roles.setdefault(role, []).append((form, value)) + + known_classes = {context.class_id for context in skeleton.contexts} + currencies = [] + for location, (phrase, code, class_id, currency_roles) in read_rows(data_dir, "money/currency.tsv", 4): + if class_id not in known_classes: + raise ValueError(f"{location}: unknown class {class_id!r}") + names = tuple(name for name in currency_roles.split(",") if name) + currencies.append(CurrencyContext(phrase, code, class_id, names)) + + cues = {field: [] for field in CUE_KINDS.values()} + for location, (word, kind) in read_rows(data_dir, "money/cues.tsv", 2): + cues[lookup(CUE_KINDS, kind, location, "kind")].append(word) + + variants = OrderedDict() + for _, (word, variant) in read_rows(data_dir, "numbers/spelling_variants.tsv", 2): + variants.setdefault(word, []).append(variant) + + profile = replace( + skeleton, + contexts=tuple( + replace(context, **{field: tuple(forms) for field, forms in contexts[context.class_id].items()}) + for context in skeleton.contexts + ), + places=tuple(replace(place, forms=tuple(places[str(place.power)])) for place in skeleton.places), + counted_scales=tuple( + replace(scale, **{field: tuple(forms) for field, forms in scales[scale.plural].items()}) + for scale in skeleton.counted_scales + ), + exact_boundaries=tuple(tuple(row) for _, row in read_rows(data_dir, "numbers/exact.tsv", 2)), + roles=tuple((role, tuple(forms)) for role, forms in roles.items()), + currencies=tuple(currencies), + money_cues=MoneyCues(**{field: tuple(words) for field, words in cues.items()}), + held_back_words=tuple(row[0] for _, row in read_rows(data_dir, "numbers/held_back.tsv", 1)), + ) + if variants: + profile = with_spelling_variants(profile, {word: tuple(v) for word, v in variants.items()}) + return profile diff --git a/nemo_text_processing/inverse_text_normalization/toy/__init__.py b/nemo_text_processing/inverse_text_normalization/toy/__init__.py new file mode 100644 index 000000000..eb60428b9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/__init__.py b/nemo_text_processing/inverse_text_normalization/toy/data/__init__.py new file mode 100644 index 000000000..eb60428b9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/money/__init__.py b/nemo_text_processing/inverse_text_normalization/toy/data/money/__init__.py new file mode 100644 index 000000000..eb60428b9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/money/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/money/cues.tsv b/nemo_text_processing/inverse_text_normalization/toy/data/money/cues.tsv new file mode 100644 index 000000000..73f1393f9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/money/cues.tsv @@ -0,0 +1,3 @@ +lipo stem +dalo word +olipon not diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/money/currency.tsv b/nemo_text_processing/inverse_text_normalization/toy/data/money/currency.tsv new file mode 100644 index 000000000..36594f785 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/money/currency.tsv @@ -0,0 +1,5 @@ +koin XTS 4 short-tens +dolari USD 4 +paundi GBP 4 +yuro EUR 6 +yeni JPY 8 diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/numbers/__init__.py b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/__init__.py new file mode 100644 index 000000000..eb60428b9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/numbers/exact.tsv b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/exact.tsv new file mode 100644 index 000000000..a33d66765 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/exact.tsv @@ -0,0 +1 @@ +tausi 1000 diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/numbers/held_back.tsv b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/held_back.tsv new file mode 100644 index 000000000..171e78a3a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/held_back.tsv @@ -0,0 +1 @@ +halfu diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/numbers/places.tsv b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/places.tsv new file mode 100644 index 000000000..252102e5b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/places.tsv @@ -0,0 +1,35 @@ +pideka 2 1 +kodeka 3 1 +dedeka 4 1 +fudeka 5 1 +seideka 6 1 +sitideka 7 1 +otodeka 8 1 +nonadeka 9 1 +hanto 1 2 +hanto pi 2 2 +hanto ko 3 2 +hanto de 4 2 +hanto fu 5 2 +hanto sei 6 2 +hanto siti 7 2 +hanto oto 8 2 +hanto nona 9 2 +milo 1 3 +milo pi 2 3 +milo ko 3 3 +milo de 4 3 +milo fu 5 3 +milo sei 6 3 +milo siti 7 3 +milo oto 8 3 +milo nona 9 3 +omuna 1 4 +emina epi 2 4 +emina eko 3 4 +emina ede 4 4 +emina efu 5 4 +emina sei 6 4 +emina siti 7 4 +emina oto 8 4 +emina nona 9 4 diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/numbers/roles.tsv b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/roles.tsv new file mode 100644 index 000000000..7ba9e4c91 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/roles.tsv @@ -0,0 +1 @@ +mapi 20 short-tens diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/numbers/scale_words.tsv b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/scale_words.tsv new file mode 100644 index 000000000..43b76cfae --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/scale_words.tsv @@ -0,0 +1,28 @@ +mina tens deka 1 +mina tens pideka 2 +mina tens kodeka 3 +mina tens dedeka 4 +mina tens fudeka 5 +mina tens seideka 6 +mina tens sitideka 7 +mina tens otodeka 8 +mina tens nonadeka 9 +bumega tens deka 1 +bumega tens pideka 2 +bumega tens kodeka 3 +bumega tens dedeka 4 +bumega tens fudeka 5 +bumega tens seideka 6 +bumega tens sitideka 7 +bumega tens otodeka 8 +bumega tens nonadeka 9 +bumega ones kaima 1 +bumega hundreds hanto 1 +bumega hundreds hanto pi 2 +bumega hundreds hanto ko 3 +bumega hundreds hanto de 4 +bumega hundreds hanto fu 5 +bumega hundreds hanto sei 6 +bumega hundreds hanto siti 7 +bumega hundreds hanto oto 8 +bumega hundreds hanto nona 9 diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/numbers/spelling_variants.tsv b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/spelling_variants.tsv new file mode 100644 index 000000000..1de714fdd --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/spelling_variants.tsv @@ -0,0 +1 @@ +pideka piideka diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/numbers/units.tsv b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/units.tsv new file mode 100644 index 000000000..a69d7a7a7 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/units.tsv @@ -0,0 +1,41 @@ +ima 1 1 atomic +bapi 2 2 any +bako 3 2 any +bade 4 2 any +bafu 5 2 any +sei 6 2 any +siti 7 2 any +oto 8 2 any +nona 9 2 any +deka 10 2 atomic +baima 1 2 final +zipi 2 4 any +ziko 3 4 any +zide 4 4 any +zifu 5 4 any +sei 6 4 any +siti 7 4 any +oto 8 4 any +nona 9 4 any +deka 10 4 atomic +zima 1 4 final +mapi 2 6 any +mako 3 6 any +made 4 6 any +mafu 5 6 any +sei 6 6 any +siti 7 6 any +oto 8 6 any +nona 9 6 any +deka 10 6 atomic +dekafu 15 6 teen +epi 2 8 any +eko 3 8 any +ede 4 8 any +efu 5 8 any +sei 6 8 any +siti 7 8 any +oto 8 8 any +nona 9 8 any +deka 10 8 atomic +ema 1 8 final diff --git a/nemo_text_processing/inverse_text_normalization/toy/settings.py b/nemo_text_processing/inverse_text_normalization/toy/settings.py new file mode 100644 index 000000000..accaffe1c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/settings.py @@ -0,0 +1,115 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A small made-up language for testing the shared Bantu kernel on its own. + +Every word here is invented. The language is shaped like a Bantu language so that +each kernel feature is exercised (Luganda equivalents in brackets): +- noun classes with agreeing number words: bakopa bapi / zikopa zipi = 2 [abantu babiri / emiti ebiri] +- 6-9 and ten do not agree: sei, siti, oto, nona, deka [mukaaga ... kkumi] +- teens join with "ke", larger parts with "vo" [na, mu] +- "vo" is also an ordinary word, "in" [mu] +- a currency phrase that licenses a special reading: koin mapi = 20 XTS, but makopa mapi = 2 [abiri] +- other currency phrases, each agreeing with its own class: dolari zipi = 2 USD, + yuro mafu = 5 EUR, yeni efu = 5 JPY +- a counted scale of ten-thousands: mina + count [mitwalo]; after a listed noun, + emina counts that noun, also as an ordinary number place for 1-9: + bakopa emina epi = 20,000, bakopa emina pideka = 200,000 [abantu emitwalo ebiri / abiri] +- implied currency: bare "mina" is always money; word + "emina" counts that word, + except after a money verb [mitwalo / emitwalo] +- a held-back word that continues a number but is not handled: halfu [n'ekitundu] +- a count class with vowel-initial forms, so ke + epi contracts to k'epi [n'ebiri] +- a larger scale of millions: bumega + count, with augmented forms that count a noun, + "one" forms, hundreds inside the count, and money at the start [obukadde / akakadde] +- a fixed phrase (tausi = 1,000), a special teen form (dekafu = 15) and a spelling + variant (piideka = pideka) [akakadde, kkumi nataano, abiiri] + +A failing test with this language points at the kernel, not at real language data. +XTS is the ISO 4217 code reserved for testing. + +The word lists are in data/; this file holds the shape of the grammar: noun classes, +join words, places, counted scales and their switches. Hidden from the command line. +""" +import os + +from nemo_text_processing.inverse_text_normalization.bantu.data_loader import load_profile +from nemo_text_processing.inverse_text_normalization.bantu.profile import ( + CardinalProfile, + CountedScale, + NounContext, + Place, +) + +DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") + +SKELETON = CardinalProfile( + language="toy", + contexts=( + NounContext("1", "kopa", (), (), compounds=False), + NounContext("2", "bakopa", (), ()), + NounContext("4", "zikopa", (), ()), + NounContext("6", "makopa", (), ()), + NounContext("8", "ekopa", (), ()), + ), + ten="deka", + teen_join=" ke ", + places=(Place(1, (), " vo "), Place(2, (), " vo "), Place(3, (), " vo "), Place(4, (), " vo ")), + exact_boundaries=(), + roles=(), + max_value=999_999_999, + counted_scales=( + # Ten-thousands [mitwalo]: bare mina is money anywhere; emina counts a word. + CountedScale( + singular="muna", + singular_count="ema", + plural="mina", + power=4, + count_class="8", + tens=(), + remainder_class="4", + contracted_teen_joins=("k'", "k"), + implied_currency="XTS", + counting_plural="emina", + implied_anywhere=True, + counts_any_word=True, + after_counted_nouns=True, + ), + # Millions [obukadde]: augmented forms are also money at the start of input, + # count listed nouns with agreement, and need "one" to count (akamega kaima). + CountedScale( + singular="kamega", + singular_count="kaima", + plural="bumega", + power=6, + count_class="2", + tens=(), + remainder_class="4", + implied_currency="XTS", + counting_plural="obumega", + counting_ones=("akamega kaima",), + money_only_ones=("akamega",), + implied_anywhere=True, + counting_money_at_start=True, + counts_any_word=True, + after_counted_nouns=True, + ), + ), +) + + +def load() -> CardinalProfile: + """The test language's full profile: the skeleton above filled from data/.""" + return load_profile(DATA_DIR, SKELETON) diff --git a/tests/nemo_text_processing/bantu/test_data_loader.py b/tests/nemo_text_processing/bantu/test_data_loader.py new file mode 100644 index 000000000..28e0a2f21 --- /dev/null +++ b/tests/nemo_text_processing/bantu/test_data_loader.py @@ -0,0 +1,185 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The shared loader that fills a language's profile skeleton from its TSV files.""" + +import pytest + +from nemo_text_processing.inverse_text_normalization.bantu.data_loader import load_profile +from nemo_text_processing.inverse_text_normalization.bantu.profile import ( + CardinalProfile, + CountedScale, + CurrencyContext, + MoneyCues, + NounContext, + Place, +) + +SKELETON = CardinalProfile( + language="t", + contexts=(NounContext("2", "bakopa", (), ()),), + ten="deka", + teen_join=" ke ", + places=(Place(1, (), " vo "),), + exact_boundaries=(), + roles=(), + max_value=99, + counted_scales=( + CountedScale( + singular="muna", + singular_count="ema", + plural="mina", + power=4, + count_class="2", + tens=(), + remainder_class="2", + ), + ), +) + + +def write(root, files): + for name, rows in files.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join("\t".join(row) + "\n" for row in rows), encoding="utf-8") + return str(root) + + +BASE = { + "numbers/units.tsv": [("bapi", "2", "2", "any"), ("deka", "10", "2", "atomic"), ("baima", "1", "2", "final")], + "numbers/places.tsv": [("pideka", "2", "1")], +} + + +class TestBantuDataLoader: + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_units_fill_atomic_and_final(self, tmp_path): + profile = load_profile(write(tmp_path, BASE), SKELETON) + context = profile.contexts[0] + assert context.atomic_units == (("bapi", "2"), ("deka", "10")) + assert context.final_units == (("bapi", "2"), ("baima", "1")) + assert profile.places[0].forms == (("pideka", "2"),) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_missing_files_are_empty(self, tmp_path): + profile = load_profile(write(tmp_path, BASE), SKELETON) + assert profile.currencies == () + assert profile.money_cues == MoneyCues() + assert profile.held_back_words == () + assert profile.roles == () + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_blank_lines_are_skipped(self, tmp_path): + root = write(tmp_path, BASE) + (tmp_path / "numbers/held_back.tsv").write_text("halfu\n\n", encoding="utf-8") + assert load_profile(root, SKELETON).held_back_words == ("halfu",) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_bad_column_count(self, tmp_path): + root = write(tmp_path, {**BASE, "numbers/units.tsv": [("bapi", "2", "2")]}) + with pytest.raises(ValueError, match=r"units\.tsv:1"): + load_profile(root, SKELETON) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_unknown_where(self, tmp_path): + root = write(tmp_path, {**BASE, "numbers/units.tsv": [("bapi", "2", "2", "sometimes")]}) + with pytest.raises(ValueError, match=r"units\.tsv:1.*sometimes"): + load_profile(root, SKELETON) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_unknown_class(self, tmp_path): + root = write(tmp_path, {**BASE, "numbers/units.tsv": [("bapi", "2", "99", "any")]}) + with pytest.raises(ValueError, match=r"units\.tsv:1.*99"): + load_profile(root, SKELETON) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_unknown_power(self, tmp_path): + root = write(tmp_path, {**BASE, "numbers/places.tsv": [("pideka", "2", "7")]}) + with pytest.raises(ValueError, match=r"places\.tsv:1.*7"): + load_profile(root, SKELETON) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_unknown_scale(self, tmp_path): + root = write(tmp_path, {**BASE, "numbers/scale_words.tsv": [("bumega", "tens", "deka", "1")]}) + with pytest.raises(ValueError, match=r"scale_words\.tsv:1.*bumega"): + load_profile(root, SKELETON) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_scale_words(self, tmp_path): + rows = [("mina", "tens", "deka", "1"), ("mina", "ones", "kaima", "1"), ("mina", "hundreds", "hanto", "1")] + profile = load_profile(write(tmp_path, {**BASE, "numbers/scale_words.tsv": rows}), SKELETON) + scale = profile.counted_scales[0] + assert (scale.tens, scale.count_ones, scale.count_hundreds) == ( + (("deka", "1"),), + (("kaima", "1"),), + (("hanto", "1"),), + ) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_currency_roles(self, tmp_path): + rows = [("koin", "XTS", "2", "short-tens"), ("dolari", "USD", "2", "")] + profile = load_profile(write(tmp_path, {**BASE, "money/currency.tsv": rows}), SKELETON) + assert profile.currencies == ( + CurrencyContext("koin", "XTS", "2", ("short-tens",)), + CurrencyContext("dolari", "USD", "2", ()), + ) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_roles_cues_exact_and_variants(self, tmp_path): + files = { + **BASE, + "numbers/roles.tsv": [("mapi", "20", "short-tens")], + "numbers/exact.tsv": [("tausi", "1000")], + "numbers/spelling_variants.tsv": [("pideka", "piideka")], + "money/cues.tsv": [("lipo", "stem"), ("dalo", "word"), ("olipon", "not")], + } + profile = load_profile(write(tmp_path, files), SKELETON) + assert profile.roles == (("short-tens", (("mapi", "20"),)),) + assert profile.exact_boundaries == (("tausi", "1000"),) + assert profile.money_cues == MoneyCues(stems=("lipo",), words=("dalo",), not_cues=("olipon",)) + assert profile.places[0].forms == (("pideka", "2"), ("piideka", "2")) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_unknown_cue_kind(self, tmp_path): + root = write(tmp_path, {**BASE, "money/cues.tsv": [("lipo", "verb")]}) + with pytest.raises(ValueError, match=r"cues\.tsv:1.*verb"): + load_profile(root, SKELETON) + + # The test language's own data loads and builds a classifier. + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_toy_profile_loads(self): + from nemo_text_processing.inverse_text_normalization.bantu.taggers.tokenize_and_classify import ( + build_classifier, + ) + from nemo_text_processing.inverse_text_normalization.toy.settings import load + + profile = load() + assert [c.noun for c in profile.contexts] == ["kopa", "bakopa", "zikopa", "makopa", "ekopa"] + assert ("piideka", "2") in profile.places[0].forms + assert build_classifier(profile).num_states() > 0 From 5a4c7359af6bbdbf3590bea33949678dfb8d62b9 Mon Sep 17 00:00:00 2001 From: Cal3bdev Date: Thu, 5 Nov 2026 13:37:09 +0300 Subject: [PATCH 8/9] Bantu ITN: test language as a standard hidden NeMo language The test language 'toy' now has NeMo's standard ClassifyFst(cache_dir, whitelist, overwrite_cache, input_case) and VerbalizeFinalFst(), and is registered in InverseNormalizer but not offered on the command line. Its case tests move to tests/nemo_text_processing/toy/ and run through InverseNormalizer(lang='toy'); the build_normalizer test helper and toy_profile.py are removed. Kernel options that a case file cannot switch stay in bantu/test_kernel_options.py. Output is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Cal3bdev --- Jenkinsfile | 2 +- .../inverse_normalize.py | 5 + .../toy/taggers/__init__.py | 14 ++ .../toy/taggers/tokenize_and_classify.py | 63 +++++++ .../toy/verbalizers/__init__.py | 14 ++ .../toy/verbalizers/verbalize_final.py | 29 ++++ .../bantu/test_data_loader.py | 7 + ...test_profile.py => test_kernel_options.py} | 58 ++++++- .../nemo_text_processing/bantu/toy_profile.py | 154 ------------------ tests/nemo_text_processing/bantu/utils.py | 45 ----- tests/nemo_text_processing/toy/__init__.py | 14 ++ .../test_cases_cardinal.txt | 0 .../test_cases_money.txt | 0 .../test_cases_word.txt | 0 .../{bantu => toy}/test_cardinal.py | 11 +- .../{bantu => toy}/test_money.py | 11 +- .../{bantu => toy}/test_verbalizer.py | 8 +- .../{bantu => toy}/test_word.py | 11 +- tests/nemo_text_processing/toy/utils.py | 21 +++ 19 files changed, 243 insertions(+), 224 deletions(-) create mode 100644 nemo_text_processing/inverse_text_normalization/toy/taggers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/toy/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/inverse_text_normalization/toy/verbalizers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/toy/verbalizers/verbalize_final.py rename tests/nemo_text_processing/bantu/{test_profile.py => test_kernel_options.py} (65%) delete mode 100644 tests/nemo_text_processing/bantu/toy_profile.py delete mode 100644 tests/nemo_text_processing/bantu/utils.py create mode 100644 tests/nemo_text_processing/toy/__init__.py rename tests/nemo_text_processing/{bantu => toy}/data_inverse_text_normalization/test_cases_cardinal.txt (100%) rename tests/nemo_text_processing/{bantu => toy}/data_inverse_text_normalization/test_cases_money.txt (100%) rename tests/nemo_text_processing/{bantu => toy}/data_inverse_text_normalization/test_cases_word.txt (100%) rename tests/nemo_text_processing/{bantu => toy}/test_cardinal.py (74%) rename tests/nemo_text_processing/{bantu => toy}/test_money.py (74%) rename tests/nemo_text_processing/{bantu => toy}/test_verbalizer.py (85%) rename tests/nemo_text_processing/{bantu => toy}/test_word.py (74%) create mode 100644 tests/nemo_text_processing/toy/utils.py diff --git a/Jenkinsfile b/Jenkinsfile index 30bfb3213..40c414947 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -485,7 +485,7 @@ pipeline { } stage('L1: Run Bantu kernel ITN tests') { steps { - sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/bantu/ -m "not pleasefixme" --cpu' + sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/bantu/ tests/nemo_text_processing/toy/ -m "not pleasefixme" --cpu' } } } diff --git a/nemo_text_processing/inverse_text_normalization/inverse_normalize.py b/nemo_text_processing/inverse_text_normalization/inverse_normalize.py index 9a6fcc64c..0db0aea07 100644 --- a/nemo_text_processing/inverse_text_normalization/inverse_normalize.py +++ b/nemo_text_processing/inverse_text_normalization/inverse_normalize.py @@ -146,6 +146,11 @@ def __init__( from nemo_text_processing.inverse_text_normalization.ko.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) + elif lang == 'toy': # test language for the shared Bantu kernel; hidden from the CLI + from nemo_text_processing.inverse_text_normalization.toy.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.inverse_text_normalization.toy.verbalizers.verbalize_final import ( + VerbalizeFinalFst, + ) else: raise NotImplementedError(f"Language {lang} has not been supported yet.") diff --git a/nemo_text_processing/inverse_text_normalization/toy/taggers/__init__.py b/nemo_text_processing/inverse_text_normalization/toy/taggers/__init__.py new file mode 100644 index 000000000..eb60428b9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/taggers/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/toy/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/toy/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..c7b256a0b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/taggers/tokenize_and_classify.py @@ -0,0 +1,63 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini + +from nemo_text_processing.inverse_text_normalization.bantu.taggers.tokenize_and_classify import build_classifier +from nemo_text_processing.inverse_text_normalization.toy.settings import load +from nemo_text_processing.text_normalization.en.graph_utils import INPUT_LOWER_CASED, GraphFst, generator_main +from nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars for the test language + of the shared Bantu kernel. The grammar is built by the kernel from this language's + data/ and settings.py; see bantu.taggers.tokenize_and_classify.build_classifier. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + whitelist: not used; the test language has no whitelist + overwrite_cache: set to True to overwrite .far files + input_case: accepting either "lower_cased" or "cased" input. The kernel reads + lower-cased words and passes other words through, so both build the same grammar. + """ + + def __init__( + self, + cache_dir: str = None, + whitelist: str = None, + overwrite_cache: bool = False, + input_case: str = INPUT_LOWER_CASED, + ): + super().__init__(name="tokenize_and_classify", kind="classify") + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"toy_itn_{input_case}.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logger.info("Creating ClassifyFst grammars.") + self.fst = build_classifier(load()) + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logger.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/toy/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/toy/verbalizers/__init__.py new file mode 100644 index 000000000..eb60428b9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/verbalizers/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/toy/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/toy/verbalizers/verbalize_final.py new file mode 100644 index 000000000..85fb22f09 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/verbalizers/verbalize_final.py @@ -0,0 +1,29 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.verbalize_final import build_final_verbalizer +from nemo_text_processing.inverse_text_normalization.toy.settings import SKELETON +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence of the test language, + e.g. tokens { name: "kasa" } tokens { cardinal { integer: "1000" } } -> kasa 1,000 + """ + + def __init__(self): + super().__init__(name="verbalize_final", kind="verbalize") + self.fst = build_final_verbalizer(SKELETON.max_value, SKELETON.thousands_separator) diff --git a/tests/nemo_text_processing/bantu/test_data_loader.py b/tests/nemo_text_processing/bantu/test_data_loader.py index 28e0a2f21..06081c475 100644 --- a/tests/nemo_text_processing/bantu/test_data_loader.py +++ b/tests/nemo_text_processing/bantu/test_data_loader.py @@ -147,6 +147,13 @@ def test_currency_roles(self, tmp_path): CurrencyContext("dolari", "USD", "2", ()), ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_currency_unknown_class(self, tmp_path): + root = write(tmp_path, {**BASE, "money/currency.tsv": [("koin", "XTS", "99", "")]}) + with pytest.raises(ValueError, match=r"currency\.tsv:1.*99"): + load_profile(root, SKELETON) + @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_roles_cues_exact_and_variants(self, tmp_path): diff --git a/tests/nemo_text_processing/bantu/test_profile.py b/tests/nemo_text_processing/bantu/test_kernel_options.py similarity index 65% rename from tests/nemo_text_processing/bantu/test_profile.py rename to tests/nemo_text_processing/bantu/test_kernel_options.py index 386540b93..9d4b2424b 100644 --- a/tests/nemo_text_processing/bantu/test_profile.py +++ b/tests/nemo_text_processing/bantu/test_kernel_options.py @@ -13,8 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Kernel behaviour that depends on the profile itself: validation, optional features, caching.""" +""" +Kernel options that a case file cannot switch: invalid profiles, optional features, +caching, and how the test language is exposed. Variants of the test language are +built from its profile with some settings changed. +""" +import sys from dataclasses import replace import pytest @@ -24,12 +29,24 @@ from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import build_role from nemo_text_processing.inverse_text_normalization.bantu.taggers.scale import build_scale_amount from nemo_text_processing.inverse_text_normalization.bantu.taggers.tokenize_and_classify import build_classifier +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer, parse_args +from nemo_text_processing.inverse_text_normalization.toy.settings import load +from nemo_text_processing.inverse_text_normalization.toy.taggers.tokenize_and_classify import ClassifyFst -from .toy_profile import TOY_PROFILE -from .utils import build_normalizer +from ..utils import CACHE_DIR +TOY_PROFILE = load() +SAMPLES = ("bakopa pideka vo baima", "kasa emina epi", "lipoka obumega bapi", "bakopa bapi vo kasa") -class TestBantuKernelProfile: + +def build_normalizer(profile) -> InverseNormalizer: + """The test language's runtime with its classifier rebuilt from a changed profile.""" + normalizer = InverseNormalizer(lang='toy', cache_dir=CACHE_DIR) + normalizer.tagger.fst = build_classifier(profile) + return normalizer + + +class TestBantuKernelOptions: # A broken profile is rejected when the grammar is built, with a clear error. @pytest.mark.run_only_on('CPU') @pytest.mark.unit @@ -98,3 +115,36 @@ def test_money_takes_the_cardinal(self): money = MoneyFst(cardinal) assert money.has_implied_currency assert money.fst.num_states() > 0 + + # The compiled classifier is saved to and restored from a FAR file with the same results. + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_far_cache_roundtrip(self, tmp_path): + built = ClassifyFst(cache_dir=str(tmp_path), overwrite_cache=True) + assert (tmp_path / "toy_itn_lower_cased.far").exists() + restored = ClassifyFst(cache_dir=str(tmp_path)) + normalizer = InverseNormalizer(lang='toy', cache_dir=CACHE_DIR) + for spoken in SAMPLES: + normalizer.tagger = built + expected = normalizer.inverse_normalize(spoken, verbose=False) + normalizer.tagger = restored + assert normalizer.inverse_normalize(spoken, verbose=False) == expected + + # Cased input builds the same grammar (the kernel reads lower-cased words), cached under its own name. + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_cased_input_case(self, tmp_path): + ClassifyFst(cache_dir=str(tmp_path), input_case="cased") + assert (tmp_path / "toy_itn_cased.far").exists() + cased = InverseNormalizer(lang='toy', cache_dir=str(tmp_path), input_case='cased') + lower = InverseNormalizer(lang='toy', cache_dir=CACHE_DIR) + for spoken in SAMPLES: + assert cased.inverse_normalize(spoken, verbose=False) == lower.inverse_normalize(spoken, verbose=False) + + # The test language is not offered on the command line. + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_hidden_from_cli(self, monkeypatch): + monkeypatch.setattr(sys, "argv", ["inverse_normalize.py", "--language", "toy", "--text", "kasa"]) + with pytest.raises(SystemExit): + parse_args() diff --git a/tests/nemo_text_processing/bantu/toy_profile.py b/tests/nemo_text_processing/bantu/toy_profile.py deleted file mode 100644 index c1d0e8bc5..000000000 --- a/tests/nemo_text_processing/bantu/toy_profile.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# Copyright (c) 2026, Crane Labs. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -A small made-up language for testing the shared Bantu kernel on its own. - -Every word here is invented. The language is shaped like a Bantu language so that -each kernel feature is exercised (Luganda equivalents in brackets): -- noun classes with agreeing number words: bakopa bapi / zikopa zipi = 2 [abantu babiri / emiti ebiri] -- 6-9 and ten do not agree: sei, siti, oto, nona, deka [mukaaga ... kkumi] -- teens join with "ke", larger parts with "vo" [na, mu] -- "vo" is also an ordinary word, "in" [mu] -- a currency phrase that licenses a special reading: koin mapi = 20 XTS, but makopa mapi = 2 [abiri] -- other currency phrases, each agreeing with its own class: dolari zipi = 2 USD, - yuro mafu = 5 EUR, yeni efu = 5 JPY -- a counted scale of ten-thousands: mina + count [mitwalo]; after a listed noun, - emina counts that noun, also as an ordinary number place for 1-9: - bakopa emina epi = 20,000, bakopa emina pideka = 200,000 [abantu emitwalo ebiri / abiri] -- implied currency: bare "mina" is always money; word + "emina" counts that word, - except after a money verb [mitwalo / emitwalo] -- a held-back word that continues a number but is not handled: halfu [n'ekitundu] -- a count class with vowel-initial forms, so ke + epi contracts to k'epi [n'ebiri] -- a larger scale of millions: bumega + count, with augmented forms that count a noun, - "one" forms, hundreds inside the count, and money at the start [obukadde / akakadde] -- a fixed phrase (tausi = 1,000), a special teen form (dekafu = 15) and a spelling - variant (piideka = pideka) [akakadde, kkumi nataano, abiiri] - -A failing test with this profile points at the kernel, not at real language data. -XTS is the ISO 4217 code reserved for testing. -""" -from nemo_text_processing.inverse_text_normalization.bantu.profile import ( - CardinalProfile, - CountedScale, - CurrencyContext, - MoneyCues, - NounContext, - Place, - with_spelling_variants, -) - -INVARIANT = (("sei", "6"), ("siti", "7"), ("oto", "8"), ("nona", "9")) -STEMS = (("pi", "2"), ("ko", "3"), ("de", "4"), ("fu", "5")) -TENS = ( - ("pideka", "2"), - ("kodeka", "3"), - ("dedeka", "4"), - ("fudeka", "5"), - ("seideka", "6"), - ("sitideka", "7"), - ("otodeka", "8"), - ("nonadeka", "9"), -) - - -# Count forms agreeing with class 8 (ekopa, and the ten-thousands noun mina): 2-9. -EIGHT_COUNTS = tuple(("e" + stem, value) for stem, value in STEMS) + INVARIANT - - -def plural(class_id, noun, prefix, final_one=None, teen_variants=()): - units = tuple((prefix + stem, value) for stem, value in STEMS) + INVARIANT - final = units + (((final_one, "1"),) if final_one else ()) - return NounContext(class_id, noun, units + (("deka", "10"),), final, teen_variants=teen_variants) - - -def multiples(scale): - words = (("pi", "2"), ("ko", "3"), ("de", "4"), ("fu", "5")) + INVARIANT - return ((scale, "1"),) + tuple((f"{scale} {word}", value) for word, value in words) - - -TOY_PROFILE = with_spelling_variants( - CardinalProfile( - language="toy", - contexts=( - NounContext("1", "kopa", (("ima", "1"),), (), compounds=False), - plural("2", "bakopa", "ba", final_one="baima"), - plural("4", "zikopa", "zi", final_one="zima"), - plural("6", "makopa", "ma", teen_variants=(("dekafu", "15"),)), - plural("8", "ekopa", "e", final_one="ema"), - ), - ten="deka", - teen_join=" ke ", - places=( - Place(1, TENS, " vo "), - Place(2, multiples("hanto"), " vo "), - Place(3, multiples("milo"), " vo "), - Place(4, (("omuna", "1"),) + tuple((f"emina {word}", value) for word, value in EIGHT_COUNTS), " vo "), - ), - exact_boundaries=(("tausi", "1000"),), - roles=(("short-tens", (("mapi", "20"),)),), - max_value=999_999_999, - currencies=( - CurrencyContext("koin", "XTS", agreement_class="4", roles=("short-tens",)), - # Base cases for common currencies; the words are made up like the rest. - CurrencyContext("dolari", "USD", agreement_class="4"), - CurrencyContext("paundi", "GBP", agreement_class="4"), - CurrencyContext("yuro", "EUR", agreement_class="6"), - CurrencyContext("yeni", "JPY", agreement_class="8"), - ), - counted_scales=( - # Ten-thousands [mitwalo]: bare mina is money anywhere; emina counts a word. - CountedScale( - singular="muna", - singular_count="ema", - plural="mina", - power=4, - count_class="8", - tens=(("deka", "1"),) + TENS, - remainder_class="4", - contracted_teen_joins=("k'", "k"), - implied_currency="XTS", - counting_plural="emina", - implied_anywhere=True, - counts_any_word=True, - after_counted_nouns=True, - ), - # Millions [obukadde]: augmented forms are also money at the start of input, - # count listed nouns with agreement, and need "one" to count (akamega kaima). - CountedScale( - singular="kamega", - singular_count="kaima", - plural="bumega", - power=6, - count_class="2", - tens=(("deka", "1"),) + TENS, - remainder_class="4", - count_ones=(("kaima", "1"),), - count_hundreds=multiples("hanto"), - implied_currency="XTS", - counting_plural="obumega", - counting_ones=("akamega kaima",), - money_only_ones=("akamega",), - implied_anywhere=True, - counting_money_at_start=True, - counts_any_word=True, - after_counted_nouns=True, - ), - ), - money_cues=MoneyCues(stems=("lipo",), words=("dalo",), not_cues=("olipon",)), - held_back_words=("halfu",), - ), - {"pideka": ("piideka",)}, -) diff --git a/tests/nemo_text_processing/bantu/utils.py b/tests/nemo_text_processing/bantu/utils.py deleted file mode 100644 index c6f65ebc5..000000000 --- a/tests/nemo_text_processing/bantu/utils.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# Copyright (c) 2026, Crane Labs. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from types import SimpleNamespace - -from pynini.lib import rewrite - -from nemo_text_processing.inverse_text_normalization.bantu.taggers.tokenize_and_classify import build_classifier -from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.verbalize_final import build_final_verbalizer -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer -from nemo_text_processing.text_normalization.token_parser import TokenParser - - -def build_normalizer(profile) -> InverseNormalizer: - """ - NeMo's own inverse-normalization runtime with the kernel's tagger and verbalizer - built directly from a profile. The kernel is not a selectable language, so it is - not constructed through InverseNormalizer(lang=...); the attributes set here - mirror the end of InverseNormalizer.__init__. - """ - normalizer = InverseNormalizer.__new__(InverseNormalizer) - normalizer.lang = profile.language - normalizer.tagger = SimpleNamespace(fst=build_classifier(profile)) - normalizer.verbalizer = SimpleNamespace(fst=build_final_verbalizer(profile.max_value, profile.thousands_separator)) - normalizer.parser = TokenParser() - normalizer.max_number_of_permutations_per_split = 729 # InverseNormalizer default - normalizer.post_processor = None - return normalizer - - -def best_tags(normalizer: InverseNormalizer, text: str) -> set: - """All equally best tagger outputs. The kernel must give exactly one for any input.""" - return set(rewrite.optimal_rewrites(text, normalizer.tagger.fst)) diff --git a/tests/nemo_text_processing/toy/__init__.py b/tests/nemo_text_processing/toy/__init__.py new file mode 100644 index 000000000..eb60428b9 --- /dev/null +++ b/tests/nemo_text_processing/toy/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/toy/data_inverse_text_normalization/test_cases_cardinal.txt similarity index 100% rename from tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_cardinal.txt rename to tests/nemo_text_processing/toy/data_inverse_text_normalization/test_cases_cardinal.txt diff --git a/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/toy/data_inverse_text_normalization/test_cases_money.txt similarity index 100% rename from tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_money.txt rename to tests/nemo_text_processing/toy/data_inverse_text_normalization/test_cases_money.txt diff --git a/tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/toy/data_inverse_text_normalization/test_cases_word.txt similarity index 100% rename from tests/nemo_text_processing/bantu/data_inverse_text_normalization/test_cases_word.txt rename to tests/nemo_text_processing/toy/data_inverse_text_normalization/test_cases_word.txt diff --git a/tests/nemo_text_processing/bantu/test_cardinal.py b/tests/nemo_text_processing/toy/test_cardinal.py similarity index 74% rename from tests/nemo_text_processing/bantu/test_cardinal.py rename to tests/nemo_text_processing/toy/test_cardinal.py index 2c46c403f..8cbade61f 100644 --- a/tests/nemo_text_processing/bantu/test_cardinal.py +++ b/tests/nemo_text_processing/toy/test_cardinal.py @@ -16,16 +16,17 @@ import pytest from parameterized import parameterized -from ..utils import parse_test_case_file -from .toy_profile import TOY_PROFILE -from .utils import best_tags, build_normalizer +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file +from .utils import best_tags class TestCardinal: - inverse_normalizer = build_normalizer(TOY_PROFILE) + inverse_normalizer = InverseNormalizer(lang='toy', cache_dir=CACHE_DIR, overwrite_cache=False) - @parameterized.expand(parse_test_case_file('bantu/data_inverse_text_normalization/test_cases_cardinal.txt')) + @parameterized.expand(parse_test_case_file('toy/data_inverse_text_normalization/test_cases_cardinal.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm(self, test_input, expected): diff --git a/tests/nemo_text_processing/bantu/test_money.py b/tests/nemo_text_processing/toy/test_money.py similarity index 74% rename from tests/nemo_text_processing/bantu/test_money.py rename to tests/nemo_text_processing/toy/test_money.py index b92ec49ec..1e26e981d 100644 --- a/tests/nemo_text_processing/bantu/test_money.py +++ b/tests/nemo_text_processing/toy/test_money.py @@ -16,16 +16,17 @@ import pytest from parameterized import parameterized -from ..utils import parse_test_case_file -from .toy_profile import TOY_PROFILE -from .utils import best_tags, build_normalizer +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file +from .utils import best_tags class TestMoney: - inverse_normalizer = build_normalizer(TOY_PROFILE) + inverse_normalizer = InverseNormalizer(lang='toy', cache_dir=CACHE_DIR, overwrite_cache=False) - @parameterized.expand(parse_test_case_file('bantu/data_inverse_text_normalization/test_cases_money.txt')) + @parameterized.expand(parse_test_case_file('toy/data_inverse_text_normalization/test_cases_money.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm(self, test_input, expected): diff --git a/tests/nemo_text_processing/bantu/test_verbalizer.py b/tests/nemo_text_processing/toy/test_verbalizer.py similarity index 85% rename from tests/nemo_text_processing/bantu/test_verbalizer.py rename to tests/nemo_text_processing/toy/test_verbalizer.py index 2f0ed0618..9c8767581 100644 --- a/tests/nemo_text_processing/bantu/test_verbalizer.py +++ b/tests/nemo_text_processing/toy/test_verbalizer.py @@ -13,17 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Kernel verbalizers on their own: formatting only, values are resolved by the classifier.""" +"""The test language's verbalizer on its own: formatting only, values are resolved by the classifier.""" import pytest from parameterized import parameterized from pynini.lib import rewrite -from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.verbalize_final import build_final_verbalizer +from nemo_text_processing.inverse_text_normalization.toy.verbalizers.verbalize_final import VerbalizeFinalFst -from .toy_profile import TOY_PROFILE - -VERBALIZER = build_final_verbalizer(TOY_PROFILE.max_value, TOY_PROFILE.thousands_separator) +VERBALIZER = VerbalizeFinalFst().fst class TestBantuKernelVerbalizer: diff --git a/tests/nemo_text_processing/bantu/test_word.py b/tests/nemo_text_processing/toy/test_word.py similarity index 74% rename from tests/nemo_text_processing/bantu/test_word.py rename to tests/nemo_text_processing/toy/test_word.py index a4eccd707..4913ec46a 100644 --- a/tests/nemo_text_processing/bantu/test_word.py +++ b/tests/nemo_text_processing/toy/test_word.py @@ -16,16 +16,17 @@ import pytest from parameterized import parameterized -from ..utils import parse_test_case_file -from .toy_profile import TOY_PROFILE -from .utils import best_tags, build_normalizer +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file +from .utils import best_tags class TestWord: - inverse_normalizer = build_normalizer(TOY_PROFILE) + inverse_normalizer = InverseNormalizer(lang='toy', cache_dir=CACHE_DIR, overwrite_cache=False) - @parameterized.expand(parse_test_case_file('bantu/data_inverse_text_normalization/test_cases_word.txt')) + @parameterized.expand(parse_test_case_file('toy/data_inverse_text_normalization/test_cases_word.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm(self, test_input, expected): diff --git a/tests/nemo_text_processing/toy/utils.py b/tests/nemo_text_processing/toy/utils.py new file mode 100644 index 000000000..6242f1bea --- /dev/null +++ b/tests/nemo_text_processing/toy/utils.py @@ -0,0 +1,21 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib import rewrite + + +def best_tags(normalizer, text: str) -> set: + """All equally best tagger outputs. The kernel must give exactly one for any input.""" + return set(rewrite.optimal_rewrites(text, normalizer.tagger.fst)) From ac6899c1d1aed9ab34ec42d6f802eb0d9b181c00 Mon Sep 17 00:00:00 2001 From: Cal3bdev Date: Thu, 5 Nov 2026 13:42:37 +0300 Subject: [PATCH 9/9] Bantu ITN: stricter checks when loading TSVs load_profile now rejects, with the file and line: empty fields, a currency whose class has no compounding numbers, and a currency role missing from roles.tsv. A missing data folder is an error instead of an empty language. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Cal3bdev --- .../bantu/data_loader.py | 25 ++++++++++---- .../bantu/test_data_loader.py | 33 ++++++++++++++++++- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/bantu/data_loader.py b/nemo_text_processing/inverse_text_normalization/bantu/data_loader.py index 961d0dc63..f852e2237 100644 --- a/nemo_text_processing/inverse_text_normalization/bantu/data_loader.py +++ b/nemo_text_processing/inverse_text_normalization/bantu/data_loader.py @@ -56,8 +56,11 @@ CUE_KINDS = {"stem": "stems", "word": "words", "not": "not_cues"} -def read_rows(data_dir: str, name: str, columns: int) -> List[Tuple[str, List[str]]]: - """Non-blank rows of one TSV as (location, row); location is "path:line" for error messages.""" +def read_rows(data_dir: str, name: str, columns: int, last_optional: bool = False) -> List[Tuple[str, List[str]]]: + """ + Non-blank rows of one TSV as (location, row); location is "path:line" for error messages. + Every cell must be non-empty, except the last one when last_optional is set. + """ path = os.path.join(data_dir, name) if not os.path.exists(path): return [] @@ -68,6 +71,9 @@ def read_rows(data_dir: str, name: str, columns: int) -> List[Tuple[str, List[st continue if len(row) != columns: raise ValueError(f"{path}:{line}: expected {columns} columns, found {len(row)}") + required = row[:-1] if last_optional else row + if not all(required): + raise ValueError(f"{path}:{line}: empty field") rows.append((f"{path}:{line}", row)) return rows @@ -86,6 +92,8 @@ def load_profile(data_dir: str, skeleton: CardinalProfile) -> CardinalProfile: data_dir: the language's data folder skeleton: the language's profile with empty word lists (from its settings.py) """ + if not os.path.isdir(data_dir): + raise ValueError(f"no data folder at {data_dir}") contexts = { context.class_id: {"atomic_units": [], "final_units": [], "teen_variants": []} for context in skeleton.contexts } @@ -112,12 +120,17 @@ def load_profile(data_dir: str, skeleton: CardinalProfile) -> CardinalProfile: for _, (form, value, role) in read_rows(data_dir, "numbers/roles.tsv", 3): roles.setdefault(role, []).append((form, value)) - known_classes = {context.class_id for context in skeleton.contexts} + # An amount after a currency agrees with a class whose numbers compound. + compounding = {context.class_id for context in skeleton.contexts if context.compounds} currencies = [] - for location, (phrase, code, class_id, currency_roles) in read_rows(data_dir, "money/currency.tsv", 4): - if class_id not in known_classes: - raise ValueError(f"{location}: unknown class {class_id!r}") + rows = read_rows(data_dir, "money/currency.tsv", 4, last_optional=True) + for location, (phrase, code, class_id, currency_roles) in rows: + if class_id not in compounding: + raise ValueError(f"{location}: unknown or non-compounding class {class_id!r}") names = tuple(name for name in currency_roles.split(",") if name) + for name in names: + if name not in roles: + raise ValueError(f"{location}: unknown role {name!r}") currencies.append(CurrencyContext(phrase, code, class_id, names)) cues = {field: [] for field in CUE_KINDS.values()} diff --git a/tests/nemo_text_processing/bantu/test_data_loader.py b/tests/nemo_text_processing/bantu/test_data_loader.py index 06081c475..f108ddf8d 100644 --- a/tests/nemo_text_processing/bantu/test_data_loader.py +++ b/tests/nemo_text_processing/bantu/test_data_loader.py @@ -15,6 +15,8 @@ """The shared loader that fills a language's profile skeleton from its TSV files.""" +from dataclasses import replace + import pytest from nemo_text_processing.inverse_text_normalization.bantu.data_loader import load_profile @@ -141,7 +143,8 @@ def test_scale_words(self, tmp_path): @pytest.mark.unit def test_currency_roles(self, tmp_path): rows = [("koin", "XTS", "2", "short-tens"), ("dolari", "USD", "2", "")] - profile = load_profile(write(tmp_path, {**BASE, "money/currency.tsv": rows}), SKELETON) + files = {**BASE, "money/currency.tsv": rows, "numbers/roles.tsv": [("mapi", "20", "short-tens")]} + profile = load_profile(write(tmp_path, files), SKELETON) assert profile.currencies == ( CurrencyContext("koin", "XTS", "2", ("short-tens",)), CurrencyContext("dolari", "USD", "2", ()), @@ -190,3 +193,31 @@ def test_toy_profile_loads(self): assert [c.noun for c in profile.contexts] == ["kopa", "bakopa", "zikopa", "makopa", "ekopa"] assert ("piideka", "2") in profile.places[0].forms assert build_classifier(profile).num_states() > 0 + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_currency_unknown_role(self, tmp_path): + root = write(tmp_path, {**BASE, "money/currency.tsv": [("koin", "XTS", "2", "nosuchrole")]}) + with pytest.raises(ValueError, match=r"currency\.tsv:1.*nosuchrole"): + load_profile(root, SKELETON) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_currency_class_must_compound(self, tmp_path): + skeleton = replace(SKELETON, contexts=SKELETON.contexts + (NounContext("1", "kopa", (), (), compounds=False),)) + root = write(tmp_path, {**BASE, "money/currency.tsv": [("koin", "XTS", "1", "")]}) + with pytest.raises(ValueError, match=r"currency\.tsv:1.*class '1'"): + load_profile(root, skeleton) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_empty_field(self, tmp_path): + root = write(tmp_path, {**BASE, "numbers/units.tsv": [("", "3", "2", "any")]}) + with pytest.raises(ValueError, match=r"units\.tsv:1.*empty"): + load_profile(root, SKELETON) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_missing_data_dir(self, tmp_path): + with pytest.raises(ValueError, match="no data folder"): + load_profile(str(tmp_path / "nope"), SKELETON)