diff --git a/Jenkinsfile b/Jenkinsfile index f5d114f84..40c414947 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -483,6 +483,11 @@ pipeline { sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/ko/ -m "not pleasefixme" --cpu --tn_cache_dir ${KO_TN_CACHE}' } } + stage('L1: Run Bantu kernel ITN tests') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/bantu/ tests/nemo_text_processing/toy/ -m "not pleasefixme" --cpu' + } + } } } diff --git a/nemo_text_processing/inverse_text_normalization/bantu/__init__.py b/nemo_text_processing/inverse_text_normalization/bantu/__init__.py new file mode 100644 index 000000000..eb60428b9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/bantu/data_loader.py b/nemo_text_processing/inverse_text_normalization/bantu/data_loader.py new file mode 100644 index 000000000..f852e2237 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/data_loader.py @@ -0,0 +1,163 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fills a language's profile skeleton with the word lists in its data folder. + +Each language writes the shape of its grammar in settings.py: a CardinalProfile whose +word lists are empty (noun classes, places, counted scales with their head words and +switches). Its word lists live in tab-separated files under data/; every file is +optional, and a missing file means an empty list: + + numbers/units.tsv form, value, class, where (any, atomic, final, teen) + numbers/places.tsv form, coefficient, power + numbers/exact.tsv form, value + numbers/roles.tsv form, value, role + numbers/scale_words.tsv scale plural, part (tens, ones, hundreds), form, value + numbers/held_back.tsv word + numbers/spelling_variants.tsv word, variant + money/currency.tsv phrase, code, class, roles (comma-separated, may be empty) + money/cues.tsv word, kind (stem, word, not) + +A "where" of any puts a unit form both inside compounds (atomic) and at the end of a +number (final); teen forms are whole teen words such as a contracted 15. +""" +import csv +import os +from collections import OrderedDict +from dataclasses import replace +from typing import Dict, List, Tuple + +from nemo_text_processing.inverse_text_normalization.bantu.profile import ( + CardinalProfile, + CurrencyContext, + MoneyCues, + with_spelling_variants, +) + +WHERE = { + "any": ("atomic_units", "final_units"), + "atomic": ("atomic_units",), + "final": ("final_units",), + "teen": ("teen_variants",), +} +SCALE_PARTS = {"tens": "tens", "ones": "count_ones", "hundreds": "count_hundreds"} +CUE_KINDS = {"stem": "stems", "word": "words", "not": "not_cues"} + + +def read_rows(data_dir: str, name: str, columns: int, last_optional: bool = False) -> List[Tuple[str, List[str]]]: + """ + Non-blank rows of one TSV as (location, row); location is "path:line" for error messages. + Every cell must be non-empty, except the last one when last_optional is set. + """ + path = os.path.join(data_dir, name) + if not os.path.exists(path): + return [] + rows = [] + with open(path, encoding="utf-8", newline="") as tsv: + for line, row in enumerate(csv.reader(tsv, delimiter="\t"), start=1): + if not row: + continue + if len(row) != columns: + raise ValueError(f"{path}:{line}: expected {columns} columns, found {len(row)}") + required = row[:-1] if last_optional else row + if not all(required): + raise ValueError(f"{path}:{line}: empty field") + rows.append((f"{path}:{line}", row)) + return rows + + +def lookup(table: Dict[str, str], key: str, location: str, what: str) -> str: + if key not in table: + raise ValueError(f"{location}: unknown {what} {key!r}") + return table[key] + + +def load_profile(data_dir: str, skeleton: CardinalProfile) -> CardinalProfile: + """ + Returns the skeleton with its word lists filled from data_dir. + + Args: + data_dir: the language's data folder + skeleton: the language's profile with empty word lists (from its settings.py) + """ + if not os.path.isdir(data_dir): + raise ValueError(f"no data folder at {data_dir}") + contexts = { + context.class_id: {"atomic_units": [], "final_units": [], "teen_variants": []} for context in skeleton.contexts + } + for location, (form, value, class_id, where) in read_rows(data_dir, "numbers/units.tsv", 4): + lists = contexts.get(class_id) + if lists is None: + raise ValueError(f"{location}: unknown class {class_id!r}") + for field in lookup(WHERE, where, location, "where"): + lists[field].append((form, value)) + + places = {str(place.power): [] for place in skeleton.places} + for location, (form, value, power) in read_rows(data_dir, "numbers/places.tsv", 3): + if power not in places: + raise ValueError(f"{location}: unknown power {power!r}") + places[power].append((form, value)) + + scales = {scale.plural: {field: [] for field in SCALE_PARTS.values()} for scale in skeleton.counted_scales} + for location, (plural, part, form, value) in read_rows(data_dir, "numbers/scale_words.tsv", 4): + if plural not in scales: + raise ValueError(f"{location}: unknown scale {plural!r}") + scales[plural][lookup(SCALE_PARTS, part, location, "part")].append((form, value)) + + roles = OrderedDict() + for _, (form, value, role) in read_rows(data_dir, "numbers/roles.tsv", 3): + roles.setdefault(role, []).append((form, value)) + + # An amount after a currency agrees with a class whose numbers compound. + compounding = {context.class_id for context in skeleton.contexts if context.compounds} + currencies = [] + rows = read_rows(data_dir, "money/currency.tsv", 4, last_optional=True) + for location, (phrase, code, class_id, currency_roles) in rows: + if class_id not in compounding: + raise ValueError(f"{location}: unknown or non-compounding class {class_id!r}") + names = tuple(name for name in currency_roles.split(",") if name) + for name in names: + if name not in roles: + raise ValueError(f"{location}: unknown role {name!r}") + currencies.append(CurrencyContext(phrase, code, class_id, names)) + + cues = {field: [] for field in CUE_KINDS.values()} + for location, (word, kind) in read_rows(data_dir, "money/cues.tsv", 2): + cues[lookup(CUE_KINDS, kind, location, "kind")].append(word) + + variants = OrderedDict() + for _, (word, variant) in read_rows(data_dir, "numbers/spelling_variants.tsv", 2): + variants.setdefault(word, []).append(variant) + + profile = replace( + skeleton, + contexts=tuple( + replace(context, **{field: tuple(forms) for field, forms in contexts[context.class_id].items()}) + for context in skeleton.contexts + ), + places=tuple(replace(place, forms=tuple(places[str(place.power)])) for place in skeleton.places), + counted_scales=tuple( + replace(scale, **{field: tuple(forms) for field, forms in scales[scale.plural].items()}) + for scale in skeleton.counted_scales + ), + exact_boundaries=tuple(tuple(row) for _, row in read_rows(data_dir, "numbers/exact.tsv", 2)), + roles=tuple((role, tuple(forms)) for role, forms in roles.items()), + currencies=tuple(currencies), + money_cues=MoneyCues(**{field: tuple(words) for field, words in cues.items()}), + held_back_words=tuple(row[0] for _, row in read_rows(data_dir, "numbers/held_back.tsv", 1)), + ) + if variants: + profile = with_spelling_variants(profile, {word: tuple(v) for word, v in variants.items()}) + return profile diff --git a/nemo_text_processing/inverse_text_normalization/bantu/graph_utils.py b/nemo_text_processing/inverse_text_normalization/bantu/graph_utils.py new file mode 100644 index 000000000..15e852e54 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/graph_utils.py @@ -0,0 +1,162 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers shared by the Bantu taggers: digit strings, number-word runs, token formats and the money-signal filter. + +The classifier reads the whole sentence at once. Maximal runs of number words are +marked first, so each run is read as a single number in the context of the word +before it (a counted noun, a currency phrase or a money cue). Weights only prefer a +normalization over leaving words unchanged. All numeric readings have equal weight, +so structure alone must make the reading unique; the kernel tests check that every +test input has exactly one best reading. +""" +import string + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile, Forms +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_NOT_SPACE, + NEMO_SPACE, + NEMO_WHITE_SPACE, +) + +# Values are composed digit by digit inside the FSTs; there is no runtime arithmetic. +NONZERO_DIGITS = string.digits[1:] +DIGIT = pynini.union(*string.digits).optimize() +NONZERO = pynini.union(*NONZERO_DIGITS).optimize() + + +def mapping(forms: Forms) -> pynini.Fst: + return pynini.string_map(forms).optimize() if forms else pynini.Fst() + + +def integer_domain(max_value: int) -> pynini.Fst: + """Canonical positive decimal strings <= bound, without integer enumeration.""" + if isinstance(max_value, bool) or not isinstance(max_value, int) or max_value < 1: + raise ValueError("max_value must be a positive integer") + bound = str(max_value) + branches = [pynini.accep(bound)] + if len(bound) > 1: + branches.append(NONZERO + pynini.closure(DIGIT, 0, len(bound) - 2)) + for index, limit in enumerate(bound): + minimum = 1 if index == 0 else 0 + if int(limit) > minimum: + lower = pynini.union(*(str(n) for n in range(minimum, int(limit)))) + branches.append( + pynini.accep(bound[:index]) + + lower + + pynini.closure(DIGIT, len(bound) - index - 1, len(bound) - index - 1) + ) + return pynini.union(*branches).optimize() + + +def format_integer(max_value: int, thousands_separator: str) -> "pynini.FstLike": + """Groups digits in threes, e.g. 25000 -> 25,000. Rejects values outside 1..max_value.""" + first = pynini.closure(DIGIT, 1, 3) + triple = pynini.closure(DIGIT, 3, 3) + grouped = first + pynini.closure(pynutil.insert(thousands_separator) + triple) + return (integer_domain(max_value) @ grouped).optimize() + + +# Internal markers, removed before output. Control characters, so they never clash with input text. +RUN_START = "\x01" # start of a run of number words +RUN_END = "\x02" # end of a run of number words +IMPLIED = "\x03" # implied-currency amount: allowed at the start of the input or right after a money cue +CUE_ONLY = "\x04" # implied-currency amount: allowed only right after a money cue +MARKERS = pynini.union(RUN_START, RUN_END, IMPLIED, CUE_ONLY).optimize() + +# Any input word: every character except whitespace. An apostrophe may be part of a +# number word (Luganda n'ebiri). Words with capitals, digits or punctuation are +# ordinary words and pass through unchanged. +WORD_CHAR = pynini.difference(NEMO_NOT_SPACE, MARKERS).optimize() +WORD = pynini.closure(WORD_CHAR, 1).optimize() +# Runs of whitespace between words become a single space. +SINGLE_SPACES = pynini.closure(WORD_CHAR | pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), NEMO_SPACE)).optimize() + + +def word_token(graph: pynini.Fst) -> pynini.Fst: + return pynutil.insert('tokens { name: "') + graph + pynutil.insert('" }') + + +def token(graph: pynini.Fst) -> pynini.Fst: + """Wraps a class token, e.g. cardinal { ... }, as one item of the sentence.""" + return pynutil.insert("tokens { ") + graph + pynutil.insert(" }") + + +def run(graph: pynini.Fst) -> pynini.Fst: + """Reads a whole marked run of number words with graph.""" + return pynutil.delete(RUN_START) + graph + pynutil.delete(RUN_END) + + +def join_aware_runs(number_words, join_words): + """Marks maximal runs of number words; a join word followed by an ordinary word ends the run. + + Joins such as Luganda mu also have nonnumeric uses ("in"). In + "ente bbiri mu nnimiro" the run is "bbiri"; "mu nnimiro" stays as words. + A run may still end on a join at the end of the input, where it is malformed + and falls back whole (a cut-off amount is not guessed). A join between two + number words still extends the run, so malformed internals fall back whole. + + Returns (number word, ordinary word or join, segmenter). + """ + numeric_word = pynini.union(*sorted(number_words)).optimize() + join_word = pynini.union(*sorted(join_words)).optimize() + other_word = pynini.difference(WORD, numeric_word).optimize() + plain_number = pynini.difference(numeric_word, join_word).optimize() + any_word = (other_word | join_word).optimize() + # Between or after runs: words and joins, containing at least one ordinary word. + others = (pynini.closure(any_word + NEMO_SPACE) + other_word + pynini.closure(NEMO_SPACE + any_word)).optimize() + numbers = numeric_word + pynini.closure(NEMO_SPACE + numeric_word) + before_others = pynini.closure(numeric_word + NEMO_SPACE) + plain_number + + def marked(graph): + return pynutil.insert(RUN_START) + graph + pynutil.insert(RUN_END) + + runs = pynini.closure(marked(before_others) + NEMO_SPACE + others + NEMO_SPACE) + ( + marked(before_others) + NEMO_SPACE + others | marked(numbers) + ) + segmenter = (others | pynini.closure(others + NEMO_SPACE, 0, 1) + runs).optimize() + return numeric_word, any_word, segmenter + + +def money_cue_word(profile: CardinalProfile) -> pynini.Fst: + """Words that signal a payment (e.g. Luganda money verbs); empty if the profile has none.""" + cues = profile.money_cues + letters = pynini.closure(WORD_CHAR) + options = [letters + pynini.union(*cues.stems) + letters] if cues.stems else [] + options += [pynini.union(*cues.words)] if cues.words else [] + if not options: + return pynini.Fst() + cue = pynini.union(*options).optimize() + if cues.not_cues: + cue = pynini.difference(cue, pynini.union(*cues.not_cues).optimize()) + return cue.optimize() + + +def only_with_money_signal(profile: CardinalProfile) -> pynini.Fst: + """Keeps marked implied-currency amounts only where they are allowed, then drops the markers. + + An amount marked IMPLIED is money at the start of the input or right after a money + cue; one marked CUE_ONLY only right after a money cue. Otherwise the tagging is + rejected and another reading (the words unchanged, or a counted noun) is used. + """ + plain = pynini.difference(NEMO_CHAR, pynini.union(IMPLIED, CUE_ONLY)).optimize() + after_cue = 'tokens { name: "' + money_cue_word(profile) + '" }' + NEMO_SPACE + pynini.union(IMPLIED, CUE_ONLY) + allowed = (pynini.closure(pynini.accep(IMPLIED), 0, 1) + pynini.closure(plain | after_cue)).optimize() + strip = pynini.closure(plain | pynutil.delete(IMPLIED) | pynutil.delete(CUE_ONLY)) + return (allowed @ strip.optimize()).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/profile.py b/nemo_text_processing/inverse_text_normalization/bantu/profile.py new file mode 100644 index 000000000..fd3efe85e --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/profile.py @@ -0,0 +1,194 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The settings a language supplies to the shared Bantu kernel: its number words, agreement and scales.""" +from dataclasses import dataclass, replace +from typing import Dict, FrozenSet, List, Tuple + +Forms = Tuple[Tuple[str, str], ...] + + +@dataclass(frozen=True) +class NounContext: + class_id: str + noun: str + atomic_units: Forms + final_units: Forms + compounds: bool = True + teen_variants: Forms = () + + +@dataclass(frozen=True) +class Place: + # Each form outputs its nonzero coefficient, not the complete place value. + power: int + forms: Forms + remainder_join: str + + +@dataclass(frozen=True) +class CurrencyContext: + # A spoken currency phrase sets the amount's agreement, like a counted noun. + phrase: str + code: str + agreement_class: str + roles: Tuple[str, ...] = () # extra readings licensed only after this phrase + + +@dataclass(frozen=True) +class CountedScale: + """A scale noun followed by its count, e.g. Luganda mitwalo abiri = 20 x 10,000.""" + + singular: str # head counted once, e.g. mutwalo + singular_count: str # its agreeing "one", e.g. gumu + plural: str # e.g. mitwalo + power: int # 4 for ten-thousands + count_class: str # agreement class of unit words inside the count + tens: Forms # tens words inside a count, coefficient 1-9 + remainder_class: str # agreement class of the amount after the scale + contracted_teen_joins: Tuple[str, ...] = () # teen join before a vowel, e.g. n' + implied_currency: str = "" # payment policy: a bare amount means this currency + count_ones: Forms = () # a "one" ending a count, e.g. kkumi na kamu (11) + count_hundreds: Forms = () # hundreds inside a count, e.g. obukadde kikumi (100) + # Heads with the augment (Luganda e-/a-/o-) count the noun before them; the bare + # heads above are money. + counting_plural: str = "" # e.g. emitwalo, obukadde + counting_ones: Tuple[str, ...] = () # e.g. akakadde kamu + money_only_ones: Tuple[str, ...] = () # one, as money only: akakadde (not ebitabo akakadde) + implied_anywhere: bool = False # bare heads are money wherever they appear + counting_money_at_start: bool = False # augmented heads are money at the start of input + counts_any_word: bool = False # word + augmented head counts that word, listed or not + after_counted_nouns: bool = False # augmented heads count listed nouns, with agreement + + +@dataclass(frozen=True) +class MoneyCues: + """Words that signal a payment just before an amount (e.g. Luganda money verbs).""" + + stems: Tuple[str, ...] = () # match anywhere inside a word + words: Tuple[str, ...] = () # match only the whole word + not_cues: Tuple[str, ...] = () # whole words containing a stem that are not cues + + +@dataclass(frozen=True) +class CardinalProfile: + language: str + contexts: Tuple[NounContext, ...] + ten: str + teen_join: str + places: Tuple[Place, ...] + exact_boundaries: Forms + roles: Tuple[Tuple[str, Forms], ...] + max_value: int + currencies: Tuple[CurrencyContext, ...] = () + counted_scales: Tuple[CountedScale, ...] = () + money_cues: MoneyCues = MoneyCues() + # Words that continue a number but are not handled yet (e.g. "and a half"). + # They join the number's run, so the whole phrase stays unchanged. + held_back_words: Tuple[str, ...] = () + # Vowels, for contracting a join before a vowel-initial word (Luganda na + ebiri -> n'ebiri). + vowels: str = "aeiou" + # Written separator between digit groups in the output (25,000). + thousands_separator: str = "," + + def join_words(self) -> FrozenSet[str]: + """Words that join number parts; they may also be ordinary words (e.g. locative mu).""" + joins = [self.teen_join] + [place.remainder_join for place in self.places] + return frozenset(word for join in joins for word in join.split()) + + def number_words(self) -> FrozenSet[str]: + """Every word that can be part of a number, including joins and role-only words.""" + surfaces = [self.ten, self.teen_join] + for place in self.places: + surfaces.append(place.remainder_join) + surfaces.extend(form for form, _ in place.forms) + for context in self.contexts: + surfaces.extend(form for form, _ in context.atomic_units + context.final_units + context.teen_variants) + surfaces.extend(form for form, _ in self.exact_boundaries) + for _, forms in self.roles: + surfaces.extend(form for form, _ in forms) + for scale in self.counted_scales: + surfaces.extend( + (scale.singular, scale.singular_count, scale.plural) + scale.counting_ones + scale.money_only_ones + ) + if scale.counting_plural: + surfaces.append(scale.counting_plural) + surfaces.extend(form for form, _ in scale.tens + scale.count_ones + scale.count_hundreds) + surfaces.extend(scale_teen_words(self, scale)) + surfaces.extend(self.held_back_words) + return frozenset(word for form in surfaces for word in form.split()) + + +def scale_teen_words(profile: CardinalProfile, scale: CountedScale) -> List[str]: + """Contracted teen words such as n'ebiri / nebiri (one word each).""" + context = agreement_context(profile, scale.count_class) + return [ + prefix + form + for form, _ in context.final_units + scale.count_ones + if form[0] in profile.vowels + for prefix in scale.contracted_teen_joins + ] + + +def agreement_context(profile: CardinalProfile, class_id: str) -> NounContext: + context = next((c for c in profile.contexts if c.class_id == class_id and c.compounds), None) + if context is None: + raise ValueError(f"no compound agreement context for class {class_id}") + return context + + +def invariant_context(profile: CardinalProfile) -> NounContext: + """Unit forms that are the same in every class (Luganda 6-9), for unknown nouns. + + A number after a noun the profile does not list may only end in these, so + nothing that must agree with that noun is guessed. + """ + contexts = [c for c in profile.contexts if c.compounds] + shared = set(contexts[0].final_units) + for context in contexts[1:]: + shared &= set(context.final_units) + units = tuple(sorted(shared, key=lambda form: form[1])) + return NounContext("invariant", "", units + ((profile.ten, "10"),), units) + + +def with_spelling_variants(profile: CardinalProfile, variants: Dict[str, Tuple[str, ...]]) -> CardinalProfile: + """Accept extra spellings of whole words (e.g. abiiri for abiri) in every form.""" + + def spell(forms: Forms) -> Forms: + extra = [] + for form, value in forms: + words = form.split() + for index, word in enumerate(words): + for variant in variants.get(word, ()): + extra.append((" ".join(words[:index] + [variant] + words[index + 1 :]), value)) + return forms + tuple(extra) + + return replace( + profile, + contexts=tuple( + replace( + c, + atomic_units=spell(c.atomic_units), + final_units=spell(c.final_units), + teen_variants=spell(c.teen_variants), + ) + for c in profile.contexts + ), + places=tuple(replace(p, forms=spell(p.forms)) for p in profile.places), + roles=tuple((name, spell(forms)) for name, forms in profile.roles), + counted_scales=tuple( + replace(s, tens=spell(s.tens), count_hundreds=spell(s.count_hundreds)) for s in profile.counted_scales + ), + ) diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/__init__.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/__init__.py new file mode 100644 index 000000000..eb60428b9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py new file mode 100644 index 000000000..ae462929c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/cardinal.py @@ -0,0 +1,136 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import ( + DIGIT, + NONZERO, + NONZERO_DIGITS, + integer_domain, + mapping, + money_cue_word, + run, + token, + word_token, +) +from nemo_text_processing.inverse_text_normalization.bantu.profile import ( + CardinalProfile, + NounContext, + invariant_context, +) +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE, GraphFst + + +def build_cardinal(profile: CardinalProfile, context: NounContext) -> pynini.Fst: + """Compose fixed-width places, supply skipped zeros, then canonicalize once.""" + atomic = mapping(context.atomic_units) + if not context.compounds: + return (atomic @ integer_domain(profile.max_value)).optimize() + units = mapping(context.final_units) + lower = units + expected_power = 1 + for place in profile.places: + if place.power != expected_power: + raise ValueError("places must be consecutive ascending decimal powers") + if any(value not in NONZERO_DIGITS or len(value) != 1 for _, value in place.forms): + raise ValueError("place coefficients must be single nonzero digits") + # A present high place either consumes a smaller remainder or emits zeros. + high = mapping(place.forms) + ( + pynutil.delete(place.remainder_join) + lower | pynutil.insert("0" * place.power) + ) + if place.power == 1: + high |= pynini.cross(profile.ten, "10") + high |= pynini.cross(profile.ten, "1") + pynutil.delete(profile.teen_join) + units + high |= mapping(context.teen_variants) + lower = (pynutil.insert("0") + lower | high).optimize() + expected_power += 1 + canonical = pynutil.delete(pynini.closure("0")) + NONZERO + pynini.closure(DIGIT) + compounds = lower @ canonical + # A final-one form need not be licensed as a standalone plural count. + compounds @= NONZERO + pynini.closure(DIGIT, 1) + result = atomic | compounds | mapping(profile.exact_boundaries) + return (result @ integer_domain(profile.max_value)).optimize() + + +def build_role(profile: CardinalProfile, role: str) -> pynini.Fst: + """Explicit role probe; not a standalone sentence or currency grammar.""" + for name, forms in profile.roles: + if name == role: + return mapping(forms) + raise KeyError(role) + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying a counted number: a noun followed by a + run of number words that agree with it. The noun stays a word; the number is + fully resolved here, e.g. + bakopa pideka vo baima -> tokens { name: "bakopa" } tokens { cardinal { integer: "21" } } + + Any other word followed by a counting scale (Luganda embuzi emitwalo ebiri, + 20,000 goats) is also a counted number. Here the count may only end in forms + that are the same in every class, so no agreement with an unlisted noun is guessed. + + The graphs other classes reuse are kept as attributes: + by_class: the agreeing number graph for each noun class that compounds + roles: each role's graph (e.g. a currency-only reading), bounded by the profile maximum + + Args: + profile: the language profile + ordinary_word: any word that is not a number word + """ + + def __init__(self, profile: CardinalProfile, ordinary_word: pynini.Fst): + super().__init__(name="cardinal", kind="classify") + # Imported here: the scale builders use build_cardinal from this module. + from nemo_text_processing.inverse_text_normalization.bantu.taggers.scale import ( + build_noun_scale_amount, + build_scale_amount, + ) + + self.profile = profile + numbers = [build_cardinal(profile, context) for context in profile.contexts] + self.by_class = {} + for context, number in zip(profile.contexts, numbers): + if context.compounds: + self.by_class.setdefault(context.class_id, number) + self.roles = {} + for name, _ in profile.roles: + self.roles.setdefault(name, (build_role(profile, name) @ integer_domain(profile.max_value)).optimize()) + + def cardinal_token(number): + return token(self.add_tokens(pynutil.insert('integer: "') + number + pynutil.insert('"'))) + + graphs = [] + for context, number in zip(profile.contexts, numbers): + number = number | build_noun_scale_amount(profile, context) + graphs.append(word_token(pynini.accep(context.noun)) + NEMO_SPACE + run(cardinal_token(number))) + + # Listed nouns are counted by their own cardinal above: a profile covers large + # amounts after them with its places (Luganda abantu emitwalo ebiri) or with + # after_counted_nouns. After a money cue or a currency word the amount is money, + # handled by MoneyFst. + excluded = [context.noun for context in profile.contexts] + excluded += [currency.phrase.split()[-1] for currency in profile.currencies] + excluded = pynini.union(*excluded, money_cue_word(profile)).optimize() + any_noun = pynini.difference(ordinary_word, excluded).optimize() + for scale in profile.counted_scales: + if scale.counts_any_word: + amount = build_scale_amount(profile, scale, "counting", invariant_context(profile)) + graphs.append(word_token(any_noun) + NEMO_SPACE + run(cardinal_token(amount))) + + self.fst = pynini.union(*graphs).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/money.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/money.py new file mode 100644 index 000000000..78ad943e0 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/money.py @@ -0,0 +1,84 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import CUE_ONLY, IMPLIED, run, token +from nemo_text_processing.inverse_text_normalization.bantu.profile import CurrencyContext, agreement_context +from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.bantu.taggers.scale import build_scale_amount +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE, GraphFst + + +def build_amount(cardinal: CardinalFst, currency: CurrencyContext) -> pynini.Fst: + """Digits for an amount after this currency phrase: agreeing numbers, licensed roles and counted scales.""" + profile = cardinal.profile + if currency.agreement_class not in cardinal.by_class: + agreement_context(profile, currency.agreement_class) # raises a clear error + amount = cardinal.by_class[currency.agreement_class].copy() # |= below edits in place + for role in currency.roles: + amount |= cardinal.roles[role] + for scale in profile.counted_scales: + amount |= build_scale_amount(profile, scale) + return amount.optimize() + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money, e.g. + koin mina epi -> tokens { money { integer_part: "20000" currency: "XTS" } } + + An amount after a spoken currency phrase is always money. A counted scale may + also imply a currency without one (Luganda mitwalo ebiri, 20,000 UGX): + - bare scale heads (mitwalo) are money anywhere, or only after a money cue + when the scale's implied_anywhere is not set; + - heads with the augment (emitwalo, akakadde) are money only after a money cue, + or also at the start of the input when counting_money_at_start is set. + Implied amounts carry a marker, checked and removed by only_with_money_signal + once the whole sentence is tagged. + + Args: + cardinal: the CardinalFst, whose number graphs the amounts reuse + """ + + def __init__(self, cardinal: CardinalFst): + super().__init__(name="money", kind="classify") + profile = cardinal.profile + + def money_token(amount, code, marker=""): + fields = pynutil.insert('integer_part: "') + amount + pynutil.insert(f'" currency: "{code}"') + return run(pynutil.insert(marker) + token(self.add_tokens(fields))) + + graphs = [] + for currency in profile.currencies: + graphs.append( + pynutil.delete(currency.phrase + NEMO_SPACE) + + money_token(build_amount(cardinal, currency), currency.code) + ) + + self.has_implied_currency = False + for scale in profile.counted_scales: + if not scale.implied_currency: + continue + self.has_implied_currency = True + bare = build_scale_amount(profile, scale, "bare") + graphs.append(money_token(bare, scale.implied_currency, "" if scale.implied_anywhere else IMPLIED)) + if scale.counting_plural or scale.counting_ones or scale.money_only_ones: + augmented = build_scale_amount(profile, scale, "counting_money") + marker = IMPLIED if scale.counting_money_at_start else CUE_ONLY + graphs.append(money_token(augmented, scale.implied_currency, marker)) + + self.fst = pynini.union(*graphs).optimize() if graphs else pynini.Fst() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/scale.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/scale.py new file mode 100644 index 000000000..aaa269163 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/scale.py @@ -0,0 +1,141 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Counted scales: a scale noun, its count (1-999), then an optional smaller remainder. + +The remainder may itself be a smaller counted scale: +obukadde bubiri mu mitwalo ataano = 2 x 1,000,000 + 50 x 10,000. + +Example (Luganda): mitwalo abiri mu etaano mu nkumi ttaano = 25 x 10,000 + 5,000. +Unit words in the count agree with the scale noun; the remainder agrees with what +is counted. A remainder must be at least 10, so a single unit after the join always +belongs to the count (6-9 have the same form in every class). +""" +from typing import Optional, Tuple + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import DIGIT, NONZERO, integer_domain, mapping +from nemo_text_processing.inverse_text_normalization.bantu.profile import ( + CardinalProfile, + CountedScale, + NounContext, + agreement_context, +) +from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import build_cardinal +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE + + +def build_count(profile: CardinalProfile, scale: CountedScale) -> Tuple[pynini.Fst, pynini.Fst]: + """Digits for a plural count: 02-99, or 100-999 when the scale has count_hundreds. + + Returns (count, round_hundreds): round_hundreds is the part of count that is + hundreds alone (obukadde kikumi), which a remainder below 100 may not follow. + """ + context = agreement_context(profile, scale.count_class) + final_units = context.final_units + scale.count_ones + units = mapping(final_units) # 1-9 + plural_units = mapping(tuple((f, v) for f, v in context.atomic_units if v != "10")) + tens = mapping(scale.tens) + higher_tens = mapping(tuple((f, v) for f, v in scale.tens if v != "1")) + join = profile.places[0].remainder_join + teen_units = [] + for form, value in final_units: + if form[0] in profile.vowels: + spoken = [NEMO_SPACE + prefix + form for prefix in scale.contracted_teen_joins] + else: + spoken = [profile.teen_join + form] + teen_units.extend(pynini.cross(s, value) for s in spoken) + teens = pynini.cross(profile.ten, "1") + pynini.union(*teen_units) + two_digits = ( + pynutil.insert("0") + plural_units + | tens + pynutil.insert("0") + | teens + | higher_tens + pynutil.delete(join) + units + ).optimize() + if not scale.count_hundreds: + return two_digits, pynini.Fst() + # obukadde kikumi mu ataano = 150: hundreds, then an optional mu + 01-99. + below_hundred = two_digits | pynutil.insert("0") + units + hundreds = mapping(scale.count_hundreds) + count = two_digits | hundreds + pynutil.delete(join) + below_hundred + return count.optimize(), (hundreds + pynutil.insert("00")).optimize() + + +HEADS = ("bare", "counting", "counting_money", "all") + + +def build_scale_amount( + profile: CardinalProfile, scale: CountedScale, heads: str = "all", remainder_context: Optional[NounContext] = None +) -> pynini.Fst: + """Digits for a counted-scale amount, bounded by the profile maximum. + + heads picks the spoken heads: "bare" (mitwalo, kakadde kamu), "counting" (with + the augment: emitwalo, akakadde kamu), "counting_money" (counting + money-only + ones such as akakadde), or "all". Empty if the scale has no such heads. + remainder_context sets the agreement after the join (default: remainder_class), + e.g. the counted noun in "ebitabo obukadde bubiri". + """ + if heads not in HEADS: + raise ValueError(f"unknown heads {heads!r}") + join = profile.places[0].remainder_join + count, round_hundreds = build_count(profile, scale) + ones, plurals = [], [] + if heads in ("bare", "all"): + ones.append(f"{scale.singular} {scale.singular_count}") + plurals.append(scale.plural) + if heads != "bare": + ones.extend(scale.counting_ones) + plurals.extend([scale.counting_plural] if scale.counting_plural else []) + if heads in ("counting_money", "all"): + ones.extend(scale.money_only_ones) + if not ones and not plurals: + return pynini.Fst() + plural = pynini.union(*(pynutil.delete(p + NEMO_SPACE) for p in plurals)) if plurals else pynini.Fst() + head = plural + count + if ones: + head |= pynini.union(*(pynini.cross(one, "01") for one in ones)) + width = scale.power + + def padded(smallest_digits): + return pynini.union( + *( + pynutil.insert("0" * (width - k)) + NONZERO + pynini.closure(DIGIT, k - 1, k - 1) + for k in range(smallest_digits, width + 1) + ) + ) + + context = remainder_context or agreement_context(profile, scale.remainder_class) + remainder = build_cardinal(profile, context) + for smaller in profile.counted_scales: + if smaller.power < scale.power: + remainder |= build_scale_amount(profile, smaller, "all", context) + amount = head + (pynutil.insert("0" * width) | pynutil.delete(join) + (remainder @ padded(2))) + # After round hundreds, tens/units words continue the count: + # obukadde lwenda mu kyenda mu mwenda = 999 million, never 900 million + 99. + amount |= plural + round_hundreds + (pynutil.insert("0" * width) | pynutil.delete(join) + (remainder @ padded(3))) + canonical = pynutil.delete(pynini.closure("0")) + NONZERO + pynini.closure(DIGIT) + return (amount @ canonical @ integer_domain(profile.max_value)).optimize() + + +def build_noun_scale_amount(profile: CardinalProfile, context: NounContext) -> pynini.Fst: + """Scale amounts that may count a noun (ebitabo obukadde bubiri), agreeing with it.""" + amounts = [ + build_scale_amount(profile, scale, "counting", context) + for scale in profile.counted_scales + if scale.after_counted_nouns and context.compounds + ] + return pynini.union(*amounts).optimize() if amounts else pynini.Fst() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..ea0551555 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/tokenize_and_classify.py @@ -0,0 +1,57 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini + +from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import ( + SINGLE_SPACES, + join_aware_runs, + only_with_money_signal, +) +from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile +from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.bantu.taggers.money import MoneyFst +from nemo_text_processing.inverse_text_normalization.bantu.taggers.word import WordFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE + + +def build_classifier(profile: CardinalProfile) -> pynini.Fst: + """ + Sentence-level classifier for a Bantu language, built from its profile. Each + language's ClassifyFst calls this and handles caching. + + Unlike most languages, the whole sentence is read at once: number words are + resolved in their context (the counted noun, a currency phrase, a money cue, + or a join word followed by an ordinary word), and the value is fully resolved + here. The verbalizer only formats it. Output is the standard token format, e.g. + bakopa bapi -> tokens { name: "bakopa" } tokens { cardinal { integer: "2" } } + + The kernel reads lower-cased input. Other words, including words with capitals, + digits or punctuation, pass through unchanged. + + Args: + profile: the language profile (word lists, agreement, scales, currencies) + """ + number_word, ordinary_word, segmenter = join_aware_runs(profile.number_words(), profile.join_words()) + cardinal = CardinalFst(profile, ordinary_word) + money = MoneyFst(cardinal) + word = WordFst(number_word, ordinary_word) + + token = pynini.union(cardinal.fst, money.fst, word.fst) + sentence = token + pynini.closure(NEMO_SPACE + token) + graph = SINGLE_SPACES @ segmenter @ sentence + if money.has_implied_currency: + graph @= only_with_money_signal(profile) + return graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/taggers/word.py b/nemo_text_processing/inverse_text_normalization/bantu/taggers/word.py new file mode 100644 index 000000000..d5e5f7291 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/taggers/word.py @@ -0,0 +1,42 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import run, word_token +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying words left unchanged, e.g. + kasa -> tokens { name: "kasa" } + + A run of number words that cannot be read as a number in its context also stays + unchanged as a whole, word by word; it is never partly converted. Both carry a + weight, so any valid number reading is preferred. + + Args: + number_word: any word that can be part of a number + ordinary_word: any other word, including a join word used as an ordinary word + """ + + def __init__(self, number_word: pynini.Fst, ordinary_word: pynini.Fst): + super().__init__(name="word", kind="classify") + + unread_run = run(word_token(number_word) + pynini.closure(NEMO_SPACE + word_token(number_word))) + graph = word_token(ordinary_word) | unread_run + self.fst = pynutil.add_weight(graph, 1).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/__init__.py new file mode 100644 index 000000000..eb60428b9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py new file mode 100644 index 000000000..386ff23c9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/cardinal.py @@ -0,0 +1,36 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import format_integer +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing a resolved cardinal, e.g. + cardinal { integer: "25000" } -> 25,000 + + Args: + max_value: largest value the language profile allows + thousands_separator: written separator between digit groups + """ + + def __init__(self, max_value: int, thousands_separator: str): + super().__init__(name="cardinal", kind="verbalize") + number = format_integer(max_value, thousands_separator) + graph = pynutil.delete("integer:") + delete_space + pynutil.delete('"') + number + pynutil.delete('"') + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/money.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/money.py new file mode 100644 index 000000000..6edc64835 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/money.py @@ -0,0 +1,51 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import format_integer +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_UPPER, GraphFst, delete_space, insert_space + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing a resolved amount as " ", e.g. + money { integer_part: "20000" currency: "UGX" } -> 20,000 UGX + + Args: + max_value: largest value the language profile allows + thousands_separator: written separator between digit groups + """ + + def __init__(self, max_value: int, thousands_separator: str): + super().__init__(name="money", kind="verbalize") + number = format_integer(max_value, thousands_separator) + code = pynini.closure(NEMO_UPPER, 3, 3) + graph = ( + pynutil.delete("integer_part:") + + delete_space + + pynutil.delete('"') + + number + + pynutil.delete('"') + + delete_space + + pynutil.delete("currency:") + + delete_space + + pynutil.delete('"') + + insert_space + + code + + pynutil.delete('"') + ) + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize.py new file mode 100644 index 000000000..35d162710 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize.py @@ -0,0 +1,33 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini + +from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.money import MoneyFst + + +def build_verbalizer(max_value: int, thousands_separator: str) -> pynini.Fst: + """ + Composes the verbalizer grammars. Values arrive fully resolved from the + classifier, so verbalizing is formatting only. + + Args: + max_value: largest value the language allows + thousands_separator: written separator between digit groups + """ + cardinal = CardinalFst(max_value, thousands_separator).fst + money = MoneyFst(max_value, thousands_separator).fst + return (cardinal | money).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize_final.py new file mode 100644 index 000000000..61f8c6cbe --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/verbalize_final.py @@ -0,0 +1,43 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.verbalize import build_verbalizer +from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.word import WordFst +from nemo_text_processing.text_normalization.en.graph_utils import delete_extra_space, delete_space + + +def build_final_verbalizer(max_value: int, thousands_separator: str) -> pynini.Fst: + """ + Verbalizes an entire sentence. Each language's VerbalizeFinalFst calls this. + + Args: + max_value: largest value the language allows + thousands_separator: written separator between digit groups + """ + types = build_verbalizer(max_value, thousands_separator) | WordFst().fst + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + return graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/word.py new file mode 100644 index 000000000..c2382e72b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/bantu/verbalizers/word.py @@ -0,0 +1,37 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst, delete_space + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing a plain word, e.g. + tokens { name: "ente" } -> ente + """ + + def __init__(self): + super().__init__(name="word", kind="verbalize") + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_SPACE, 1) + + pynutil.delete('"') + ) + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/inverse_normalize.py b/nemo_text_processing/inverse_text_normalization/inverse_normalize.py index 9a6fcc64c..0db0aea07 100644 --- a/nemo_text_processing/inverse_text_normalization/inverse_normalize.py +++ b/nemo_text_processing/inverse_text_normalization/inverse_normalize.py @@ -146,6 +146,11 @@ def __init__( from nemo_text_processing.inverse_text_normalization.ko.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) + elif lang == 'toy': # test language for the shared Bantu kernel; hidden from the CLI + from nemo_text_processing.inverse_text_normalization.toy.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.inverse_text_normalization.toy.verbalizers.verbalize_final import ( + VerbalizeFinalFst, + ) else: raise NotImplementedError(f"Language {lang} has not been supported yet.") diff --git a/nemo_text_processing/inverse_text_normalization/toy/__init__.py b/nemo_text_processing/inverse_text_normalization/toy/__init__.py new file mode 100644 index 000000000..eb60428b9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/__init__.py b/nemo_text_processing/inverse_text_normalization/toy/data/__init__.py new file mode 100644 index 000000000..eb60428b9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/money/__init__.py b/nemo_text_processing/inverse_text_normalization/toy/data/money/__init__.py new file mode 100644 index 000000000..eb60428b9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/money/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/money/cues.tsv b/nemo_text_processing/inverse_text_normalization/toy/data/money/cues.tsv new file mode 100644 index 000000000..73f1393f9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/money/cues.tsv @@ -0,0 +1,3 @@ +lipo stem +dalo word +olipon not diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/money/currency.tsv b/nemo_text_processing/inverse_text_normalization/toy/data/money/currency.tsv new file mode 100644 index 000000000..36594f785 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/money/currency.tsv @@ -0,0 +1,5 @@ +koin XTS 4 short-tens +dolari USD 4 +paundi GBP 4 +yuro EUR 6 +yeni JPY 8 diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/numbers/__init__.py b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/__init__.py new file mode 100644 index 000000000..eb60428b9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/numbers/exact.tsv b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/exact.tsv new file mode 100644 index 000000000..a33d66765 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/exact.tsv @@ -0,0 +1 @@ +tausi 1000 diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/numbers/held_back.tsv b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/held_back.tsv new file mode 100644 index 000000000..171e78a3a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/held_back.tsv @@ -0,0 +1 @@ +halfu diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/numbers/places.tsv b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/places.tsv new file mode 100644 index 000000000..252102e5b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/places.tsv @@ -0,0 +1,35 @@ +pideka 2 1 +kodeka 3 1 +dedeka 4 1 +fudeka 5 1 +seideka 6 1 +sitideka 7 1 +otodeka 8 1 +nonadeka 9 1 +hanto 1 2 +hanto pi 2 2 +hanto ko 3 2 +hanto de 4 2 +hanto fu 5 2 +hanto sei 6 2 +hanto siti 7 2 +hanto oto 8 2 +hanto nona 9 2 +milo 1 3 +milo pi 2 3 +milo ko 3 3 +milo de 4 3 +milo fu 5 3 +milo sei 6 3 +milo siti 7 3 +milo oto 8 3 +milo nona 9 3 +omuna 1 4 +emina epi 2 4 +emina eko 3 4 +emina ede 4 4 +emina efu 5 4 +emina sei 6 4 +emina siti 7 4 +emina oto 8 4 +emina nona 9 4 diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/numbers/roles.tsv b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/roles.tsv new file mode 100644 index 000000000..7ba9e4c91 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/roles.tsv @@ -0,0 +1 @@ +mapi 20 short-tens diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/numbers/scale_words.tsv b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/scale_words.tsv new file mode 100644 index 000000000..43b76cfae --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/scale_words.tsv @@ -0,0 +1,28 @@ +mina tens deka 1 +mina tens pideka 2 +mina tens kodeka 3 +mina tens dedeka 4 +mina tens fudeka 5 +mina tens seideka 6 +mina tens sitideka 7 +mina tens otodeka 8 +mina tens nonadeka 9 +bumega tens deka 1 +bumega tens pideka 2 +bumega tens kodeka 3 +bumega tens dedeka 4 +bumega tens fudeka 5 +bumega tens seideka 6 +bumega tens sitideka 7 +bumega tens otodeka 8 +bumega tens nonadeka 9 +bumega ones kaima 1 +bumega hundreds hanto 1 +bumega hundreds hanto pi 2 +bumega hundreds hanto ko 3 +bumega hundreds hanto de 4 +bumega hundreds hanto fu 5 +bumega hundreds hanto sei 6 +bumega hundreds hanto siti 7 +bumega hundreds hanto oto 8 +bumega hundreds hanto nona 9 diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/numbers/spelling_variants.tsv b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/spelling_variants.tsv new file mode 100644 index 000000000..1de714fdd --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/spelling_variants.tsv @@ -0,0 +1 @@ +pideka piideka diff --git a/nemo_text_processing/inverse_text_normalization/toy/data/numbers/units.tsv b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/units.tsv new file mode 100644 index 000000000..a69d7a7a7 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/data/numbers/units.tsv @@ -0,0 +1,41 @@ +ima 1 1 atomic +bapi 2 2 any +bako 3 2 any +bade 4 2 any +bafu 5 2 any +sei 6 2 any +siti 7 2 any +oto 8 2 any +nona 9 2 any +deka 10 2 atomic +baima 1 2 final +zipi 2 4 any +ziko 3 4 any +zide 4 4 any +zifu 5 4 any +sei 6 4 any +siti 7 4 any +oto 8 4 any +nona 9 4 any +deka 10 4 atomic +zima 1 4 final +mapi 2 6 any +mako 3 6 any +made 4 6 any +mafu 5 6 any +sei 6 6 any +siti 7 6 any +oto 8 6 any +nona 9 6 any +deka 10 6 atomic +dekafu 15 6 teen +epi 2 8 any +eko 3 8 any +ede 4 8 any +efu 5 8 any +sei 6 8 any +siti 7 8 any +oto 8 8 any +nona 9 8 any +deka 10 8 atomic +ema 1 8 final diff --git a/nemo_text_processing/inverse_text_normalization/toy/settings.py b/nemo_text_processing/inverse_text_normalization/toy/settings.py new file mode 100644 index 000000000..accaffe1c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/settings.py @@ -0,0 +1,115 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A small made-up language for testing the shared Bantu kernel on its own. + +Every word here is invented. The language is shaped like a Bantu language so that +each kernel feature is exercised (Luganda equivalents in brackets): +- noun classes with agreeing number words: bakopa bapi / zikopa zipi = 2 [abantu babiri / emiti ebiri] +- 6-9 and ten do not agree: sei, siti, oto, nona, deka [mukaaga ... kkumi] +- teens join with "ke", larger parts with "vo" [na, mu] +- "vo" is also an ordinary word, "in" [mu] +- a currency phrase that licenses a special reading: koin mapi = 20 XTS, but makopa mapi = 2 [abiri] +- other currency phrases, each agreeing with its own class: dolari zipi = 2 USD, + yuro mafu = 5 EUR, yeni efu = 5 JPY +- a counted scale of ten-thousands: mina + count [mitwalo]; after a listed noun, + emina counts that noun, also as an ordinary number place for 1-9: + bakopa emina epi = 20,000, bakopa emina pideka = 200,000 [abantu emitwalo ebiri / abiri] +- implied currency: bare "mina" is always money; word + "emina" counts that word, + except after a money verb [mitwalo / emitwalo] +- a held-back word that continues a number but is not handled: halfu [n'ekitundu] +- a count class with vowel-initial forms, so ke + epi contracts to k'epi [n'ebiri] +- a larger scale of millions: bumega + count, with augmented forms that count a noun, + "one" forms, hundreds inside the count, and money at the start [obukadde / akakadde] +- a fixed phrase (tausi = 1,000), a special teen form (dekafu = 15) and a spelling + variant (piideka = pideka) [akakadde, kkumi nataano, abiiri] + +A failing test with this language points at the kernel, not at real language data. +XTS is the ISO 4217 code reserved for testing. + +The word lists are in data/; this file holds the shape of the grammar: noun classes, +join words, places, counted scales and their switches. Hidden from the command line. +""" +import os + +from nemo_text_processing.inverse_text_normalization.bantu.data_loader import load_profile +from nemo_text_processing.inverse_text_normalization.bantu.profile import ( + CardinalProfile, + CountedScale, + NounContext, + Place, +) + +DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") + +SKELETON = CardinalProfile( + language="toy", + contexts=( + NounContext("1", "kopa", (), (), compounds=False), + NounContext("2", "bakopa", (), ()), + NounContext("4", "zikopa", (), ()), + NounContext("6", "makopa", (), ()), + NounContext("8", "ekopa", (), ()), + ), + ten="deka", + teen_join=" ke ", + places=(Place(1, (), " vo "), Place(2, (), " vo "), Place(3, (), " vo "), Place(4, (), " vo ")), + exact_boundaries=(), + roles=(), + max_value=999_999_999, + counted_scales=( + # Ten-thousands [mitwalo]: bare mina is money anywhere; emina counts a word. + CountedScale( + singular="muna", + singular_count="ema", + plural="mina", + power=4, + count_class="8", + tens=(), + remainder_class="4", + contracted_teen_joins=("k'", "k"), + implied_currency="XTS", + counting_plural="emina", + implied_anywhere=True, + counts_any_word=True, + after_counted_nouns=True, + ), + # Millions [obukadde]: augmented forms are also money at the start of input, + # count listed nouns with agreement, and need "one" to count (akamega kaima). + CountedScale( + singular="kamega", + singular_count="kaima", + plural="bumega", + power=6, + count_class="2", + tens=(), + remainder_class="4", + implied_currency="XTS", + counting_plural="obumega", + counting_ones=("akamega kaima",), + money_only_ones=("akamega",), + implied_anywhere=True, + counting_money_at_start=True, + counts_any_word=True, + after_counted_nouns=True, + ), + ), +) + + +def load() -> CardinalProfile: + """The test language's full profile: the skeleton above filled from data/.""" + return load_profile(DATA_DIR, SKELETON) diff --git a/nemo_text_processing/inverse_text_normalization/toy/taggers/__init__.py b/nemo_text_processing/inverse_text_normalization/toy/taggers/__init__.py new file mode 100644 index 000000000..eb60428b9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/taggers/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/toy/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/toy/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..c7b256a0b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/taggers/tokenize_and_classify.py @@ -0,0 +1,63 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini + +from nemo_text_processing.inverse_text_normalization.bantu.taggers.tokenize_and_classify import build_classifier +from nemo_text_processing.inverse_text_normalization.toy.settings import load +from nemo_text_processing.text_normalization.en.graph_utils import INPUT_LOWER_CASED, GraphFst, generator_main +from nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars for the test language + of the shared Bantu kernel. The grammar is built by the kernel from this language's + data/ and settings.py; see bantu.taggers.tokenize_and_classify.build_classifier. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + whitelist: not used; the test language has no whitelist + overwrite_cache: set to True to overwrite .far files + input_case: accepting either "lower_cased" or "cased" input. The kernel reads + lower-cased words and passes other words through, so both build the same grammar. + """ + + def __init__( + self, + cache_dir: str = None, + whitelist: str = None, + overwrite_cache: bool = False, + input_case: str = INPUT_LOWER_CASED, + ): + super().__init__(name="tokenize_and_classify", kind="classify") + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"toy_itn_{input_case}.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logger.info("Creating ClassifyFst grammars.") + self.fst = build_classifier(load()) + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logger.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/toy/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/toy/verbalizers/__init__.py new file mode 100644 index 000000000..eb60428b9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/verbalizers/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/toy/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/toy/verbalizers/verbalize_final.py new file mode 100644 index 000000000..85fb22f09 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/toy/verbalizers/verbalize_final.py @@ -0,0 +1,29 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.bantu.verbalizers.verbalize_final import build_final_verbalizer +from nemo_text_processing.inverse_text_normalization.toy.settings import SKELETON +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence of the test language, + e.g. tokens { name: "kasa" } tokens { cardinal { integer: "1000" } } -> kasa 1,000 + """ + + def __init__(self): + super().__init__(name="verbalize_final", kind="verbalize") + self.fst = build_final_verbalizer(SKELETON.max_value, SKELETON.thousands_separator) diff --git a/tests/nemo_text_processing/bantu/__init__.py b/tests/nemo_text_processing/bantu/__init__.py new file mode 100644 index 000000000..eb60428b9 --- /dev/null +++ b/tests/nemo_text_processing/bantu/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/bantu/test_data_loader.py b/tests/nemo_text_processing/bantu/test_data_loader.py new file mode 100644 index 000000000..f108ddf8d --- /dev/null +++ b/tests/nemo_text_processing/bantu/test_data_loader.py @@ -0,0 +1,223 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The shared loader that fills a language's profile skeleton from its TSV files.""" + +from dataclasses import replace + +import pytest + +from nemo_text_processing.inverse_text_normalization.bantu.data_loader import load_profile +from nemo_text_processing.inverse_text_normalization.bantu.profile import ( + CardinalProfile, + CountedScale, + CurrencyContext, + MoneyCues, + NounContext, + Place, +) + +SKELETON = CardinalProfile( + language="t", + contexts=(NounContext("2", "bakopa", (), ()),), + ten="deka", + teen_join=" ke ", + places=(Place(1, (), " vo "),), + exact_boundaries=(), + roles=(), + max_value=99, + counted_scales=( + CountedScale( + singular="muna", + singular_count="ema", + plural="mina", + power=4, + count_class="2", + tens=(), + remainder_class="2", + ), + ), +) + + +def write(root, files): + for name, rows in files.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join("\t".join(row) + "\n" for row in rows), encoding="utf-8") + return str(root) + + +BASE = { + "numbers/units.tsv": [("bapi", "2", "2", "any"), ("deka", "10", "2", "atomic"), ("baima", "1", "2", "final")], + "numbers/places.tsv": [("pideka", "2", "1")], +} + + +class TestBantuDataLoader: + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_units_fill_atomic_and_final(self, tmp_path): + profile = load_profile(write(tmp_path, BASE), SKELETON) + context = profile.contexts[0] + assert context.atomic_units == (("bapi", "2"), ("deka", "10")) + assert context.final_units == (("bapi", "2"), ("baima", "1")) + assert profile.places[0].forms == (("pideka", "2"),) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_missing_files_are_empty(self, tmp_path): + profile = load_profile(write(tmp_path, BASE), SKELETON) + assert profile.currencies == () + assert profile.money_cues == MoneyCues() + assert profile.held_back_words == () + assert profile.roles == () + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_blank_lines_are_skipped(self, tmp_path): + root = write(tmp_path, BASE) + (tmp_path / "numbers/held_back.tsv").write_text("halfu\n\n", encoding="utf-8") + assert load_profile(root, SKELETON).held_back_words == ("halfu",) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_bad_column_count(self, tmp_path): + root = write(tmp_path, {**BASE, "numbers/units.tsv": [("bapi", "2", "2")]}) + with pytest.raises(ValueError, match=r"units\.tsv:1"): + load_profile(root, SKELETON) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_unknown_where(self, tmp_path): + root = write(tmp_path, {**BASE, "numbers/units.tsv": [("bapi", "2", "2", "sometimes")]}) + with pytest.raises(ValueError, match=r"units\.tsv:1.*sometimes"): + load_profile(root, SKELETON) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_unknown_class(self, tmp_path): + root = write(tmp_path, {**BASE, "numbers/units.tsv": [("bapi", "2", "99", "any")]}) + with pytest.raises(ValueError, match=r"units\.tsv:1.*99"): + load_profile(root, SKELETON) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_unknown_power(self, tmp_path): + root = write(tmp_path, {**BASE, "numbers/places.tsv": [("pideka", "2", "7")]}) + with pytest.raises(ValueError, match=r"places\.tsv:1.*7"): + load_profile(root, SKELETON) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_unknown_scale(self, tmp_path): + root = write(tmp_path, {**BASE, "numbers/scale_words.tsv": [("bumega", "tens", "deka", "1")]}) + with pytest.raises(ValueError, match=r"scale_words\.tsv:1.*bumega"): + load_profile(root, SKELETON) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_scale_words(self, tmp_path): + rows = [("mina", "tens", "deka", "1"), ("mina", "ones", "kaima", "1"), ("mina", "hundreds", "hanto", "1")] + profile = load_profile(write(tmp_path, {**BASE, "numbers/scale_words.tsv": rows}), SKELETON) + scale = profile.counted_scales[0] + assert (scale.tens, scale.count_ones, scale.count_hundreds) == ( + (("deka", "1"),), + (("kaima", "1"),), + (("hanto", "1"),), + ) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_currency_roles(self, tmp_path): + rows = [("koin", "XTS", "2", "short-tens"), ("dolari", "USD", "2", "")] + files = {**BASE, "money/currency.tsv": rows, "numbers/roles.tsv": [("mapi", "20", "short-tens")]} + profile = load_profile(write(tmp_path, files), SKELETON) + assert profile.currencies == ( + CurrencyContext("koin", "XTS", "2", ("short-tens",)), + CurrencyContext("dolari", "USD", "2", ()), + ) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_currency_unknown_class(self, tmp_path): + root = write(tmp_path, {**BASE, "money/currency.tsv": [("koin", "XTS", "99", "")]}) + with pytest.raises(ValueError, match=r"currency\.tsv:1.*99"): + load_profile(root, SKELETON) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_roles_cues_exact_and_variants(self, tmp_path): + files = { + **BASE, + "numbers/roles.tsv": [("mapi", "20", "short-tens")], + "numbers/exact.tsv": [("tausi", "1000")], + "numbers/spelling_variants.tsv": [("pideka", "piideka")], + "money/cues.tsv": [("lipo", "stem"), ("dalo", "word"), ("olipon", "not")], + } + profile = load_profile(write(tmp_path, files), SKELETON) + assert profile.roles == (("short-tens", (("mapi", "20"),)),) + assert profile.exact_boundaries == (("tausi", "1000"),) + assert profile.money_cues == MoneyCues(stems=("lipo",), words=("dalo",), not_cues=("olipon",)) + assert profile.places[0].forms == (("pideka", "2"), ("piideka", "2")) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_unknown_cue_kind(self, tmp_path): + root = write(tmp_path, {**BASE, "money/cues.tsv": [("lipo", "verb")]}) + with pytest.raises(ValueError, match=r"cues\.tsv:1.*verb"): + load_profile(root, SKELETON) + + # The test language's own data loads and builds a classifier. + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_toy_profile_loads(self): + from nemo_text_processing.inverse_text_normalization.bantu.taggers.tokenize_and_classify import ( + build_classifier, + ) + from nemo_text_processing.inverse_text_normalization.toy.settings import load + + profile = load() + assert [c.noun for c in profile.contexts] == ["kopa", "bakopa", "zikopa", "makopa", "ekopa"] + assert ("piideka", "2") in profile.places[0].forms + assert build_classifier(profile).num_states() > 0 + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_currency_unknown_role(self, tmp_path): + root = write(tmp_path, {**BASE, "money/currency.tsv": [("koin", "XTS", "2", "nosuchrole")]}) + with pytest.raises(ValueError, match=r"currency\.tsv:1.*nosuchrole"): + load_profile(root, SKELETON) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_currency_class_must_compound(self, tmp_path): + skeleton = replace(SKELETON, contexts=SKELETON.contexts + (NounContext("1", "kopa", (), (), compounds=False),)) + root = write(tmp_path, {**BASE, "money/currency.tsv": [("koin", "XTS", "1", "")]}) + with pytest.raises(ValueError, match=r"currency\.tsv:1.*class '1'"): + load_profile(root, skeleton) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_empty_field(self, tmp_path): + root = write(tmp_path, {**BASE, "numbers/units.tsv": [("", "3", "2", "any")]}) + with pytest.raises(ValueError, match=r"units\.tsv:1.*empty"): + load_profile(root, SKELETON) + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_missing_data_dir(self, tmp_path): + with pytest.raises(ValueError, match="no data folder"): + load_profile(str(tmp_path / "nope"), SKELETON) diff --git a/tests/nemo_text_processing/bantu/test_kernel_options.py b/tests/nemo_text_processing/bantu/test_kernel_options.py new file mode 100644 index 000000000..9d4b2424b --- /dev/null +++ b/tests/nemo_text_processing/bantu/test_kernel_options.py @@ -0,0 +1,150 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Kernel options that a case file cannot switch: invalid profiles, optional features, +caching, and how the test language is exposed. Variants of the test language are +built from its profile with some settings changed. +""" + +import sys +from dataclasses import replace + +import pytest + +from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import integer_domain +from nemo_text_processing.inverse_text_normalization.bantu.profile import MoneyCues, Place +from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import build_role +from nemo_text_processing.inverse_text_normalization.bantu.taggers.scale import build_scale_amount +from nemo_text_processing.inverse_text_normalization.bantu.taggers.tokenize_and_classify import build_classifier +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer, parse_args +from nemo_text_processing.inverse_text_normalization.toy.settings import load +from nemo_text_processing.inverse_text_normalization.toy.taggers.tokenize_and_classify import ClassifyFst + +from ..utils import CACHE_DIR + +TOY_PROFILE = load() +SAMPLES = ("bakopa pideka vo baima", "kasa emina epi", "lipoka obumega bapi", "bakopa bapi vo kasa") + + +def build_normalizer(profile) -> InverseNormalizer: + """The test language's runtime with its classifier rebuilt from a changed profile.""" + normalizer = InverseNormalizer(lang='toy', cache_dir=CACHE_DIR) + normalizer.tagger.fst = build_classifier(profile) + return normalizer + + +class TestBantuKernelOptions: + # A broken profile is rejected when the grammar is built, with a clear error. + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_invalid_profiles_are_rejected(self): + with pytest.raises(ValueError, match="no compound agreement context"): + build_classifier( + replace(TOY_PROFILE, currencies=(replace(TOY_PROFILE.currencies[0], agreement_class="99"),)) + ) + with pytest.raises(ValueError, match="consecutive ascending"): + build_classifier(replace(TOY_PROFILE, places=TOY_PROFILE.places[1:])) + with pytest.raises(ValueError, match="single nonzero digits"): + build_classifier( + replace(TOY_PROFILE, places=(Place(1, (("pideka", "20"),), " vo "),) + TOY_PROFILE.places[1:]) + ) + with pytest.raises(ValueError, match="positive integer"): + integer_domain(0) + with pytest.raises(KeyError): + build_role(TOY_PROFILE, "no-such-role") + with pytest.raises(ValueError, match="unknown heads"): + build_scale_amount(TOY_PROFILE, TOY_PROFILE.counted_scales[0], "no-such-heads") + + # Optional features can be left out: no money cues, and scales with no implied currency. + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_profile_without_money_cues_or_implied_currency(self): + plain = replace( + TOY_PROFILE, + money_cues=MoneyCues(), + counted_scales=tuple(replace(s, implied_currency="") for s in TOY_PROFILE.counted_scales), + ) + normalizer = build_normalizer(plain) + assert normalizer.inverse_normalize("mina epi", verbose=False) == "mina epi" + assert normalizer.inverse_normalize("koin mina epi", verbose=False) == "20,000 XTS" + assert normalizer.inverse_normalize("kasa emina epi", verbose=False) == "kasa 20,000" + assert normalizer.inverse_normalize("lipoka emina epi", verbose=False) == "lipoka 20,000" + + # Implied currency without any money cues: only the start-of-input and bare-form rules apply. + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_implied_currency_without_money_cues(self): + normalizer = build_normalizer(replace(TOY_PROFILE, money_cues=MoneyCues())) + assert normalizer.inverse_normalize("obumega bapi", verbose=False) == "2,000,000 XTS" + assert normalizer.inverse_normalize("kasa mina epi", verbose=False) == "kasa 20,000 XTS" + assert normalizer.inverse_normalize("lipoka emina epi", verbose=False) == "lipoka 20,000" + + # A scale without counting heads simply has no counting reading. + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_scale_without_counting_heads(self): + bare_only = replace(TOY_PROFILE.counted_scales[0], counting_plural="", counting_ones=()) + assert build_scale_amount(TOY_PROFILE, bare_only, "counting").num_states() == 0 + + # Money reuses the cardinal's graphs, as in other NeMo languages. + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_money_takes_the_cardinal(self): + from nemo_text_processing.inverse_text_normalization.bantu.graph_utils import join_aware_runs + from nemo_text_processing.inverse_text_normalization.bantu.taggers.cardinal import CardinalFst + from nemo_text_processing.inverse_text_normalization.bantu.taggers.money import MoneyFst + + _, ordinary_word, _ = join_aware_runs(TOY_PROFILE.number_words(), TOY_PROFILE.join_words()) + cardinal = CardinalFst(TOY_PROFILE, ordinary_word) + assert cardinal.profile is TOY_PROFILE + assert set(cardinal.by_class) == {"2", "4", "6", "8"} + assert set(cardinal.roles) == {"short-tens"} + money = MoneyFst(cardinal) + assert money.has_implied_currency + assert money.fst.num_states() > 0 + + # The compiled classifier is saved to and restored from a FAR file with the same results. + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_far_cache_roundtrip(self, tmp_path): + built = ClassifyFst(cache_dir=str(tmp_path), overwrite_cache=True) + assert (tmp_path / "toy_itn_lower_cased.far").exists() + restored = ClassifyFst(cache_dir=str(tmp_path)) + normalizer = InverseNormalizer(lang='toy', cache_dir=CACHE_DIR) + for spoken in SAMPLES: + normalizer.tagger = built + expected = normalizer.inverse_normalize(spoken, verbose=False) + normalizer.tagger = restored + assert normalizer.inverse_normalize(spoken, verbose=False) == expected + + # Cased input builds the same grammar (the kernel reads lower-cased words), cached under its own name. + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_cased_input_case(self, tmp_path): + ClassifyFst(cache_dir=str(tmp_path), input_case="cased") + assert (tmp_path / "toy_itn_cased.far").exists() + cased = InverseNormalizer(lang='toy', cache_dir=str(tmp_path), input_case='cased') + lower = InverseNormalizer(lang='toy', cache_dir=CACHE_DIR) + for spoken in SAMPLES: + assert cased.inverse_normalize(spoken, verbose=False) == lower.inverse_normalize(spoken, verbose=False) + + # The test language is not offered on the command line. + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_hidden_from_cli(self, monkeypatch): + monkeypatch.setattr(sys, "argv", ["inverse_normalize.py", "--language", "toy", "--text", "kasa"]) + with pytest.raises(SystemExit): + parse_args() diff --git a/tests/nemo_text_processing/toy/__init__.py b/tests/nemo_text_processing/toy/__init__.py new file mode 100644 index 000000000..eb60428b9 --- /dev/null +++ b/tests/nemo_text_processing/toy/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/toy/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/toy/data_inverse_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..be1e7318e --- /dev/null +++ b/tests/nemo_text_processing/toy/data_inverse_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,60 @@ +kopa ima~kopa 1 +zikopa zipi~zikopa 2 +zikopa ziko~zikopa 3 +zikopa zide~zikopa 4 +zikopa zifu~zikopa 5 +zikopa sei~zikopa 6 +zikopa siti~zikopa 7 +zikopa oto~zikopa 8 +zikopa nona~zikopa 9 +zikopa deka~zikopa 10 +zikopa deka ke zima~zikopa 11 +zikopa deka ke zipi~zikopa 12 +zikopa deka ke ziko~zikopa 13 +zikopa deka ke zide~zikopa 14 +zikopa deka ke zifu~zikopa 15 +zikopa deka ke sei~zikopa 16 +zikopa deka ke siti~zikopa 17 +zikopa deka ke oto~zikopa 18 +zikopa deka ke nona~zikopa 19 +zikopa pideka~zikopa 20 +bakopa bapi~bakopa 2 +makopa mapi~makopa 2 +bakopa zipi~bakopa zipi +bakopa ima~bakopa ima +bakopa sei~bakopa 6 +makopa deka~makopa 10 +bakopa pideka vo baima~bakopa 21 +zikopa hanto pi vo kodeka vo ziko~zikopa 233 +bakopa milo~bakopa 1,000 +bakopa milo vo hanto~bakopa 1,100 +zikopa milo nona vo hanto nona vo nonadeka vo nona~zikopa 9,999 +bakopa tausi~bakopa 1,000 +makopa dekafu~makopa 15 +bakopa piideka~bakopa 20 +bakopa piideka vo baima~bakopa 21 +koin piideka~20 XTS +bakopa hanto vo milo~bakopa hanto vo milo +bakopa deka ke~bakopa deka ke +bakopa bade bako~bakopa bade bako +bakopa bafu vo kasa~bakopa 5 vo kasa +zikopa pideka vo kasa~zikopa 20 vo kasa +rana bakopa bako vo kasa lena~rana bakopa 3 vo kasa lena +rana bakopa bade lena zikopa ziko~rana bakopa 4 lena zikopa 3 +bakopa bako zikopa zifu~bakopa 3 zikopa 5 +bakopa bafu~bakopa 5 +bakopa sei !~bakopa 6 ! +zikopa ziko, bakopa bade~zikopa ziko, bakopa 4 +ŋoma bakopa siti~ŋoma bakopa 7 +bakopa omuna~bakopa 10,000 +bakopa emina epi~bakopa 20,000 +zikopa emina nona vo milo fu vo hanto~zikopa 95,100 +ekopa emina eko vo kasa~ekopa 30,000 vo kasa +bakopa emina pideka~bakopa 200,000 +bakopa emina deka k'epi~bakopa 120,000 +bakopa emina pideka vo ema~bakopa 210,000 +bakopa emina pideka vo milo fu~bakopa 205,000 +bakopa emina pideka vo baima~bakopa emina pideka vo baima +bakopa obumega bako vo emina dedeka vo efu vo milo siti vo hanto sei vo pideka vo baima~bakopa 3,457,621 +bakopa akamega kaima vo milo fu vo kodeka~bakopa 1,005,030 +zikopa obumega bafu vo emina eko vo kodeka~zikopa 5,030,030 diff --git a/tests/nemo_text_processing/toy/data_inverse_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/toy/data_inverse_text_normalization/test_cases_money.txt new file mode 100644 index 000000000..c4e0a2990 --- /dev/null +++ b/tests/nemo_text_processing/toy/data_inverse_text_normalization/test_cases_money.txt @@ -0,0 +1,52 @@ +koin zipi~2 XTS +koin mapi~20 XTS +makopa mapi~makopa 2 +koin milo vo hanto~1,100 XTS +koin~koin +koin kasa~koin kasa +muna ema~10,000 XTS +mina epi~20,000 XTS +mina pideka vo efu vo milo fu~255,000 XTS +koin mina epi~20,000 XTS +mina zipi~mina zipi +mina deka k'epi~120,000 XTS +mina deka kepi~120,000 XTS +mina deka ke epi~mina deka ke epi +kamega kaima~1,000,000 XTS +bumega bapi~2,000,000 XTS +bumega deka ke kaima~11,000,000 XTS +bumega hanto vo fudeka~150,000,000 XTS +bumega hanto nona vo nonadeka vo nona~999,000,000 XTS +bumega hanto vo milo fu~100,005,000 XTS +bumega bapi vo mina fudeka~2,500,000 XTS +kasa mina epi~kasa 20,000 XTS +kasa emina epi~kasa 20,000 +lipoka emina epi~lipoka 20,000 XTS +dalo emina epi~dalo 20,000 XTS +olipon emina epi~olipon 20,000 +emina epi~emina epi +obumega bapi~2,000,000 XTS +akamega~1,000,000 XTS +akamega kaima~1,000,000 XTS +kasa bumega bapi~kasa 2,000,000 XTS +kasa obumega bapi~kasa 2,000,000 +kasa akamega kaima~kasa 1,000,000 +kasa akamega~kasa akamega +bakopa obumega bapi~bakopa 2,000,000 +bakopa akamega kaima~bakopa 1,000,000 +lipoka obumega bapi~lipoka 2,000,000 XTS +mina epi halfu~mina epi halfu +lipoka mina epi halfu~lipoka mina epi halfu +dolari zide~4 USD +dolari pideka vo zima~21 USD +dolari mina epi~20,000 USD +paundi zifu~5 GBP +paundi milo vo hanto~1,100 GBP +yuro mafu~5 EUR +yuro deka ke mapi~12 EUR +yeni efu~5 JPY +yeni bumega bako~3,000,000 JPY +lipoka dolari ziko~lipoka 3 USD +dolari mapi~dolari mapi +yuro kasa~yuro kasa +yeni~yeni diff --git a/tests/nemo_text_processing/toy/data_inverse_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/toy/data_inverse_text_normalization/test_cases_word.txt new file mode 100644 index 000000000..0e4cbaaff --- /dev/null +++ b/tests/nemo_text_processing/toy/data_inverse_text_normalization/test_cases_word.txt @@ -0,0 +1,9 @@ +bapi~bapi +rana bapi lena~rana bapi lena +kasa vo rana~kasa vo rana +Bakopa bapi~Bakopa bapi +bakopa bapi.~bakopa bapi. +bakopa 2~bakopa 2 +bakopa-bapi~bakopa-bapi +kasa é~kasa é +50% #1 ~50% #1 diff --git a/tests/nemo_text_processing/toy/test_cardinal.py b/tests/nemo_text_processing/toy/test_cardinal.py new file mode 100644 index 000000000..8cbade61f --- /dev/null +++ b/tests/nemo_text_processing/toy/test_cardinal.py @@ -0,0 +1,36 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file +from .utils import best_tags + + +class TestCardinal: + + inverse_normalizer = InverseNormalizer(lang='toy', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('toy/data_inverse_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + # Exactly one best reading: the kernel never leaves a tie for NeMo's runtime to pick from. + assert len(best_tags(self.inverse_normalizer, test_input)) == 1 + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/toy/test_money.py b/tests/nemo_text_processing/toy/test_money.py new file mode 100644 index 000000000..1e26e981d --- /dev/null +++ b/tests/nemo_text_processing/toy/test_money.py @@ -0,0 +1,36 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file +from .utils import best_tags + + +class TestMoney: + + inverse_normalizer = InverseNormalizer(lang='toy', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('toy/data_inverse_text_normalization/test_cases_money.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + # Exactly one best reading: the kernel never leaves a tie for NeMo's runtime to pick from. + assert len(best_tags(self.inverse_normalizer, test_input)) == 1 + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/toy/test_verbalizer.py b/tests/nemo_text_processing/toy/test_verbalizer.py new file mode 100644 index 000000000..9c8767581 --- /dev/null +++ b/tests/nemo_text_processing/toy/test_verbalizer.py @@ -0,0 +1,54 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The test language's verbalizer on its own: formatting only, values are resolved by the classifier.""" + +import pytest +from parameterized import parameterized +from pynini.lib import rewrite + +from nemo_text_processing.inverse_text_normalization.toy.verbalizers.verbalize_final import VerbalizeFinalFst + +VERBALIZER = VerbalizeFinalFst().fst + + +class TestBantuKernelVerbalizer: + @parameterized.expand( + [ + ('tokens { cardinal { integer: "2" } }', "2"), + ('tokens { cardinal { integer: "999999999" } }', "999,999,999"), + ('tokens { money { integer_part: "25000" currency: "XTS" } }', "25,000 XTS"), + ('tokens { name: "kasa" } tokens { cardinal { integer: "1000" } }', "kasa 1,000"), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_formatting(self, tokens, expected): + assert rewrite.top_rewrite(tokens, VERBALIZER) == expected + + # Values outside 1..max_value, non-canonical digits and lowercase codes are rejected. + @parameterized.expand( + [ + ('tokens { cardinal { integer: "0" } }',), + ('tokens { cardinal { integer: "02" } }',), + ('tokens { cardinal { integer: "1000000000" } }',), + ('tokens { money { integer_part: "20" currency: "xts" } }',), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_rejects_invalid(self, tokens): + with pytest.raises(rewrite.Error): + rewrite.top_rewrite(tokens, VERBALIZER) diff --git a/tests/nemo_text_processing/toy/test_word.py b/tests/nemo_text_processing/toy/test_word.py new file mode 100644 index 000000000..4913ec46a --- /dev/null +++ b/tests/nemo_text_processing/toy/test_word.py @@ -0,0 +1,36 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file +from .utils import best_tags + + +class TestWord: + + inverse_normalizer = InverseNormalizer(lang='toy', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('toy/data_inverse_text_normalization/test_cases_word.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + # Exactly one best reading: the kernel never leaves a tie for NeMo's runtime to pick from. + assert len(best_tags(self.inverse_normalizer, test_input)) == 1 + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/toy/utils.py b/tests/nemo_text_processing/toy/utils.py new file mode 100644 index 000000000..6242f1bea --- /dev/null +++ b/tests/nemo_text_processing/toy/utils.py @@ -0,0 +1,21 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, Crane Labs. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib import rewrite + + +def best_tags(normalizer, text: str) -> set: + """All equally best tagger outputs. The kernel must give exactly one for any input.""" + return set(rewrite.optimal_rewrites(text, normalizer.tagger.fst))