Skip to content
Merged
5 changes: 5 additions & 0 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -483,6 +483,11 @@ pipeline {
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/ko/ -m "not pleasefixme" --cpu --tn_cache_dir ${KO_TN_CACHE}'
}
}
stage('L1: Run Bantu kernel ITN tests') {
steps {
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/bantu/ tests/nemo_text_processing/toy/ -m "not pleasefixme" --cpu'
}
}
}
}

Expand Down
14 changes: 14 additions & 0 deletions nemo_text_processing/inverse_text_normalization/bantu/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# Copyright (c) 2026, Crane Labs. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
163 changes: 163 additions & 0 deletions nemo_text_processing/inverse_text_normalization/bantu/data_loader.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# Copyright (c) 2026, Crane Labs. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Fills a language's profile skeleton with the word lists in its data folder.

Each language writes the shape of its grammar in settings.py: a CardinalProfile whose
word lists are empty (noun classes, places, counted scales with their head words and
switches). Its word lists live in tab-separated files under data/; every file is
optional, and a missing file means an empty list:

numbers/units.tsv form, value, class, where (any, atomic, final, teen)
numbers/places.tsv form, coefficient, power
numbers/exact.tsv form, value
numbers/roles.tsv form, value, role
numbers/scale_words.tsv scale plural, part (tens, ones, hundreds), form, value
numbers/held_back.tsv word
numbers/spelling_variants.tsv word, variant
money/currency.tsv phrase, code, class, roles (comma-separated, may be empty)
money/cues.tsv word, kind (stem, word, not)

A "where" of any puts a unit form both inside compounds (atomic) and at the end of a
number (final); teen forms are whole teen words such as a contracted 15.
"""
import csv
import os
from collections import OrderedDict
from dataclasses import replace
from typing import Dict, List, Tuple

from nemo_text_processing.inverse_text_normalization.bantu.profile import (
CardinalProfile,
CurrencyContext,
MoneyCues,
with_spelling_variants,
)

WHERE = {
"any": ("atomic_units", "final_units"),
"atomic": ("atomic_units",),
"final": ("final_units",),
"teen": ("teen_variants",),
}
SCALE_PARTS = {"tens": "tens", "ones": "count_ones", "hundreds": "count_hundreds"}
CUE_KINDS = {"stem": "stems", "word": "words", "not": "not_cues"}


def read_rows(data_dir: str, name: str, columns: int, last_optional: bool = False) -> List[Tuple[str, List[str]]]:
"""
Non-blank rows of one TSV as (location, row); location is "path:line" for error messages.
Every cell must be non-empty, except the last one when last_optional is set.
"""
path = os.path.join(data_dir, name)
if not os.path.exists(path):
return []
rows = []
with open(path, encoding="utf-8", newline="") as tsv:
for line, row in enumerate(csv.reader(tsv, delimiter="\t"), start=1):
if not row:
continue
if len(row) != columns:
raise ValueError(f"{path}:{line}: expected {columns} columns, found {len(row)}")
required = row[:-1] if last_optional else row
if not all(required):
raise ValueError(f"{path}:{line}: empty field")
rows.append((f"{path}:{line}", row))
return rows


def lookup(table: Dict[str, str], key: str, location: str, what: str) -> str:
if key not in table:
raise ValueError(f"{location}: unknown {what} {key!r}")
return table[key]


def load_profile(data_dir: str, skeleton: CardinalProfile) -> CardinalProfile:
"""
Returns the skeleton with its word lists filled from data_dir.

Args:
data_dir: the language's data folder
skeleton: the language's profile with empty word lists (from its settings.py)
"""
if not os.path.isdir(data_dir):
raise ValueError(f"no data folder at {data_dir}")
contexts = {
context.class_id: {"atomic_units": [], "final_units": [], "teen_variants": []} for context in skeleton.contexts
}
for location, (form, value, class_id, where) in read_rows(data_dir, "numbers/units.tsv", 4):
lists = contexts.get(class_id)
if lists is None:
raise ValueError(f"{location}: unknown class {class_id!r}")
for field in lookup(WHERE, where, location, "where"):
lists[field].append((form, value))

places = {str(place.power): [] for place in skeleton.places}
for location, (form, value, power) in read_rows(data_dir, "numbers/places.tsv", 3):
if power not in places:
raise ValueError(f"{location}: unknown power {power!r}")
places[power].append((form, value))

scales = {scale.plural: {field: [] for field in SCALE_PARTS.values()} for scale in skeleton.counted_scales}
for location, (plural, part, form, value) in read_rows(data_dir, "numbers/scale_words.tsv", 4):
if plural not in scales:
raise ValueError(f"{location}: unknown scale {plural!r}")
scales[plural][lookup(SCALE_PARTS, part, location, "part")].append((form, value))

roles = OrderedDict()
for _, (form, value, role) in read_rows(data_dir, "numbers/roles.tsv", 3):
roles.setdefault(role, []).append((form, value))

# An amount after a currency agrees with a class whose numbers compound.
compounding = {context.class_id for context in skeleton.contexts if context.compounds}
currencies = []
rows = read_rows(data_dir, "money/currency.tsv", 4, last_optional=True)
for location, (phrase, code, class_id, currency_roles) in rows:
if class_id not in compounding:
raise ValueError(f"{location}: unknown or non-compounding class {class_id!r}")
names = tuple(name for name in currency_roles.split(",") if name)
for name in names:
if name not in roles:
raise ValueError(f"{location}: unknown role {name!r}")
currencies.append(CurrencyContext(phrase, code, class_id, names))

cues = {field: [] for field in CUE_KINDS.values()}
for location, (word, kind) in read_rows(data_dir, "money/cues.tsv", 2):
cues[lookup(CUE_KINDS, kind, location, "kind")].append(word)

variants = OrderedDict()
for _, (word, variant) in read_rows(data_dir, "numbers/spelling_variants.tsv", 2):
variants.setdefault(word, []).append(variant)

profile = replace(
skeleton,
contexts=tuple(
replace(context, **{field: tuple(forms) for field, forms in contexts[context.class_id].items()})
for context in skeleton.contexts
),
places=tuple(replace(place, forms=tuple(places[str(place.power)])) for place in skeleton.places),
counted_scales=tuple(
replace(scale, **{field: tuple(forms) for field, forms in scales[scale.plural].items()})
for scale in skeleton.counted_scales
),
exact_boundaries=tuple(tuple(row) for _, row in read_rows(data_dir, "numbers/exact.tsv", 2)),
roles=tuple((role, tuple(forms)) for role, forms in roles.items()),
currencies=tuple(currencies),
money_cues=MoneyCues(**{field: tuple(words) for field, words in cues.items()}),
held_back_words=tuple(row[0] for _, row in read_rows(data_dir, "numbers/held_back.tsv", 1)),
)
if variants:
profile = with_spelling_variants(profile, {word: tuple(v) for word, v in variants.items()})
return profile
162 changes: 162 additions & 0 deletions nemo_text_processing/inverse_text_normalization/bantu/graph_utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
Comment thread
tbartley94 marked this conversation as resolved.
Resolved
# Copyright (c) 2026, Crane Labs. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Helpers shared by the Bantu taggers: digit strings, number-word runs, token formats and the money-signal filter.

The classifier reads the whole sentence at once. Maximal runs of number words are
marked first, so each run is read as a single number in the context of the word
before it (a counted noun, a currency phrase or a money cue). Weights only prefer a
normalization over leaving words unchanged. All numeric readings have equal weight,
so structure alone must make the reading unique; the kernel tests check that every
test input has exactly one best reading.
"""
import string

import pynini
from pynini.lib import pynutil

from nemo_text_processing.inverse_text_normalization.bantu.profile import CardinalProfile, Forms
from nemo_text_processing.text_normalization.en.graph_utils import (
NEMO_CHAR,
NEMO_NOT_SPACE,
NEMO_SPACE,
NEMO_WHITE_SPACE,
)

# Values are composed digit by digit inside the FSTs; there is no runtime arithmetic.
NONZERO_DIGITS = string.digits[1:]
DIGIT = pynini.union(*string.digits).optimize()
NONZERO = pynini.union(*NONZERO_DIGITS).optimize()


def mapping(forms: Forms) -> pynini.Fst:
return pynini.string_map(forms).optimize() if forms else pynini.Fst()


def integer_domain(max_value: int) -> pynini.Fst:
"""Canonical positive decimal strings <= bound, without integer enumeration."""
if isinstance(max_value, bool) or not isinstance(max_value, int) or max_value < 1:
raise ValueError("max_value must be a positive integer")
bound = str(max_value)
branches = [pynini.accep(bound)]
if len(bound) > 1:
branches.append(NONZERO + pynini.closure(DIGIT, 0, len(bound) - 2))
for index, limit in enumerate(bound):
minimum = 1 if index == 0 else 0
if int(limit) > minimum:
lower = pynini.union(*(str(n) for n in range(minimum, int(limit))))
branches.append(
pynini.accep(bound[:index])
+ lower
+ pynini.closure(DIGIT, len(bound) - index - 1, len(bound) - index - 1)
)
return pynini.union(*branches).optimize()


def format_integer(max_value: int, thousands_separator: str) -> "pynini.FstLike":
"""Groups digits in threes, e.g. 25000 -> 25,000. Rejects values outside 1..max_value."""
first = pynini.closure(DIGIT, 1, 3)
triple = pynini.closure(DIGIT, 3, 3)
grouped = first + pynini.closure(pynutil.insert(thousands_separator) + triple)
return (integer_domain(max_value) @ grouped).optimize()


# Internal markers, removed before output. Control characters, so they never clash with input text.
RUN_START = "\x01" # start of a run of number words
RUN_END = "\x02" # end of a run of number words
IMPLIED = "\x03" # implied-currency amount: allowed at the start of the input or right after a money cue
CUE_ONLY = "\x04" # implied-currency amount: allowed only right after a money cue
MARKERS = pynini.union(RUN_START, RUN_END, IMPLIED, CUE_ONLY).optimize()

# Any input word: every character except whitespace. An apostrophe may be part of a
# number word (Luganda n'ebiri). Words with capitals, digits or punctuation are
# ordinary words and pass through unchanged.
WORD_CHAR = pynini.difference(NEMO_NOT_SPACE, MARKERS).optimize()
WORD = pynini.closure(WORD_CHAR, 1).optimize()
# Runs of whitespace between words become a single space.
SINGLE_SPACES = pynini.closure(WORD_CHAR | pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), NEMO_SPACE)).optimize()


def word_token(graph: pynini.Fst) -> pynini.Fst:
return pynutil.insert('tokens { name: "') + graph + pynutil.insert('" }')


def token(graph: pynini.Fst) -> pynini.Fst:
"""Wraps a class token, e.g. cardinal { ... }, as one item of the sentence."""
return pynutil.insert("tokens { ") + graph + pynutil.insert(" }")


def run(graph: pynini.Fst) -> pynini.Fst:
"""Reads a whole marked run of number words with graph."""
return pynutil.delete(RUN_START) + graph + pynutil.delete(RUN_END)


def join_aware_runs(number_words, join_words):
"""Marks maximal runs of number words; a join word followed by an ordinary word ends the run.

Joins such as Luganda mu also have nonnumeric uses ("in"). In
"ente bbiri mu nnimiro" the run is "bbiri"; "mu nnimiro" stays as words.
A run may still end on a join at the end of the input, where it is malformed
and falls back whole (a cut-off amount is not guessed). A join between two
number words still extends the run, so malformed internals fall back whole.

Returns (number word, ordinary word or join, segmenter).
"""
numeric_word = pynini.union(*sorted(number_words)).optimize()
join_word = pynini.union(*sorted(join_words)).optimize()
other_word = pynini.difference(WORD, numeric_word).optimize()
plain_number = pynini.difference(numeric_word, join_word).optimize()
any_word = (other_word | join_word).optimize()
# Between or after runs: words and joins, containing at least one ordinary word.
others = (pynini.closure(any_word + NEMO_SPACE) + other_word + pynini.closure(NEMO_SPACE + any_word)).optimize()
numbers = numeric_word + pynini.closure(NEMO_SPACE + numeric_word)
before_others = pynini.closure(numeric_word + NEMO_SPACE) + plain_number

def marked(graph):
return pynutil.insert(RUN_START) + graph + pynutil.insert(RUN_END)

runs = pynini.closure(marked(before_others) + NEMO_SPACE + others + NEMO_SPACE) + (
marked(before_others) + NEMO_SPACE + others | marked(numbers)
)
segmenter = (others | pynini.closure(others + NEMO_SPACE, 0, 1) + runs).optimize()
return numeric_word, any_word, segmenter


def money_cue_word(profile: CardinalProfile) -> pynini.Fst:
"""Words that signal a payment (e.g. Luganda money verbs); empty if the profile has none."""
cues = profile.money_cues
letters = pynini.closure(WORD_CHAR)
options = [letters + pynini.union(*cues.stems) + letters] if cues.stems else []
options += [pynini.union(*cues.words)] if cues.words else []
if not options:
return pynini.Fst()
cue = pynini.union(*options).optimize()
if cues.not_cues:
cue = pynini.difference(cue, pynini.union(*cues.not_cues).optimize())
return cue.optimize()


def only_with_money_signal(profile: CardinalProfile) -> pynini.Fst:
"""Keeps marked implied-currency amounts only where they are allowed, then drops the markers.

An amount marked IMPLIED is money at the start of the input or right after a money
cue; one marked CUE_ONLY only right after a money cue. Otherwise the tagging is
rejected and another reading (the words unchanged, or a counted noun) is used.
"""
plain = pynini.difference(NEMO_CHAR, pynini.union(IMPLIED, CUE_ONLY)).optimize()
after_cue = 'tokens { name: "' + money_cue_word(profile) + '" }' + NEMO_SPACE + pynini.union(IMPLIED, CUE_ONLY)
allowed = (pynini.closure(pynini.accep(IMPLIED), 0, 1) + pynini.closure(plain | after_cue)).optimize()
strip = pynini.closure(plain | pynutil.delete(IMPLIED) | pynutil.delete(CUE_ONLY))
return (allowed @ strip.optimize()).optimize()
Loading
Loading