From ddfaf2d752b53bce9f7a4c623cc31bd21332baf8 Mon Sep 17 00:00:00 2001 From: Dezhen Lu Date: Fri, 13 Nov 2026 18:27:16 +0100 Subject: [PATCH] Handle empty input in normalize_list and normalize_manifest normalize_list() computed batch = min(len(texts), batch_size), which is 0 for an empty list, and range(0, 0, 0) raised "ValueError: range() arg 3 must not be zero". inverse_normalize_list() and the normalize.py and inverse_normalize.py scripts with an empty --input_file failed the same way. Return [] for empty input. normalize_manifest() had the same batch computation and failed the same way for an empty manifest. Use a batch size of at least 1, so an empty manifest gives an empty output file. Signed-off-by: Dezhen Lu --- .../text_normalization/normalize.py | 5 ++++- .../nemo_text_processing/en/test_cardinal.py | 22 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/nemo_text_processing/text_normalization/normalize.py b/nemo_text_processing/text_normalization/normalize.py index c6a256eb5..3df5d72e9 100644 --- a/nemo_text_processing/text_normalization/normalize.py +++ b/nemo_text_processing/text_normalization/normalize.py @@ -262,6 +262,9 @@ def _process_batch(batch, verbose, punct_pre_process, punct_post_process, **kwar ] return normalized_lines + if not texts: + return [] + # to save intermediate results to a file batch = min(len(texts), batch_size) @@ -520,7 +523,7 @@ def _process_batch( logger.warning(f'Normalizing {len(lines)} line(s) of {manifest}...') # to save intermediate results to a file - batch = min(len(lines), batch_size) + batch = max(1, min(len(lines), batch_size)) tmp_dir = "/tmp/parts" if os.path.exists(tmp_dir): diff --git a/tests/nemo_text_processing/en/test_cardinal.py b/tests/nemo_text_processing/en/test_cardinal.py index f40e0d1f6..dd73a9c56 100644 --- a/tests/nemo_text_processing/en/test_cardinal.py +++ b/tests/nemo_text_processing/en/test_cardinal.py @@ -68,3 +68,25 @@ def test_norm(self, test_input, expected): punct_post_process=False, ) assert expected in pred_non_deterministic, f"input: {test_input}" + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_normalize_list_empty(self): + assert self.normalizer_en.normalize_list([]) == [] + assert self.inverse_normalizer_en.inverse_normalize_list([]) == [] + + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_normalize_manifest_empty(self, tmp_path): + manifest = tmp_path / "empty.json" + manifest.write_text("") + output = tmp_path / "empty_normalized.json" + self.normalizer_en.normalize_manifest( + manifest=str(manifest), + n_jobs=1, + punct_pre_process=True, + punct_post_process=True, + batch_size=10, + output_filename=str(output), + ) + assert output.read_text() == ""