diff --git a/AGENTS.md b/AGENTS.md index 3c2808de..bd549c27 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -271,7 +271,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **Before adding a short/common word to `PARTICLES` globally**, test it mid-string against realistic 3-token names, not just check for English-word collisions: Korean/Vietnamese given names put a short syllable in the middle slot (`Park In Hwan`, `Nguyen To Nga`), and Western names put a bare initial there (`John V. Smith`). A word that looks safe ("nobody is named 'to'") can still swallow a real middle name/initial into the last name once it's a global prefix — confirmed regressions for `to`/`in`/`an`/`ten`/`then` and bare `v` this way (PR #191). -**Adding a curated sub-set of an existing vocabulary set** (a marker over a base set, as `GIVEN_NAME_TITLES` is over `TITLES`) — it is a behavioral PROPERTY of a reading, not a second word list: a row in `tools/vocab/build.py`'s `BEHAVIORAL` table, a branch in `compile_pool` adding the words whose reading carries it, a constant in its `CONSTANTS` roster, and the `NAME = _compiled.NAME` assignment in the config module. The subset relation then holds by construction, since the property cannot exist without its reading, so it needs no check. A relation construction does not give (`∩ == ∅` with a set the sub-set cannot logically overlap; a sub-set member that is also a title is silently inert, title handling consuming it first) goes in `check_pool`, with a mutation in `tests/v2/test_vocab.py`'s table. Do **not** forbid `titles ∩ particles` — that overlap is intentional (`freiherr`, `st`). On the v1 side the sub-set is a `_SetManagerAttribute` on `Constants` (like `first_name_titles` — the v1 ATTRIBUTE names did not move in #293, only the module constants did), **not** a `_CachedUnionMember` — only `prefixes`/`suffix_acronyms`/`suffix_not_acronyms`/`titles` feed the `_pst` hot-path cache, so a sub-set costs nothing at runtime and stays out of `is_rootname`. A runtime `.add()` to the sub-set's `SetManager` does **not** propagate to the parent's (same as `first_name_titles`→`titles`), so a caller adding a brand-new word adds it to both. +**Adding a curated sub-set of an existing vocabulary set** (a marker over a base set, as `GIVEN_NAME_TITLES` is over `TITLES`) — it is a behavioral PROPERTY of a reading, not a second word list: a row in `tools/vocab/build.py`'s `BEHAVIORAL` table, a branch in `compile_pool` adding the words whose reading carries it, a constant in its `CONSTANTS` roster and a key of the same name in its `Compiled` TypedDict (an import-time assert holds the two to one list), and the `NAME = _compiled.NAME` assignment in the config module. The subset relation then holds by construction, since the property cannot exist without its reading, so it needs no check. A relation construction does not give (`∩ == ∅` with a set the sub-set cannot logically overlap; a sub-set member that is also a title is silently inert, title handling consuming it first) goes in `check_pool`, with a mutation in `tests/v2/test_vocab.py`'s table. Do **not** forbid `titles ∩ particles` — that overlap is intentional (`freiherr`, `st`). On the v1 side the sub-set is a `_SetManagerAttribute` on `Constants` (like `first_name_titles` — the v1 ATTRIBUTE names did not move in #293, only the module constants did), **not** a `_CachedUnionMember` — only `prefixes`/`suffix_acronyms`/`suffix_not_acronyms`/`titles` feed the `_pst` hot-path cache, so a sub-set costs nothing at runtime and stays out of `is_rootname`. A runtime `.add()` to the sub-set's `SetManager` does **not** propagate to the parent's (same as `first_name_titles`→`titles`), so a caller adding a brand-new word adds it to both. **Adding a flag-gated post-parse transform** (reorder/adjust) — add a `Constants` boolean (default `False`), implement a `handle_*()` method, and call it in `post_process()` after `handle_firstnames()` and before `handle_capitalization()`, gated on the flag. Default-off keeps existing parses byte-for-byte unchanged. Two shipped examples: `patronymic_name_order` gates both `handle_east_slavic_patronymic_name_order()` (#85) and `handle_turkic_patronymic_name_order()` (#185) — one flag driving two independent handlers, added in the same `post_process()` slot; `middle_name_as_last` gates `handle_middle_name_as_last()` (#133), which folds `middle_list` into `last_list`. diff --git a/tests/v2/test_vocab.py b/tests/v2/test_vocab.py index 3e89db85..b006c824 100644 --- a/tests/v2/test_vocab.py +++ b/tests/v2/test_vocab.py @@ -149,6 +149,7 @@ def _w(body: str) -> str: ("reserved-top-level-key", {"und.toml": "default = false\n" + OK}, "reserved for phase 3"), ("file-name-not-a-tag", {"english.toml": OK}, "BCP 47 tag"), ("file-name-not-canonical-case", {"pt-br.toml": OK}, "BCP 47 tag"), + ("file-with-no-entries", {"en.toml": OK, "und.toml": "schema = 1\n"}, "no \\[\\[word\\]\\] entries"), ("reserved-reading-property", {"und.toml": _w('[[word]]\ntext = "mba"\ncredential = { bare_only = true }')}, "reserved for phase 2"), ("unknown-reading-property", {"und.toml": _w('[[word]]\ntext = "mba"\ncredential = { acronymm = true }')}, "unknown key 'acronymm'"), ("unknown-word-key", {"und.toml": _w('[[word]]\ntext = "jr"\ngeneration = {}\nlang = "en"')}, "unknown key 'lang'"), @@ -157,6 +158,17 @@ def _w(body: str) -> str: ("text-not-normalized", {"en.toml": _w('[[word]]\ntext = "Jnr."\ngeneration = {}')}, "normalized form"), ("text-double-spaced", {"und.toml": _w('[[word]]\ntext = "z domu"\nbirth_marker = {}')}, "normalized form"), ("text-missing", {"und.toml": _w("[[word]]\ngeneration = {}")}, "non-empty `text`"), + # a text no token can equal: the tokenizer divides at a comma and a + # nakaguro, strips a bidi mark, and keeps an invisible character the + # file does not show + ("text-with-a-comma", {"en.toml": _w('[[word]]\ntext = "jr,"\ngeneration = {}')}, "divides a name at"), + ("text-with-a-nakaguro", {"ja.toml": _w('[[word]]\ntext = "さん・"\nhonorific = {}')}, "divides a name at"), + ("text-with-a-bidi-mark", {"he.toml": _w('[[word]]\ntext = "dr\\u200f"\ntitle = {}')}, "invisible character U\\+200F"), + ("text-with-a-zero-width-space", {"en.toml": _w('[[word]]\ntext = "dr\\u200b"\ntitle = {}')}, "invisible character U\\+200B"), + ("text-with-a-control-character", {"en.toml": _w('[[word]]\ntext = "dr\\u0007"\ntitle = {}')}, "invisible character U\\+0007"), + ("text-with-an-emoji", {"en.toml": _w('[[word]]\ntext = "dr★"\ntitle = {}')}, "strips by default"), + ("text-with-an-east-asian-interpunct", {"zh.toml": _w('[[word]]\ntext = "先·生"\nhonorific = {}')}, "rules/T3"), + ("phrase-word-with-an-edge-period", {"de.toml": _w('[[word]]\ntext = "dr. med"\ntitle = { addresses = "given" }')}, "write 'dr med'"), ("text-twice-in-a-file", {"en.toml": _w('[[word]]\ntext = "jr"\ngeneration = {}\n\n[[word]]\ntext = "jr"\nstatus = {}')}, "listed twice"), # 3: kinds, properties and values ("unregistered-kind", {"hr.toml": _w('[[word]]\ntext = "mlađi"\ngeneraton = {}')}, "unknown key 'generaton'"), @@ -188,6 +200,10 @@ def _w(body: str) -> str: ("flag-not-a-bool", {"und.toml": _w('[[word]]\ntext = "van"\nparticle = { may_be_given = "yes" }')}, "true or false"), ("issue-not-a-number", {"en.toml": _w('[[word]]\ntext = "jr"\ngeneration = { issue = "661" }')}, "issue number"), ("decided-not-a-date", {"en.toml": _w('[[word]]\ntext = "jr"\ngeneration = { decided = "2026-10-18" }')}, "TOML date"), + # tomllib reads a date-time as a datetime, which isinstance calls a date + ("decided-a-date-time", {"en.toml": _w('[[word]]\ntext = "jr"\ngeneration = { decided = 2026-10-18T09:00:00 }')}, "not a string or a date-time"), + ("as-of-a-date-time", {"fr.toml": _w('[[word]]\ntext = "fils"\nfamily = { us_census_2010 = 767, as_of = 2026-10-16T00:00:00Z }\nexcluded = { generation = "x" }')}, "as_of must be a TOML date"), + ("as-of-without-a-count", {"fr.toml": _w('[[word]]\ntext = "fils"\nfamily = { as_of = 2026-10-16 }\nexcluded = { generation = "x" }')}, "as_of dates a count"), ("gender-out-of-range", {"en.toml": _w('[[word]]\ntext = "jr"\ngeneration = { gender = "male" }')}, "gender must be"), ("count-without-as-of", {"fr.toml": _w('[[word]]\ntext = "fils"\nfamily = { us_census_2010 = 767 }\nexcluded = { generation = "x" }')}, "needs `as_of`"), ("count-source-typo", {"fr.toml": _w('[[word]]\ntext = "fils"\nfamily = { us_census = 767, as_of = 2026-10-16 }\nexcluded = { generation = "x" }')}, "unknown key 'us_census'"), @@ -212,15 +228,19 @@ def _w(body: str) -> str: ("excluded-kind-sidestepped-by-a-sibling-kind", {"fr.toml": _w('[[word]]\ntext = "fils"\nexcluded = { generation = "Marie Fils" }'), "en.toml": _w('[[word]]\ntext = "fils"\nstatus = {}')}, "Marie Fils"), ("excluded-credential-sidestepped-by-an-acronym", {"und.toml": _w('[[word]]\ntext = "rai"\nexcluded = { credential = "a surname" }'), "en.toml": _w('[[word]]\ntext = "rai"\ncourtesy = { acronym = true }')}, "a surname"), ("excluded-category-in-another-file", {"und.toml": _w('[[word]]\ntext = "rai"\nexcluded = { suffix = "a surname" }'), "en.toml": _w('[[word]]\ntext = "rai"\ncredential = {}')}, "a surname"), + # a reason written as a table reaches the refusal too + ("excluded-kind-with-a-table-reason", {"fr.toml": _w('[[word]]\ntext = "fils"\nexcluded = { generation = { reason = "TABLE REASON", issue = 684 } }'), "en.toml": _w('[[word]]\ntext = "fils"\ngeneration = {}')}, "TABLE REASON"), ("excluded-property-in-another-file", {"ja.toml": _w('[[word]]\ntext = "博士"\ncredential = { acronym = false, excluded = { attached = "Hiroshi" } }'), "zh.toml": _w('[[word]]\ntext = "博士"\ncredential = { acronym = false, attached = { side = "after" } }')}, "Hiroshi"), # 7: same_as and cased ("same-as-missing-target", {"en.toml": _w('[[word]]\ntext = "jnr"\ngeneration = { same_as = "jr" }')}, "same_as 'jr'"), + ("same-as-its-own-word", {"en.toml": _w('[[word]]\ntext = "jr"\ngeneration = { same_as = "jr" }')}, "names the word itself"), ("same-as-other-category", {"en.toml": _w('[[word]]\ntext = "jr"\ntitle = {}\n\n[[word]]\ntext = "jnr"\ngeneration = { same_as = "jr" }')}, "no suffix reading"), ("cased-spells-other-letters", {"und.toml": _w('[[word]]\ntext = "phd"\ncredential = { cased = "PhDr" }')}, "case mask"), ("two-cased-values", {"und.toml": _w('[[word]]\ntext = "ó"\nparticle = { may_be_given = false, cased = "Ó" }'), "ga.toml": _w('[[word]]\ntext = "ó"\ntitle = { cased = "ó" }')}, "one cased mask per word"), ("cased-disagrees-across-files", {"und.toml": _w('[[word]]\ntext = "ó"\nparticle = { may_be_given = false, cased = "Ó" }'), "ga.toml": _w('[[word]]\ntext = "ó"\nparticle = { may_be_given = false, cased = "ó" }')}, "cased is"), # 8: phrases ("phrase-title-addressing-the-family", {"und.toml": _w('[[word]]\ntext = "deputy secretary"\ntitle = {}')}, "phrase can never match"), + ("phrase-with-cased", {"und.toml": _w('[[word]]\ntext = "sir and dame"\ntitle = { addresses = "given", cased = "Sir and Dame" }')}, "looked up per word"), ("phrase-particle", {"und.toml": _w('[[word]]\ntext = "de la"\nparticle = { may_be_given = false }')}, "phrase can never match"), # 9: a bound given name that is a particle may be given ("bound-given-particle-never-given", {"und.toml": _w('[[word]]\ntext = "abu"\nparticle = { may_be_given = false }\nbound_given = {}')}, "may_be_given = true"), @@ -250,6 +270,15 @@ def test_every_mutation_starts_from_a_pool_the_checks_accept() -> None: {"und.toml": _w('[[word]]\ntext = "d\'"\nparticle = { may_be_given = false }\n\n' '[[word]]\ntext = "d’"\nparticle = { may_be_given = false }')}, {"und.toml": _w('[[word]]\ntext = "sir and dame"\ntitle = { addresses = "given" }')}, + # a zero-width joiner or non-joiner is spelling, and a token keeps it + {"fa.toml": _w('[[word]]\ntext = "ab\\u200ccd"\ntitle = {}\n\n' + '[[word]]\ntext = "ef\\u200dgh"\ntitle = {}')}, + # an interpunct is part of the word except between two East Asian + # characters, and an East Asian text may end in one + {"ca.toml": _w('[[word]]\ntext = "l·l"\ntitle = {}'), + "zh.toml": _w('[[word]]\ntext = "先·"\nhonorific = {}')}, + # a phrase word may hold an interior period; a date is a date + {"und.toml": _w('[[word]]\ntext = "lt.col and dame"\ntitle = { addresses = "given", decided = 2026-10-18 }')}, {"fr.toml": _w('[[word]]\ntext = "fils"\nfamily = { us_census_2010 = 767, as_of = 2026-10-16 }\n' 'excluded = { generation = { reason = "Marie Fils", issue = 684 } }')}, # the name kinds refuse only themselves: refused as a given name, a @@ -274,6 +303,36 @@ def test_every_mutation_starts_from_a_pool_the_checks_accept() -> None: _compile(files) +def test_every_character_the_tokenizer_ignores_is_refused() -> None: + # check 2's hand copies are held to the pipeline, and the sweep is + # driven by the tokenizer's own predicate, so a separator or strip + # class added there fails here until the build refuses it too + from nameparser._pipeline import _tokenize + from nameparser._pipeline._state import COMMA_CHARS, ParseState + from nameparser._policy import Policy, _SCRIPT_RANGES + assert build.DIVIDERS == COMMA_CHARS | _tokenize._NAME_DOT_SEPARATORS + assert build.EMOJI_RANGES == _tokenize._EMOJI_RANGES + assert build.EAST_ASIAN_RANGES == tuple( + span for spans in _SCRIPT_RANGES.values() for span in spans) + assert build.INTERPUNCT == _tokenize._INTERPUNCT + state = ParseState(original="", lexicon=Lexicon.default(), policy=Policy(), + segmenter=None) + ignored = [chr(c) for c in range(0x110000) + # a space is the one ignored character a text holds: the + # separator between a phrase's words + if not 0xD800 <= c <= 0xDFFF and c != 0x20 + and (_tokenize._ignorable(chr(c), state) or chr(c) in COMMA_CHARS)] + assert len(ignored) > 100, len(ignored) # whitespace, bidi, emoji, dots, commas + for ch in ignored: + escaped = f"\\U{ord(ch):08X}" # a TOML escape: no raw control in the file + # refused by check 2 itself, not by whichever later check happens + # to trip, which would leave a new class guarded by accident + with pytest.raises(build.VocabError, + match="divides a name|strips by default|rules/T3|" + "invisible character|not in normalized form"): + build.load({"und.toml": _w(f'[[word]]\ntext = "dr{escaped}x"\ntitle = {{}}')}) + + def test_a_name_reading_on_nothing_warns() -> None: files = {"en.toml": _w('[[word]]\ntext = "smith"\nfamily = { us_census_2010 = 2442977, as_of = 2026-10-16 }')} with pytest.warns(build.VocabWarning, match="inert"): @@ -306,6 +365,9 @@ def test_query_tells_a_refused_reading_from_a_refused_property() -> None: assert [r.text for r in query.select(pool, excluded="honorific")] == ["君"] assert [r.text for r in query.select(pool, excluded="attached")] == ["殿"] assert [r.text for r in query.select(pool, excluded="suffix")] == ["君"] + # the label is for display only, built from the same fields + assert sorted(r.label for r in query.rows(pool)) == [ + "excluded:honorific", "honorific", "honorific excluded:attached"] def _load_query() -> ModuleType: @@ -334,6 +396,7 @@ def _load_query() -> ModuleType: ({"kind": "suffix"}, ["さん", "殿", "jr"]), # a category matches its kinds ({"kind": "honorific", "excluded": "attached"}, ["殿"]), ({"excluded": "suffix"}, ["senior"]), + ({"excluded": "attach"}, []), # a property by its whole name only ({"file": "und.toml"}, ["dr"]), ({"word": "jr", "kind": "title"}, []), ]) diff --git a/tools/vocab/build.py b/tools/vocab/build.py index c0d2f26b..32c77a14 100644 --- a/tools/vocab/build.py +++ b/tools/vocab/build.py @@ -28,11 +28,17 @@ at fault: 1. `schema` present and supported, no unknown key anywhere (a name a - later phase will use says so), the file name a BCP 47 tag; -2. `text` normalized, non-empty, once per file; + later phase will use says so), the file name a BCP 47 tag, at least + one entry in the file; +2. `text` normalized, non-empty, once per file, each word of a phrase + normalized on its own, and a text some token can equal: no comma or + nakaguro, no character the tokenizer strips by default (an emoji), + no interpunct between two East Asian characters, no invisible + character but the two joiners; 3. every kind registered, every property allowed on its kind with an - allowed value, an acronym's text free of periods (it is matched with - them removed), and no entry without a reading or an exclusion; + allowed value (a date, never a date-time; `as_of` only beside a + count), an acronym's text free of periods (it is matched with them + removed), and no entry without a reading or an exclusion; 4. one suffix kind per word, across files; 5. the same kind of the same word agrees on every behavioral property in every file; @@ -42,11 +48,16 @@ has unwritten on a reading of the excluding kind, on any reading of its category (`cased`: of the word); run before 4 and 5, so the recorded reason is what a contributor sees; -7. `same_as` names a word of the same category, `cased` spells its - word's letters (#459), at most one `cased` per word; -8. a phrase only as a birth_marker or an addresses = "given" title; +7. `same_as` names another word of the same category, `cased` spells + its word's letters (#459), at most one `cased` per word; +8. a phrase only as a birth_marker or an addresses = "given" title, and + never with a `cased`, which is looked up per word; 9. a bound given name that is also a particle has may_be_given; +numbered by subject, not by when they run: 1-3, 8 and 7's case mask +run per entry as each file loads, and 4-7 and 9 over the whole pool in +check_pool, 6 first; + plus the data invariants that were asserts: may_be_initial only on a cased single letter, and an elided particle in both apostrophes (#700). No post-nominal is both an acronym and a word, by checks 4 and 5; a @@ -64,12 +75,13 @@ import sys import tomllib import traceback +import unicodedata import warnings from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from pathlib import Path from types import ModuleType -from typing import Any +from typing import Any, TypedDict ROOT = Path(__file__).resolve().parents[2] @@ -193,7 +205,7 @@ class VocabWarning(UserWarning): """A finding that compiles but is probably a mistake.""" -@dataclass +@dataclass(frozen=True) class Reading: file: str text: str @@ -204,24 +216,61 @@ class Reading: def category(self) -> str: return CATEGORY[self.kind] + # the defaults of the two behavioral properties whose unwritten value + # is not false, each defined once: checks, compile_pool and + # _behavior all read them here + @property + def acronym(self) -> bool: + """Matched with its periods removed: unwritten, true only for a + credential.""" + return bool(self.props.get("acronym", self.kind == "credential")) + + def flag(self, prop: str) -> bool: + """A `may_be_*` flag: unwritten, false.""" + return bool(self.props.get(prop, False)) + + @property + def addresses(self) -> object: + """Which name a title addresses: unwritten, the family name. An + object, not a str, because it is read before check 3 has + refused a value that is not one of the allowed strings.""" + return self.props.get("addresses", "family") + def where(self) -> str: return f"{self.file}: {self.text!r} {self.kind}" -@dataclass +def _exclusion_where(file: str, text: str, kind: str, prop: str | None) -> str: + what = f"excluded {kind}" if prop is None else f"{kind} excluded {prop}" + return f"{file}: {text!r} {what}" + + +@dataclass(frozen=True) class Exclusion: """A refused reading (`prop` None: `kind` is a kind or a category) or - a refused property of the `kind` reading the word keeps.""" + a refused property of the `kind` reading the word keeps. + `props["reason"]` is always a non-empty string: __post_init__ + refuses any other, so _exclusion builds one even from a value with + no reason and lets the refusal come from here.""" file: str text: str kind: str prop: str | None - reason: str props: dict[str, Any] # the value's table: reason, issue, decided, note + def __post_init__(self) -> None: + reason = self.props.get("reason") + if not isinstance(reason, str) or not reason: + raise VocabError(f"{self.where()}: an exclusion needs a reason, as a " + f"string or a table with `reason`") + + @property + def reason(self) -> str: + reason: str = self.props["reason"] + return reason + def where(self) -> str: - what = f"excluded {self.kind}" if self.prop is None else f"{self.kind} excluded {self.prop}" - return f"{self.file}: {self.text!r} {what}" + return _exclusion_where(self.file, self.text, self.kind, self.prop) @dataclass @@ -242,21 +291,23 @@ def _reserved_or_unknown(where: str, key: str, allowed: Iterable[str]) -> VocabE def _exclusion(file: str, text: str, kind: str, prop: str | None, value: object) -> Exclusion: """An `excluded` value: a reason string, or a table with `reason`.""" - ex = Exclusion(file, text, kind, prop, "", {}) + where = _exclusion_where(file, text, kind, prop) if isinstance(value, str) and value: - ex.reason, ex.props = value, {"reason": value} - return ex + return Exclusion(file, text, kind, prop, {"reason": value}) if isinstance(value, dict): for k in value: if k not in EXCLUSION_KEYS: - raise _reserved_or_unknown(ex.where(), k, EXCLUSION_KEYS) - _check_descriptive(ex.where(), value) - reason = value.get("reason") - if isinstance(reason, str) and reason: - ex.reason, ex.props = reason, value - return ex - raise VocabError(f"{ex.where()}: an exclusion needs a reason, as a " - f"string or a table with `reason`") + raise _reserved_or_unknown(where, k, EXCLUSION_KEYS) + _check_descriptive(where, value) + return Exclusion(file, text, kind, prop, value) + # a bare value that is no reason string: Exclusion's own refusal + return Exclusion(file, text, kind, prop, {}) + + +def _is_date(value: object) -> bool: + # tomllib reads a TOML date-time as a datetime, which IS a date to + # isinstance: a recorded day must not carry an hour it never had + return isinstance(value, datetime.date) and not isinstance(value, datetime.datetime) def _check_descriptive(where: str, props: Mapping[str, Any]) -> None: @@ -269,8 +320,9 @@ def _check_descriptive(where: str, props: Mapping[str, Any]) -> None: if not vs or not all(isinstance(i, int) and not isinstance(i, bool) and i > 0 for i in vs): raise VocabError(f"{where}: issue must be an issue number or a list of them") - if "decided" in props and not isinstance(props["decided"], datetime.date): - raise VocabError(f"{where}: decided must be a TOML date (2026-10-18), not a string") + if "decided" in props and not _is_date(props["decided"]): + raise VocabError(f"{where}: decided must be a TOML date (2026-10-18), " + f"not a string or a date-time") if "expands" in props: v = props["expands"] vs = v if isinstance(v, list) else [v] @@ -318,6 +370,11 @@ def load(files: Mapping[str, str]) -> Pool: words = data.get("word", []) if not isinstance(words, list): raise VocabError(f"{name}: `word` must be an array of tables ([[word]])") + if not words: + # a file with the header and nothing else -- a botched move, + # every entry cut and none pasted -- compiles to nothing + raise VocabError(f"{name}: no [[word]] entries; delete the file " + f"or add its words") seen: set[str] = set() for entry in words: if not isinstance(entry, dict): @@ -326,6 +383,83 @@ def load(files: Mapping[str, str]) -> Pool: return pool +#: Hand copies of what the tokenizer divides or strips at under the +#: default Policy (this tool cannot import the pipeline without the +#: package), each held to its source by tests/v2/test_vocab.py, which +#: also sweeps every codepoint the tokenizer ignores. +#: +#: The separators: nameparser/_pipeline/_state.py's COMMA_CHARS and +#: _tokenize.py's _NAME_DOT_SEPARATORS. The main stream divides at both; +#: inside a delimited clause a comma is kept as written (rules/N1), and +#: there rules/M3 reads a marker followed by a comma as no marker, which +#: holds only while the vocabulary stores no comma. +DIVIDERS = frozenset({",", "\u060c", "\uff0c", "\u30fb", "\uff65"}) +#: _tokenize.py's _EMOJI_RANGES, stripped under strip_emoji (default on). +EMOJI_RANGES = ((0x1F300, 0x1F64F), (0x1F680, 0x1F6FF), + (0x2600, 0x26FF), (0x2700, 0x27BF)) +#: The spans of nameparser/_policy.py's _SCRIPT_RANGES (Han, Hangul, +#: hiragana, katakana): an interpunct between two of these characters +#: divides the name (rules/T3), anywhere else it is part of the word. +EAST_ASIAN_RANGES = ((0x3005, 0x3006), (0x3400, 0x4DBF), (0x4E00, 0x9FFF), + (0xF900, 0xFAFF), (0x20000, 0x323AF), (0xAC00, 0xD7A3), + (0x3040, 0x309F), (0x30A0, 0x30FF), (0xFF65, 0xFF9F)) +INTERPUNCT = "\u00b7" +#: The format characters a word may hold: the zero-width non-joiner and +#: joiner are spelling in Persian and the Indic scripts, and a token +#: keeps them. +JOINERS = frozenset({"\u200c", "\u200d"}) + + +def _in(ch: str, ranges: tuple[tuple[int, int], ...]) -> bool: + return any(lo <= ord(ch) <= hi for lo, hi in ranges) + + +def _check_tokenizable(name: str, text: str) -> None: + """Build check 2's second half. The normalized-form test passes each + of these. An emoji and an interpunct between East Asian characters + match no token under the default Policy; a comma could only match + inside a delimited clause, where rules/M3 needs the vocabulary to + hold none; an invisible character is a spelling the file does not + show; and a phrase word with an edge period is a second spelling of + the key Lexicon folds it to.""" + for i, ch in enumerate(text): + if ch in DIVIDERS: + raise VocabError( + f"{name}: {text!r} holds {ch!r} (U+{ord(ch):04X}), which the " + f"tokenizer divides a name at; no vocabulary word holds one") + if _in(ch, EMOJI_RANGES): + raise VocabError( + f"{name}: {text!r} holds U+{ord(ch):04X}, which the tokenizer " + f"strips by default, so no token can equal it") + if (ch == INTERPUNCT and 0 < i < len(text) - 1 + and _in(text[i - 1], EAST_ASIAN_RANGES) + and _in(text[i + 1], EAST_ASIAN_RANGES)): + raise VocabError( + f"{name}: {text!r} holds an interpunct between two East Asian " + f"characters, where the tokenizer divides the name (rules/T3), " + f"so no token can equal it") + category = unicodedata.category(ch) + if (category == "Cf" and ch not in JOINERS) or category == "Cc": + # a bidi mark is stripped by default; a zero-width space, word + # joiner, BOM or control character is kept but unseen, so the + # entry reads as the word without it and matches only text + # carrying the same invisible character + raise VocabError( + f"{name}: {text!r} holds the invisible character " + f"U+{ord(ch):04X} ({unicodedata.name(ch, 'a control character')}); " + f"remove it") + if " " in text: + # Lexicon folds each word of a phrase on its own (_title_key), so + # 'z. domu' is stored and matched as 'z domu': the entry works, + # but the compiled set would hold a spelling Lexicon never keeps. + # One key, one spelling, as check 2 asks of every text + words = [_normalize(w) for w in text.split(" ")] + if " ".join(words) != text: + raise VocabError( + f"{name}: phrase {text!r} is not in normalized form, which " + f"folds each word; write {' '.join(w for w in words if w)!r}") + + def _load_entry(name: str, e: dict[str, Any], seen: set[str], pool: Pool) -> None: text = e.get("text") if not isinstance(text, str) or not text: @@ -335,6 +469,7 @@ def _load_entry(name: str, e: dict[str, Any], seen: set[str], pool: Pool) -> Non f"{name}: text {text!r} is not in normalized form; write " f"{' '.join(_normalize(text).split())!r} (lowercase, NFC, no edge " f"periods, single-spaced)") + _check_tokenizable(name, text) if text in seen: raise VocabError(f"{name}: {text!r} is listed twice in one file") seen.add(text) @@ -376,8 +511,14 @@ def _check_reading(r: Reading, pool: Pool) -> None: if not isinstance(p[src], int) or isinstance(p[src], bool) or p[src] < 1: raise VocabError(f"{where}: {src} must be a positive count " f"(absent means below the source's floor)") - if COUNT_SOURCES & p.keys() and not isinstance(p.get("as_of"), datetime.date): + if COUNT_SOURCES & p.keys() and "as_of" not in p: raise VocabError(f"{where}: a count needs `as_of`, a TOML date") + if "as_of" in p and not _is_date(p["as_of"]): + raise VocabError(f"{where}: as_of must be a TOML date (2026-10-16), " + f"not a string or a date-time") + if "as_of" in p and not COUNT_SOURCES & p.keys(): + raise VocabError(f"{where}: as_of dates a count, and this reading " + f"has none ({', '.join(sorted(COUNT_SOURCES))})") _check_descriptive(where, p) pool.readings.append(r) return @@ -390,14 +531,14 @@ def _check_reading(r: Reading, pool: Pool) -> None: if flag in p and not isinstance(p[flag], bool): raise VocabError(f"{where}: {flag} must be true or false") if r.category == "title": - addresses = p.get("addresses", "family") + addresses = r.addresses if addresses in ("full", "any"): raise VocabError(f"{where}: addresses = {addresses!r} is reserved " f"for a later phase; phase 1 has 'family' and 'given'") if addresses not in ("family", "given"): raise VocabError(f"{where}: addresses must be 'family' or 'given'") if r.category == "suffix": - acronym = _acronym(r) + acronym = r.acronym if acronym and "." in r.text: # suffix_as_written looks an acronym up with EVERY period # removed, so a dotted entry is never matched -- and it slips @@ -430,13 +571,17 @@ def _check_reading(r: Reading, pool: Pool) -> None: # silently pick one side of that raise VocabError(f"{where}: a particle states may_be_given = true or " f"false; under uncertainty, true (vocabulary-collisions C-i)") - if r.category == "conjunction" and p.get("may_be_initial"): + if r.category == "conjunction" and r.flag("may_be_initial"): if len(r.text) != 1 or r.text.upper() == r.text.lower(): raise VocabError( f"{where}: may_be_initial needs a CASED single letter; the " f"fork reads only a cased one-letter token, so this is inert") if "cased" in p: cased = p["cased"] + if " " in r.text: + # capitalization_exceptions is looked up one word at a time + raise VocabError(f"{where}: cased is looked up per word, so a " + f"phrase's is never read; put it on each word") if not isinstance(cased, str) or _alnum(_normalize(cased)) != _alnum(r.text): raise VocabError(f"{where}: cased {cased!r} must spell {r.text!r}'s " f"letters and digits, recased (a case mask, #459)") @@ -454,17 +599,13 @@ def _check_reading(r: Reading, pool: Pool) -> None: # build check 8: a phrase is matched whole only as a given-addressing # title run or a birth marker; anywhere else it can never match if " " in r.text and not (r.kind == "birth_marker" or ( - r.category == "title" and p.get("addresses") == "given")): + r.category == "title" and r.addresses == "given")): raise VocabError(f"{where}: a phrase can never match as a {r.kind}; " f"only birth_marker and addresses = 'given' titles are " f"matched as a whole run") pool.readings.append(r) -def _acronym(r: Reading) -> bool: - return bool(r.props.get("acronym", r.kind == "credential")) - - def _refused_by(r: Reading, excluded: str) -> bool: """Whether a word-level exclusion of `excluded` refuses reading r. @@ -484,7 +625,7 @@ def _refused_by(r: Reading, excluded: str) -> bool: return True if CATEGORY[excluded] != r.category or r.category == "name": return False - return r.category != "suffix" or _acronym(r) == (excluded == "credential") + return r.category != "suffix" or r.acronym == (excluded == "credential") def _shown(value: object) -> str: @@ -569,9 +710,13 @@ def check_pool(pool: Pool) -> None: f"{text!r} {r.kind}: {prop} is {a!r} in {seen.file} " f"but {b!r} in {r.file}; one word reads one way") - # 7: same_as names a word of the same category; one cased per word + # 7: same_as names another word of the same category; one cased + # per word for r in pool.readings: target = r.props.get("same_as") + if target == r.text: + raise VocabError(f"{r.where()}: same_as names the word itself; it " + f"names the word this one is a spelling of") if target is not None and not any( t.category == r.category for t in by_text.get(target, ())): raise VocabError(f"{r.where()}: same_as {target!r} has no " @@ -587,7 +732,7 @@ def check_pool(pool: Pool) -> None: for text, rs in by_text.items(): if any(r.kind == "bound_given" for r in rs): for r in rs: - if r.category == "particle" and not r.props.get("may_be_given"): + if r.category == "particle" and not r.flag("may_be_given"): raise VocabError(f"{r.where()}: also a bound given name, so " f"the particle needs may_be_given = true") @@ -613,18 +758,19 @@ def check_pool(pool: Pool) -> None: def _behavior(r: Reading, prop: str) -> object: if prop == "acronym": - return _acronym(r) + return r.acronym if prop == "addresses": - return r.props.get("addresses", "family") + return r.addresses if prop == "attached": att = r.props.get("attached") return None if att is None else att.get("side") - return r.props.get(prop, False if prop.startswith("may_be") else None) + if prop.startswith("may_be"): + return r.flag(prop) + return r.props.get(prop) #: The constants _compiled.py defines, in its order, each with the -#: nameparser.config module that re-exports it. All are frozensets but -#: the ones in MAPPINGS. +#: nameparser.config module that re-exports it. CONSTANTS = ( ("GIVEN_NAME_TITLES", "titles"), ("TITLES", "titles"), @@ -640,52 +786,95 @@ def _behavior(r: Reading, prop: str) -> object: ("SUFFIX_ACRONYMS", "suffixes"), ("CAPITALIZATION_EXCEPTIONS", "capitalization"), ) -MAPPINGS = frozenset({"CAPITALIZATION_EXCEPTIONS"}) -def compile_pool(pool: Pool) -> dict[str, Any]: +class Compiled(TypedDict): + """What compile_pool returns: one key per constant, in CONSTANTS + order, typed as _compiled.py declares it.""" + GIVEN_NAME_TITLES: frozenset[str] + TITLES: frozenset[str] + NON_GIVEN_NAME_PARTICLES: frozenset[str] + PARTICLES: frozenset[str] + CONJUNCTIONS: frozenset[str] + CONJUNCTIONS_AMBIGUOUS: frozenset[str] + BOUND_GIVEN_NAMES: frozenset[str] + MAIDEN_MARKERS: frozenset[str] + SUFFIX_WORDS: frozenset[str] + GLUED_HONORIFICS: frozenset[str] + SUFFIX_ACRONYMS_AMBIGUOUS: frozenset[str] + SUFFIX_ACRONYMS: frozenset[str] + CAPITALIZATION_EXCEPTIONS: dict[str, str] + + +# the one tie between the two: mypy holds compile_pool to Compiled's +# keys, and this holds Compiled's keys to the roster render writes +assert tuple(Compiled.__annotations__) == tuple(name for name, _ in CONSTANTS), ( + "Compiled and CONSTANTS list different constants") + + +def compile_pool(pool: Pool) -> Compiled: """Fold the readings into today's constants.""" check_pool(pool) - s: dict[str, set[str]] = { - name: set() for name, _ in CONSTANTS if name not in MAPPINGS} + titles: set[str] = set() + given_titles: set[str] = set() + particles: set[str] = set() + never_given: set[str] = set() + conjunctions: set[str] = set() + initial_conjunctions: set[str] = set() + bound: set[str] = set() + markers: set[str] = set() + words: set[str] = set() + glued: set[str] = set() + ambiguous: set[str] = set() + acronyms: set[str] = set() cased: dict[str, str] = {} for r in pool.readings: t, p = r.text, r.props if "cased" in p: cased[t] = p["cased"] if r.category == "title": - s["TITLES"].add(t) - if p.get("addresses") == "given": - s["GIVEN_NAME_TITLES"].add(t) + titles.add(t) + if r.addresses == "given": + given_titles.add(t) elif r.category == "suffix": - if _acronym(r): - s["SUFFIX_ACRONYMS"].add(t) - if p.get("may_be_name"): - s["SUFFIX_ACRONYMS_AMBIGUOUS"].add(t) + if r.acronym: + acronyms.add(t) + if r.flag("may_be_name"): + ambiguous.add(t) else: - s["SUFFIX_WORDS"].add(t) + words.add(t) if "attached" in p: - s["GLUED_HONORIFICS"].add(t) + glued.add(t) elif r.category == "particle": - s["PARTICLES"].add(t) + particles.add(t) + # never-given where any reading has may_be_given = false; + # check 5 makes every reading of one word agree, so "any" + # is "all" + if not r.flag("may_be_given"): + never_given.add(t) elif r.category == "conjunction": - s["CONJUNCTIONS"].add(t) - if p.get("may_be_initial"): - s["CONJUNCTIONS_AMBIGUOUS"].add(t) + conjunctions.add(t) + if r.flag("may_be_initial"): + initial_conjunctions.add(t) elif r.category == "bound_given": - s["BOUND_GIVEN_NAMES"].add(t) + bound.add(t) elif r.category == "relation": - s["MAIDEN_MARKERS"].add(t) - # a particle is never-given where any of its readings has - # may_be_given = false; check 5 makes every reading of one word - # agree, so "any" is "all" - s["NON_GIVEN_NAME_PARTICLES"] = { - r.text for r in pool.readings - if r.category == "particle" and not r.props.get("may_be_given")} - out: dict[str, Any] = {k: frozenset(v) for k, v in s.items()} - out["CAPITALIZATION_EXCEPTIONS"] = dict(sorted(cased.items())) - if out.keys() != {name for name, _ in CONSTANTS}: - raise RuntimeError(f"compile_pool built {sorted(out)}, not the CONSTANTS roster") + markers.add(t) + out = Compiled( + GIVEN_NAME_TITLES=frozenset(given_titles), + TITLES=frozenset(titles), + NON_GIVEN_NAME_PARTICLES=frozenset(never_given), + PARTICLES=frozenset(particles), + CONJUNCTIONS=frozenset(conjunctions), + CONJUNCTIONS_AMBIGUOUS=frozenset(initial_conjunctions), + BOUND_GIVEN_NAMES=frozenset(bound), + MAIDEN_MARKERS=frozenset(markers), + SUFFIX_WORDS=frozenset(words), + GLUED_HONORIFICS=frozenset(glued), + SUFFIX_ACRONYMS_AMBIGUOUS=frozenset(ambiguous), + SUFFIX_ACRONYMS=frozenset(acronyms), + CAPITALIZATION_EXCEPTIONS=dict(sorted(cased.items())), + ) # One form per post-nominal: the word test strips only edge periods # and the acronym test strips all of them, so a word in both is # matched twice by two rules, and which one fired is unreadable from @@ -733,7 +922,7 @@ def read_dir(path: Path = VOCAB) -> dict[str, str]: return files -def compile_files(files: Mapping[str, str]) -> dict[str, Any]: +def compile_files(files: Mapping[str, str]) -> Compiled: return compile_pool(load(files)) @@ -749,17 +938,21 @@ def compile_files(files: Mapping[str, str]) -> dict[str, Any]: ''' -def render(compiled: Mapping[str, Any]) -> str: +def render(compiled: Compiled) -> str: """One sorted entry per line, so a one-word change is a one-line diff.""" out = [HEADER] + # a TypedDict is read by literal key; the roster loop reads it as + # the Mapping it also is + values: Mapping[str, object] = compiled for name, _ in CONSTANTS: - v = compiled[name] + v = values[name] out.append("") if isinstance(v, dict): out.append(f"{name}: dict[str, str] = {{") out += [f" {k!r}: {v[k]!r}," for k in sorted(v)] out.append("}") else: + assert isinstance(v, frozenset), (name, type(v)) out.append(f"{name}: frozenset[str] = frozenset({{") out += [f" {w!r}," for w in sorted(v)] out.append("})") diff --git a/tools/vocab/query.py b/tools/vocab/query.py index 83fa8f0a..8924901f 100644 --- a/tools/vocab/query.py +++ b/tools/vocab/query.py @@ -49,15 +49,30 @@ import build # noqa: E402 -@dataclass +@dataclass(frozen=True) class Row: + """A reading, or an exclusion: of a reading (`prop` None) or of a + property of the `kind` reading the word keeps.""" file: str text: str - label: str kind: str excluded: bool + prop: str | None props: dict[str, Any] + def __post_init__(self) -> None: + # a reading refuses no property + assert self.excluded or self.prop is None, self + + @property + def label(self) -> str: + """How the row prints; select reads the fields, never this.""" + if not self.excluded: + return self.kind + if self.prop is None: + return f"excluded:{self.kind}" + return f"{self.kind} excluded:{self.prop}" + def issues(self) -> set[int]: found: set[int] = set() for table in (self.props, self.props.get("attached")): @@ -68,10 +83,8 @@ def issues(self) -> set[int]: def rows(pool: build.Pool) -> list[Row]: - out = [Row(r.file, r.text, r.kind, r.kind, False, r.props) for r in pool.readings] - for x in pool.exclusions: - label = f"excluded:{x.kind}" if x.prop is None else f"{x.kind} excluded:{x.prop}" - out.append(Row(x.file, x.text, label, x.kind, True, x.props)) + out = [Row(r.file, r.text, r.kind, False, None, r.props) for r in pool.readings] + out += [Row(x.file, x.text, x.kind, True, x.prop, x.props) for x in pool.exclusions] return out @@ -96,10 +109,9 @@ def select(pool: build.Pool, *, issue: int | None = None, kind: str | None = Non # PROPERTY matches only the property's name, so `--excluded # honorific` is the words refused the honorific reading, not the # honorifics refused the glued peel - word_level = row.label.startswith("excluded:") if excluded is not None and not (row.excluded and ( - (word_level and _is(row.kind, excluded)) - or (not word_level and row.label.endswith(f":{excluded}")))): + (row.prop is None and _is(row.kind, excluded)) + or row.prop == excluded)): continue if word is not None and row.text != word: continue