"""Immutable vocabulary configuration for the 2.0 API.
Layering: may import nameparser.config DATA modules (the imports in
_default_lexicon() are the authoritative list) as the single source of
vocabulary during 2.x -- never nameparser.config itself, never
nameparser.parser. Enforced by tests/v2/test_layering.py.
"""
from __future__ import annotations
import dataclasses
import functools
import sys
import unicodedata
import warnings
from collections.abc import Iterable, Mapping, Set
from dataclasses import dataclass, field
from types import FrameType, MappingProxyType
from typing import cast
#: Vocabulary set fields, in declaration order. add()/remove() operate
#: on exactly these and reject capitalization_exceptions (its entries
#: are pairs -- use dataclasses.replace); __or__ unions these AND merges
#: capitalization_exceptions right-biased.
_VOCAB_FIELDS = (
"titles", "given_name_titles", "suffix_acronyms", "suffix_words",
"suffix_acronyms_ambiguous", "particles", "particles_ambiguous",
"conjunctions", "conjunctions_ambiguous", "bound_given_names",
"maiden_markers", "surnames", "honorific_tails",
)
#: (marker, base, why) triples. Each marker QUALIFIES how entries of
#: its base vocabulary are read and carries no vocabulary of its own,
#: so an entry outside the base is a configuration mistake -- but the
#: mistake differs per pair, and the reason is recorded here rather
#: than generalized, because an orphan is NOT simply inert. Nor is the
#: qualification one-directional: the first two NARROW their base (an
#: entry is read as vocabulary in fewer places), while honorific_tails
#: WIDENS it, granting a suffix word the glued position on top of the
#: whole-token match every suffix word already gets.
#:
#: * particles_ambiguous: _assign keys on the tag alone, so an orphan
#: makes the parse emit a spurious particle-or-given ambiguity.
#: * suffix_acronyms_ambiguous: _vocab returns True on the ambiguous
#: set before testing suffix_acronyms, so an orphan silently turns a
#: word into a period-gated suffix.
#: * honorific_tails: script_segment peels the tail into its own token
#: before classify ever runs, so an orphan splits the name and leaves
#: the fragment stranded inside it -- worse than not peeling at all.
#: Its base is deliberately NARROWER than what actually claims the
#: peeled piece: suffix_as_written ORs suffix_words with the
#: non-ambiguous suffix_acronyms, so a tail listed only as an acronym
#: would classify fine yet is rejected here. Accepted, and a decision
#: rather than an oversight -- the three-term predicate is easy to
#: get wrong in the dangerous direction (an ambiguous acronym admitted
#: as a tail would peel a period-gated word off a real name), and
#: nothing needs the acronym half: the shipped tails are CJK
#: honorifics, which are words.
#: The same relation is asserted a second time in config/suffixes.py,
#: over the raw GLUED_HONORIFICS/SUFFIX_WORDS constants at
#: import. The two are not redundant in the way they look: that one
#: is an `assert`, stripped under `python -O`, while the check here
#: raises unconditionally -- so under -O this is what still holds the
#: SHIPPED vocabulary to the invariant, as it is the only thing that
#: ever held a caller's own.
#:
#: Two fields are deliberately NOT here and have no check of their own.
#: given_name_titles -- see the note in __post_init__ for why every
#: attempt at one rejected working configurations. conjunctions_ambiguous
#: -- an orphan is never consulted because both the classify fork and
#: its ambiguity emitter require the base entry as well, so an orphan
#: decides nothing, and AGENTS.md's invariants rule guards harm, not
#: no-ops.
_SUBSET_FIELDS = (
("particles_ambiguous", "particles",
"an orphan emits a spurious particle-or-given ambiguity"),
("suffix_acronyms_ambiguous", "suffix_acronyms",
"an orphan silently becomes a period-gated suffix"),
("honorific_tails", "suffix_words",
"an orphan splits the name and leaves the tail inside it"),
)
#: The fields whose entries may be PHRASES: stored space-joined and
#: folded per word (_title_key), and exempt from the dead-entry warning
#: below. Every other field is matched one word at a time, where a
#: multi-word entry can never match.
#:
#: The two are not the same mechanism, and the difference is why the
#: exemption is a list rather than a rule. A given_name_titles run is
#: identified per word FIRST -- 'lt' and 'col' are each title
#: vocabulary -- and only then looked up, as the whole run's key or as
#: its LAST word's (_run_addresses_by_given, #489). The whole-run arm
#: is the one a phrase entry is matched by; the last-word arm is a
#: single word and cannot reach a phrase, so storage stays what it was
#: (the two are asked together, not in precedence). A maiden marker
#: phrase has no such per-word foothold: 'z' and 'domu' are not markers
#: individually, and adding them separately (which this warning used to
#: advise) reads 'Maria Kowalska z domu Nowak' as maiden 'domu Nowak'
#: and strips 'Anna z Nowak' of its family name. So markers are matched
#: by a genuine multi-token lookahead instead --
#: _pipeline._vocab.maiden_marker_run, longest first -- and only the
#: STORAGE rule is shared with titles.
_PHRASE_FIELDS = ("given_name_titles", "maiden_markers")
#: Every character the parser reads as a full stop at a WORD'S EDGE
#: (#322/#323): the ASCII period, the fullwidth FULL STOP U+FF0E, the
#: IDEOGRAPHIC FULL STOP U+3002 and its halfwidth form U+FF61. One
#: string, four readers -- the lookup fold below, _vocab's script
#: classification fold, _script_segment's surname site and honorific
#: tail match -- so that "which characters are a period" is answered
#: once; the bundle's fifth site, _pieces' opening-abbreviation shape,
#: reads no stop set at all but the no-initials repertoire
#: (_policy._NO_INITIALS) that licenses every reader here. NFKC is
#: NOT a substitute for listing them: it folds U+FF0E to '.' and
#: U+FF61 to U+3002, and leaves U+3002 as it is.
#: Not every period test reads this set, and the ones that do not are
#: named so the claim is checkable: the interior-period shapes -- the
#: dotted acronym 'M.A.' and the split 'Ph. D.' (_vocab) -- and FOUR
#: ASCII-only edge tests on Latin shapes, across THREE modules -- the
#: initial 'J.' (_vocab), the bracketed credential '(Mgr.)' and the
#: word-internal apostrophe rule (_extract), and the trailing
#: middle-initial carve-out 'V.' (_assign) -- stay ASCII. That is a
#: fact about those TESTS and not about this set's reach: the fold
#: below is script-agnostic, so a LATIN word wearing a wide stop
#: reaches vocabulary the ASCII veto would have taken it out of.
#: 'V。' folds to 'v' and does not end in the ASCII period the
#: trailing-suffix carve-out tests (_assign, not is_initial), so
#: it is roman five -- suffix -- where 'V.' stays a middle
#: initial through that same carve-out; 'Jr。' simply reaches its
#: entry, and 'Dr。' its title.
#: decisions.md#cjk-full-stops carries the measurements.
FULL_STOPS = ".．。｡"
def _normalize(word: str) -> str:
"""Lowercase, NFC-compose, strip whitespace and EDGE full stops
(FULL_STOPS, not the ASCII period alone -- #322). Interior periods
survive on purpose: 'J.R.' must not collapse to 'jr' and hit the
periodless vocabulary (v1 parity, pinned live 2026-07-17).
Suffix-ACRONYM membership alone uses the period-free form (see
_vocab.suffix_as_written), mirroring v1's is_suffix, which removed
periods only for the acronym test.
lower(), NOT casefold(): casefold's caseless-matching folds mutate
the stored vocabulary itself -- 'κος' becomes the misspelling 'κοσ'
(final sigma flattened) and 'großfürst' becomes 'grossfürst' --
while lower() applies Unicode SpecialCasing contextually and keeps
both as authored. This function is the single fold for storage AND
match-time lookups, so matching stays symmetric either way; lower()
is what v1's lc() used, preserving which cross-spellings match.
NFC, since #322, and only for a non-ASCII word (ASCII is already
NFC, and this runs per token on the parse path --
decisions.md#parse-cost). NFD hangul decomposes to jamo and missed
every vocabulary entry; composing at lookup AND at storage keeps
the two symmetric, and no shipped entry changes under it (measured
2026-09-09: fold every string member of every Lexicon.default()
field through NFC and count the changes; zero of 1735). This is
the same NFC composition script classification already applies
(_vocab._normalized_for_script); token text is never rewritten.
Strips to a FIXED POINT. A single strip().strip(FULL_STOPS) leaves
stops-around-whitespace half done ('. a .' -> ' a '), so a value
that is re-normalized later -- on unpickle, or by a second add() --
would change under its owner. v1 never re-normalized, so this only
matters now that storage and match-time share one fold."""
word = word.lower()
if not word.isascii():
word = unicodedata.normalize("NFC", word)
while True:
stripped = word.strip().strip(FULL_STOPS)
if stripped == word:
return word
word = stripped
# rules.md#P7: "A one-letter word written with its period is an
# initial and not a particle"
def _spells_an_initial(n: str, text: str) -> bool:
"""Whether `text` is ONE letter written with its period ('Ó.',
'j.'), which makes it an initial whatever particle the letter also
spells (#604). `n` is `_normalize(text)`, folded once by the caller.
Every particle test asks this AFTER its membership hit, so a word
no particle set holds never pays the frame. It lives here rather
than in _pipeline._vocab because the two views that re-read the
particle vocabulary over a parsed name -- _render's case repair and
the facade's initials and last-name split -- must ask it too, and
neither may import the pipeline.
The ASCII period only, as `_vocab.is_initial` takes it: a CJK full
stop marks no initial ('Ó。' is the particle). The other half of
is_initial, the scripts that write no initials (#320), is NOT asked
-- its script table is _policy's, which this module may not import
-- so a caller's one-character particle in such a script, written
with a period, is vetoed where is_initial would call it no initial
(decisions.md#P7). No shipped particle is one character of those
scripts."""
return len(n) == 1 and text[-1] == "."
def _fold_words(words: Iterable[str]) -> list[str]:
"""The words of a title run, folded for storage and lookup.
A multi-word title is matched as one key ('lt col'), so the fold has
to run per word -- _normalize on the whole phrase would leave
interior periods.
Words that fold away are DROPPED, not kept as empty. Keeping the
gap makes the key non-idempotent -- 'lt .' would store 'lt ', which
match time can never build: a run CAN carry a lone '.' (the
conjunction merge puts one there), but the fold drops the empty
word, so the key is 'lt' and the stored entry is inert. Storage
re-runs this fold on unpickle and on every dataclasses.replace, so a
value that changes under a second pass is one Lexicon later rejects as
"not written by this version". _normalize converges for the same
reason; so must anything built on top of it.
A LIST, so that _run_addresses_by_given's last-word arm can be the
last word of this fold rather than a re-split of the joined key --
"the last word of the FOLDED key" is then structural, and the two
arms read one fold between them.
map/filter rather than a comprehension: both are C calls where a
comprehension is a Python frame on 3.11, and this runs on the parse
path for every name carrying a title run (decisions.md#parse-cost).
"""
return list(filter(None, map(_normalize, words)))
def _title_key(words: Iterable[str]) -> str:
"""The given_name_titles lookup key for a run of title words: the
folded words, space-joined.
Defined once because it is built at match time
(_run_addresses_by_given, which is how post_rules' H1 and group's
P5 licence both reach it) and at translation time (_config_shim's
first_name_titles), and a divergence between them fails silently:
the entry simply stops matching."""
return " ".join(_fold_words(words))
def _run_addresses_by_given(words: Iterable[str],
vocabulary: Set[str]) -> bool:
"""Whether a run of title words addresses by the GIVEN name.
Two sites ask it and they must agree -- post_rules for H1, group
for the P5 licence -- because a run read two ways is a rule
contradicting itself (decisions.md#P5, the 2026-08-22 #369 entry).
One predicate, so they cannot drift.
The WHOLE run's key, or that key's LAST word (#489): a run is
written as several titles and addresses the way its final one does,
so 'Her Majesty Queen' addresses by given name because 'queen'
does. The two arms are asked of one key and neither is the other's
fallback -- the `or` short-circuits but decides nothing, since for
a one-word run the key IS its last word. Both arms read one
_fold_words list, so the last word is the last word of the FOLDED
key by construction rather than by agreement, and a run token that
folds away cannot empty that arm: the conjunction merge can put a
lone '.' in the run ('Sir and . John'), and the fold drops it.
What the drop leaves as the last word can then be the CONJUNCTION
-- 'Sir and . John' keys 'sir and' and reads family 'John', where
'Sir and Dame John' keys 'sir and dame' and reads given. Harmless
on the shipped vocabulary, which holds no entry ending in a
connective, and a caller who stored one would be asking for it
(measured 2026-09-09).
The whole-run arm is what keeps a caller's multi-word phrase entry
working: 'lt col' is stored as one key and matched as one run. Over
the SHIPPED vocabulary it is dead -- every shipped entry is a single
word (asserted in test_lexicon.py), so it can only ever match a
one-word run, which is a run the last-word arm reads the same way.
H2's unlisted abbreviations ride in the run. One can never match as
the last-word key, being in no vocabulary by definition -- written
as the inputs that produce those runs, 'Xyz. Sir John' keys 'xyz
sir', matches on 'sir' and reads given 'John', while 'Sir Xyz.
John' keys 'sir xyz', matches on neither arm and reads family
'John'. It CAN sit inside a whole-run key that matches, because
given_name_titles is deliberately not validated against titles: a
caller who stores 'sir xyz' makes 'Sir Xyz. John' read given.
The vocabulary is passed in rather than read off a default: a
caller's own Lexicon is the one that has to be consulted, and this
module is where Lexicon is defined."""
folded = _fold_words(words)
if not folded:
# every word folded away, so there is no key and no last word:
# the joined spelling built "" here, which matched nothing
return False
return " ".join(folded) in vocabulary or folded[-1] in vocabulary
def _reject_buffer(value: object, label: str, plural: str) -> None:
"""Binary sequences iterate to INTS, so every downstream entry check
reports a byte value -- "must be strings, got 100" for b'dr', where
100 is 'd'. That names neither the cause nor the fix. v1 shipped a
decode hint for this (#238), and parse() and the facade both carry
one, so no config entry point should be the odd one out.
"""
if isinstance(value, (bytes, bytearray, memoryview)):
raise TypeError(
f"{label} must be an iterable of {plural}, not "
f"{type(value).__name__} -- decode first, e.g. "
f"raw.decode('utf-8')"
)
def _warn_dead_entry(message: str) -> None:
# A fixed stacklevel always lands on library internals: the call
# depth differs per entry point (Lexicon(), add(), unpickle,
# dataclasses.replace, and the v1 shim's lazy snapshot -- built on
# the first parse after a Constants mutation, several facade frames
# below the user's own add()). Walk out of this module (and
# dataclasses' replace frames, and the facade layer that builds
# lexicons on the caller's behalf) so the warning points at the
# caller's own line.
level = 2
frame: FrameType | None = sys._getframe(1)
while frame is not None and frame.f_globals.get("__name__") in (
__name__, "dataclasses",
"nameparser._config_shim", "nameparser._facade"):
frame, level = frame.f_back, level + 1
warnings.warn(message, UserWarning, stacklevel=level)
def _normset(
entries: Iterable[str], field_name: str, warn: bool = True,
) -> frozenset[str]:
# Reject a bare str before iterating: iterating "dr" would silently
# yield the single characters {'d', 'r'} -- the set(str) footgun on
# the primary customization surface.
if isinstance(entries, str):
raise TypeError(
f"Lexicon.{field_name} must be an iterable of strings, "
f"not a bare string"
)
_reject_buffer(entries, f"Lexicon.{field_name}", "strings")
# A Mapping would silently contribute only its keys; a dict here
# almost always means the caller confused this field with
# capitalization_exceptions.
if isinstance(entries, Mapping):
raise TypeError(
f"Lexicon.{field_name} must be an iterable of strings, not a "
f"mapping (only capitalization_exceptions holds key->value pairs)"
)
try:
items = tuple(entries) # materialize once; entries may be a generator
except TypeError:
raise TypeError(
f"Lexicon.{field_name} must be an iterable of strings, "
f"got {entries!r}"
) from None
normalized = set()
for w in items:
if not isinstance(w, str):
raise TypeError(
f"Lexicon.{field_name} entries must be strings, got {w!r}"
)
# A phrase field's entries are matched as a multi-word run, so
# they fold per word: stored as the same key the match site
# builds, or 'lt. col' (and 'z. domu') would be kept verbatim
# and never match anything -- a silent no-op on the config
# surface. Every other field holds single words, where the two
# folds agree. See _PHRASE_FIELDS for how the two differ once
# stored.
n = _title_key(w.split()) if field_name in _PHRASE_FIELDS \
else _normalize(w)
# "." or "" is a data bug (stray split artifact, empty CSV
# cell); dropping it silently would also let a data-module typo
# vanish instead of failing CI.
if not n:
raise ValueError(
f"Lexicon.{field_name} entry {w!r} normalizes to empty "
f"(lowercase + strip full stops/whitespace leaves nothing)"
)
# Every field outside _PHRASE_FIELDS is matched one word at a
# time, so a multi-word entry can never match -- the library
# itself shipped eight such dead entries for years (repaired
# 2026-07-26). Warn, never raise: an inert entry produces
# nothing, and the given_name_titles precedent says a raise
# here costs working configurations (see __post_init__).
# warn=False is _edit()'s pass (both ops): add() warns via the
# new instance's __post_init__; remove() stores nothing, so
# warning there would name entries the caller is trying to get
# RID of, with "split it" advice that makes no sense for a
# no-op. Both invariants are indifferent to WHICH fields are
# exempt: the exemption only decides whether a warning exists to
# be emitted once or suppressed.
if (warn and field_name not in _PHRASE_FIELDS
# interior whitespace test; split() covers all Unicode
# whitespace
and n != "".join(n.split())):
_warn_dead_entry(
f"Lexicon.{field_name} entries are matched one word at "
f"a time; multi-word entry {w!r} can never match. "
f"Split it into separate entries")
normalized.add(n)
return frozenset(normalized)
def _alnum(text: str) -> str:
return "".join(c for c in text if c.isalnum())
def _offered_mask(normalized_key: str) -> str:
"""The working value a mismatched-mask error offers (carried to the
v1 shim's re-raise on _MaskValueError, so both name one fix): the
key upper-cased, or the key unchanged -- an identity mask, always
legal -- where .upper() changes the letter count ('straße' ->
'STRASSE') and would fail the check if pasted back in."""
upper = normalized_key.upper()
return upper if _alnum(_normalize(upper)) == _alnum(
normalized_key) else normalized_key
class _MaskValueError(ValueError):
"""_normpairs' error for a value that does not spell its key.
Built in the UnicodeDecodeError idiom -- structure in, message
out: `key`, `normalized_key` and `value` are the offending inputs;
`offered` (the v1 shim's re-spelled fix, and the v2 one quoted in
the message below) is computed from them, not carried separately.
__reduce__ is explicit because the inherited one re-calls
__init__ with self.args, which holds the message alone, not the
three fields __init__ needs -- and a ProcessPoolExecutor pickles a
worker's exception to deliver it, surfacing a construction failure
as BrokenProcessPool. Passing self.__dict__ as reduce's third
element (restored via a plain __dict__.update, there being no
__setstate__ here) carries along anything added after construction
-- e.g. add_note() -- that reconstructing from the three fields
alone would not. (An exception, so _types.py's frozen-dataclass
pickle guards do not apply.)"""
def __init__(self, key: str, normalized_key: str, value: str) -> None:
self.key = key
self.normalized_key = normalized_key
self.value = value
self.offered = _offered_mask(normalized_key)
super().__init__(
f"capitalization_exceptions value {value!r} for key {key!r} "
f"does not spell the key's letters and digits: a value "
f"is a case mask, the key's own letters and digits "
f"recased -- e.g. "
f"capitalization_exceptions=(({normalized_key!r}, "
f"{self.offered!r}),)")
def __reduce__(
self,
) -> tuple[type[_MaskValueError], tuple[str, str, str], dict[str, object]]:
return (type(self), (self.key, self.normalized_key, self.value),
self.__dict__)
def _normpairs(
raw: Mapping[str, str] | Iterable[tuple[str, str]],
) -> tuple[tuple[str, str], ...]:
"""Canonicalize capitalization_exceptions input: _normset's sibling
for the one pair-valued field. Dedupes on the NORMALIZED key so the
tuple and the derived map always agree ("PHD" and "phd" collide
after normalization; "Ph.D." does not, the fold keeping interior
periods, so it is a key of its own, "ph.d"); last occurrence wins,
matching dict semantics and the right-bias rule used elsewhere."""
if isinstance(raw, str):
raise TypeError(
"capitalization_exceptions must be a mapping or an "
"iterable of (key, value) pairs, not a bare string"
)
_reject_buffer(raw, "capitalization_exceptions", "(key, value) pairs")
pairs = raw.items() if isinstance(raw, Mapping) else raw
try:
pairs = iter(pairs)
except TypeError:
raise TypeError(
"capitalization_exceptions must be a mapping or an "
f"iterable of (key, value) pairs, got {raw!r}"
) from None
deduped: dict[str, str] = {}
for entry in pairs:
# A 2-char str entry would unpack "ab" into ("a", "b")
# silently, so reject str outright; other mis-shapes would
# otherwise surface as bare unpack errors.
if isinstance(entry, str):
raise TypeError(
f"capitalization_exceptions entries must be "
f"(key, value) pairs, got {entry!r}"
)
try:
k, v = entry
except (TypeError, ValueError):
raise TypeError(
f"capitalization_exceptions entries must be "
f"(key, value) pairs, got {entry!r}"
) from None
if not isinstance(k, str) or not isinstance(v, str):
raise TypeError(
f"capitalization_exceptions entries must be "
f"str -> str, got {k!r}: {v!r}"
)
normalized_key = _normalize(k)
if not normalized_key:
raise ValueError(
f"capitalization_exceptions key {k!r} normalizes to "
f"empty (lowercase + strip full stops/whitespace leaves "
f"nothing)"
)
# Stored NFC-composed, case kept (unicodedata, not _normalize):
# one stored spelling per value, whichever form the caller
# wrote. _apply_mask reads both forms alike since #542
# (_render._beside looks past combining marks); before it, a
# decomposed letter read as a base letter split off beside a
# non-alpha combining mark. The mask check below
# passes either spelling, _normalize composing both of its
# sides. `written` keeps the caller's own spelling -- composed
# or not -- for the mismatch error below: a decomposed value
# (e.g. 'e' + a combining acute) and its NFC form print
# identically to a reader's eye but are different str objects,
# so echoing the composed spelling back would show the caller
# a value they did not write.
written = v
v = unicodedata.normalize("NFC", v)
# capitalized() looks words up one at a time (the _WORD regex
# never yields spaces), so a multi-word key is unreachable --
# checked FIRST, ahead of the mask check below, because an
# invariant guards harm and a value that can never be read
# does no harm no matter what it says (AGENTS.md: "a check
# guards harm, not no-ops"). Raising the mask error for such a
# key -- as a naive mask-first order does -- would refuse a
# config that was always going to be a no-op.
# interior whitespace test; split() covers all Unicode whitespace
if normalized_key != "".join(normalized_key.split()):
_warn_dead_entry(
f"capitalization_exceptions keys are matched one word "
f"at a time; multi-word key {k!r} can never match. "
f"Split it into per-word entries")
deduped[normalized_key] = v
continue
# A value is a case MASK (#459): the key's own letters and
# digits recased, compared through the key's own fold. Its
# punctuation marks which letters are joined into one run
# (_render._apply_mask reads that) and is never written into
# the word. A raise, not a warning: a mismatched value used to
# be SUBSTITUTED for the word, and rules.md#R4: "Repair
# changes case and nothing else", so there is no reading of
# one that repair can honor.
if _alnum(_normalize(v)) != _alnum(normalized_key):
# key is the RAW key, not normalized_key: the v1 shim's
# hint must overwrite the offending dict entry, which is
# stored under the raw spelling -- writing the normalized
# spelling would add a second entry and still raise
# (measured with {"PHD": "Junior"}: the shim's suggested
# constants.capitalization_exceptions['PHD'] = ... has to
# match the key already there, not 'phd'). `written`, not
# `v`: the caller's own spelling, not the NFC-composed one.
raise _MaskValueError(k, normalized_key, written)
deduped[normalized_key] = v
return tuple(sorted(deduped.items()))
[docs]
@dataclass(frozen=True, slots=True)
class Lexicon:
"""The vocabulary a parser matches against: which words are
titles, particles, suffixes, and so on. Immutable and hashable.
Start from :meth:`default` (the shipped vocabulary) or
:meth:`empty`, derive variants with :meth:`add` / :meth:`remove` /
``|`` (union), and pass the result to ``Parser(lexicon=...)``.
Entries are normalized at construction -- lowercased, NFC-composed,
edge full stops stripped -- so matching is case-insensitive.
Vocabulary entries are single words -- a multi-word entry warns at
construction and can never match. Two fields are exempt, and they
differ in HOW they match: ``given_name_titles`` is looked up
against the run of words the parse has ALREADY read as titles --
the whole run space-joined, or that run's last word -- while
``maiden_markers`` is matched by lookahead, longest first, over
words that need not be markers on their own (``"z domu"``).
Field docs below show examples, not full
contents; inspect any field's shipped vocabulary directly, e.g.
``Lexicon.default().conjunctions``."""
#: Pre-nominal titles ("dr", "sir", "capt", ...). Full default
#: list: :data:`~nameparser.config.titles.TITLES`.
titles: frozenset[str] = frozenset()
#: Titles whose single following name reads as a GIVEN name
#: ("sheikh", "sister", ...) rather than a family name. Full
#: default list: :data:`~nameparser.config.titles.GIVEN_NAME_TITLES`.
given_name_titles: frozenset[str] = frozenset()
#: Post-nominal acronym suffixes, matched with or without periods
#: ("phd" matches "PhD" and "Ph.D."). Full default list:
#: :data:`~nameparser.config.suffixes.SUFFIX_ACRONYMS`.
suffix_acronyms: frozenset[str] = frozenset()
#: Post-nominal word suffixes ("jr", "esquire", "iii", ...). Full
#: default list:
#: :data:`~nameparser.config.suffixes.SUFFIX_WORDS`.
suffix_words: frozenset[str] = frozenset()
#: Subset of suffix_acronyms whose bare forms are also common
#: surnames ("ma", "do"). Written with periods one is a suffix
#: outright; written bare, capitals decide in a mixed-case name
#: ("Jack MA" is the credential, "Jack Ma" keeps his family name)
#: and otherwise the word counts as a suffix only where a name
#: word is left beside it ("John Smith ma"), rules.md#S2. Full
#: default list:
#: :data:`~nameparser.config.suffixes.SUFFIX_ACRONYMS_AMBIGUOUS`.
suffix_acronyms_ambiguous: frozenset[str] = frozenset()
#: Family-name particles that chain onto the following piece
#: ("van", "de", "bin", ...). Full default list:
#: :data:`~nameparser.config.particles.PARTICLES`.
particles: frozenset[str] = frozenset()
#: Subset of particles that can also BE a given name ("Van
#: Johnson", but also "Van Buren"). Membership decides nothing
#: about chaining: the prefix chain skips the name's first piece
#: and never consults this set, so it leaves a leading particle a
#: piece of its own whether listed or not -- "de Mesnil" groups
#: into two pieces exactly as "van Gogh" does, and since #367 the
#: NAME in "Dr. de Mesnil" and "Dr. Van Johnson" groups into those
#: same two pieces behind the title piece, a title not being
#: part of the name it precedes. What membership
#: decides is what becomes of that piece afterwards. Under ANY
#: ``name_order`` a member records a particle-or-given ambiguity
#: and a non-member records none, and a non-member is additionally
#: folded back into the family name once roles exist WHERE IT OPENS
#: THE NAME, so the whole name is the surname ("de Mesnil" -- a bare "de", with nothing to
#: fold into, is left alone). That fold is order-independent too
#: (#359) -- not because the word could not be a given name, which
#: it can where position forces it (``parse("de")`` reports given
#: "de"), but because a particle OPENING the name has the rest of
#: the name to join forward to, and that evidence is positional
#: rather than vocabulary, so no declared order contradicts it. Which field a MEMBER's piece
#: lands in is ``name_order``'s question, not this set's.
#: No constant of its own -- the default derives
#: as particles minus
#: :data:`~nameparser.config.particles.NON_GIVEN_NAME_PARTICLES`
#: (which marks the opposite, never-given subset).
particles_ambiguous: frozenset[str] = frozenset()
#: Words or characters that join surrounding pieces into one
#: ("and", "&", "y", "и", ...). Full default list:
#: :data:`~nameparser.config.conjunctions.CONJUNCTIONS`.
conjunctions: frozenset[str] = frozenset()
#: Subset of conjunctions that read as an INITIAL rather than a
#: connective in a name written wholly in one case ("e": "jose e
#: maria santos" reads middle "e maria", where "juan garcia y
#: lopez" joins because "y" is not a member). Mixed-case input is
#: decided by the writing instead and never consults this set, and
#: neither does a caseless letter (Arabic و), which has no case to
#: read. A member additionally reports
#: :attr:`~nameparser.AmbiguityKind.CONJUNCTION_OR_INITIAL`; a
#: non-member reports nothing, its reading not being in doubt.
#: Full default list:
#: :data:`~nameparser.config.conjunctions.CONJUNCTIONS_AMBIGUOUS`.
#: Entries need not also be in ``conjunctions``; one that is not is
#: simply never consulted, so it is inert rather than an error.
conjunctions_ambiguous: frozenset[str] = frozenset()
#: Given-name prefixes that bind to the following word to form one
#: given name ("abdul" -> "Abdul Salam"); never standalone names.
#: Full default list:
#: :data:`~nameparser.config.bound_given_names.BOUND_GIVEN_NAMES`.
bound_given_names: frozenset[str] = frozenset()
#: Marker words introducing a birth surname, routed to the maiden
#: field ("née", "geb.", "rozená", ...). An entry may be a PHRASE
#: ("z domu"): entries are matched by lookahead, longest first, so
#: a phrase wins where it matches and a word entry that starts one
#: still matches on its own everywhere else. A phrase matches only
#: where its words stand together -- a bracketed clause or a comma
#: between them ends the run. Full default list:
#: :data:`~nameparser.config.maiden_markers.MAIDEN_MARKERS`.
maiden_markers: frozenset[str] = frozenset()
#: Family names for the unspaced-name segmentation stage (#271),
#: matched longest-first against the start of the FIRST token
#: written wholly in a script :attr:`Policy.segment_scripts
#: <nameparser.Policy.segment_scripts>` activates. The default
#: carries the Korean census list
#: (:data:`~nameparser.config.surnames.KOREAN_SURNAMES`); Chinese
#: surnames ship in locales.ZH because Han segmentation is opt-in.
surnames: frozenset[str] = frozenset()
#: Honorifics that may be peeled off the END of a name token
#: (#308), matched longest-first: 田中さん splits into 田中 and さん
#: before the tokens are classified. Every entry must also be a
#: :attr:`suffix_words` entry -- the peeled tail is claimed by
#: suffix classification like any other post-nominal. Deliberately
#: NOT gated on :attr:`Policy.segment_scripts
#: <nameparser.Policy.segment_scripts>` (unlike :attr:`surnames`
#: above): 田中さん peels under the default policy, where HAN is in
#: no activation set, because a tail entry carries its own license
#: to fire. Entries are matched against the RAW token text, and
#: only within a name containing a non-ASCII character, so an ASCII
#: or mixed-case entry is at best conditionally active -- a ``"Jr"``
#: entry is stored ``"jr"`` and matches only lowercase text. The
#: field is effectively CJK-scoped in 2.1, which is what the shipped
#: vocabulary is. Full default list:
#: :data:`~nameparser.config.suffixes.GLUED_HONORIFICS`.
honorific_tails: frozenset[str] = frozenset()
#: Lowercase word -> case mask used by capitalized(): the word's
#: own letters and digits, each in the case it takes ("phd" ->
#: "PhD"), laid over the word as written, so the one entry
#: repairs "ph.d." to "Ph.D.". The mask's own punctuation marks
#: where its letters are joined into one run versus split apart,
#: and is never written into the word -- repair keeps the writer's
#: own punctuation. A value that does not spell the key's letters
#: and digits raises ValueError. Pair-valued: change it with
#: dataclasses.replace(), not add()/remove(); read it as a mapping
#: via capitalization_exceptions_map. Full default mapping:
#: :data:`~nameparser.config.capitalization.CAPITALIZATION_EXCEPTIONS`.
# Canonical storage: sorted tuple of (key, value) pairs. The
# constructor tolerates any Mapping (or pair iterable) at runtime and
# canonicalizes here; this closes the caller-aliasing hole and keeps
# Lexicon hashable. Read via capitalization_exceptions_map.
capitalization_exceptions: tuple[tuple[str, str], ...] = ()
_cap_map: Mapping[str, str] = field(
init=False, repr=False, compare=False, hash=False,
default_factory=lambda: MappingProxyType({}))
def __post_init__(self) -> None:
for name in _VOCAB_FIELDS:
object.__setattr__(self, name, _normset(getattr(self, name), name))
canonical = _normpairs(self.capitalization_exceptions)
object.__setattr__(self, "capitalization_exceptions", canonical)
object.__setattr__(self, "_cap_map", MappingProxyType(dict(canonical)))
for marker, base, why in _SUBSET_FIELDS:
orphans = getattr(self, marker) - getattr(self, base)
if orphans:
raise ValueError(
f"{marker} marks a subset of {base}; "
f"not in {base}: {', '.join(sorted(orphans))} "
f"({why}). Add them to {base}, or drop them from "
f"{marker}"
)
# NOT validated: given_name_titles against titles. The lookup
# key is the space-joined run of Role.TITLE tokens, built by the
# parse rather than drawn from this vocabulary, and a
# conjunction inside a run is itself tagged Role.TITLE -- so
# "sir and dame" is a matchable key whose middle word is in
# conjunctions, not titles. A whole-entry check rejected
# multi-word entries; a per-word check rejected that one. An
# unreachable entry here is inert: nothing consults it, nothing
# misparses. That is the cheap failure, and guarding it proved
# the expensive one -- three working configurations broken
# across two attempts. Do not add a third.
#
# The v2 form of particles.py's NON_GIVEN_NAME_PARTICLES-disjoint-
# from-BOUND_GIVEN_NAMES assertion. That module guards its own
# data at import; this guards vocabulary a caller supplies.
contradictory = (
self.bound_given_names & self.particles) - self.particles_ambiguous
if contradictory:
raise ValueError(
f"bound_given_names entries that are also particles must be "
f"in particles_ambiguous; not in particles_ambiguous: "
f"{', '.join(sorted(contradictory))}. A particle that never "
f"starts a given name cannot also bind one — add them to "
f"particles_ambiguous, or drop them from bound_given_names"
)
# suffix_as_written ORs the acronym and word branches, so a word
# membership bypasses the period gate the ambiguous set exists
# to impose: listing 'ma' in suffix_words would make "Jack Ma"
# read as a suffix and lose the family name.
gate_bypassed = self.suffix_acronyms_ambiguous & self.suffix_words
if gate_bypassed:
raise ValueError(
f"an ambiguous suffix acronym must not also be a suffix "
f"word; in both: {', '.join(sorted(gate_bypassed))}. The "
f"word branch matches without periods, which bypasses the "
f"period gate suffix_acronyms_ambiguous exists to impose"
)
# -- constructors ----------------------------------------------------
@classmethod
def empty(cls) -> Lexicon:
return cls()
@classmethod
def default(cls) -> Lexicon:
return _default_lexicon()
# -- dunders ----------------------------------------------------------
def _deltas_from(self, baseline: Lexicon) -> list[tuple[str, int, int]]:
deltas = []
for name in _VOCAB_FIELDS + ("capitalization_exceptions",):
mine = set(getattr(self, name))
theirs = set(getattr(baseline, name))
added, removed = len(mine - theirs), len(theirs - mine)
if added or removed:
deltas.append((name, added, removed))
return deltas
def __repr__(self) -> str:
# Bounded: renders only which fields deviate from the nearer of
# the two named constructors and by how many entries -- never the
# entries themselves (design rule, see nameparser._types module
# docstring). Diffing empty()-built lexicons against default()
# would tell the wrong story ("default minus the entire
# default vocabulary").
if self == Lexicon.default():
return "Lexicon(default)"
if self == Lexicon.empty():
return "Lexicon(empty)"
candidates = [(label, self._deltas_from(baseline))
for label, baseline in (("default", Lexicon.default()),
("empty", Lexicon.empty()))]
label, deltas = min(
candidates, key=lambda c: sum(a + r for _, a, r in c[1]))
rendered = ", ".join(
name + ": " + "".join(
part for part, n in ((f"+{a}", a), (f"-{r}", r)) if n)
for name, a, r in deltas)
return f"Lexicon({label} + {rendered})"
def __getstate__(self) -> dict[str, object]:
# _cap_map is a MappingProxyType, which pickle rejects; ship every
# other slot and rebuild the proxy from the canonical tuple on load.
return {f.name: getattr(self, f.name)
for f in dataclasses.fields(self) if f.name != "_cap_map"}
def __setstate__(self, state: dict[str, object]) -> None:
# Fail at the unpickle site if the state comes from a different
# Lexicon field layout (version skew) -- silently loading it
# would defer the failure to some distant attribute read.
# Message kept in sync with _types._guarded_setstate by design
# (layering keeps this module import-free of _types).
expected = {f.name for f in dataclasses.fields(Lexicon)} - {"_cap_map"}
if set(state) != expected:
missing = ", ".join(sorted(expected - set(state))) or "none"
unexpected = ", ".join(sorted(set(state) - expected)) or "none"
raise ValueError(
f"incompatible Lexicon pickle: missing fields: {missing}; "
f"unexpected fields: {unexpected}"
)
for name, value in state.items():
object.__setattr__(self, name, value)
# Re-run construction validation rather than trusting the blob.
# The layout check above catches SHAPE skew; this catches
# CONTENT skew, which is likelier -- particles_ambiguous flipped
# meaning between v1's never-given set and v2's may-be-given set
# without changing its name, so an old pickle would otherwise
# load a semantically inverted lexicon in silence. Rebuilds
# _cap_map for free, which this used to do by hand.
self.__post_init__()
# __post_init__ also normalizes, and quietly accepting a
# rewritten value would make this a place where caller data
# changes without a word. A pickle this library wrote is always
# normalized already (_normalize converges), so a difference
# here means the state came from somewhere else -- say so while
# the offending entries can still be named.
# One pass per field, in _VOCAB_FIELDS declaration order. cast is
# safe: __post_init__ just ran _normset over each of these and
# raised for anything that was not an iterable of str.
drifted = []
for name in _VOCAB_FIELDS:
lost = frozenset(
cast("Iterable[str]", state[name])) - getattr(self, name)
if lost:
drifted.append(f"{name}: {', '.join(sorted(lost))}")
# the pair field too, or ten fields raise and the eleventh is
# quietly re-canonicalized (keys normalized, sorted, deduped)
given_pairs = cast("Iterable[tuple[str, str]]",
state["capitalization_exceptions"])
if tuple(given_pairs) != self.capitalization_exceptions:
drifted.append("capitalization_exceptions")
if drifted:
raise ValueError(
"incompatible Lexicon pickle: entries are not normalized "
f"({'; '.join(drifted)}); this state was not written by "
"this version of nameparser"
)
def __or__(self, other: Lexicon) -> Lexicon:
if not isinstance(other, Lexicon):
return NotImplemented
updates: dict[str, object] = {
name: getattr(self, name) | getattr(other, name)
for name in _VOCAB_FIELDS
}
# right-biased on key conflicts, mirroring later-wins for scalars
merged = dict(self._cap_map) | dict(other._cap_map)
updates["capitalization_exceptions"] = tuple(sorted(merged.items()))
return dataclasses.replace(self, **updates) # type: ignore[arg-type]
# -- properties -------------------------------------------------------
@property
def capitalization_exceptions_map(self) -> Mapping[str, str]:
return self._cap_map
# -- editing ----------------------------------------------------------
def _edit(self, op: str, entries: Mapping[str, Iterable[str]]) -> Lexicon:
updates: dict[str, frozenset[str]] = {}
for name, words in entries.items():
if name == "capitalization_exceptions":
raise TypeError(
"capitalization_exceptions holds key->value pairs; "
"use dataclasses.replace(lexicon, "
"capitalization_exceptions=(*lexicon."
"capitalization_exceptions, ('key', 'KEY'))) "
"instead of "
f"{op}()"
)
if name not in _VOCAB_FIELDS:
raise TypeError(
f"unknown Lexicon field {name!r}; valid fields: "
f"{', '.join(_VOCAB_FIELDS)}"
)
current: frozenset[str] = getattr(self, name)
# warn=False: this pass only computes the new set membership,
# never stores it directly. add() still warns exactly once,
# from the replaced instance's own __post_init__ -> _normset;
# remove() never reaches __post_init__ with the dead entry
# (it is subtracted out here), so it stays silent -- correct,
# since a removal stores nothing a warning could be about.
# That silence covers the entry BEING REMOVED only: a
# different multi-word entry still stored re-warns from the
# derived instance's __post_init__, since the warning is
# per-construction by design.
normalized = _normset(words, name, warn=False)
updates[name] = (current | normalized if op == "add"
else current - normalized)
# mypy's dataclasses.replace() typing checks a **dict's single
# value type against every field's type (it can't see which keys
# are actually present behind the unpack), so a homogeneous
# frozenset[str] dict is flagged against the tuple/Mapping-typed
# capitalization_exceptions/_cap_map fields even though this dict
# never contains those keys (guarded above).
return dataclasses.replace(self, **updates) # type: ignore[arg-type]
def add(self, **entries: Iterable[str]) -> Lexicon:
return self._edit("add", entries)
def remove(self, **entries: Iterable[str]) -> Lexicon:
return self._edit("remove", entries)
@functools.cache
def _default_lexicon() -> Lexicon:
# v1 data modules are the single source of vocabulary through 2.x.
from nameparser.config.bound_given_names import BOUND_GIVEN_NAMES
from nameparser.config.capitalization import CAPITALIZATION_EXCEPTIONS
from nameparser.config.conjunctions import (
CONJUNCTIONS, CONJUNCTIONS_AMBIGUOUS)
from nameparser.config.maiden_markers import MAIDEN_MARKERS
from nameparser.config.particles import NON_GIVEN_NAME_PARTICLES, PARTICLES
from nameparser.config.suffixes import (
GLUED_HONORIFICS, SUFFIX_ACRONYMS, SUFFIX_ACRONYMS_AMBIGUOUS,
SUFFIX_WORDS,
)
from nameparser.config.surnames import KOREAN_SURNAMES
from nameparser.config.titles import GIVEN_NAME_TITLES, TITLES
# every vocabulary constant is a frozenset since #293, so each one
# feeds its strictly-typed frozenset[str] field as it stands -- and
# this cache reading them ONCE is the reason they are frozen. A
# mutated module set always reached a freshly built Constants, and
# reached this Lexicon only when the edit landed before the first
# call; after it, the cache was already built and the same edit was
# invisible here. Which of the two a program got was not something
# the code doing the mutating could see.
# keep in sync with _config_shim.Constants._snapshot() (pinned by the
# default-Constants equality test in tests/v2/test_config_shim.py)
return Lexicon(
titles=TITLES,
given_name_titles=GIVEN_NAME_TITLES,
suffix_acronyms=SUFFIX_ACRONYMS,
suffix_words=SUFFIX_WORDS,
suffix_acronyms_ambiguous=SUFFIX_ACRONYMS_AMBIGUOUS,
particles=PARTICLES,
# FLIPPED from v1: v1 marks the never-given subset; v2 marks the
# may-be-given subset (migration: complement translation).
particles_ambiguous=PARTICLES - NON_GIVEN_NAME_PARTICLES,
conjunctions=CONJUNCTIONS,
conjunctions_ambiguous=CONJUNCTIONS_AMBIGUOUS,
bound_given_names=BOUND_GIVEN_NAMES,
maiden_markers=MAIDEN_MARKERS,
surnames=KOREAN_SURNAMES,
honorific_tails=GLUED_HONORIFICS,
# pass canonical pair-tuples so this strictly-typed call site never
# feeds a Mapping to the tuple-annotated field; __post_init__
# still tolerates a Mapping at runtime for interactive use
capitalization_exceptions=tuple(sorted(CAPITALIZATION_EXCEPTIONS.items())),
)