From 1f223d6a7f775b430db7ea44f46b86b3efb7c567 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 9 Nov 2026 23:36:06 -0800 Subject: [PATCH] Write one classify tag for the words H5's trailing chain takes (#1171) Classify writes CHAIN_TITLE_TAG, after H9's phrase test, on a word carrying JOINED_TITLE_TAG or wearing the trailing abbreviation shape with "vocab:title" or TRAILING_TITLE_TAG. trailing_titles, is_trailing_title_word and _weighed read the tag instead of spelling the test three times; assemble drops it. No reading moves. Co-Authored-By: Claude Opus 5.5 --- docs/design/AGENTS.md | 2 +- .../design/decisions/H5-trailing-title-run.md | 1 + nameparser/_pipeline/_assemble.py | 8 +- nameparser/_pipeline/_classify.py | 34 +++++++-- nameparser/_pipeline/_group.py | 5 +- nameparser/_pipeline/_pieces.py | 74 ++++++++----------- nameparser/_pipeline/_state.py | 28 ++++--- tests/v2/pipeline/test_classify.py | 32 +++++++- tests/v2/pipeline/test_pieces.py | 3 +- tests/v2/test_regex_sync.py | 5 +- 10 files changed, 120 insertions(+), 72 deletions(-) diff --git a/docs/design/AGENTS.md b/docs/design/AGENTS.md index b7b810a01..b71080374 100644 --- a/docs/design/AGENTS.md +++ b/docs/design/AGENTS.md @@ -24,7 +24,7 @@ Two shapes, needing different instruments. CROSS-SESSION distillation is what th Skip what the suite already enforces. `tests/v2/test_rules_doc.py`, `tests/v2/test_doc_citations.py` and `tests/v2/test_design_docs.py` machine-check example lines and their `deviates:` today-values, boundaries, the one-of `implemented:`/`tracked:` rule, `implemented:` against the modules that actually cite the rule, the rule IDs a rule names anywhere in its prose, cross-doc anchor IDs, citation excerpts quoted verbatim, legacy citation forms (the pre-#713 file-anchor form among them), that every rule and decisions file is indexed once, under a title matching its heading and file name, each decisions file's `## Standing` block against its live entries and against the index's Reference lists, the one kind of file that carries none, a rule file reusing an ID a decisions file records `Retired:` and a misspelled `Retired:` line, the README's copy of rules/D2 against the file and its split-candidate grep against the rules' Signal lines, that the NFD-typed examples stay decomposed, a denylist of spellings that shipped wrong (`tests/v2/test_doc_spellings.py`, over nameparser/ and docs/), each rule's `Signal:` line (its presence, spelling and place) and the README's list of the tags those lines use, a hub's rules against the bullets under its Then, both ways, a comment naming an Except or Precedence bullet against that bullet's bold lead, the skeleton grammar in `tests/v2/test_rules_skeleton.py` (labels, field order, fences, the footer, one paragraph each for When, Tolerated and Then, the two sentence limits — Then one sentence, Because three at most — and an example under every Except bullet), and the refusal of a rule in the retired block format. Assume they pass; a finding one of them would catch is a false positive. Read those files rather than trusting this sentence — it named five of the nine guards when written, and goes stale as guards are added. -1. **Recompute every number.** Any count, ratio or "N names" in the diff gets recomputed by a script written now, never read. (A filter described as returning one name returned three; a rule's stated blast radius listed two names its own scope excluded.) +1. **Recompute every number.** Any count, ratio or "N names" in the diff gets recomputed by a script written now, never read, and a before/after figure against the commit the change actually sits on, named in the entry: a rebase moves the base under it. (A filter described as returning one name returned three; a rule's stated blast radius listed two names its own scope excluded; #1171's frame figures, taken before a rebase onto #1298, named a base the commit no longer sat on, and the recompute `call_count.py --against` offers then mixed #1298's −7 into the change's +1.) 2. **Your detector is a second, unreviewed implementation.** It must read the same vocabulary and boundaries the rule reads. (A particle-run detector that walked only the never-given half — the rule chains through ANY particle — reported 50 false movers where the true count was 1, printing cleanly both times.) 3. **Rule vs. its own examples.** (P1 said the fold takes "the particle and the one name word"; its example `de la Vega` is two particles onto one word.) An Except bullet's fence is examples of that bullet, not of the rule: a row there must be one where Then does NOT hold, so read each row in an Except fence against its own bullet's bold lead and move a row Then still decides out of it — to the main fence, or to the Accepted bullet that records it as a cost. Nothing checks this — the runner asserts a row's output, never which fence it sits in — and the #761 migrations, which sorted every rule's examples into fences by hand, put rows where Then held into an Except fence twice that review caught: H1's given-name rows (PR #779, back to the listing-record Accepted bullet) and S3's switched-off `Msc.Ed.` row (PR #783, to the main fence). 4. **Any claim about a document vs. what that document says about itself.** Two directions, both bitten. Inward: follow every `history:` and read the entry (P6's rationale asserted what decisions/vocabulary-collisions denies, and the rule's own guard exists because the assertion is false). Outward: when prose anywhere — this file, a code comment, another design doc — describes what one of these files contains or does, read that file's preamble or entry conventions first (an AGENTS.md bullet claimed the rules name the pipeline stages; their preamble forbids exactly that — "Statements are implementation-free: no stage names, no function names, no regexes"). Those declarations sit at the top of files people cite without reading. Mind their scope before calling a violation: the rules' ban binds STATEMENTS, which `_statements()` in test_doc_citations.py defines — a rule's fields, When, Tolerated, Then and its bullets, Because, and the Except, Precedence and Accepted bullets, without fences or footer — so Backgrounds and Not-in-scope may name functions and do. diff --git a/docs/design/decisions/H5-trailing-title-run.md b/docs/design/decisions/H5-trailing-title-run.md index 2cd93a0ab..6e841cfc6 100644 --- a/docs/design/decisions/H5-trailing-title-run.md +++ b/docs/design/decisions/H5-trailing-title-run.md @@ -76,3 +76,4 @@ Open: none in this file. - **Pinned** by tests/v2/pipeline/test_pieces.py's test_a_listed_joined_title_reads_as_prof_does, one shape per reader of the mark (P13's connective, the maiden take, the particle run's weighed word), each of which passed the suite with its read removed until the code review's mutants found it. - **Measured** against `34812254` (2026-11-06, `tools/differential/measure.py --base`): the committed corpora move only rules/H5's own example (`Maria Rossi Dott.ssa`); CLDR, GND and Wikidata agreement match their records; no local corpus moves. A rare writing, then, settled for its consistency with the listing, which already read it as a title in front. In brackets only the academic five read as the title (`John Smith (Dott.ssa)`, `(Prof.ssa)`, `(Univ.Prof)`, `(Dipl.-Ing)`, `(Priv.-Doz)`); the salutations and styles read as a nickname there on both trees (`Maria Rossi (Sig.ra)` → nickname 'Sig.ra', as `(Mrs)` does), so for those eleven the bare and bracketed spellings disagree as title and nickname where they disagreed as family and nickname, found by the design-docs review and left to S1, with #1104. - 2026-11-07 #1131 — A WORD EVERY CHUNK OF WHICH IS A TITLE CHAINS WITHOUT A FINAL PERIOD. The question the 2026-11-06 #1104 entry left for brackets and the end of a name, answered with decisions/S3's entry of this date: every chunk, not any. Classify marks a word with interior periods every chunk of which is title vocabulary with `JOINED_TITLE_TAG`, as it marks a title listed whole (`_vocab.period_joined_title`), so the chain, `_weighed` and P13's trailing title take `John Smith Lt.Gov` (title 'Lt.Gov', where 1.4.0 through 2.8.2 read family 'Lt.Gov') and `Smith, John Lt.Col`, and in brackets rules/S1 reads it as the title (`John Smith (Lt.Gov)`). A word a title only by some chunk (`Dr.PhD`, `St.John`) still needs the final period, the chunk beside the title being no title. ctgov-officials' `Reva PROF.DR` reads title 'PROF.DR' (measured with decisions/S3's entry). +- 2026-11-09 #1171 — ONE TAG FOR THE CHAIN'S WORD TEST. Which words rules/H5's chain takes was spelled at three sites kept in step by comments -- `_pieces.trailing_titles` inline, `_pieces.is_trailing_title_word` (the maiden take, and P13's walk back since #1168 deleted the fourth copy, `_group._is_trailing_title`, which had drifted, decisions/P13) and `_pieces._weighed`. Classify now writes it once, as `CHAIN_TITLE_TAG`: a word carrying `JOINED_TITLE_TAG`, or one wearing `_vocab._TRAILING_TITLE_SHAPE` that carries "vocab:title" or `TRAILING_TITLE_TAG`; each site reads the tag. It is written after rules/H9's phrase test, which settles "vocab:title" (`Associate Dean.` carries it, `John Smith Dean.` does not), and against the text as written, so a glued delimiter (`Prof.;`) wears no shape, as before; assemble drops it. No reading moves: the suite and `snapshot.jsonl` hold byte-identical. Cost, measured against `1c5a220c` on py3.11 (2026-11-09): an ordinary name no longer pays the shape regex; every title word that wears the shape, or that carries the joined mark, pays one frame for its write, in any position, so the benchmark's reference row (`tools/perf/call_count.py`) goes 284.80 → 285.80 for its leading `Dr.` and a title run pays per word (`Dr. Prof. John Smith` 223 → 225); a name ending in a plain listed title saves the `is_title_piece` call the chain made, netting zero (`John Smith Prof.` 186 → 186) or one fewer where a second reader asked it (`John Smith y Prof.` 228 → 227), while one ending in a word with the written-short or joined mark, which made no such call, pays the frame (`John Smith Maj.` 198 → 199). Pinned by tests/v2/pipeline/test_classify.py's test_chain_title_tag_marks_the_words_h5s_chain_takes; with the tag's written-short arm removed, 23 tests fail. diff --git a/nameparser/_pipeline/_assemble.py b/nameparser/_pipeline/_assemble.py index b99f387a1..bdc148c7b 100644 --- a/nameparser/_pipeline/_assemble.py +++ b/nameparser/_pipeline/_assemble.py @@ -56,7 +56,7 @@ SHARED_NAME_TAG, PARTICLE_GIVEN_TAG, BOUND_PHRASE_TAG, FIRST_SURNAME_TAG, SURNAME_PAIR_TAG, SPLIT_CREDENTIAL_TAG, SPLIT_MEMBER_TAG, PAST_REACH_TAG, - DOTTED_TITLE_TAG, JOINED_TITLE_TAG, TITLE_AMBIGUOUS_TAG, TRAILING_TITLE_TAG, Beside, Deferred, ParseState, PendingAmbiguity, + CHAIN_TITLE_TAG, DOTTED_TITLE_TAG, JOINED_TITLE_TAG, TITLE_AMBIGUOUS_TAG, TRAILING_TITLE_TAG, Beside, Deferred, ParseState, PendingAmbiguity, Structure, WorkToken, comma_bucket, reported_suffix_fork, ) from nameparser._pipeline._vocab import ( @@ -266,12 +266,12 @@ def _split_partner(tokens: tuple[WorkToken, ...], dropped: set[int], #: TITLE_AMBIGUOUS_TAG; and the shared-name and particle-given marks #: rules/O6 and O7 read (#704), the bound-phrase mark rules/P12 #: reads (#694), the past-reach mark on a particle chain past rules/P1's -#: fold (#828), and the joined-title mark rules/H5's chain reads -#: (#1104). +#: fold (#828), and the joined-title mark (#1104) and the +#: chain-title mark (#1171) rules/H5's chain reads. _KIND_TAGS = frozenset({GENERATION_WORD_TAG, CREDENTIAL_WORD_TAG, HONORIFIC_WORD_TAG, PATRONYMIC_TAG, ATTACHED_TAG, TRAILING_TITLE_TAG, SHARED_NAME_TAG, PARTICLE_GIVEN_TAG, PAST_REACH_TAG, - BOUND_PHRASE_TAG, JOINED_TITLE_TAG, + BOUND_PHRASE_TAG, JOINED_TITLE_TAG, CHAIN_TITLE_TAG, SURNAME_PAIR_TAG}) #: The second-surname mark O6 or W5 writes (rules/R10) and the diff --git a/nameparser/_pipeline/_classify.py b/nameparser/_pipeline/_classify.py index c0f88119b..cade81e21 100644 --- a/nameparser/_pipeline/_classify.py +++ b/nameparser/_pipeline/_classify.py @@ -48,10 +48,10 @@ "vocab:suffix", "vocab:suffix-word", "vocab:suffix-ambiguous", "vocab:suffix-word-ambiguous", "vocab:agnomen", "vocab:generation-word", "vocab:credential-word", -"vocab:honorific-word", "vocab:patronymic", "vocab:trailing-title" -and "vocab:joined-title" (read by the later stages and dropped by -assemble, Token.reading carrying the kind out, save the last two, -which mark no kind), "vocab:title-case-name" (a suffix word whose +"vocab:honorific-word", "vocab:patronymic", "vocab:trailing-title", +"vocab:joined-title" and "vocab:chain-title" (read by the later stages +and dropped by assemble, Token.reading carrying the kind out, save the +last three, which mark no kind), "vocab:title-case-name" (a suffix word whose Title case leans to the name, rules/S6, kept on the published token), "vocab:particle-ambiguous", "vocab:bound-given", "vocab:given-abbreviation", "vocab:shared-name" (which rules/P10, O6 @@ -97,13 +97,15 @@ class (rules/S12). GENERATION_WORD_TAG, GIVEN_ABBREVIATION_TAG, HONORIFIC_WORD_TAG, BOUND_PHRASE_TAG, PARTICLE_GIVEN_TAG, PATRONYMIC_TAG, SHAPE_ACRONYM_TAG, SHARED_NAME_TAG, SPLIT_CREDENTIAL_TAG, SPLIT_MEMBER_TAG, TITLE_AMBIGUOUS_TAG, TITLE_CASE_NAME_TAG, - DOTTED_TITLE_TAG, JOINED_TITLE_TAG, TRAILING_TITLE_TAG, ParseState, + CHAIN_TITLE_TAG, DOTTED_TITLE_TAG, JOINED_TITLE_TAG, TRAILING_TITLE_TAG, + ParseState, Deferred, PendingAmbiguity, PendingReading, WorkToken, copy_with, ) from nameparser._types import AmbiguityKind, ReadingKind, ReadingSource from nameparser._pipeline._vocab import ( - BUILTIN_GLUED_DELIMITERS, SPLIT_CHUNK, SPLIT_PAIR_OPENS, _INITIAL, _JOINED_WORD, caps_shape_candidate, + BUILTIN_GLUED_DELIMITERS, SPLIT_CHUNK, SPLIT_PAIR_OPENS, _INITIAL, _JOINED_WORD, + _TRAILING_TITLE_SHAPE, caps_shape_candidate, composed_len, glued_cores, glued_stem, is_cased_paired_initials, is_dotted_title, is_initial, is_one_case, period_joined_title, period_joined_vocab, @@ -653,6 +655,26 @@ def classify(state: ParseState) -> ParseState: for i, tag in split.items(): written[i] = copy_with(written[i], tags=written[i].tags | {tag}) tokens = tuple(written) + # rules/H5: "successive single words that wear the abbreviation + # shape and are title vocabulary chain into the title from the end" + # -- which words those are, written once as CHAIN_TITLE_TAG for + # every reader of the chain (#1171). After H9's phrase test, which + # settles "vocab:title"; against the text as written, which the + # readers saw before the tag ('Prof.;' wears no shape, C10's stem + # notwithstanding). The tag tests first, in C, so only a title word + # pays the regex, and only one wearing the shape pays a frame, in + # any position ('Dr.' too: position is the readers' question). + chain: list[WorkToken] | None = None + for i, token in enumerate(tokens): + tags = token.tags + if (JOINED_TITLE_TAG in tags + or ("vocab:title" in tags or TRAILING_TITLE_TAG in tags) + and _TRAILING_TITLE_SHAPE.match(token.text) is not None): + if chain is None: + chain = list(tokens) + chain[i] = copy_with(token, tags=tags | {CHAIN_TITLE_TAG}) + if chain is not None: + tokens = tuple(chain) ambiguities = list(state.ambiguities) for i, token in enumerate(tokens): # #383/#479: narrows the fork's own "initial" tag with the same diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 7b4e9ee23..2e8b2e0bd 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -559,7 +559,7 @@ def _link_the_last_letter(pieces: list[list[int]], ptags: list[set[str]], # trailing title H5 reads" -- H5's own word test, asked rather # than copied, so the walk steps over every title the chain # takes (decisions/P13, #1168) - if not (is_trailing_title_word(piece, ptags[e], tokens) + if not (is_trailing_title_word(piece, tokens) or is_suffix_piece(piece, ptags[e], tokens) or (e > 0 and len(piece) == 1 # reads_as_generation's own first test, asked at C @@ -591,8 +591,7 @@ def _link_the_last_letter(pieces: list[list[int]], ptags: list[set[str]], and text.lower() != text.upper()) and "vocab:suffix" not in word.tags and not (len(text) > 1 and e + 1 < len(pieces) - and is_trailing_title_word( - pieces[e + 1], ptags[e + 1], tokens))): + and is_trailing_title_word(pieces[e + 1], tokens))): return if (at - 1 >= leading_titles(pieces, ptags, tokens) + keep_a_name and _is_name_word(pieces[at - 1], ptags[at - 1], tokens)): diff --git a/nameparser/_pipeline/_pieces.py b/nameparser/_pipeline/_pieces.py index 3ee5f7c2e..790e533e9 100644 --- a/nameparser/_pipeline/_pieces.py +++ b/nameparser/_pipeline/_pieces.py @@ -60,11 +60,11 @@ from nameparser._pipeline._state import ( AMBIGUOUS_ACRONYM_TAG, AMBIGUOUS_WORD_TAG, GENERATION_WORD_TAG, TITLE_CASE_NAME_TAG, GIVEN_ABBREVIATION_TAG, NAME_ROLES, SHAPE_ACRONYM_TAG, SUFFIX_OR_UNREAD, AmbiguityKind, PendingAmbiguity, - DOTTED_TITLE_TAG, JOINED_TITLE_TAG, PendingReading, TRAILING_TITLE_TAG, + CHAIN_TITLE_TAG, DOTTED_TITLE_TAG, PendingReading, WorkToken, ) from nameparser._pipeline._vocab import ( - _DIGRAPH_INITIALS, _INDIC_BRAHMIC, _PERIOD_ABBREV, _ROMAN, _TRAILING_TITLE_SHAPE, + _DIGRAPH_INITIALS, _INDIC_BRAHMIC, _PERIOD_ABBREV, _ROMAN, Lean, ambiguous_lean, in_initialless_script, is_listed_numeral, is_single_letter_numeral, numeral_leans_to_the_name, @@ -1888,8 +1888,9 @@ def credential_run(rest: Sequence[int], peel: Peel, p: int, # that predicate carries H2's unlisted-abbreviation inference, which is # the LEADING slot's shape rule and has no trailing counterpart, so # with it 'John Smith Xyz.' would lose its family name to a title -# (decisions/H5). The vocabulary read is is_title_piece's, shared -# with the leading run so the two cannot disagree about what a title +# (decisions/H5). Classify's CHAIN_TITLE_TAG reads a word's title +# vocabulary through the same "vocab:title" tag is_title_piece reads, +# so the leading and trailing runs cannot disagree about what a title # WORD is while disagreeing, deliberately, about what a title SHAPE is # -- save the word H5's written-short clause adds, classify's # TRAILING_TITLE_TAG, which no leading read admits (#999). @@ -1924,38 +1925,26 @@ def trailing_titles(rest: Sequence[int], pieces: Sequence[Sequence[int]], Every parse with a name word to place enters this frame -- assign asks the question here rather than answering a cheaper version of it inline (mechanisms.md#ONE-PREDICATE-PER-QUESTION) -- so what it - costs an ordinary name is one frame, one tag test and one regex - match. The + costs an ordinary name is one frame and one tag test. The exceptions return before it: a segment that is all title, and a comma part read wholly as a credential run, have no name piece to hand this (52 of the 1289 corpus parses, measured 2026-09-09 -- 'Coach', 'Lord of the Universe', 'Smith, Jr.', 'MD, PHD'). - The tag and shape tests run BEFORE the vocabulary call to keep it - at that: classify's JOINED_TITLE_TAG is a set membership test and - _TRAILING_TITLE_SHAPE a compiled regex (C calls, no Python frame) - where is_title_piece is a call, and almost no name ends in a - period-marked word, so the ordinary parse pays the two tests and - stops (decisions/parse-cost). + The word test is classify's CHAIN_TITLE_TAG, a set membership test + (a C call, no Python frame), so the ordinary parse pays one tag + test and stops (decisions/parse-cost, #1171). """ k = len(rest) if end is None else end while k > 1: idx = rest[k - 1] piece = pieces[idx] - # no #323 veto on the shape here, unlike is_leading_title's: - # the shape is ANDed with a vocabulary read, so the word is - # listed, and a listed CJK title wearing a stop should read as - # a title. rules/H5: "a title that may be a given name is - # title vocabulary there where its title reading is written - # short" -- classify's TRAILING_TITLE_TAG, tested ahead of - # is_title_piece, a C-level read (decisions/H5, #999) + # classify's CHAIN_TITLE_TAG is the word test, a C-level read + # (#1171); no #323 veto on the shape there, unlike + # is_leading_title's: the shape is ANDed with a vocabulary + # read, so the word is listed, and a listed CJK title wearing a + # stop should read as a title if len(piece) == 1 and ( - # rules/H5: "a title the vocabulary lists whole with a - # period inside it chains without a final one" -- - # classify's tag, a C-level read (decisions/H5, #1104) - JOINED_TITLE_TAG in tokens[piece[0]].tags - or (_TRAILING_TITLE_SHAPE.match(tokens[piece[0]].text) - and (TRAILING_TITLE_TAG in tokens[piece[0]].tags - or is_title_piece(piece, ptags[idx], tokens))) + CHAIN_TITLE_TAG in tokens[piece[0]].tags # rules/H8: "the given part after it ends in a word whose # title reading is a salutation" -- the salutation chains # as a period-marked title does ("It reads as the @@ -1976,24 +1965,21 @@ def trailing_titles(rest: Sequence[int], pieces: Sequence[Sequence[int]], # asking it of a piece the chain did not walk to: the maiden take, # choosing the words the clause-free view is read over (#601), and # P13's walk back to a connective ending the name words (#1168). -# `trailing_titles` asks the same five tests INLINE, in the same order, -# for the frame budget its docstring states, and adds its `salutations` -# arm, which no caller of this one asks; keep the two in step. -def is_trailing_title_word(piece: Sequence[int], ptags: Set[str], +# Both read classify's CHAIN_TITLE_TAG, the one statement of the word +# test (#1171); `trailing_titles` reads it inline, for the frame budget +# its docstring states, and adds its `salutations` arm, which no +# caller of this one asks. +def is_trailing_title_word(piece: Sequence[int], tokens: Sequence[WorkToken]) -> bool: """Whether one piece is a word H5's trailing chain takes: a lone token wearing the abbreviation shape that the title vocabulary lists, or that classify marks as a title written short that may be a given name (TRAILING_TITLE_TAG), or a title the vocabulary lists whole with a period inside it, which classify marks - (JOINED_TITLE_TAG) and which needs no final period. Position is - the caller's question -- this answers only what the word is.""" - return (len(piece) == 1 - and (JOINED_TITLE_TAG in tokens[piece[0]].tags - or _TRAILING_TITLE_SHAPE.match(tokens[piece[0]].text) - is not None - and (TRAILING_TITLE_TAG in tokens[piece[0]].tags - or is_title_piece(piece, ptags, tokens)))) + (JOINED_TITLE_TAG) and which needs no final period -- the words + classify marks CHAIN_TITLE_TAG. Position is the caller's question + -- this answers only what the word is.""" + return len(piece) == 1 and CHAIN_TITLE_TAG in tokens[piece[0]].tags #: lone-word tags `trailing_candidates` admits on: the ambiguous class, @@ -2040,7 +2026,7 @@ def trailing_candidates(lo: int, pieces: Sequence[Sequence[int]], while c - 1 > lo: k = c - 1 piece = pieces[k] - title = is_trailing_title_word(piece, ptags[k], tokens) + title = is_trailing_title_word(piece, tokens) if not (title or "suffix" in ptags[k] or "split" in ptags[k] or is_suffix_piece(piece, ptags[k], tokens) or (len(piece) == 1 and ( @@ -2481,15 +2467,13 @@ def _weighed(piece: Sequence[int], tokens: Sequence[WorkToken]) -> bool: tok = tokens[piece[0]] return (not tok.tags.isdisjoint(_WEIGHED_TAGS) or _ROMAN.match(stem := tok.text.rstrip(".")) is not None - and stem != "" - or ("vocab:title" in tok.tags or TRAILING_TITLE_TAG in tok.tags) - and _TRAILING_TITLE_SHAPE.match(tok.text) is not None) + and stem != "") #: the tags any one of which makes `_weighed` True for a one-word -#: piece: the trailing reading's lone-word tags, an initial, and a -#: title listed whole with a period inside it (#1104) -_WEIGHED_TAGS = _TRAILING_WORD_TAGS | {"initial", JOINED_TITLE_TAG} +#: piece: the trailing reading's lone-word tags, an initial, and a word +#: H5's chain takes (#999, #1104, #1171) +_WEIGHED_TAGS = _TRAILING_WORD_TAGS | {"initial", CHAIN_TITLE_TAG} def read_trailing_run(pieces: Sequence[Sequence[int]], diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py index acd175e2d..5405bd5c0 100644 --- a/nameparser/_pipeline/_state.py +++ b/nameparser/_pipeline/_state.py @@ -188,10 +188,9 @@ def reported_suffix_fork(ambiguities: Sequence[PendingAmbiguity], #: Written by classify on every occurrence of a word of #: `Lexicon.titles_ambiguous` whose title reading is written short #: (`Lexicon.title_abbreviations`, `maj`), whichever way the phrase -#: test read it; read by rules/H5's trailing chain -#: (`_pieces.trailing_titles`, `is_trailing_title_word`), which takes -#: the word where it wears the abbreviation shape, by `_pieces._weighed` -#: and by group's P5 decline. Dropped from the published token like +#: test read it; read by classify's CHAIN_TITLE_TAG below, rules/H5's +#: trailing chain taking the word where it wears the abbreviation +#: shape, and by group's P5 decline. Dropped from the published token like #: the kind tags above; the published token keeps TITLE_AMBIGUOUS_TAG. TRAILING_TITLE_TAG = "vocab:trailing-title" @@ -201,13 +200,24 @@ def reported_suffix_fork(ambiguities: Sequence[PendingAmbiguity], #: 'Priv.-Doz'), and on a word with interior periods every chunk of #: which is a title ('Lt.Gov', #1131), never on one read as a title #: through some chunk only ('Dr.PhD', 'St.John') nor on one a hyphen -#: alone joins ('Co-Founder'); read by -#: rules/H5's trailing chain (`_pieces.trailing_titles`, -#: `is_trailing_title_word`), which takes it without a final period, -#: by `_pieces._weighed` and by P13's trailing title (#1104). Dropped -#: from the published token like the kind tags above. +#: alone joins ('Co-Founder'); read by classify's CHAIN_TITLE_TAG +#: below, rules/H5's trailing chain taking it without a final period +#: (#1104). Dropped from the published token like the kind tags above. JOINED_TITLE_TAG = "vocab:joined-title" +#: Written by classify, after rules/H9's phrase test has settled which +#: words keep "vocab:title", on every word rules/H5's trailing chain +#: takes: a word carrying JOINED_TITLE_TAG, or one wearing the trailing +#: abbreviation shape (`_vocab._TRAILING_TITLE_SHAPE`) that carries +#: "vocab:title" or TRAILING_TITLE_TAG. The one statement of that test, +#: read as a set membership by the chain (`_pieces.trailing_titles`), +#: by its one-word form (`_pieces.is_trailing_title_word`, which the +#: maiden take and P13's walk back ask) and by `_pieces._weighed` +#: (#1171). Whether the word stands where the chain reaches is the +#: reader's question. Dropped from the published token like the kind +#: tags above. +CHAIN_TITLE_TAG = "vocab:chain-title" + #: On a run of two or more single letters joined by full stops whose #: spelling without them is a title written `dotted` #: (`Lexicon.dotted_titles`: 'U.S.', 'P.D.', #490), a word the diff --git a/tests/v2/pipeline/test_classify.py b/tests/v2/pipeline/test_classify.py index 339eb209c..8e20baa01 100644 --- a/tests/v2/pipeline/test_classify.py +++ b/tests/v2/pipeline/test_classify.py @@ -12,7 +12,7 @@ from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._segment import segment from nameparser._pipeline._state import ( - SHAPE_ACRONYM_TAG, ParseState, WorkToken, + CHAIN_TITLE_TAG, SHAPE_ACRONYM_TAG, ParseState, WorkToken, ) from nameparser._pipeline._tokenize import tokenize from nameparser._pipeline._vocab import ( @@ -108,6 +108,36 @@ def test_given_title_tag() -> None: assert {"vocab:title", "vocab:given-title"} <= _tags(out, "Sir") +@pytest.mark.parametrize(("text", "word", "takes"), [ + # a listed title wearing the trailing shape, plain and joined + ("John Smith Prof.", "Prof.", True), + ("Hans Meier Priv.-Doz.", "Priv.-Doz.", True), + ("John Smith Lt.Gov.", "Lt.Gov.", True), + # listed whole with a period inside it: no final period needed + ("Maria Rossi Dott.ssa", "Dott.ssa", True), + # a gated title written short, whichever way H9 read it (#999) + ("John Smith Maj.", "Maj.", True), + # a gated title H9 reads in its phrase, and the same word read out + ("Associate Dean. Smith", "Dean.", True), + ("John Smith Dean.", "Dean.", False), + # a bare title word, an unlisted abbreviation, a name word + ("John Smith Prof", "Prof", False), + ("John Smith Xyz.", "Xyz.", False), + ("John Smith", "Smith", False), + # a glued delimiter after the comma: tagged as its stem (C10), but + # the text as written wears no shape + ("Smith, John Prof.;", "Prof.;", False), +]) +def test_chain_title_tag_marks_the_words_h5s_chain_takes( + text: str, word: str, takes: bool) -> None: + """CHAIN_TITLE_TAG is the one statement of H5's word test, read by + `_pieces.trailing_titles`, `is_trailing_title_word` and `_weighed` + (#1171), so it must be written after H9's phrase test has settled + "vocab:title" ('Dean.' in and out of its phrase).""" + tags = _tags_by_text(text, lexicon=Lexicon.default())[word] + assert (CHAIN_TITLE_TAG in tags) is takes + + def test_initial_tag() -> None: out = _classified("John A. B Smith") assert "initial" in _tags(out, "A.") diff --git a/tests/v2/pipeline/test_pieces.py b/tests/v2/pipeline/test_pieces.py index 6ff2ccb3d..f233bb3a7 100644 --- a/tests/v2/pipeline/test_pieces.py +++ b/tests/v2/pipeline/test_pieces.py @@ -1390,7 +1390,8 @@ def counting(rest: Sequence[int], units: Sequence[int]) -> int: # H5's listed joined title (decisions/H5, #1104): the three readers of -# classify's JOINED_TITLE_TAG besides the chain itself (which the +# classify's CHAIN_TITLE_TAG, which a JOINED_TITLE_TAG word carries +# (#1171), besides the chain itself (which the # tests/v2/cases.py rows pin), each asked through the shape its reader serves: P13's # trailing title ('y'), the maiden take's candidates (`trailing_candidates` # via `is_trailing_title_word`) and the particle run's weighed word diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index 533030873..438cfb93b 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -76,8 +76,9 @@ def test_dotted_initial_is_the_period_alternative_of_initial() -> None: # rules/H5's trailing shape, _PERIOD_ABBREV widened to an # abbreviation written joined or hyphenated (#1116) ("_vocab", "_TRAILING_TITLE_SHAPE"): None, - # SAME object, imported for H5's trailing chain - ("_pieces", "_TRAILING_TITLE_SHAPE"): ("_vocab", "_TRAILING_TITLE_SHAPE"), + # SAME object, imported for classify's CHAIN_TITLE_TAG, the word + # test of H5's trailing chain (#1171) + ("_classify", "_TRAILING_TITLE_SHAPE"): ("_vocab", "_TRAILING_TITLE_SHAPE"), # rules/H5's listed joined title (#1104's narrow half) ("_vocab", "_JOINED_WORD"): None, # SAME object, imported for classify's JOINED_TITLE_TAG