From eb2d0d299a76446e173363ec0a367d7696d1a5a0 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 5 Sep 2026 13:10:36 -0700 Subject: [PATCH 1/5] feat(tests): the period-honorific rows declare tolerated, and shapes 6/7 refuse ASCII punctuation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2026-09-01 CJK demotion swept the case table for a comma or an ASCII letter. A trailing ASCII period is neither, so three fix(#320) probe rows -- '田中さん 様.', '김민준 씨.' and '김민준 양.', each the spaced twin of a comma row that WAS demoted -- stayed on the contract tier. A period standing after a CJK honorific is a listing artifact no writing system produces, which is exactly the class that demotion names; the rows carry tolerated=True now. The criterion follows the doctrine rather than sampling it: a shape 6/7 tag admits no ASCII character but the space between two name words, so a period, a digit and a bracket are refused beside the comma and the Latin letter already refused. That gate is a property of the shape TAG, not of CJK text -- an untagged row may carry ASCII, and the four mixed-script rows d562164 reviewed by name keep the contract tier untouched. Measured 2026-09-05 by a wider sweep (CJK-bearing untolerated rows carrying any non-space ASCII, grouped by kind): eight texts before this commit, five after -- the period class gone, the letter class whole, and '山田 太郎 (マイケル・ジャクソン)' deliberately kept, its ASCII parentheses being one spelling of a delimiter the writing system does produce. corpus_cjk.jsonl 73 -> 70 and corpus_cjk_tolerated.jsonl 26 -> 29, the identity holding at 99 either side. _CORPUS_FLOORS lowered 70 -> 67 deliberately; the tolerated floor of 22 still stands under 29. W2's example line for '田中さん 様.' goes with the flag rather than with the clause it witnessed, which lands in the next commit: the plan held it back, but test_the_tolerated_corpus_is_disjoint_from_the_contract_ones refuses a text that is tolerated in the case table while a contract corpus still holds it -- the "demoted and still enforced" state nobody reads for. So corpus_rules.jsonl 248 -> 247 here, and the demotion is complete at this commit rather than half-made. W2 keeps the clause and loses its witness for one commit. Gate at four baselines: 352 / 247 / 155 / 14 intentional, unexplained 0, radar unclassified 0, exit 0, no MOVED SHAPE and no NOT CHECKED. compare.py's #501 block comment still calls '田中さん 様.' contract-tier and is stale from here; it is re-measured two commits on. Co-Authored-By: Claude Fable 5.1 --- docs/design/rules.md | 1 - tests/v2/cases.py | 73 ++++++++++++++++--- tests/v2/test_cases.py | 41 ++++++++++- tools/differential/compare.py | 32 ++++++-- tools/differential/corpus_cjk.jsonl | 3 - tools/differential/corpus_cjk_tolerated.jsonl | 3 + tools/differential/corpus_rules.jsonl | 1 - 7 files changed, 130 insertions(+), 24 deletions(-) diff --git a/docs/design/rules.md b/docs/design/rules.md index f6cc6523..5ed9747a 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -1068,7 +1068,6 @@ W2. Rationale: some East Asian honorifics glue directly onto the end not name text as the name's end. "田中さん" → suffix="さん" "김민준씨" → suffix="씨" - "田中さん 様." → suffix="さん, 様." "马丁·路德·金씨" → suffix="씨" "김지양" → suffix="" · boundary "王君" → family="王君" · boundary diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 29e3f00e..ffe3c274 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -50,15 +50,28 @@ def _has_ascii_letter(text: str) -> bool: """True when text contains an ASCII a-z/A-Z letter. Shapes 6/7's - purity check calls this ALONGSIDE a separate comma test -- this - function tests neither a comma nor a non-ASCII Latin letter on its - own. The ASCII restriction is deliberate: a diacritic or a letter + purity check calls this ALONGSIDE a comma test and a residue test + that takes every OTHER non-space ASCII character -- this function + tests neither a comma nor a non-ASCII Latin letter on its own. The + ASCII restriction is deliberate: a diacritic or a letter outside a-z/A-Z is not what a Latin WRAPPER around CJK text looks like in the corpus today (title/credential vocabulary is ASCII), and widening this is a call for whichever future row needs it.""" return any(c.isascii() and c.isalpha() for c in text) +def _stray_ascii(text: str) -> str: + """The distinct ASCII characters in text other than a space, + sorted, as one string ("" when there are none). The residue arm of + shapes 6/7's purity test: the comma and the ASCII letter keep + messages of their own because each names a composed form with its + own doctrine, and this takes everything else -- a period, a digit, + a parenthesis. A space is the one ASCII character a pure CJK + arrangement writes (rules.md#W4's '山田 太郎'), so it is the one + exemption.""" + return "".join(sorted({c for c in text if c.isascii() and c != " "})) + + #: Shape 7's other admission besides an explicit divider: a #: transcription written wholly in katakana with no dividing #: punctuation at all (e.g. "マイケルジャクソン" or the spaced @@ -113,7 +126,8 @@ class Case: #: tag -- mutually exclusive with `shape`, since a shape ADMITS a #: text to the contract and tolerated deliberately does not. Every #: composed/wrapped CJK form (a comma listing, a Latin title or - #: credential around a CJK name) is this table's ground for it, + #: credential around a CJK name, and since 2026-09-05 a trailing + #: ASCII period on an honorific) is this table's ground for it, #: not shapes 6/7's. Restricted to CJK-bearing text (`_has_cjk`): #: it exists to demote composed/wrapped CJK forms specifically, and #: a Latin row asking for it is a smell until some future arc @@ -217,8 +231,13 @@ def _check_latin_shape_order(self) -> None: def _check_cjk_shape_purity(self) -> None: """Shapes 6/7 (2026-09-01): the CJK arrangements, admitted - wholly classified-script text only -- no comma, no Latin - letter. Every composed/wrapped form is tolerated=True's + wholly classified-script text only -- no ASCII character at + all except the space between two name words (WIDENED + 2026-09-05: the check read 'no comma, no Latin letter', which + admitted the trailing-period honorifics '田中さん 様.' and + '김민준 씨.' -- a listing artifact no writing system produces, + and the same class as the forms it was already refusing). + Every composed/wrapped form is tolerated=True's ground, not a shape tag's, so this REFUSES rather than requires a particular arrangement beyond that purity test (plus shape 7's divider/katakana requirement, and shape 6's @@ -255,6 +274,19 @@ def _check_cjk_shape_purity(self) -> None: f"{self.id}: shape {self.shape} refuses a Latin " f"letter; Latin-wrapped compositions belong under " f"tolerated=True, not a shape tag") + # The residue, after the two forms with doctrine of their own: + # a space is the only ASCII character a pure CJK arrangement + # writes, so anything else ASCII came in with a convention + # from elsewhere -- a trailing period, a digit, a bracket -- + # and is a composed form whatever it is called. + stray = _stray_ascii(self.text) + if stray: + raise ValueError( + f"{self.id}: shape {self.shape} refuses the ASCII " + f"{stray!r} ({self.text!r}); a space is the only " + f"ASCII character a pure arrangement carries, and " + f"composed forms belong under tolerated=True, not a " + f"shape tag") # U+00B7 (间隔号) marks a name transcription in SOURCE order # (W1 Accepted) -- shape 7's ground, not shape 6's family- # first one. The fullwidth nakaguro U+30FB is NOT a source- @@ -3298,7 +3330,15 @@ def _check_cjk_shape_purity(self) -> None: "this has one -- which is no longer what separates " "them.) 1.4.0 read this first " "田中さん / last '様.', which is what 2.0 produced until " - "#320: parity before, a classified change after"), + "#320: parity before, a classified change after. " + "TOLERATED since 2026-09-05: a trailing ASCII period on " + "a CJK honorific is a listing artifact no writing " + "system produces -- the same class as a comma listing " + "or a Latin credential, and the class the 2026-09-01 " + "sweep's 'comma or ASCII letter' criterion could not " + "see. The row still pins #320's mechanism at HEAD; " + "what moves is which corpus file carries the text", + tolerated=True), Case("ja_honorific_period_does_not_stop_the_peel", "田中さん, 様.", {"family": "田中", "suffix": "さん, 様."}, classification="fix(#320)", @@ -3425,7 +3465,14 @@ def _check_cjk_shape_purity(self) -> None: "middle 민준 / family '씨.'; the fields above are #320's, " "not the segmenter's, so the row is classified to it -- " "as ko_honorific_ssi is classified to #307 without " - "naming the same segmentation it also depends on"), + "naming the same segmentation it also depends on. " + "TOLERATED since 2026-09-05 with its two period twins: " + "a trailing ASCII period on a CJK honorific is a " + "listing artifact no writing system produces, and the " + "2026-09-01 sweep's 'comma or ASCII letter' criterion " + "could not see the class. The row still pins #320's " + "mechanism at HEAD", + tolerated=True), Case("ko_honorific_glued_teacher", "김선생님", {"family": "김", "suffix": "선생님"}, classification="fix(#307) + fix(#271)", @@ -4019,7 +4066,15 @@ def _check_cjk_shape_purity(self) -> None: "pin nothing these two do not. Classified to #320 like " "its 씨 counterpart: 1.4.0 read this first 김민준 / last " "'양.', and the fields above are the ones this change " - "produced, not the segmenter's"), + "produced, not the segmenter's. TOLERATED since " + "2026-09-05 for the same reason as that counterpart -- " + "a trailing ASCII period on a CJK honorific is a " + "listing artifact no writing system produces, missed by " + "the 2026-09-01 sweep's 'comma or ASCII letter' " + "criterion -- and the pair moves tiers together the way " + "it moves fields together. The row still pins #320's " + "mechanism at HEAD", + tolerated=True), Case("ko_surname_yang_leads_a_segmentable_given", "양 지훈", {"family": "양", "given": "지훈"}, classification="fix(#271)", diff --git a/tests/v2/test_cases.py b/tests/v2/test_cases.py index f1e74101..1c2c1b51 100644 --- a/tests/v2/test_cases.py +++ b/tests/v2/test_cases.py @@ -74,10 +74,14 @@ def test_the_family_partitions_into_particles_and_base( f"particles={pn.family_particles!r} + base={pn.family_base!r}") -#: Case.__post_init__'s shape checks, each probed for the one message it -#: alone raises. A row here is a Case that must fail to construct, not +#: Case.__post_init__'s shape checks, each probed for the message that +#: identifies it. A row here is a Case that must fail to construct, not #: one that ever joins CASES -- unlike test_case above, this exercises -#: the dataclass's own validation rather than the parser. +#: the dataclass's own validation rather than the parser. One message +#: is shared by three rows and deliberately: the residue arm of the +#: purity check (2026-09-05) refuses every non-space ASCII character +#: the comma and Latin-letter arms do not, so its probes differ in the +#: character that trips it rather than in what they are told. @pytest.mark.parametrize("kwargs, match", [ pytest.param( dict(text="Beethoven, Ludwig van", shape=2, locale="nl_NL"), @@ -113,6 +117,30 @@ def test_the_family_partitions_into_particles_and_base( dict(text="김민준 V", shape=6), "refuses a Latin letter", id="shape-6-refuses-a-latin-letter"), + pytest.param( + # The 2026-09-05 widening, and the text that motivated it: the + # comma and Latin-letter arms above both said no of every + # composed form anyone had written down, and this one carries + # neither. It is a tolerated row today + # (ja_honorific_with_a_period_no_comma); the tag it must not be + # able to take back is what this probe holds. + dict(text="田中さん 様.", shape=6), + "refuses the ASCII", + id="shape-6-refuses-a-trailing-period"), + pytest.param( + dict(text="김민준 2", shape=6), + "refuses the ASCII", + id="shape-6-refuses-a-digit"), + pytest.param( + # Refused for its parentheses, BEFORE the transcription test + # this text would also fail -- the residue arm runs first, so + # the message names the ASCII rather than the divider. The + # nickname row this text belongs to (fix(#272)) stays contract + # and untagged: the purity gate is a property of a SHAPE tag, + # not of the corpus. + dict(text="山田 太郎 (マイケル・ジャクソン)", shape=7), + "refuses the ASCII", + id="shape-7-refuses-ascii-parentheses"), pytest.param( dict(text="김민준·지훈", shape=6), "belongs to shape 7", @@ -176,7 +204,10 @@ def test_case_construction_rejects_a_bad_shape_tag( #: ja_nakaguro_han_takes_the_han_order), an interpunct-divided shape-7 #: row, a SPACED wholly-katakana shape-7 row (the subtler admission -- #: a transcription with no U+00B7 at all is still a shape, not a -#: demotion, as long as every non-space character is katakana), and a +#: demotion, as long as every non-space character is katakana), a +#: SPACED HONORIFIC written without the period the residue arm refuses +#: (the boundary the 2026-09-05 widening had to leave standing: what +#: the demoted text loses is its period, not its arrangement), and a #: tolerated row built from the SAME text a shape probe above refuses #: as a comma -- the boundary reading the pair as intended: what a #: shape tag refuses, tolerated=True admits. Each must construct @@ -184,6 +215,8 @@ def test_case_construction_rejects_a_bad_shape_tag( #: that admits nothing. @pytest.mark.parametrize("kwargs", [ pytest.param(dict(text="김민준", shape=6), id="pure-shape-6-constructs"), + pytest.param(dict(text="田中さん 様", shape=6), + id="spaced-honorific-without-a-period-constructs"), pytest.param(dict(text="高橋・一郎", shape=6), id="han-nakaguro-shape-6-constructs"), pytest.param(dict(text="威廉·莎士比亚", shape=7), diff --git a/tools/differential/compare.py b/tools/differential/compare.py index 7b4830ed..a73c6903 100644 --- a/tools/differential/compare.py +++ b/tools/differential/compare.py @@ -569,18 +569,25 @@ def _legal_orders() -> frozenset[str]: #: decision when a corpus is added, the way the Script tables do. _CORPUS_FLOORS = { "corpus.jsonl": 480, # 486 today, from v1's banks at a pinned ref - "corpus_cjk.jsonl": 70, # 73 today, generated from the case table. + "corpus_cjk.jsonl": 67, # 70 today, generated from the case table. # LOWERED 95 -> 70 on 2026-09-01, # deliberately: the CJK comma demotion # moved 25 tolerated texts out of this # file into corpus_cjk_tolerated.jsonl # below. Nothing left the harness -- # the names are compared and classified - # exactly as before, on the radar tier - "corpus_cjk_tolerated.jsonl": 22, # 26 today, the tolerated half of + # exactly as before, on the radar tier. + # LOWERED again 70 -> 67 on 2026-09-05, + # the same way and for the same class: + # the trailing-period honorifics + # ('田中さん 様.' and its two twins) are + # the listing artifact the first + # sweep's criterion could not see + "corpus_cjk_tolerated.jsonl": 22, # 29 today, the tolerated half of # the same generator: composed and # wrapped CJK forms (comma listings, - # Latin titles and credentials) whose + # Latin titles and credentials, + # trailing ASCII periods) whose # handling the contract stopped # promising on 2026-09-01. 25 on the # day it was created; the 26th is @@ -589,9 +596,22 @@ def _legal_orders() -> frozenset[str]: # and the rules corpus stopped # carrying it -- the row was written # so the text moved tiers instead of - # leaving the harness + # leaving the harness. 29 since + # 2026-09-05, the three period rows. + # Floor left at 22: it guards against + # the file emptying, and this half only + # grows as the contract narrows "corpus_issues.jsonl": 370, # 381 today, harvested and append-only - "corpus_rules.jsonl": 150, # 248 today, generated from rules.md. + "corpus_rules.jsonl": 150, # 247 today, generated from rules.md. + # 248 until 2026-09-05, when W2's + # trailing-period example moved into + # the tolerated W3 and the builder + # stopped harvesting it -- the seventh + # text a CJK demotion has taken out of + # this file, and (measured 2026-09-05) + # the last CJK example anywhere outside + # W3 that carried a non-space ASCII + # character. # 252 until 2026-09-01, when W3 took # rules.md's `tolerated:` marker and # build_rules_corpus.py stopped diff --git a/tools/differential/corpus_cjk.jsonl b/tools/differential/corpus_cjk.jsonl index b45a00d7..e48b6482 100644 --- a/tools/differential/corpus_cjk.jsonl +++ b/tools/differential/corpus_cjk.jsonl @@ -38,7 +38,6 @@ "田中 殿" "田中『ハナ』花子" "田中さん" -"田中さん 様." "田中博士" "諸葛亮" "阿明" @@ -55,9 +54,7 @@ "김민준 박사 씨" "김민준 박사님" "김민준 씨" -"김민준 씨." "김민준 양" -"김민준 양." "김민준님" "김민준박사님" "김민준씨" diff --git a/tools/differential/corpus_cjk_tolerated.jsonl b/tools/differential/corpus_cjk_tolerated.jsonl index 64630834..f3ef161d 100644 --- a/tools/differential/corpus_cjk_tolerated.jsonl +++ b/tools/differential/corpus_cjk_tolerated.jsonl @@ -6,6 +6,7 @@ "田中, 太郎さん" "田中さん II" "田中さん V." +"田中さん 様." "田中さん, Dr." "田中さん, Ph. D." "田中さん, PhD" @@ -14,6 +15,8 @@ "田中さん, 様." "김, 민준씨" "김, 민준씨 (Jimmy)" +"김민준 씨." +"김민준 양." "김민준, 씨" "김민준, 씨." "김민준씨 (Jimmy)" diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index e341ac40..e13074fb 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -235,7 +235,6 @@ "王君" "田中、太郎" "田中さん" -"田中さん 様." "马丁·路德·金씨" "高橋 みなみ" "高橋・一郎" From 214a1edc9b863913200be3fa8f648381d07094fe Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 5 Sep 2026 13:12:33 -0700 Subject: [PATCH 2/5] docs(design): W2's punctuation clause and its witness move to W3's tolerated zone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit W2 read "The split-off ignores surrounding punctuation, but never treats a part that is not name text as the name's end". Native CJK writing puts no punctuation around a name for a split-off to ignore, so every input the clause describes is a listing form -- W3's ground, and a description rather than a promise. The clause had no pure witness and could not be given one: its example, '田中さん 様.', left in the previous commit with the flag, and the two boundary lines that might have looked like witnesses for the second half ('김지양', '王君') are about the vocabulary's own limits, not about punctuation. The second half was already stated at more length in W3 ("A part that is nothing but suffix words is not the name's end: it is declined as the site"), so both halves went. W3 gains the example line, one descriptive sentence, and a rationale clause naming the punctuation a listing carries in beside the comma itself; its `tolerated:` marker now counts five illustrated-not-promised lines where it counted four. W2 keeps its glued-honorific statement and five examples, none carrying ASCII -- measured 2026-09-05, W2's line was the only CJK example outside W3 that carried an ASCII character at all. Both `interacts:` lines still hold: W2 points at W3, where the crossing and now the punctuation reading live. _script_segment.py's W2 citation is trimmed to the surviving statement in the same commit (test_citations_are_verbatim_excerpts catches it) and says where the punctuation half went. corpus_rules.jsonl is unchanged at 247: build_rules_corpus.py skips a tolerated rule's examples whole, so the line arriving in W3 adds nothing back. decisions.md#cjk-comma-demotion takes the record -- the criterion the 2026-09-01 sweep used, the wider one that replaces it, the parenthesized-nickname row kept and the delimiter measurement behind that keep, and the recompute for the sweep itself. Gate at four baselines: summary lines byte-identical to the previous commit's -- 352 / 247 / 155 / 14 intentional, unexplained 0, radar unclassified 0, exit 0. Co-Authored-By: Claude Fable 5.1 --- docs/design/decisions.md | 10 ++++++++++ docs/design/rules.md | 18 +++++++++++------- nameparser/_pipeline/_script_segment.py | 14 ++++++++------ tests/v2/test_ledger_guards.py | 8 +++++--- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index d130f445..f2ff75f2 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -447,6 +447,16 @@ Closes #469, and continues the corpus-tier arc below rather than standing apart - COUNTS, 2026-09-01, with the recompute beside each. 29 case rows carry `tolerated=True` over 26 distinct texts — the flag is per ROW and the corpus is per TEXT, which is why the two numbers differ (recompute: read `CASES` and count `c.tolerated`). The 26th text arrived with the W3 demotion itself: the given-side listing `지훈, 남궁민수` was a rules.md example with no case row, and would have left the harness altogether when the rules corpus stopped carrying W3, so it was given a tolerated row rather than dropped. `corpus_cjk.jsonl` went 98 → 73 and the radar-tier `corpus_cjk_tolerated.jsonl` was created at 25, then 26; the identity is the check worth keeping, since 73 + 25 = 98 exactly says the split MOVED names and dropped none, and the 99th is that one rescued row. `corpus_rules.jsonl` went 252 → 248: six comma-bearing example texts left (`田中さん, Dr.`, `田中さん, PhD`, `남궁민수, 지훈`, `지훈, 남궁민수` when the builder skipped W3 whole, plus `김, 민준씨` and `田中さん, V.` from W2's swap) and two pure ones arrived (`김민준씨`, `田中さん 様.`). `corpus_shapes.jsonl` stands at 37, with the new shapes covered 5 names (shape 6) and 2 (shape 7). And the stop condition held: intentional diffs are UNMOVED at 226/205/113/0 across 1.4.0/2.0.0/2.1.0/2.2.0, with unexplained 0, radar unclassified 0 and exit 0 at each — 1113 names compare at 1.4.0 (7 skipped, shapes 4/5) and 1120 above it. That the counts do not move is the substance and not a formality: a classified diff counts identically on either tier, so demoting a name changes which file it loads from and nothing about what the gate makes of it. +- 2026-09-05 — THE SWEEP'S CRITERION WAS NARROWER THAN THE DOCTRINE IT IMPLEMENTED, and three rows sat in the gap. The 2026-09-01 sweep asked, over the CJK-bearing case rows, for a comma or an ASCII letter. A trailing ASCII period is neither, so `田中さん 様.`, `김민준 씨.` and `김민준 양.` — the three `fix(#320)` probes, each the spaced twin of a comma row that WAS demoted — stayed contract; and `田中さん 様.` was then chosen as rules.md#W2's replacement example for its "ignores surrounding punctuation" clause on the strength of passing that same test, so the miss propagated into the normative document the same day. A period standing after a CJK honorific is a listing artifact no writing system produces, which is this entry's own class in its own words: the admission test judges the FORMAT, and `様.` is no more an arrangement Japanese writes than `田中さん, PhD` is. The three rows carry `tolerated=True` now, and nothing about the parse moves — the rows still pin #320's mechanism at HEAD, and the texts are still compared and classified at every baseline, from the radar file instead of the contract one. + +- 2026-09-05 — THE CRITERION IS RESTATED AS THE DOCTRINE rather than as a sample of it. A SHAPE-TAGGED row (shapes 6/7, `Case.__post_init__` in tests/v2/cases.py) now admits no ASCII character but the space between two name words, so a period, a digit and a bracket are refused beside the comma and the Latin letter the check already refused, and the next composed form does not need a fresh criterion to catch it. The gate is a property of the shape TAG and not of CJK text, which is the half that keeps it honest: an UNTAGGED row may carry ASCII, and several deliberately do. The four mixed-script rows commit `d562164` reviewed one by one — `John 王`, `王·Smith`, `Andersonさん`, `Anderson선생님` — keep the contract tier on the reasons recorded there, untouched by this. + +- 2026-09-05 — `山田 太郎 (マイケル・ジャクソン)` IS KEPT contract: it is the one text the wider sweep surfaces and this entry declines to demote. Its ASCII parentheses are not a convention carried in from elsewhere — they are one SPELLING of a delimiter Japanese writing does produce, and the parser reads the spellings identically. Measured 2026-09-05 under the default facade: `(マイケル・ジャクソン)`, the fullwidth `（マイケル・ジャクソン）` and the corner-bracketed `「マイケル・ジャクソン」` all give family 山田, given 太郎, nickname `マイケル ジャクソン` (recompute: parse the three; ASCII square brackets are NOT in that set and give family `ジャクソン]`, which is what makes this a delimiter roster rather than a rule about brackets). What the row pins is the nakaguro INSIDE a delimited nickname (`fix(#272)`), and which of the three delimiters spells it is incidental to that. + +- 2026-09-05 — W2's PUNCTUATION CLAUSE MOVED to W3 with its example, because it had no pure witness and could not be given one. The clause read "The split-off ignores surrounding punctuation, but never treats a part that is not name text as the name's end", and native CJK writing puts no punctuation around a name for a split-off to ignore — so every input the clause describes is a listing form, which is W3's ground. Both halves went, the second being already stated at more length in W3 ("A part that is nothing but suffix words is not the name's end: it is declined as the site…"), and the two boundary lines that might have looked like witnesses for it (`김지양`, `王君`) are about the vocabulary's own limits, not about punctuation. W2 keeps its glued-honorific statement and five examples, none carrying ASCII, and its `interacts: W3` still names where the punctuation reading lives; W3 gained the example line, one descriptive sentence, and a `tolerated:` marker counting five illustrated-not-promised lines where it counted four. + +- COUNTS, 2026-09-05, with the recompute beside each. `corpus_cjk.jsonl` went 73 → 70 and `corpus_cjk_tolerated.jsonl` 26 → 29 — the same identity as the first demotion, 99 names either side, so the split MOVED three texts and dropped none. `corpus_rules.jsonl` went 248 → 247, the single example line W2 gave up. 32 case rows carry `tolerated=True` over 29 distinct texts (recompute: read `CASES` and count `c.tolerated`, per row and per text). `_CORPUS_FLOORS["corpus_cjk.jsonl"]` is lowered 70 → 67 deliberately, and the tolerated file's floor of 22 still stands under 29. The stop condition held again: intentional diffs UNMOVED at 352 / 247 / 155 / 14 across 1.4.0 / 2.0.0 / 2.1.0 / 2.2.0, unexplained 0, radar unclassified 0 and exit 0 at each. RECOMPUTE THE SWEEP, which is the part worth having rather than the digits: it is the CJK-bearing (`_has_cjk`) rows that do NOT declare `tolerated`, carrying any ASCII character other than a space, grouped by the kind of character that trips them and joined to the corpus files holding each text. Over the tree before this bundle it returns eight distinct texts — the three period rows, the parenthesized nickname, and the four mixed-script rows — and over this one, five: the period class is gone and the letter class is untouched, which is the shape of the answer to look for rather than the count. + ### P5 — bound given names - 2026-06-30 (first-name-prefix-join design; v1-era, carried into the v2 port) — the join is vocabulary-driven and deliberately tiny. diff --git a/docs/design/rules.md b/docs/design/rules.md index 5ed9747a..7cb39539 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -1063,9 +1063,7 @@ W2. Rationale: some East Asian honorifics glue directly onto the end never itself end a name, so the listed vocabulary carries its own license and needs no other gate. A listed honorific glued to the end of the name's last name - word splits off once and reads as a suffix. The split-off - ignores surrounding punctuation, but never treats a part that is - not name text as the name's end. + word splits off once and reads as a suffix. "田中さん" → suffix="さん" "김민준씨" → suffix="씨" "马丁·路德·金씨" → suffix="씨" @@ -1078,8 +1076,10 @@ W3. Rationale: a family name declared by a comma is the writer's drew — but none of the East Asian writing systems declares a family name that way (the Background above), so every input this rule reads is a listing convention wrapped around a name whose - own script has already arranged it. What follows describes what - the parser does with such input; it does not promise it. + own script has already arranged it — the comma that declares the + family name, and the punctuation such a listing carries in with + it. What follows describes what the parser does with such input; + it does not promise it. Under a family comma the pre-comma text is the family by declaration and never divides, and the post-comma side is given text with no family to find; only the honorific split-off (W2) @@ -1097,13 +1097,17 @@ W3. Rationale: a family name declared by a comma is the writer's name's end, and a glued honorific before the comma stays glued. The vocabulary question is C1's own, asked without C1's word-count condition: the two differ in what else they require, - not in what they ask of the words. + not in what they ask of the words. The split-off ignores the + punctuation a listing leaves around the name — a period trailing + an honorific is the same artifact as the comma, no writing system + putting one there — and reads across it to the word behind. "남궁민수" → family="남궁" "지훈, 남궁민수" → given="남궁민수" "남궁민수, 지훈" → family="남궁민수" · boundary "田中さん, Dr." → family="田中さん" "田中さん, PhD" → suffix="さん, PhD" - tolerated: native CJK writing has no family-comma convention, so the four comma lines above illustrate current behavior — changeable without notice — rather than promise it; the comma-free line beside them is W1's claim, which is normative. All four comma names stay watched at every released baseline on the differential's radar tier (tools/differential/corpus_cjk_tolerated.jsonl, projected from the `tolerated` rows of tests/v2/cases.py) instead of its contract tier, and those rows pin them at HEAD. + "田中さん 様." → suffix="さん, 様." + tolerated: native CJK writing has neither a family-comma convention nor a period standing after an honorific, so the four comma lines above and the period line under them illustrate current behavior — changeable without notice — rather than promise it; the line carrying neither, beside them, is W1's claim, which is normative. All five stay watched at every released baseline on the differential's radar tier (tools/differential/corpus_cjk_tolerated.jsonl, projected from the `tolerated` rows of tests/v2/cases.py) instead of its contract tier, and those rows pin them at HEAD. history: decisions.md#W3 · interacts: W1, W2, C1 · implemented: nameparser/_pipeline/_script_segment.py W4. Rationale: Chinese, Japanese and Korean all write the family diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index a28e6f03..914ddbb8 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -272,14 +272,16 @@ def _peel_site(state: ParseState, flat: Sequence[int], # rules.md#W2: "a listed honorific glued to the end of the name's -# last name word splits off once and reads as a suffix. The -# split-off ignores surrounding punctuation, but never treats a -# part that is not name text as the name's end." (history: +# last name word splits off once and reads as a suffix." (history: # decisions.md#W2) -# That the peel also reaches ACROSS a family comma is stated at +# That the peel also reaches ACROSS a family comma, and across the +# punctuation a listing leaves around the name, is stated at # rules.md#W3 instead, which is a tolerated rule since the -# 2026-09-01 comma demotion -- the crossing is what the parser does -# today, not something W2 promises. +# 2026-09-01 comma demotion -- both readings are what the parser +# does today, not something W2 promises. The punctuation half moved +# there on 2026-09-05 (decisions.md#cjk-comma-demotion): it had no +# witness native writing could supply, every input it describes +# being a listing form. def _peel_honorific_tail(state: ParseState) -> ParseState: """#308: split a listed honorific off the END of the name's last NON-POST-NOMINAL token -- 田中さん -> 田中 + さん -- and let diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 0cc14ec0..13be8ab6 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -1043,8 +1043,10 @@ def test_a_tolerated_rules_examples_are_still_watched_somewhere() -> None: comparison, and which file carries it is the projection's business (a future tolerated rule outside the CJK sections would have no business in a CJK file at all). Today every one of them arrives - through corpus_cjk_tolerated.jsonl -- W3's four comma texts, its - fifth example being W1's normative one and subtracted here -- and + through corpus_cjk_tolerated.jsonl -- W3's four comma texts and, + since 2026-09-05, the trailing-period one W2 handed over, its + remaining example being W1's normative one and subtracted here -- + and that file's equality with the case table is pinned separately by test_tolerated_cjk_corpus_matches_the_case_table. """ @@ -1058,7 +1060,7 @@ def test_a_tolerated_rules_examples_are_still_watched_somewhere() -> None: # assertion below is a truth about the empty set. assert demoted, ( "no rule in rules.md carries a `tolerated:` marker with an " - "example text of its own; W3 has carried four since " + "example text of its own; W3 has carried at least four since " "2026-09-01. If a demotion was reversed, delete this guard in " "that commit rather than leaving it asserting nothing") unwatched = sorted(demoted - set(_CORPUS_NAMES)) From 012577bc1fa64fecb1cad6d82c648bda5b495fc6 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 5 Sep 2026 13:14:56 -0700 Subject: [PATCH 3/5] tooling(differential): the #501 block's tier claim, re-measured MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four #501 contest rows closing the 2.0.0 watched section carried "Three are contract tier (corpus_cjk.jsonl; the first also corpus_rules.jsonl), so a move on them fails the run". Two are, now: '김민준 박사님' and '선생님'. '田中さん 様.' left both contract files in this bundle's first two commits and reads radar, so a shape move on it prints and no longer fails. That is the thing worth stating and not the digit: a row's fatality follows the tier its NAME reads at, which no roster in this file records, so a demotion elsewhere changes what a row here costs without touching the row -- the comment is the only thing that goes stale, and only a re-measurement finds it. Mutation controls, in memory around main() with the worktree untouched, 2026-09-05, and run as a PAIR because either alone is consistent with a dead check: '田中さん 様.' corrupted at 2.0.0 -- MOVED SHAPE (radar) expected_since_2.0.0.toml: 1 watched diff shape(s) disagree with this run. [...] This does not fail the run because the name is radar tier (_CORPUS_TIERS), which watches without promising. main() returns 0. It returned 1 before this bundle. '선생님' corrupted the same way -- MOVED SHAPE expected_since_2.0.0.toml: [...] This fails the run because the name is contract tier (_CORPUS_TIERS). main() returns 1, so the exit 0 above is the tier and not a check that stopped running. _CORPUS_TIERS's "no text in the tolerated file is held by a contract corpus" sentence is re-verified and gains what the demotion taught it: '田中さん 様.' was one of the two PURE texts the 2026-09-01 W2 swap brought in, so marking its case row was not enough on its own -- the example line had to move into the tolerated W3 in the same breath, which test_the_tolerated_corpus_is_disjoint_from_the_contract_ones enforces on every run. decisions.md's watched-shapes measurement, taken hours before the demotion, is corrected in the same commit for the same claim. tools/differential/README.md needed no edit: its "three contract-tier names" sentence is about the #498 rule-pair swap, a different three. Full suite 6507 passed, ruff clean, mypy clean over 110 files. Gate at four baselines: 352 / 247 / 155 / 14 intentional, unexplained 0, radar unclassified 0, exit 0, no MOVED SHAPE and no NOT CHECKED. Co-Authored-By: Claude Fable 5.1 --- docs/design/decisions.md | 2 +- tools/differential/compare.py | 26 +++++++++++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index f2ff75f2..08668419 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -853,7 +853,7 @@ Declined: - Pinning winners for the six #501 contests (2026-09-05) — stays #501's. The four 2.0.0 rows here and `'MD, PHD'`'s two record SHAPES, not winners: a shape move is now a finding on them, while a handover to another rule is not — file order still decides the winner, and only a winner pin sees it. #501's own objection to pinning a boundary nobody has argued stands, and the issue is scoped down by that comment, not closed. - A checked-in generator for the rows (2026-09-05) — the rows are a snapshot, and a generator that re-derives them on demand is the re-derivation hazard with a command line (mechanisms.md#RECORDED-ROSTERS). A scratch script and a recipe in the roster comment, as `_CORPUS_FLOORS` and `_CORPUS_CLAIMS` do. -The measurement, and how to redo it. `_WATCHED_DIFFS` holds 41 / 36 / 31 / 5 rows at 1.4.0 / 2.0.0 / 2.1.0 / 2.2.0, 113 in all; the 2.0.0 section is 32 sole-watched rows plus the four #501 contests (`'田中さん 様.'`, `'김민준 박사님'`, `'선생님'`, contract tier and fatal; `'田中さん, 様.'`, `corpus_cjk_tolerated.jsonl`, radar and printed). 51 distinct sole-watched names; every row a default-order shape, so `NOT CHECKED` names none of them. Four mutation controls, in memory around `main()` with the worktree untouched, re-run 2026-09-05 against the amended data commit: `'QC MP'` corrupted at 1.4.0 (its only row — it does not diff at 2.0.0) prints `MOVED SHAPE (radar) expected_since_1.4.0.toml: 1 watched diff shape(s) disagree with this run` and exits 0, `'Smith Dr'` at 2.0.0 likewise; `'선생님'` corrupted at 2.0.0 prints `MOVED SHAPE expected_since_2.0.0.toml: 1 watched diff shape(s) disagree with this run` ending "This fails the run because the name is contract tier" and exits 1; `'Ph. D., Jr.'` corrupted at 2.1.0 prints the `(radar)` block and exits 0; and `'MD, PHD'` inserted into the 1.4.0 watched section beside its contest row is refused pre-worker, before any `baseline:` line, as "1 name(s) sit in both `_RECORDED_DIFFS['expected_since_1.4.0.toml']` and `_WATCHED_DIFFS['expected_since_1.4.0.toml']`", and fails `test_the_watched_roster_is_disjoint_and_names_every_ledger` naming it — the guard half wrapping `load_tool`, since that helper executes a fresh `compare` module per call and a mutation on an imported one never reaches it. A control whose corruption changed nothing would have meant an inert row; none did. The arc moved no classification: 352 / 247 / 155 / 14 intentional diffs, 0 unexplained and 0 radar-unclassified at every baseline, byte-identical summary lines before and after every commit. The quiet-tree claim that made this the moment to snapshot: measured 2026-09-05, `git rev-list --count a965272..HEAD -- nameparser/` is 5 commits since the tier split, moving 38 inserted and 15 deleted lines across two files, `_facade.py` (the #462 initials fix and its review rounds) and `_pipeline/_script_segment.py` (`f046d5d`, the tier-split day's own W3 demotion) — a figure that goes stale by design, since the next feature is what the snapshot exists to make legible. RECOMPUTE the row counts with `{k: len(v) for k, v in compare._WATCHED_DIFFS.items()}`, the population with the script in the dict's header, the controls with `compare._WATCHED_DIFFS[ledger][name] = (...)` around `main()`, and the quiet tree with the command above plus `git diff --stat a965272..HEAD -- nameparser/`. +The measurement, and how to redo it. `_WATCHED_DIFFS` holds 41 / 36 / 31 / 5 rows at 1.4.0 / 2.0.0 / 2.1.0 / 2.2.0, 113 in all; the 2.0.0 section is 32 sole-watched rows plus the four #501 contests (`'김민준 박사님'` and `'선생님'`, contract tier and fatal; `'田中さん 様.'` and `'田中さん, 様.'`, `corpus_cjk_tolerated.jsonl`, radar and printed — the first of that pair read contract when this was measured earlier the same day, and the period-class demotion under #cjk-comma-demotion above moved it hours later, which is the point the snapshot makes better than the prose could: a row's fatality is a property of the tier its NAME reads at, and no roster in `compare.py` records it). 51 distinct sole-watched names; every row a default-order shape, so `NOT CHECKED` names none of them. Four mutation controls, in memory around `main()` with the worktree untouched, re-run 2026-09-05 against the amended data commit: `'QC MP'` corrupted at 1.4.0 (its only row — it does not diff at 2.0.0) prints `MOVED SHAPE (radar) expected_since_1.4.0.toml: 1 watched diff shape(s) disagree with this run` and exits 0, `'Smith Dr'` at 2.0.0 likewise; `'선생님'` corrupted at 2.0.0 prints `MOVED SHAPE expected_since_2.0.0.toml: 1 watched diff shape(s) disagree with this run` ending "This fails the run because the name is contract tier" and exits 1; `'Ph. D., Jr.'` corrupted at 2.1.0 prints the `(radar)` block and exits 0; and `'MD, PHD'` inserted into the 1.4.0 watched section beside its contest row is refused pre-worker, before any `baseline:` line, as "1 name(s) sit in both `_RECORDED_DIFFS['expected_since_1.4.0.toml']` and `_WATCHED_DIFFS['expected_since_1.4.0.toml']`", and fails `test_the_watched_roster_is_disjoint_and_names_every_ledger` naming it — the guard half wrapping `load_tool`, since that helper executes a fresh `compare` module per call and a mutation on an imported one never reaches it. A control whose corruption changed nothing would have meant an inert row; none did. The arc moved no classification: 352 / 247 / 155 / 14 intentional diffs, 0 unexplained and 0 radar-unclassified at every baseline, byte-identical summary lines before and after every commit. The quiet-tree claim that made this the moment to snapshot: measured 2026-09-05, `git rev-list --count a965272..HEAD -- nameparser/` is 5 commits since the tier split, moving 38 inserted and 15 deleted lines across two files, `_facade.py` (the #462 initials fix and its review rounds) and `_pipeline/_script_segment.py` (`f046d5d`, the tier-split day's own W3 demotion) — a figure that goes stale by design, since the next feature is what the snapshot exists to make legible. RECOMPUTE the row counts with `{k: len(v) for k, v in compare._WATCHED_DIFFS.items()}`, the population with the script in the dict's header, the controls with `compare._WATCHED_DIFFS[ledger][name] = (...)` around `main()`, and the quiet tree with the command above plus `git diff --stat a965272..HEAD -- nameparser/`. ### comma-suffix-arc — #291/#296/#316 (2026-07-26 → 2026-08-01) diff --git a/tools/differential/compare.py b/tools/differential/compare.py index a73c6903..ef0514df 100644 --- a/tools/differential/compare.py +++ b/tools/differential/compare.py @@ -680,7 +680,15 @@ def _legal_orders() -> frozenset[str]: # NONE do today and every text in this file reads radar. The rule # is stated as a rule, not as a caveat about the five: whatever # lands here next is demoted only once no contract corpus holds - # it. + # it. What landed next needed exactly that. '田中さん 様.' was one + # of the two pure texts the W2 swap brought IN, so the 2026-09-05 + # period-class demotion had to move that example line into the + # tolerated W3 in the same breath as marking its case row -- + # marking the row alone would have left the name enforced and + # documented as demoted. Still none today, and asked of every + # contract file on every run by + # test_the_tolerated_corpus_is_disjoint_from_the_contract_ones + # (tests/v2/test_ledger_guards.py) rather than left to a reader. "corpus_cjk_tolerated.jsonl": "radar", "corpus_issues.jsonl": "radar", "corpus_rules.jsonl": "contract", @@ -1971,10 +1979,18 @@ class _ShapeMismatch(NamedTuple): # picks the one that explains it, and nobody has argued which # should -- a shape with no winner, which is the row kind this # dict exists for. They move to _RECORDED_DIFFS the day #501 - # argues a winner, and the pin goes beside them there. Three - # are contract tier (corpus_cjk.jsonl; the first also - # corpus_rules.jsonl), so a move on them fails the run; the - # last is corpus_cjk_tolerated.jsonl, radar, and prints. + # argues a winner, and the pin goes beside them there. TWO are + # contract tier (corpus_cjk.jsonl) -- '김민준 박사님' and + # '선생님' -- so a move on those fails the run; the other two + # sit in corpus_cjk_tolerated.jsonl, radar, and print. + # Re-measured 2026-09-05: '田中さん 様.' was the third contract + # one, in corpus_cjk.jsonl and corpus_rules.jsonl both, and the + # period-class demotion (decisions.md#cjk-comma-demotion) moved + # it to the radar half of the CJK projection and took W2's + # example line with it. A row's fatality follows the tier its + # NAME reads at, which no roster in this file records -- so a + # demotion changes what a row here costs without touching the + # row, and this comment is the only thing that goes stale. "田中さん 様.": ("family", "given", "suffix"), "김민준 박사님": ("family", "given", "suffix"), "선생님": ("family", "given"), From d0faed4781486b6238fd0da5f0149e38e8ce2b82 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 5 Sep 2026 13:40:31 -0700 Subject: [PATCH 4/5] docs(design)+tests: W2 keeps its normative half; the period is not ignored, it is decisive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit W2's clause read "The split-off ignores surrounding punctuation, but never treats a part that is not name text as the name's end". The previous commit moved BOTH halves to W3 on a rationale that reaches only the first, and the second half is normative: it is what the parser does with no comma and no period anywhere in the input, and contract-tier pure names depend on it. 1. W2's SECOND HALF RESTORED, with pure witnesses. Measured 2026-09-05 by forcing _is_post_nominal to False in memory -- the predicate the site scan _peel_site reads -- and re-parsing (family, given, suffix, title): '田中さん 様' ships ('田中', '', 'さん, 様', '') and gives ('田中さん', '', '様', '') without the step; '김민준 박사님' ships ('김', '민준', '박사님', '') and gives ('김', '민준', '박사, 님', ''); '선생님' ships ('선생님', '', '', '') and gives ('선', '생', '님', ''). The post-nominal word is dissected in every case. '김민준 박사님' and '선생님' are corpus_cjk.jsonl names on the CONTRACT tier, and ko_honorific_glued_doctor_spaced is the #308/#271 regression row for the first -- under W2's remaining statement alone, 박사님 ends in the listed tail 님 and reads as peelable. The sentence goes back in W2's register and takes two witnesses whose texts corpus_cjk.jsonl already held: '김민준 박사님' -> suffix="박사님" and the boundary line '선생님' -> family="선생님". '田中さん 様' would have been the third and is NOT added -- it is not a case row, and a new contract name needs a 1.4.0 classification this bundle does not have. 2. W3's NEW DESCRIPTIVE SENTENCE REWRITTEN to what is measured. It attributed the crossing to the punctuation; the punctuation does not license it and is not ignored either. W3 now says the step past a post-nominal word is W2's and fires with no punctuation present, that a period on a SEPARATE post-nominal word rides into the suffix and moves no division, and that a period glued to the honorific's own word decides against the split-off. The '田中さん 様.' example line stays; '田中さん.' is not a case-row text, so it is described in prose rather than given an example line. 3. decisions.md#cjk-comma-demotion. The moved-clause note is rewritten as a within-branch correction: the FIRST half had no pure witness and moved to W3 as description, the SECOND is normative, pure-witnessed and was cut in error. "Each the spaced twin of a comma row that WAS demoted" is corrected -- '田中さん, 様.' and '김민준, 씨.' are rows, '김민준, 양.' is not; enumerating the CASES texts containing 양 gives 김지양, 양 미선, 김민준 양, 김민준 양. and 양 지훈, and '김민준 양.' is demoted on the class argument alone. A FOUND bullet records the period measurement: '田中さん 様' -> family 田中, suffix 'さん, 様'; '田中さん 様.' -> family 田中, suffix 'さん, 様.'; '田中さん 太郎.' -> family '太郎.', given '田中さん' (family '田中さん', given '太郎' without the period -- the order flips); '田中さん.' -> title '田中さん.'; '김민준씨.' -> title '김민준씨.'. The one input where a period changes nothing is the one where it sits on a word the split-off steps past regardless. 4. The watched-shapes quiet-tree figure is PINNED to the commit it was measured at. This branch's 214a1ed touched _script_segment.py, so a965272..HEAD is 6 / 41 / 16 where the paragraph claims 5 / 38 / 15 -- which is a965272..4dd776a, still true there. Recorded as a point measurement with the next touching commit named. 5. Two stale enumerations gain the third demoted class, matching cases.py's wording: compare.py's _CORPUS_TIERS comment and build_cjk_corpus.py's module docstring both listed only the comma listing and the Latin title/credential. 6. The three period rows' notes drop the repeated "the 2026-09-01 sweep's 'comma or ASCII letter' criterion could not see" -- provenance about a past oversight, stated three times where the decisions note states it once. The constraint half and the #320-at-HEAD half stay. _script_segment.py carries the restored sentence as a second W2 citation, on _peel_site where the scan-back lives, and the _peel_honorific_tail block is corrected to what W3 now says. corpus_rules.jsonl 247 -> 249, both arrivals already in corpus_cjk.jsonl, so the compared pool is unchanged at 1113 / 1120. Full suite 6509 passed, 0 xpass; ruff clean; mypy clean over 110 files. Gate at four baselines: 352 / 247 / 155 / 14 intentional, unexplained 0, radar unclassified 0, exit 0, no MOVED SHAPE and no NOT CHECKED. All four generated corpora regenerate byte-identical. Co-Authored-By: Claude Fable 5.1 --- docs/design/decisions.md | 10 ++++---- docs/design/rules.md | 22 +++++++++++++---- nameparser/_pipeline/_script_segment.py | 32 +++++++++++++++++++------ tests/v2/cases.py | 21 +++++++--------- tools/differential/build_cjk_corpus.py | 3 ++- tools/differential/compare.py | 13 +++++++--- tools/differential/corpus_rules.jsonl | 2 ++ 7 files changed, 70 insertions(+), 33 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 08668419..b457327f 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -447,15 +447,17 @@ Closes #469, and continues the corpus-tier arc below rather than standing apart - COUNTS, 2026-09-01, with the recompute beside each. 29 case rows carry `tolerated=True` over 26 distinct texts — the flag is per ROW and the corpus is per TEXT, which is why the two numbers differ (recompute: read `CASES` and count `c.tolerated`). The 26th text arrived with the W3 demotion itself: the given-side listing `지훈, 남궁민수` was a rules.md example with no case row, and would have left the harness altogether when the rules corpus stopped carrying W3, so it was given a tolerated row rather than dropped. `corpus_cjk.jsonl` went 98 → 73 and the radar-tier `corpus_cjk_tolerated.jsonl` was created at 25, then 26; the identity is the check worth keeping, since 73 + 25 = 98 exactly says the split MOVED names and dropped none, and the 99th is that one rescued row. `corpus_rules.jsonl` went 252 → 248: six comma-bearing example texts left (`田中さん, Dr.`, `田中さん, PhD`, `남궁민수, 지훈`, `지훈, 남궁민수` when the builder skipped W3 whole, plus `김, 민준씨` and `田中さん, V.` from W2's swap) and two pure ones arrived (`김민준씨`, `田中さん 様.`). `corpus_shapes.jsonl` stands at 37, with the new shapes covered 5 names (shape 6) and 2 (shape 7). And the stop condition held: intentional diffs are UNMOVED at 226/205/113/0 across 1.4.0/2.0.0/2.1.0/2.2.0, with unexplained 0, radar unclassified 0 and exit 0 at each — 1113 names compare at 1.4.0 (7 skipped, shapes 4/5) and 1120 above it. That the counts do not move is the substance and not a formality: a classified diff counts identically on either tier, so demoting a name changes which file it loads from and nothing about what the gate makes of it. -- 2026-09-05 — THE SWEEP'S CRITERION WAS NARROWER THAN THE DOCTRINE IT IMPLEMENTED, and three rows sat in the gap. The 2026-09-01 sweep asked, over the CJK-bearing case rows, for a comma or an ASCII letter. A trailing ASCII period is neither, so `田中さん 様.`, `김민준 씨.` and `김민준 양.` — the three `fix(#320)` probes, each the spaced twin of a comma row that WAS demoted — stayed contract; and `田中さん 様.` was then chosen as rules.md#W2's replacement example for its "ignores surrounding punctuation" clause on the strength of passing that same test, so the miss propagated into the normative document the same day. A period standing after a CJK honorific is a listing artifact no writing system produces, which is this entry's own class in its own words: the admission test judges the FORMAT, and `様.` is no more an arrangement Japanese writes than `田中さん, PhD` is. The three rows carry `tolerated=True` now, and nothing about the parse moves — the rows still pin #320's mechanism at HEAD, and the texts are still compared and classified at every baseline, from the radar file instead of the contract one. +- 2026-09-05 — THE SWEEP'S CRITERION WAS NARROWER THAN THE DOCTRINE IT IMPLEMENTED, and three rows sat in the gap. The 2026-09-01 sweep asked, over the CJK-bearing case rows, for a comma or an ASCII letter. A trailing ASCII period is neither, so `田中さん 様.`, `김민준 씨.` and `김민준 양.` — the three `fix(#320)` probes — stayed contract; and `田中さん 様.` was then chosen as rules.md#W2's replacement example for its "ignores surrounding punctuation" clause on the strength of passing that same test, so the miss propagated into the normative document the same day. A period standing after a CJK honorific is a listing artifact no writing system produces, which is this entry's own class in its own words: the admission test judges the FORMAT, and `様.` is no more an arrangement Japanese writes than `田中さん, PhD` is. The three rows carry `tolerated=True` now, and nothing about the parse moves — the rows still pin #320's mechanism at HEAD, and the texts are still compared and classified at every baseline, from the radar file instead of the contract one. CORRECTED IN REVIEW, 2026-09-05: a first draft of this bullet called all three "the spaced twin of a comma row that WAS demoted", and only two are. `田中さん, 様.` and `김민준, 씨.` are both case rows and both in `corpus_cjk_tolerated.jsonl`; `김민준, 양.` is no row at all — enumerating the CASES texts containing `양` returns `김지양`, `양 미선`, `김민준 양`, `김민준 양.` and `양 지훈`, and nothing with a comma. `김민준 양.` is demoted on the class argument alone, which is what carries the other two as well: an existing demoted twin is corroboration, never the warrant, or the class would end wherever the case table happens to stop. (Recompute: read `CASES` and collect the distinct texts containing the character.) - 2026-09-05 — THE CRITERION IS RESTATED AS THE DOCTRINE rather than as a sample of it. A SHAPE-TAGGED row (shapes 6/7, `Case.__post_init__` in tests/v2/cases.py) now admits no ASCII character but the space between two name words, so a period, a digit and a bracket are refused beside the comma and the Latin letter the check already refused, and the next composed form does not need a fresh criterion to catch it. The gate is a property of the shape TAG and not of CJK text, which is the half that keeps it honest: an UNTAGGED row may carry ASCII, and several deliberately do. The four mixed-script rows commit `d562164` reviewed one by one — `John 王`, `王·Smith`, `Andersonさん`, `Anderson선생님` — keep the contract tier on the reasons recorded there, untouched by this. - 2026-09-05 — `山田 太郎 (マイケル・ジャクソン)` IS KEPT contract: it is the one text the wider sweep surfaces and this entry declines to demote. Its ASCII parentheses are not a convention carried in from elsewhere — they are one SPELLING of a delimiter Japanese writing does produce, and the parser reads the spellings identically. Measured 2026-09-05 under the default facade: `(マイケル・ジャクソン)`, the fullwidth `（マイケル・ジャクソン）` and the corner-bracketed `「マイケル・ジャクソン」` all give family 山田, given 太郎, nickname `マイケル ジャクソン` (recompute: parse the three; ASCII square brackets are NOT in that set and give family `ジャクソン]`, which is what makes this a delimiter roster rather than a rule about brackets). What the row pins is the nakaguro INSIDE a delimited nickname (`fix(#272)`), and which of the three delimiters spells it is incidental to that. -- 2026-09-05 — W2's PUNCTUATION CLAUSE MOVED to W3 with its example, because it had no pure witness and could not be given one. The clause read "The split-off ignores surrounding punctuation, but never treats a part that is not name text as the name's end", and native CJK writing puts no punctuation around a name for a split-off to ignore — so every input the clause describes is a listing form, which is W3's ground. Both halves went, the second being already stated at more length in W3 ("A part that is nothing but suffix words is not the name's end: it is declined as the site…"), and the two boundary lines that might have looked like witnesses for it (`김지양`, `王君`) are about the vocabulary's own limits, not about punctuation. W2 keeps its glued-honorific statement and five examples, none carrying ASCII, and its `interacts: W3` still names where the punctuation reading lives; W3 gained the example line, one descriptive sentence, and a `tolerated:` marker counting five illustrated-not-promised lines where it counted four. +- 2026-09-05 — W2's PUNCTUATION HALF MOVED to W3 with its example; ITS SECOND HALF IS NORMATIVE AND STAYED, and this bullet is the within-branch correction of a first draft that took both. The clause read "The split-off ignores surrounding punctuation, but never treats a part that is not name text as the name's end". The FIRST half had no pure witness and could not be given one — native CJK writing puts no punctuation around a name for a split-off to ignore, so every input it describes is a listing form, which is W3's ground — and it moved there as description, rewritten to what is measured rather than to what it asserted (the bullet below). The SECOND half is a different claim and the reason given for cutting it does not reach it: it is what the parser does with NO comma and no period anywhere in the input, and contract-tier pure names depend on it. Measured 2026-09-05 by forcing `_is_post_nominal` to False in memory — the predicate the site scan `_peel_site` reads in nameparser/_pipeline/_script_segment.py — and re-parsing (family, given, suffix, title): `田中さん 様` ships `('田中', '', 'さん, 様', '')` and gives `('田中さん', '', '様', '')` without the step; `김민준 박사님` ships `('김', '민준', '박사님', '')` and gives `('김', '민준', '박사, 님', '')`; `선생님` ships `('선생님', '', '', '')` and gives `('선', '생', '님', '')`. In every case the post-nominal word is dissected, which is exactly what the half forbids. Two of the three, `김민준 박사님` and `선생님`, are `corpus_cjk.jsonl` names on the CONTRACT tier, and `ko_honorific_glued_doctor_spaced` is the #308/#271 regression row for the first; under W2's remaining statement alone `박사님` ends in the listed tail `님` and reads as peelable, so rules.md would have stopped contradicting the regression its own example pins. Restored in W2's register as "A part that is not name text — a post-nominal word standing on its own — is never the name's end: the split-off steps past it to the name word behind, and never dissects it." — witnessed by `김민준 박사님` → `suffix="박사님"` and the boundary line `선생님` → `family="선생님"`, both texts `corpus_cjk.jsonl` already held, so the two example lines arrive in `corpus_rules.jsonl` without adding a name to the pool. `田中さん 様` would have been the third witness and is NOT added: it is not a case row, and a new contract name needs a 1.4.0 classification this bundle does not have. `_script_segment.py` carries the restored sentence as a second W2 citation, on `_peel_site` where the scan-back lives. RECOMPUTE: rebind `_is_post_nominal` to `lambda state, i: False` on the module and re-parse the three. -- COUNTS, 2026-09-05, with the recompute beside each. `corpus_cjk.jsonl` went 73 → 70 and `corpus_cjk_tolerated.jsonl` 26 → 29 — the same identity as the first demotion, 99 names either side, so the split MOVED three texts and dropped none. `corpus_rules.jsonl` went 248 → 247, the single example line W2 gave up. 32 case rows carry `tolerated=True` over 29 distinct texts (recompute: read `CASES` and count `c.tolerated`, per row and per text). `_CORPUS_FLOORS["corpus_cjk.jsonl"]` is lowered 70 → 67 deliberately, and the tolerated file's floor of 22 still stands under 29. The stop condition held again: intentional diffs UNMOVED at 352 / 247 / 155 / 14 across 1.4.0 / 2.0.0 / 2.1.0 / 2.2.0, unexplained 0, radar unclassified 0 and exit 0 at each. RECOMPUTE THE SWEEP, which is the part worth having rather than the digits: it is the CJK-bearing (`_has_cjk`) rows that do NOT declare `tolerated`, carrying any ASCII character other than a space, grouped by the kind of character that trips them and joined to the corpus files holding each text. Over the tree before this bundle it returns eight distinct texts — the three period rows, the parenthesized nickname, and the four mixed-script rows — and over this one, five: the period class is gone and the letter class is untouched, which is the shape of the answer to look for rather than the count. +- 2026-09-05, FOUND while rewriting the clause — A PERIOD ON THE NAME'S OWN LAST WORD IS NOT IGNORED; IT IS DECISIVE, and the moved half asserted the opposite. The half said the split-off "ignores surrounding punctuation", and W3's first descriptive rewrite of it kept that shape, attributing the crossing to the punctuation. Four parses, 2026-09-05 under the default facade, say otherwise. `田中さん 様` → family `田中`, suffix `さん, 様`: the crossing happens with no punctuation in the input at all, so the vocabulary licenses it and not the period. `田中さん 様.` → family `田中`, suffix `さん, 様.`: the period rides along and moves nothing, `様.` reading post-nominal-strict exactly as `様` does. `田中さん 太郎.` → family `太郎.`, given `田中さん`: a period on a NAME word, and no crossing — and not ignored either, since the same input without it is family `田中さん`, given `太郎`, the order reading flipped by the period alone. `田中さん.` → title `田中さん.`, and `김민준씨.` → title `김민준씨.`: a period glued to the honorific's own word stands between the honorific and that word's end, no listed tail matches, the peel declines and the whole text reads as a title — the parse a period changes most, not least. Of the five inputs, the ONLY one where a period changes nothing is the one where it sits on a word the split-off was going to step past regardless. So W3 now says the step past a post-nominal word is W2's and needs no punctuation, that a period on a separate post-nominal word rides into the suffix, and that a period glued to the honorific's own word decides against the split-off. RECOMPUTE: parse the five strings, each also without its period, and read `title` alongside the name fields — a rewrite that reads only `family` and `suffix` sees two empty results and can call them a null effect. + +- COUNTS, 2026-09-05, with the recompute beside each. `corpus_cjk.jsonl` went 73 → 70 and `corpus_cjk_tolerated.jsonl` 26 → 29 — the same identity as the first demotion, 99 names either side, so the split MOVED three texts and dropped none. `corpus_rules.jsonl` went 248 → 247 → 249: W2 gave up the `田中さん 様.` line, then took the two the review round restored the second half with (`김민준 박사님`, `선생님`). Both arrivals are texts `corpus_cjk.jsonl` already held, so the file grew and the compared POOL did not — the same shape as the shapes-corpus growth of 2026-09-01, and the reason a witness had to be picked from names already inside. 32 case rows carry `tolerated=True` over 29 distinct texts (recompute: read `CASES` and count `c.tolerated`, per row and per text). `_CORPUS_FLOORS["corpus_cjk.jsonl"]` is lowered 70 → 67 deliberately, and the tolerated file's floor of 22 still stands under 29. The stop condition held again: intentional diffs UNMOVED at 352 / 247 / 155 / 14 across 1.4.0 / 2.0.0 / 2.1.0 / 2.2.0, unexplained 0, radar unclassified 0 and exit 0 at each. RECOMPUTE THE SWEEP, which is the part worth having rather than the digits: it is the CJK-bearing (`_has_cjk`) rows that do NOT declare `tolerated`, carrying any ASCII character other than a space, grouped by the kind of character that trips them and joined to the corpus files holding each text. Over the tree before this bundle it returns eight distinct texts — the three period rows, the parenthesized nickname, and the four mixed-script rows — and over this one, five: the period class is gone and the letter class is untouched, which is the shape of the answer to look for rather than the count. ### P5 — bound given names @@ -853,7 +855,7 @@ Declined: - Pinning winners for the six #501 contests (2026-09-05) — stays #501's. The four 2.0.0 rows here and `'MD, PHD'`'s two record SHAPES, not winners: a shape move is now a finding on them, while a handover to another rule is not — file order still decides the winner, and only a winner pin sees it. #501's own objection to pinning a boundary nobody has argued stands, and the issue is scoped down by that comment, not closed. - A checked-in generator for the rows (2026-09-05) — the rows are a snapshot, and a generator that re-derives them on demand is the re-derivation hazard with a command line (mechanisms.md#RECORDED-ROSTERS). A scratch script and a recipe in the roster comment, as `_CORPUS_FLOORS` and `_CORPUS_CLAIMS` do. -The measurement, and how to redo it. `_WATCHED_DIFFS` holds 41 / 36 / 31 / 5 rows at 1.4.0 / 2.0.0 / 2.1.0 / 2.2.0, 113 in all; the 2.0.0 section is 32 sole-watched rows plus the four #501 contests (`'김민준 박사님'` and `'선생님'`, contract tier and fatal; `'田中さん 様.'` and `'田中さん, 様.'`, `corpus_cjk_tolerated.jsonl`, radar and printed — the first of that pair read contract when this was measured earlier the same day, and the period-class demotion under #cjk-comma-demotion above moved it hours later, which is the point the snapshot makes better than the prose could: a row's fatality is a property of the tier its NAME reads at, and no roster in `compare.py` records it). 51 distinct sole-watched names; every row a default-order shape, so `NOT CHECKED` names none of them. Four mutation controls, in memory around `main()` with the worktree untouched, re-run 2026-09-05 against the amended data commit: `'QC MP'` corrupted at 1.4.0 (its only row — it does not diff at 2.0.0) prints `MOVED SHAPE (radar) expected_since_1.4.0.toml: 1 watched diff shape(s) disagree with this run` and exits 0, `'Smith Dr'` at 2.0.0 likewise; `'선생님'` corrupted at 2.0.0 prints `MOVED SHAPE expected_since_2.0.0.toml: 1 watched diff shape(s) disagree with this run` ending "This fails the run because the name is contract tier" and exits 1; `'Ph. D., Jr.'` corrupted at 2.1.0 prints the `(radar)` block and exits 0; and `'MD, PHD'` inserted into the 1.4.0 watched section beside its contest row is refused pre-worker, before any `baseline:` line, as "1 name(s) sit in both `_RECORDED_DIFFS['expected_since_1.4.0.toml']` and `_WATCHED_DIFFS['expected_since_1.4.0.toml']`", and fails `test_the_watched_roster_is_disjoint_and_names_every_ledger` naming it — the guard half wrapping `load_tool`, since that helper executes a fresh `compare` module per call and a mutation on an imported one never reaches it. A control whose corruption changed nothing would have meant an inert row; none did. The arc moved no classification: 352 / 247 / 155 / 14 intentional diffs, 0 unexplained and 0 radar-unclassified at every baseline, byte-identical summary lines before and after every commit. The quiet-tree claim that made this the moment to snapshot: measured 2026-09-05, `git rev-list --count a965272..HEAD -- nameparser/` is 5 commits since the tier split, moving 38 inserted and 15 deleted lines across two files, `_facade.py` (the #462 initials fix and its review rounds) and `_pipeline/_script_segment.py` (`f046d5d`, the tier-split day's own W3 demotion) — a figure that goes stale by design, since the next feature is what the snapshot exists to make legible. RECOMPUTE the row counts with `{k: len(v) for k, v in compare._WATCHED_DIFFS.items()}`, the population with the script in the dict's header, the controls with `compare._WATCHED_DIFFS[ledger][name] = (...)` around `main()`, and the quiet tree with the command above plus `git diff --stat a965272..HEAD -- nameparser/`. +The measurement, and how to redo it. `_WATCHED_DIFFS` holds 41 / 36 / 31 / 5 rows at 1.4.0 / 2.0.0 / 2.1.0 / 2.2.0, 113 in all; the 2.0.0 section is 32 sole-watched rows plus the four #501 contests (`'김민준 박사님'` and `'선생님'`, contract tier and fatal; `'田中さん 様.'` and `'田中さん, 様.'`, `corpus_cjk_tolerated.jsonl`, radar and printed — the first of that pair read contract when this was measured earlier the same day, and the period-class demotion under #cjk-comma-demotion above moved it hours later, which is the point the snapshot makes better than the prose could: a row's fatality is a property of the tier its NAME reads at, and no roster in `compare.py` records it). 51 distinct sole-watched names; every row a default-order shape, so `NOT CHECKED` names none of them. Four mutation controls, in memory around `main()` with the worktree untouched, re-run 2026-09-05 against the amended data commit: `'QC MP'` corrupted at 1.4.0 (its only row — it does not diff at 2.0.0) prints `MOVED SHAPE (radar) expected_since_1.4.0.toml: 1 watched diff shape(s) disagree with this run` and exits 0, `'Smith Dr'` at 2.0.0 likewise; `'선생님'` corrupted at 2.0.0 prints `MOVED SHAPE expected_since_2.0.0.toml: 1 watched diff shape(s) disagree with this run` ending "This fails the run because the name is contract tier" and exits 1; `'Ph. D., Jr.'` corrupted at 2.1.0 prints the `(radar)` block and exits 0; and `'MD, PHD'` inserted into the 1.4.0 watched section beside its contest row is refused pre-worker, before any `baseline:` line, as "1 name(s) sit in both `_RECORDED_DIFFS['expected_since_1.4.0.toml']` and `_WATCHED_DIFFS['expected_since_1.4.0.toml']`", and fails `test_the_watched_roster_is_disjoint_and_names_every_ledger` naming it — the guard half wrapping `load_tool`, since that helper executes a fresh `compare` module per call and a mutation on an imported one never reaches it. A control whose corruption changed nothing would have meant an inert row; none did. The arc moved no classification: 352 / 247 / 155 / 14 intentional diffs, 0 unexplained and 0 radar-unclassified at every baseline, byte-identical summary lines before and after every commit. The quiet-tree claim that made this the moment to snapshot: measured 2026-09-05 at `4dd776a`, `git rev-list --count a965272..4dd776a -- nameparser/` is 5 commits since the tier split, moving 38 inserted and 15 deleted lines across two files, `_facade.py` (the #462 initials fix and its review rounds) and `_pipeline/_script_segment.py` (`f046d5d`, the tier-split day's own W3 demotion). PINNED TO THAT COMMIT deliberately, and not to HEAD: the very next commit to touch `nameparser/` was this bundle's own docstring-citation trim in `_script_segment.py`, which takes the same range at HEAD to 6 / 41 / 16. So this is a POINT measurement of the tree the snapshot was taken on, and rereading it as a live one is the error to avoid — the figure goes stale by design, since the next feature is what the snapshot exists to make legible. RECOMPUTE the row counts with `{k: len(v) for k, v in compare._WATCHED_DIFFS.items()}`, the population with the script in the dict's header, the controls with `compare._WATCHED_DIFFS[ledger][name] = (...)` around `main()`, and the quiet tree with the command above plus `git diff --shortstat a965272..4dd776a -- nameparser/` (substitute `HEAD` for the live figure, which is a different claim). ### comma-suffix-arc — #291/#296/#316 (2026-07-26 → 2026-08-01) diff --git a/docs/design/rules.md b/docs/design/rules.md index 7cb39539..ca4df7fd 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -1063,11 +1063,16 @@ W2. Rationale: some East Asian honorifics glue directly onto the end never itself end a name, so the listed vocabulary carries its own license and needs no other gate. A listed honorific glued to the end of the name's last name - word splits off once and reads as a suffix. + word splits off once and reads as a suffix. A part that is not + name text — a post-nominal word standing on its own — is never + the name's end: the split-off steps past it to the name word + behind, and never dissects it. "田中さん" → suffix="さん" "김민준씨" → suffix="씨" + "김민준 박사님" → suffix="박사님" "马丁·路德·金씨" → suffix="씨" "김지양" → suffix="" · boundary + "선생님" → family="선생님" · boundary "王君" → family="王君" · boundary history: decisions.md#W2 · interacts: W3 · implemented: nameparser/_pipeline/_script_segment.py @@ -1097,10 +1102,17 @@ W3. Rationale: a family name declared by a comma is the writer's name's end, and a glued honorific before the comma stays glued. The vocabulary question is C1's own, asked without C1's word-count condition: the two differ in what else they require, - not in what they ask of the words. The split-off ignores the - punctuation a listing leaves around the name — a period trailing - an honorific is the same artifact as the comma, no writing system - putting one there — and reads across it to the word behind. + not in what they ask of the words. A period the listing leaves + behind is not what licenses the step past a post-nominal word, + and it is not ignored either. The step is W2's, taken on the + vocabulary alone and taken with no punctuation anywhere in the + input; a period on a SEPARATE post-nominal word rides along into + the suffix and moves no division (田中さん 様. divides where + 田中さん 様 does). A period glued to the honorific's OWN word is + read instead, and decides against the split-off: the listed tail + no longer ends that word, so nothing peels and the text goes on + whole — 田中さん. and 김민준씨. each read as a title. + decisions.md#cjk-comma-demotion carries the parses. "남궁민수" → family="남궁" "지훈, 남궁민수" → given="남궁민수" "남궁민수, 지훈" → family="남궁민수" · boundary diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 914ddbb8..a7aa0c39 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -228,6 +228,19 @@ def _is_post_nominal(state: ParseState, i: int) -> bool: return is_suffix_strict(state.tokens[i].text, state.lexicon) +# rules.md#W2: "A part that is not name text — a post-nominal word +# standing on its own — is never the name's end: the split-off steps +# past it to the name word behind, and never dissects it." (history: +# decisions.md#W2) +# The scan-back below is that clause, and it needs no punctuation to +# fire. '김민준 박사님' steps past 박사님 to 김민준, finds no listed +# tail there and returns None -- so 박사님 is left whole for suffix +# classification rather than cut into 박사 + 님, which is what the +# same input gives when the step is removed. '선생님' is post-nominal +# entire with no name word behind it, so the scan yields no site at +# all and the token stays whole. Both are contract-tier corpus names +# and rules.md#W2 example lines; decisions.md#cjk-comma-demotion +# carries the forced-predicate measurement behind them. def _peel_site(state: ParseState, flat: Sequence[int], tails: frozenset[str]) -> tuple[int, int] | None: """Where a peel would land in the token run `flat`: the index of the @@ -274,14 +287,19 @@ def _peel_site(state: ParseState, flat: Sequence[int], # rules.md#W2: "a listed honorific glued to the end of the name's # last name word splits off once and reads as a suffix." (history: # decisions.md#W2) -# That the peel also reaches ACROSS a family comma, and across the -# punctuation a listing leaves around the name, is stated at +# That the peel also reaches ACROSS a family comma is stated at # rules.md#W3 instead, which is a tolerated rule since the -# 2026-09-01 comma demotion -- both readings are what the parser -# does today, not something W2 promises. The punctuation half moved -# there on 2026-09-05 (decisions.md#cjk-comma-demotion): it had no -# witness native writing could supply, every input it describes -# being a listing form. +# 2026-09-01 comma demotion -- the crossing is what the parser does +# today, not something W2 promises. W3 also took, on 2026-09-05, the +# reading of a period a listing leaves behind +# (decisions.md#cjk-comma-demotion): a period on a SEPARATE +# post-nominal word rides into the suffix and moves nothing ('様.' +# is post-nominal-strict and the scan steps past it as it steps past +# '様'), while a period glued to the honorific's OWN token stands +# between the honorific and that token's end, so no listed tail +# matches, the peel declines, and the whole text reads as a title +# downstream ('田中さん.', '김민준씨.'). Neither is a promise; the step past the post-nominal +# word itself is W2's, above, and is. def _peel_honorific_tail(state: ParseState) -> ParseState: """#308: split a listed honorific off the END of the name's last NON-POST-NOMINAL token -- 田中さん -> 田中 + さん -- and let diff --git a/tests/v2/cases.py b/tests/v2/cases.py index ffe3c274..3002f599 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -3334,10 +3334,9 @@ def _check_cjk_shape_purity(self) -> None: "TOLERATED since 2026-09-05: a trailing ASCII period on " "a CJK honorific is a listing artifact no writing " "system produces -- the same class as a comma listing " - "or a Latin credential, and the class the 2026-09-01 " - "sweep's 'comma or ASCII letter' criterion could not " - "see. The row still pins #320's mechanism at HEAD; " - "what moves is which corpus file carries the text", + "or a Latin credential. The row still pins #320's " + "mechanism at HEAD; what moves is which corpus file " + "carries the text", tolerated=True), Case("ja_honorific_period_does_not_stop_the_peel", "田中さん, 様.", {"family": "田中", "suffix": "さん, 様."}, @@ -3468,10 +3467,8 @@ def _check_cjk_shape_purity(self) -> None: "naming the same segmentation it also depends on. " "TOLERATED since 2026-09-05 with its two period twins: " "a trailing ASCII period on a CJK honorific is a " - "listing artifact no writing system produces, and the " - "2026-09-01 sweep's 'comma or ASCII letter' criterion " - "could not see the class. The row still pins #320's " - "mechanism at HEAD", + "listing artifact no writing system produces. The row " + "still pins #320's mechanism at HEAD", tolerated=True), Case("ko_honorific_glued_teacher", "김선생님", {"family": "김", "suffix": "선생님"}, @@ -4069,11 +4066,9 @@ def _check_cjk_shape_purity(self) -> None: "produced, not the segmenter's. TOLERATED since " "2026-09-05 for the same reason as that counterpart -- " "a trailing ASCII period on a CJK honorific is a " - "listing artifact no writing system produces, missed by " - "the 2026-09-01 sweep's 'comma or ASCII letter' " - "criterion -- and the pair moves tiers together the way " - "it moves fields together. The row still pins #320's " - "mechanism at HEAD", + "listing artifact no writing system produces -- and the " + "pair moves tiers together the way it moves fields " + "together. The row still pins #320's mechanism at HEAD", tolerated=True), Case("ko_surname_yang_leads_a_segmentable_given", "양 지훈", {"family": "양", "given": "지훈"}, diff --git a/tools/differential/build_cjk_corpus.py b/tools/differential/build_cjk_corpus.py index b28511db..8f589b0a 100644 --- a/tools/differential/build_cjk_corpus.py +++ b/tools/differential/build_cjk_corpus.py @@ -27,7 +27,8 @@ one sweep, so the two halves cannot drift apart or double-count a text. Nothing about the harvest predicate changed: a composed or wrapped CJK form -- a comma listing, a Latin title or credential -around a CJK name -- is still compared at every baseline and still +around a CJK name, and since 2026-09-05 a trailing ASCII period on an +honorific -- is still compared at every baseline and still classified against the ledger. What the flag moves is which of THESE TWO FILES a text is written to, and nothing else: a text that another CONTRACT corpus also holds keeps the contract tier until it leaves diff --git a/tools/differential/compare.py b/tools/differential/compare.py index ef0514df..30fe95b8 100644 --- a/tools/differential/compare.py +++ b/tools/differential/compare.py @@ -602,7 +602,7 @@ def _legal_orders() -> frozenset[str]: # the file emptying, and this half only # grows as the contract narrows "corpus_issues.jsonl": 370, # 381 today, harvested and append-only - "corpus_rules.jsonl": 150, # 247 today, generated from rules.md. + "corpus_rules.jsonl": 150, # 249 today, generated from rules.md. # 248 until 2026-09-05, when W2's # trailing-period example moved into # the tolerated W3 and the builder @@ -611,7 +611,13 @@ def _legal_orders() -> frozenset[str]: # this file, and (measured 2026-09-05) # the last CJK example anywhere outside # W3 that carried a non-space ASCII - # character. + # character. 247 -> 249 later the same + # day, when the review round restored + # W2's second half and witnessed it + # with '김민준 박사님' and '선생님' -- + # both already in corpus_cjk.jsonl, so + # the file grew and the deduped pool + # did not. # 252 until 2026-09-01, when W3 took # rules.md's `tolerated:` marker and # build_rules_corpus.py stopped @@ -663,7 +669,8 @@ def _legal_orders() -> frozenset[str]: # case rows, not by scraping. The tier still fits, and for the # reason the flag was written -- these are composed and wrapped # CJK forms (a comma listing, a Latin title or credential around a - # CJK name) that native CJK writing does not contain, so the + # CJK name, and since 2026-09-05 a trailing ASCII period on an + # honorific) that native CJK writing does not contain, so the # differential stops answering for them. Radar is what "we still # watch it, we no longer enforce it" costs; the case rows still # assert every one of these parses in the suite. diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index e13074fb..29c1979b 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -241,7 +241,9 @@ "高橋一郎" "김 민준" "김민준" +"김민준 박사님" "김민준씨" "김지양" "남궁민수" "남궁민수 지훈" +"선생님" From 88b025b11b62eb114017a711ca3ed47a38f62448 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 5 Sep 2026 13:57:25 -0700 Subject: [PATCH 5/5] docs(design): four figures and a summary the review round wrote wrong, and W3 says what pins it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five prose corrections to what this bundle's own review round wrote, each re-measured on this tree at d0faed4. The recorded-shapes snapshot contrasted its pinned quiet-tree figure with a live one "at HEAD", and that live figure was already stale at the commit that wrote it -- the exact defect the pin exists to avoid. The contrast is removed rather than refreshed: the pinned 5 / 38 / 15 over a965272..4dd776a is now the only number, and the paragraph says instead that 214a1ed and d0faed4 touched _script_segment.py's comments and W2/W3 citations, so a HEAD recompute is larger and moves with every further comment-only commit. The RECOMPUTE line now insists on the pinned endpoint. The W2 bullet's summary of its three forced parses said "in every case the post-nominal word is dissected". Re-run with _is_post_nominal rebound to False, only two are: 박사님 -> 박사, 님 and 선생님 -> 선, 생, 님. The first, 田中さん 様, yields ('田中さん', '', '様', '') -- 様 whole, but taken as the split site, so it breaks the other clause, never the name's end. The bullet now names which clause each case witnesses. The same bullet had the polarity of its own argument backwards, saying a W2 reduced to its first sentence "would have stopped contradicting" the regression its example pins. parse('김민준 박사님').suffix is '박사님'; under the first sentence alone 박사님 ends in the listed tail 님 and reads as peelable, so the reduced W2 would have BEGUN contradicting ko_honorific_glued_doctor_spaced -- which is the argument for restoring the half, not against it. The FOUND bullet said "four parses" over a list its own later sentences call "the five inputs": four counts the sentences, one of which carries two strings. It is five now, by input, with the convention stated and the sixth string (the no-period twin 田中さん 太郎) accounted for. W3's new period sentence asserted that 田中さん. and 김민준씨. each read as a title, and neither string is a case row or in any corpus, so nothing pins those readings. W3 is tolerated and illustrates rather than promises, and a new name would need a baseline classification this bundle does not carry, so no rows are added; the sentence says instead that it reports parses measured 2026-09-05 that no row pins and that can move unannounced, pointing at decisions.md#cjk-comma-demotion. _script_segment.py's comment restating the same reading is aligned to say the same. Verified: 698 passed on the doc/rules/decisions/mechanisms/guard/citation subset, ruff clean, and the differential at --baseline 2.2.0 exits 0 with 14 intentional diffs, 0 unexplained, 0 radar unclassified. Co-Authored-By: Claude Fable 5.1 --- docs/design/decisions.md | 6 +++--- docs/design/rules.md | 6 +++++- nameparser/_pipeline/_script_segment.py | 7 +++++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index b457327f..632c8c4d 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -453,9 +453,9 @@ Closes #469, and continues the corpus-tier arc below rather than standing apart - 2026-09-05 — `山田 太郎 (マイケル・ジャクソン)` IS KEPT contract: it is the one text the wider sweep surfaces and this entry declines to demote. Its ASCII parentheses are not a convention carried in from elsewhere — they are one SPELLING of a delimiter Japanese writing does produce, and the parser reads the spellings identically. Measured 2026-09-05 under the default facade: `(マイケル・ジャクソン)`, the fullwidth `（マイケル・ジャクソン）` and the corner-bracketed `「マイケル・ジャクソン」` all give family 山田, given 太郎, nickname `マイケル ジャクソン` (recompute: parse the three; ASCII square brackets are NOT in that set and give family `ジャクソン]`, which is what makes this a delimiter roster rather than a rule about brackets). What the row pins is the nakaguro INSIDE a delimited nickname (`fix(#272)`), and which of the three delimiters spells it is incidental to that. -- 2026-09-05 — W2's PUNCTUATION HALF MOVED to W3 with its example; ITS SECOND HALF IS NORMATIVE AND STAYED, and this bullet is the within-branch correction of a first draft that took both. The clause read "The split-off ignores surrounding punctuation, but never treats a part that is not name text as the name's end". The FIRST half had no pure witness and could not be given one — native CJK writing puts no punctuation around a name for a split-off to ignore, so every input it describes is a listing form, which is W3's ground — and it moved there as description, rewritten to what is measured rather than to what it asserted (the bullet below). The SECOND half is a different claim and the reason given for cutting it does not reach it: it is what the parser does with NO comma and no period anywhere in the input, and contract-tier pure names depend on it. Measured 2026-09-05 by forcing `_is_post_nominal` to False in memory — the predicate the site scan `_peel_site` reads in nameparser/_pipeline/_script_segment.py — and re-parsing (family, given, suffix, title): `田中さん 様` ships `('田中', '', 'さん, 様', '')` and gives `('田中さん', '', '様', '')` without the step; `김민준 박사님` ships `('김', '민준', '박사님', '')` and gives `('김', '민준', '박사, 님', '')`; `선생님` ships `('선생님', '', '', '')` and gives `('선', '생', '님', '')`. In every case the post-nominal word is dissected, which is exactly what the half forbids. Two of the three, `김민준 박사님` and `선생님`, are `corpus_cjk.jsonl` names on the CONTRACT tier, and `ko_honorific_glued_doctor_spaced` is the #308/#271 regression row for the first; under W2's remaining statement alone `박사님` ends in the listed tail `님` and reads as peelable, so rules.md would have stopped contradicting the regression its own example pins. Restored in W2's register as "A part that is not name text — a post-nominal word standing on its own — is never the name's end: the split-off steps past it to the name word behind, and never dissects it." — witnessed by `김민준 박사님` → `suffix="박사님"` and the boundary line `선생님` → `family="선생님"`, both texts `corpus_cjk.jsonl` already held, so the two example lines arrive in `corpus_rules.jsonl` without adding a name to the pool. `田中さん 様` would have been the third witness and is NOT added: it is not a case row, and a new contract name needs a 1.4.0 classification this bundle does not have. `_script_segment.py` carries the restored sentence as a second W2 citation, on `_peel_site` where the scan-back lives. RECOMPUTE: rebind `_is_post_nominal` to `lambda state, i: False` on the module and re-parse the three. +- 2026-09-05 — W2's PUNCTUATION HALF MOVED to W3 with its example; ITS SECOND HALF IS NORMATIVE AND STAYED, and this bullet is the within-branch correction of a first draft that took both. The clause read "The split-off ignores surrounding punctuation, but never treats a part that is not name text as the name's end". The FIRST half had no pure witness and could not be given one — native CJK writing puts no punctuation around a name for a split-off to ignore, so every input it describes is a listing form, which is W3's ground — and it moved there as description, rewritten to what is measured rather than to what it asserted (the bullet below). The SECOND half is a different claim and the reason given for cutting it does not reach it: it is what the parser does with NO comma and no period anywhere in the input, and contract-tier pure names depend on it. Measured 2026-09-05 by forcing `_is_post_nominal` to False in memory — the predicate the site scan `_peel_site` reads in nameparser/_pipeline/_script_segment.py — and re-parsing (family, given, suffix, title): `田中さん 様` ships `('田中', '', 'さん, 様', '')` and gives `('田中さん', '', '様', '')` without the step; `김민준 박사님` ships `('김', '민준', '박사님', '')` and gives `('김', '민준', '박사, 님', '')`; `선생님` ships `('선생님', '', '', '')` and gives `('선', '생', '님', '')`. The three break DIFFERENT clauses of the half, and a first draft of this bullet said all three were dissection. The restored sentence makes two claims — the post-nominal word is never the name's END, and it is never DISSECTED — and `田中さん 様` fails only the first: `様` comes out WHOLE as the suffix, undissected, but it is what the split-off takes as its SITE, so the peel lands on the post-nominal word and `田中さん` behind it is left unpeeled, where shipped the scan steps past `様` and peels `さん` off `田中さん`. `김민준 박사님` and `선생님` are the dissection failures, `박사님` coming apart into `박사, 님` and `선생님` into `선, 생, 님`. Between them the three witness both clauses, and neither clause is witnessed twice over. Two of the three, `김민준 박사님` and `선생님`, are `corpus_cjk.jsonl` names on the CONTRACT tier, and `ko_honorific_glued_doctor_spaced` is the #308/#271 regression row for the first; that row pins `suffix="박사님"`, which is what `parse('김민준 박사님').suffix` returns under the default facade, measured 2026-09-05. Under W2's remaining FIRST statement alone — a listed honorific glued to the end of the name's last name word splits off once — `박사님` ends in the listed tail `님` and reads as peelable, giving `박사` + `님`. So a W2 cut back to that one sentence would have BEGUN contradicting the regression row its own example line pins, not stopped: the second half is what keeps rules.md agreeing with the shipped parse, and that is the argument for restoring it. Restored in W2's register as "A part that is not name text — a post-nominal word standing on its own — is never the name's end: the split-off steps past it to the name word behind, and never dissects it." — witnessed by `김민준 박사님` → `suffix="박사님"` and the boundary line `선생님` → `family="선생님"`, both texts `corpus_cjk.jsonl` already held, so the two example lines arrive in `corpus_rules.jsonl` without adding a name to the pool. `田中さん 様` would have been the third witness and is NOT added: it is not a case row, and a new contract name needs a 1.4.0 classification this bundle does not have. `_script_segment.py` carries the restored sentence as a second W2 citation, on `_peel_site` where the scan-back lives. RECOMPUTE: rebind `_is_post_nominal` to `lambda state, i: False` on the module and re-parse the three. -- 2026-09-05, FOUND while rewriting the clause — A PERIOD ON THE NAME'S OWN LAST WORD IS NOT IGNORED; IT IS DECISIVE, and the moved half asserted the opposite. The half said the split-off "ignores surrounding punctuation", and W3's first descriptive rewrite of it kept that shape, attributing the crossing to the punctuation. Four parses, 2026-09-05 under the default facade, say otherwise. `田中さん 様` → family `田中`, suffix `さん, 様`: the crossing happens with no punctuation in the input at all, so the vocabulary licenses it and not the period. `田中さん 様.` → family `田中`, suffix `さん, 様.`: the period rides along and moves nothing, `様.` reading post-nominal-strict exactly as `様` does. `田中さん 太郎.` → family `太郎.`, given `田中さん`: a period on a NAME word, and no crossing — and not ignored either, since the same input without it is family `田中さん`, given `太郎`, the order reading flipped by the period alone. `田中さん.` → title `田中さん.`, and `김민준씨.` → title `김민준씨.`: a period glued to the honorific's own word stands between the honorific and that word's end, no listed tail matches, the peel declines and the whole text reads as a title — the parse a period changes most, not least. Of the five inputs, the ONLY one where a period changes nothing is the one where it sits on a word the split-off was going to step past regardless. So W3 now says the step past a post-nominal word is W2's and needs no punctuation, that a period on a separate post-nominal word rides into the suffix, and that a period glued to the honorific's own word decides against the split-off. RECOMPUTE: parse the five strings, each also without its period, and read `title` alongside the name fields — a rewrite that reads only `family` and `suffix` sees two empty results and can call them a null effect. +- 2026-09-05, FOUND while rewriting the clause — A PERIOD ON THE NAME'S OWN LAST WORD IS NOT IGNORED; IT IS DECISIVE, and the moved half asserted the opposite. The half said the split-off "ignores surrounding punctuation", and W3's first descriptive rewrite of it kept that shape, attributing the crossing to the punctuation. Five parses, 2026-09-05 under the default facade, say otherwise — counted by INPUT STRING, the convention the rest of this bullet uses when it says "the five inputs" and "parse the five strings". A first draft counted four by counting the SENTENCES below instead, the last of which carries two inputs (`田中さん.` and `김민준씨.`). Six, if the no-period twin `田中さん 太郎` quoted inside the third sentence as its control counts as a parse of its own rather than as part of that item. `田中さん 様` → family `田中`, suffix `さん, 様`: the crossing happens with no punctuation in the input at all, so the vocabulary licenses it and not the period. `田中さん 様.` → family `田中`, suffix `さん, 様.`: the period rides along and moves nothing, `様.` reading post-nominal-strict exactly as `様` does. `田中さん 太郎.` → family `太郎.`, given `田中さん`: a period on a NAME word, and no crossing — and not ignored either, since the same input without it is family `田中さん`, given `太郎`, the order reading flipped by the period alone. `田中さん.` → title `田中さん.`, and `김민준씨.` → title `김민준씨.`: a period glued to the honorific's own word stands between the honorific and that word's end, no listed tail matches, the peel declines and the whole text reads as a title — the parse a period changes most, not least. Of the five inputs, the ONLY one where a period changes nothing is the one where it sits on a word the split-off was going to step past regardless. So W3 now says the step past a post-nominal word is W2's and needs no punctuation, that a period on a separate post-nominal word rides into the suffix, and that a period glued to the honorific's own word decides against the split-off. RECOMPUTE: parse the five strings, each also without its period, and read `title` alongside the name fields — a rewrite that reads only `family` and `suffix` sees two empty results and can call them a null effect. - COUNTS, 2026-09-05, with the recompute beside each. `corpus_cjk.jsonl` went 73 → 70 and `corpus_cjk_tolerated.jsonl` 26 → 29 — the same identity as the first demotion, 99 names either side, so the split MOVED three texts and dropped none. `corpus_rules.jsonl` went 248 → 247 → 249: W2 gave up the `田中さん 様.` line, then took the two the review round restored the second half with (`김민준 박사님`, `선생님`). Both arrivals are texts `corpus_cjk.jsonl` already held, so the file grew and the compared POOL did not — the same shape as the shapes-corpus growth of 2026-09-01, and the reason a witness had to be picked from names already inside. 32 case rows carry `tolerated=True` over 29 distinct texts (recompute: read `CASES` and count `c.tolerated`, per row and per text). `_CORPUS_FLOORS["corpus_cjk.jsonl"]` is lowered 70 → 67 deliberately, and the tolerated file's floor of 22 still stands under 29. The stop condition held again: intentional diffs UNMOVED at 352 / 247 / 155 / 14 across 1.4.0 / 2.0.0 / 2.1.0 / 2.2.0, unexplained 0, radar unclassified 0 and exit 0 at each. RECOMPUTE THE SWEEP, which is the part worth having rather than the digits: it is the CJK-bearing (`_has_cjk`) rows that do NOT declare `tolerated`, carrying any ASCII character other than a space, grouped by the kind of character that trips them and joined to the corpus files holding each text. Over the tree before this bundle it returns eight distinct texts — the three period rows, the parenthesized nickname, and the four mixed-script rows — and over this one, five: the period class is gone and the letter class is untouched, which is the shape of the answer to look for rather than the count. @@ -855,7 +855,7 @@ Declined: - Pinning winners for the six #501 contests (2026-09-05) — stays #501's. The four 2.0.0 rows here and `'MD, PHD'`'s two record SHAPES, not winners: a shape move is now a finding on them, while a handover to another rule is not — file order still decides the winner, and only a winner pin sees it. #501's own objection to pinning a boundary nobody has argued stands, and the issue is scoped down by that comment, not closed. - A checked-in generator for the rows (2026-09-05) — the rows are a snapshot, and a generator that re-derives them on demand is the re-derivation hazard with a command line (mechanisms.md#RECORDED-ROSTERS). A scratch script and a recipe in the roster comment, as `_CORPUS_FLOORS` and `_CORPUS_CLAIMS` do. -The measurement, and how to redo it. `_WATCHED_DIFFS` holds 41 / 36 / 31 / 5 rows at 1.4.0 / 2.0.0 / 2.1.0 / 2.2.0, 113 in all; the 2.0.0 section is 32 sole-watched rows plus the four #501 contests (`'김민준 박사님'` and `'선생님'`, contract tier and fatal; `'田中さん 様.'` and `'田中さん, 様.'`, `corpus_cjk_tolerated.jsonl`, radar and printed — the first of that pair read contract when this was measured earlier the same day, and the period-class demotion under #cjk-comma-demotion above moved it hours later, which is the point the snapshot makes better than the prose could: a row's fatality is a property of the tier its NAME reads at, and no roster in `compare.py` records it). 51 distinct sole-watched names; every row a default-order shape, so `NOT CHECKED` names none of them. Four mutation controls, in memory around `main()` with the worktree untouched, re-run 2026-09-05 against the amended data commit: `'QC MP'` corrupted at 1.4.0 (its only row — it does not diff at 2.0.0) prints `MOVED SHAPE (radar) expected_since_1.4.0.toml: 1 watched diff shape(s) disagree with this run` and exits 0, `'Smith Dr'` at 2.0.0 likewise; `'선생님'` corrupted at 2.0.0 prints `MOVED SHAPE expected_since_2.0.0.toml: 1 watched diff shape(s) disagree with this run` ending "This fails the run because the name is contract tier" and exits 1; `'Ph. D., Jr.'` corrupted at 2.1.0 prints the `(radar)` block and exits 0; and `'MD, PHD'` inserted into the 1.4.0 watched section beside its contest row is refused pre-worker, before any `baseline:` line, as "1 name(s) sit in both `_RECORDED_DIFFS['expected_since_1.4.0.toml']` and `_WATCHED_DIFFS['expected_since_1.4.0.toml']`", and fails `test_the_watched_roster_is_disjoint_and_names_every_ledger` naming it — the guard half wrapping `load_tool`, since that helper executes a fresh `compare` module per call and a mutation on an imported one never reaches it. A control whose corruption changed nothing would have meant an inert row; none did. The arc moved no classification: 352 / 247 / 155 / 14 intentional diffs, 0 unexplained and 0 radar-unclassified at every baseline, byte-identical summary lines before and after every commit. The quiet-tree claim that made this the moment to snapshot: measured 2026-09-05 at `4dd776a`, `git rev-list --count a965272..4dd776a -- nameparser/` is 5 commits since the tier split, moving 38 inserted and 15 deleted lines across two files, `_facade.py` (the #462 initials fix and its review rounds) and `_pipeline/_script_segment.py` (`f046d5d`, the tier-split day's own W3 demotion). PINNED TO THAT COMMIT deliberately, and not to HEAD: the very next commit to touch `nameparser/` was this bundle's own docstring-citation trim in `_script_segment.py`, which takes the same range at HEAD to 6 / 41 / 16. So this is a POINT measurement of the tree the snapshot was taken on, and rereading it as a live one is the error to avoid — the figure goes stale by design, since the next feature is what the snapshot exists to make legible. RECOMPUTE the row counts with `{k: len(v) for k, v in compare._WATCHED_DIFFS.items()}`, the population with the script in the dict's header, the controls with `compare._WATCHED_DIFFS[ledger][name] = (...)` around `main()`, and the quiet tree with the command above plus `git diff --shortstat a965272..4dd776a -- nameparser/` (substitute `HEAD` for the live figure, which is a different claim). +The measurement, and how to redo it. `_WATCHED_DIFFS` holds 41 / 36 / 31 / 5 rows at 1.4.0 / 2.0.0 / 2.1.0 / 2.2.0, 113 in all; the 2.0.0 section is 32 sole-watched rows plus the four #501 contests (`'김민준 박사님'` and `'선생님'`, contract tier and fatal; `'田中さん 様.'` and `'田中さん, 様.'`, `corpus_cjk_tolerated.jsonl`, radar and printed — the first of that pair read contract when this was measured earlier the same day, and the period-class demotion under #cjk-comma-demotion above moved it hours later, which is the point the snapshot makes better than the prose could: a row's fatality is a property of the tier its NAME reads at, and no roster in `compare.py` records it). 51 distinct sole-watched names; every row a default-order shape, so `NOT CHECKED` names none of them. Four mutation controls, in memory around `main()` with the worktree untouched, re-run 2026-09-05 against the amended data commit: `'QC MP'` corrupted at 1.4.0 (its only row — it does not diff at 2.0.0) prints `MOVED SHAPE (radar) expected_since_1.4.0.toml: 1 watched diff shape(s) disagree with this run` and exits 0, `'Smith Dr'` at 2.0.0 likewise; `'선생님'` corrupted at 2.0.0 prints `MOVED SHAPE expected_since_2.0.0.toml: 1 watched diff shape(s) disagree with this run` ending "This fails the run because the name is contract tier" and exits 1; `'Ph. D., Jr.'` corrupted at 2.1.0 prints the `(radar)` block and exits 0; and `'MD, PHD'` inserted into the 1.4.0 watched section beside its contest row is refused pre-worker, before any `baseline:` line, as "1 name(s) sit in both `_RECORDED_DIFFS['expected_since_1.4.0.toml']` and `_WATCHED_DIFFS['expected_since_1.4.0.toml']`", and fails `test_the_watched_roster_is_disjoint_and_names_every_ledger` naming it — the guard half wrapping `load_tool`, since that helper executes a fresh `compare` module per call and a mutation on an imported one never reaches it. A control whose corruption changed nothing would have meant an inert row; none did. The arc moved no classification: 352 / 247 / 155 / 14 intentional diffs, 0 unexplained and 0 radar-unclassified at every baseline, byte-identical summary lines before and after every commit. The quiet-tree claim that made this the moment to snapshot: measured 2026-09-05 at `4dd776a`, `git rev-list --count a965272..4dd776a -- nameparser/` is 5 commits since the tier split, moving 38 inserted and 15 deleted lines across two files, `_facade.py` (the #462 initials fix and its review rounds) and `_pipeline/_script_segment.py` (`f046d5d`, the tier-split day's own W3 demotion). PINNED TO THAT COMMIT deliberately, and NO LIVE HEAD FIGURE STANDS BESIDE IT — a first draft of this paragraph put one here as a contrast, and that number was already wrong at the commit that wrote it, which is precisely the defect the pin exists to avoid rather than an illustration of it. What can be said without a number that rots: the period-class bundle's commits after `4dd776a` touched `nameparser/_pipeline/_script_segment.py`'s comments and its W2/W3 citations — and the commit correcting this very sentence was one of them, after a draft that named two by hash had been made stale by the third — so the same range recomputed at HEAD is LARGER than the pinned one on all three counts, and it moves again with every further comment-only commit. Neither the count of such commits nor their hashes belongs here: `git log --oneline 4dd776a..HEAD -- nameparser/` lists them, and a figure that would have to be rewritten by each commit that changes it is not a figure prose can hold. So this is a POINT measurement of the tree the snapshot was taken on, and rereading it as a live one is the error to avoid — the figure goes stale by design, since the next feature is what the snapshot exists to make legible. RECOMPUTE the row counts with `{k: len(v) for k, v in compare._WATCHED_DIFFS.items()}`, the population with the script in the dict's header, the controls with `compare._WATCHED_DIFFS[ledger][name] = (...)` around `main()`, and the quiet tree with the command above plus `git diff --shortstat a965272..4dd776a -- nameparser/` — with the PINNED endpoint and not `HEAD`. A `HEAD` substitution answers a different question, and this paragraph deliberately does not state its answer. ### comma-suffix-arc — #291/#296/#316 (2026-07-26 → 2026-08-01) diff --git a/docs/design/rules.md b/docs/design/rules.md index ca4df7fd..79f24711 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -1111,7 +1111,11 @@ W3. Rationale: a family name declared by a comma is the writer's 田中さん 様 does). A period glued to the honorific's OWN word is read instead, and decides against the split-off: the listed tail no longer ends that word, so nothing peels and the text goes on - whole — 田中さん. and 김민준씨. each read as a title. + whole — 田中さん. and 김민준씨. each read as a title, measured + 2026-09-05. Neither string is a case row or a corpus line, so no + row pins those two readings and they can move without the suite + or the differential saying so; the sentence reports them rather + than promising them, even by this rule's standard. decisions.md#cjk-comma-demotion carries the parses. "남궁민수" → family="남궁" "지훈, 남궁민수" → given="남궁민수" diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index a7aa0c39..e17a7a01 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -298,8 +298,11 @@ def _peel_site(state: ParseState, flat: Sequence[int], # '様'), while a period glued to the honorific's OWN token stands # between the honorific and that token's end, so no listed tail # matches, the peel declines, and the whole text reads as a title -# downstream ('田中さん.', '김민준씨.'). Neither is a promise; the step past the post-nominal -# word itself is W2's, above, and is. +# downstream ('田中さん.', '김민준씨.') -- measured 2026-09-05 and +# pinned by nothing, since neither string is a case row or a corpus +# line, so that reading can move with nothing reporting it. Neither +# is a promise; the step past the post-nominal word itself is W2's, +# above, and is. def _peel_honorific_tail(state: ParseState) -> ParseState: """#308: split a listed honorific off the END of the name's last NON-POST-NOMINAL token -- 田中さん -> 田中 + さん -- and let