From 3c35b85f6a162781af02f8d0205a246b43839a2b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 11 Jul 2026 23:36:15 -0700 Subject: [PATCH 001/275] Add v2 Span, Role, and Token with construction validation Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 54 ++++++++++++++++++++++++++++++++++++++++++ tests/v2/__init__.py | 0 tests/v2/conftest.py | 16 +++++++++++++ tests/v2/test_types.py | 44 ++++++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+) create mode 100644 nameparser/_types.py create mode 100644 tests/v2/__init__.py create mode 100644 tests/v2/conftest.py create mode 100644 tests/v2/test_types.py diff --git a/nameparser/_types.py b/nameparser/_types.py new file mode 100644 index 00000000..cd759265 --- /dev/null +++ b/nameparser/_types.py @@ -0,0 +1,54 @@ +"""Core value types for the 2.0 API. + +Layering (enforced by tests/v2/test_layering.py): this module imports +nothing from nameparser -- it is the bottom of the dependency graph. +""" +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import NamedTuple + + +class Role(Enum): + # Declaration order IS the canonical field order (conventions §3): + # every listing of the seven fields anywhere derives from this. + TITLE = "title" + GIVEN = "given" + MIDDLE = "middle" + FAMILY = "family" + SUFFIX = "suffix" + NICKNAME = "nickname" + MAIDEN = "maiden" + + +class Span(NamedTuple): + """Provenance range into ParsedName.original. end is exclusive.""" + + start: int + end: int + + +#: Stable, documented tag vocabulary (API). All other tags are +#: namespaced ("vocab:...", "patronymic:...") and unstable. +STABLE_TAGS = frozenset({"particle", "conjunction", "initial"}) + + +@dataclass(frozen=True, slots=True) +class Token: + text: str + span: Span | None # None = synthetic (from replace()) + role: Role + tags: frozenset[str] = frozenset() + + def __post_init__(self) -> None: + if not self.text: + raise ValueError("Token.text must be a non-empty string") + if self.span is not None: + start, end = self.span + if start < 0 or end < start: + raise ValueError( + f"invalid span ({start}, {end}): need 0 <= start <= end" + ) + object.__setattr__(self, "span", Span(start, end)) + object.__setattr__(self, "tags", frozenset(self.tags)) diff --git a/tests/v2/__init__.py b/tests/v2/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/v2/conftest.py b/tests/v2/conftest.py new file mode 100644 index 00000000..9bb5e185 --- /dev/null +++ b/tests/v2/conftest.py @@ -0,0 +1,16 @@ +"""Neutralize the v1 suite's autouse dual-run fixture for tests/v2. + +tests/conftest.py runs every test twice (empty_attribute_default '' and +None) and deep-copy-snapshots the shared CONSTANTS around each test. +v2 code never reads shared CONSTANTS, so both behaviors are pure +overhead here. Overriding the fixture by name in this conftest replaces +the parametrized parent version for this directory. +""" +from collections.abc import Iterator + +import pytest + + +@pytest.fixture(autouse=True) +def empty_attribute_default() -> Iterator[None]: + yield diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py new file mode 100644 index 00000000..80db2bb6 --- /dev/null +++ b/tests/v2/test_types.py @@ -0,0 +1,44 @@ +import pytest + +from nameparser._types import Role, Span, Token + + +def test_role_declaration_order_is_canonical_field_order(): + assert [r.value for r in Role] == [ + "title", "given", "middle", "family", "suffix", "nickname", "maiden", + ] + + +def test_token_construction_and_span_coercion(): + t = Token("Juan", (0, 4), Role.GIVEN) + assert t.span == Span(0, 4) + assert isinstance(t.span, Span) + assert t.span.start == 0 and t.span.end == 4 + assert t.tags == frozenset() + + +def test_synthetic_token_has_no_span(): + t = Token("Jane", None, Role.GIVEN) + assert t.span is None + + +def test_token_rejects_empty_text(): + with pytest.raises(ValueError, match="non-empty"): + Token("", (0, 0), Role.GIVEN) + + +def test_token_rejects_inverted_span(): + with pytest.raises(ValueError, match="start <= end"): + Token("x", (5, 2), Role.GIVEN) + + +def test_token_rejects_negative_span(): + with pytest.raises(ValueError, match="start <= end"): + Token("x", (-1, 1), Role.GIVEN) + + +def test_token_is_frozen_and_hashable(): + t = Token("Juan", (0, 4), Role.GIVEN) + with pytest.raises(AttributeError): + t.text = "X" # type: ignore[misc] + assert hash(t) == hash(Token("Juan", (0, 4), Role.GIVEN)) From 2f70ac76476d2dc6636db933dc729af367575307 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 11 Jul 2026 23:37:55 -0700 Subject: [PATCH 002/275] Raise the Python floor to 3.11 on the 2.0 branch (#257, partial) The v2 core design uses enum.StrEnum (3.11+). Full #257 (dropping typing_extensions, CI matrix) remains tracked on the issue. Co-Authored-By: Claude Fable 5 --- .python-version | 2 +- pyproject.toml | 3 +- uv.lock | 146 +++--------------------------------------------- 3 files changed, 10 insertions(+), 141 deletions(-) diff --git a/.python-version b/.python-version index c8cfe395..2c073331 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.10 +3.11 diff --git a/pyproject.toml b/pyproject.toml index 69bfe719..1e9ffef2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "nameparser" description = "A simple Python module for parsing human names into their individual components." readme = "README.rst" -requires-python = ">=3.10" +requires-python = ">=3.11" license = {text = "LGPL"} authors = [{name = "Derek Gulbranson", email = "derek73@gmail.com"}] dynamic = ["version"] @@ -13,7 +13,6 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", diff --git a/uv.lock b/uv.lock index 64369b69..c9b8164c 100644 --- a/uv.lock +++ b/uv.lock @@ -1,11 +1,10 @@ version = 1 revision = 3 -requires-python = ">=3.10" +requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.15'", "python_full_version >= '3.12' and python_full_version < '3.15'", - "python_full_version == '3.11.*'", - "python_full_version < '3.11'", + "python_full_version < '3.12'", ] [[package]] @@ -106,22 +105,6 @@ version = "3.4.7" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/26/08/0f303cb0b529e456bb116f2d50565a482694fbb94340bf56d44677e7ed03/charset_normalizer-3.4.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d", size = 315182, upload-time = "2026-04-02T09:25:40.673Z" }, - { url = "https://files.pythonhosted.org/packages/24/47/b192933e94b546f1b1fe4df9cc1f84fcdbf2359f8d1081d46dd029b50207/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8", size = 209329, upload-time = "2026-04-02T09:25:42.354Z" }, - { url = "https://files.pythonhosted.org/packages/c2/b4/01fa81c5ca6141024d89a8fc15968002b71da7f825dd14113207113fabbd/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790", size = 231230, upload-time = "2026-04-02T09:25:44.281Z" }, - { url = "https://files.pythonhosted.org/packages/20/f7/7b991776844dfa058017e600e6e55ff01984a063290ca5622c0b63162f68/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc", size = 225890, upload-time = "2026-04-02T09:25:45.475Z" }, - { url = "https://files.pythonhosted.org/packages/20/e7/bed0024a0f4ab0c8a9c64d4445f39b30c99bd1acd228291959e3de664247/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393", size = 216930, upload-time = "2026-04-02T09:25:46.58Z" }, - { url = "https://files.pythonhosted.org/packages/e2/ab/b18f0ab31cdd7b3ddb8bb76c4a414aeb8160c9810fdf1bc62f269a539d87/charset_normalizer-3.4.7-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153", size = 202109, upload-time = "2026-04-02T09:25:48.031Z" }, - { url = "https://files.pythonhosted.org/packages/82/e5/7e9440768a06dfb3075936490cb82dbf0ee20a133bf0dd8551fa096914ec/charset_normalizer-3.4.7-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af", size = 214684, upload-time = "2026-04-02T09:25:49.245Z" }, - { url = "https://files.pythonhosted.org/packages/71/94/8c61d8da9f062fdf457c80acfa25060ec22bf1d34bbeaca4350f13bcfd07/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34", size = 212785, upload-time = "2026-04-02T09:25:50.671Z" }, - { url = "https://files.pythonhosted.org/packages/66/cd/6e9889c648e72c0ab2e5967528bb83508f354d706637bc7097190c874e13/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1", size = 203055, upload-time = "2026-04-02T09:25:51.802Z" }, - { url = "https://files.pythonhosted.org/packages/92/2e/7a951d6a08aefb7eb8e1b54cdfb580b1365afdd9dd484dc4bee9e5d8f258/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752", size = 232502, upload-time = "2026-04-02T09:25:53.388Z" }, - { url = "https://files.pythonhosted.org/packages/58/d5/abcf2d83bf8e0a1286df55cd0dc1d49af0da4282aa77e986df343e7de124/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53", size = 214295, upload-time = "2026-04-02T09:25:54.765Z" }, - { url = "https://files.pythonhosted.org/packages/47/3a/7d4cd7ed54be99973a0dc176032cba5cb1f258082c31fa6df35cff46acfc/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616", size = 227145, upload-time = "2026-04-02T09:25:55.904Z" }, - { url = "https://files.pythonhosted.org/packages/1d/98/3a45bf8247889cf28262ebd3d0872edff11565b2a1e3064ccb132db3fbb0/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a", size = 218884, upload-time = "2026-04-02T09:25:57.074Z" }, - { url = "https://files.pythonhosted.org/packages/ad/80/2e8b7f8915ed5c9ef13aa828d82738e33888c485b65ebf744d615040c7ea/charset_normalizer-3.4.7-cp310-cp310-win32.whl", hash = "sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374", size = 148343, upload-time = "2026-04-02T09:25:58.199Z" }, - { url = "https://files.pythonhosted.org/packages/35/1b/3b8c8c77184af465ee9ad88b5aea46ea6b2e1f7b9dc9502891e37af21e30/charset_normalizer-3.4.7-cp310-cp310-win_amd64.whl", hash = "sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943", size = 159174, upload-time = "2026-04-02T09:25:59.322Z" }, - { url = "https://files.pythonhosted.org/packages/be/c1/feb40dca40dbb21e0a908801782d9288c64fc8d8e562c2098e9994c8c21b/charset_normalizer-3.4.7-cp310-cp310-win_arm64.whl", hash = "sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008", size = 147805, upload-time = "2026-04-02T09:26:00.756Z" }, { url = "https://files.pythonhosted.org/packages/c2/d7/b5b7020a0565c2e9fa8c09f4b5fa6232feb326b8c20081ccded47ea368fd/charset_normalizer-3.4.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7", size = 309705, upload-time = "2026-04-02T09:26:02.191Z" }, { url = "https://files.pythonhosted.org/packages/5a/53/58c29116c340e5456724ecd2fff4196d236b98f3da97b404bc5e51ac3493/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7", size = 206419, upload-time = "2026-04-02T09:26:03.583Z" }, { url = "https://files.pythonhosted.org/packages/b2/02/e8146dc6591a37a00e5144c63f29fb7c97a734ea8a111190783c0e60ab63/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e", size = 227901, upload-time = "2026-04-02T09:26:04.738Z" }, @@ -220,20 +203,6 @@ version = "7.15.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/cc/8b/adeb62ea8951f13c4c7fef2e7a85e1a06b499c8d8237ea589d496029e53f/coverage-7.15.0.tar.gz", hash = "sha256:9ac3fe7a1435986463eaa8ee253ae2f2a268709ba4ae5c7dd1f52a05391ad78f", size = 925362, upload-time = "2026-07-02T13:10:50.535Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/97/c52dc440c390b6cfa87be9432b141a956e2d56d9b9f5fc8bd71c5f471722/coverage-7.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:50913d4bf5ddafa6ca3693da5e4dd833dd1b772e0283c99ca7f7d287db67331a", size = 220539, upload-time = "2026-07-02T13:08:19.252Z" }, - { url = "https://files.pythonhosted.org/packages/3f/26/602de8c2aec7e2e3e99ebfb8e04ba65598f746275396eea5f6794ff4673f/coverage-7.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:359e141ccd33893ce3f1ad5525f8b96083003677c82182e5907d62d4ea5799fc", size = 221058, upload-time = "2026-07-02T13:08:21.013Z" }, - { url = "https://files.pythonhosted.org/packages/fc/13/ebab0743138891c1d646d61e247ec29639afcbb6c4e1905e6a0f0c75291a/coverage-7.15.0-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3200b6204935f928c64b2ca1f923ab8c1acb7c9de45ec61569711b34d25cccaf", size = 247797, upload-time = "2026-07-02T13:08:22.474Z" }, - { url = "https://files.pythonhosted.org/packages/d3/b7/b6ffb9e042aa48dc4144a8a65529affaec8dca0685309353614a2a7386ad/coverage-7.15.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:be616bf61346883b2cfdc5178669647e03531d81ab761a7e378558b7e8bcb628", size = 249626, upload-time = "2026-07-02T13:08:23.803Z" }, - { url = "https://files.pythonhosted.org/packages/9c/06/243ff05b652333d8e3d060c11223efc2723b19cacf6605e433fa686ab5d4/coverage-7.15.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc7bafc3fe1059463a8fdd97ca79972d6e2bf819d775c7d54991b5b1971201d6", size = 251493, upload-time = "2026-07-02T13:08:25.397Z" }, - { url = "https://files.pythonhosted.org/packages/d3/2b/867faa17030a806114dae388b32a3fa929d8cd4bf39226fbc11f6e6bb705/coverage-7.15.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b713aa7fcf325a01d4184d848acb46fd84f78fdb0978470c636b23a06a753d91", size = 253406, upload-time = "2026-07-02T13:08:26.842Z" }, - { url = "https://files.pythonhosted.org/packages/94/c0/d789ce18f6605afc4895db75723424be2ef494282f77f61d8e5832923183/coverage-7.15.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e38e6fba2d56652fdfaf0231f8f78aeb805234a912de25dc291ee5cce5b8faa4", size = 248512, upload-time = "2026-07-02T13:08:28.398Z" }, - { url = "https://files.pythonhosted.org/packages/c9/b6/b2673c30739f4a2e06649a0a38ad8b093c4d865462dc7bab0e9524a2c3b1/coverage-7.15.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:884499f42e382675be80770391983b90e0c0c774d87dbeeebf5f991cf6612b20", size = 249532, upload-time = "2026-07-02T13:08:29.731Z" }, - { url = "https://files.pythonhosted.org/packages/3c/2e/acd79e9a41beabee92b623afe4f30b549916f48566271475f2907e752828/coverage-7.15.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:840481b12e083dbcbafab14794a8781a958edf327c8d3d70b4eee42f9b8253aa", size = 247537, upload-time = "2026-07-02T13:08:31.173Z" }, - { url = "https://files.pythonhosted.org/packages/12/d4/2d301c4d1b3238d7c88b70ab9d13fd53ed9505662a7ff1b46ba1e2e4e3c3/coverage-7.15.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:276646e9481703d09f854f3b2f018f24e19fd7049ae670a92570043eb97203b1", size = 251348, upload-time = "2026-07-02T13:08:32.63Z" }, - { url = "https://files.pythonhosted.org/packages/35/bb/c67708b2bc00f32e12805ec23d5fa677a0a51652f449341a89f9d6b1b715/coverage-7.15.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4de4b4d3f5545aa6c60dc4efd9c63b5b5dcc3bf00fe83146b2bdfffb8f6613bd", size = 247806, upload-time = "2026-07-02T13:08:33.931Z" }, - { url = "https://files.pythonhosted.org/packages/eb/6c/57c4f653c47a6e917748f8938e389e72fbcae44e3643cd906664f0477a13/coverage-7.15.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5c504097b2a89b1e85bc6070d920df77daec701337e3aeef2c17775a5dd0ca90", size = 248410, upload-time = "2026-07-02T13:08:35.189Z" }, - { url = "https://files.pythonhosted.org/packages/6c/94/bb083041aef828903668f134273f319f2bd49224962875359c52faa5497f/coverage-7.15.0-cp310-cp310-win32.whl", hash = "sha256:f6e80ed91f98316e86b9c137206b04b2bcfbffccbdff49bd2eb09dddb1cf14e0", size = 222588, upload-time = "2026-07-02T13:08:36.486Z" }, - { url = "https://files.pythonhosted.org/packages/ef/94/a09d8ee618956f626741b0734854bac4425a00e10c0565f5abca64e7e751/coverage-7.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:b3b3e22030f3f6f5e01a5ce69936552a5c0f6992b7698777377b99041961031f", size = 223214, upload-time = "2026-07-02T13:08:37.885Z" }, { url = "https://files.pythonhosted.org/packages/ae/23/82e910835ef4b8391047025e1d53aa48d66029f444eb8b25373c849bf503/coverage-7.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:003fff99412ea848c0aaebcc78ed2b6ce7d8a1227ed17e68470672770b78a02a", size = 220662, upload-time = "2026-07-02T13:08:39.205Z" }, { url = "https://files.pythonhosted.org/packages/6d/0d/c7b213dde2f1579de5231062b386d8413f79c11667eb58c39319b25991da/coverage-7.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5cbd804bf2784ce7b45114516050f346ecd50f960c4bb630a7ee9e1d78fa2118", size = 221168, upload-time = "2026-07-02T13:08:40.471Z" }, { url = "https://files.pythonhosted.org/packages/33/77/d000aeedfac085088337b3c7becdad328474b1f8a9e4c9368a0c99605d68/coverage-7.15.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8773e15c23305b58882a4611fb9b2755977eae0dc2e515366a1b6c98866cc4c2", size = 251587, upload-time = "2026-07-02T13:08:42.033Z" }, @@ -326,44 +295,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, ] -[[package]] -name = "docutils" -version = "0.21.2" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11'", -] -sdist = { url = "https://files.pythonhosted.org/packages/ae/ed/aefcc8cd0ba62a0560c3c18c33925362d46c6075480bfa4df87b28e169a9/docutils-0.21.2.tar.gz", hash = "sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f", size = 2204444, upload-time = "2024-04-23T18:57:18.24Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408, upload-time = "2024-04-23T18:57:14.835Z" }, -] - [[package]] name = "docutils" version = "0.22.4" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.15'", - "python_full_version >= '3.12' and python_full_version < '3.15'", - "python_full_version == '3.11.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/ae/b6/03bb70946330e88ffec97aefd3ea75ba575cb2e762061e0e62a213befee8/docutils-0.22.4.tar.gz", hash = "sha256:4db53b1fde9abecbb74d91230d32ab626d94f6badfc575d6db9194a49df29968", size = 2291750, upload-time = "2025-12-18T19:00:26.443Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl", hash = "sha256:d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de", size = 633196, upload-time = "2025-12-18T19:00:18.077Z" }, ] -[[package]] -name = "exceptiongroup" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, -] - [[package]] name = "furo" version = "2025.12.19" @@ -372,8 +312,7 @@ dependencies = [ { name = "accessible-pygments" }, { name = "beautifulsoup4" }, { name = "pygments" }, - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "sphinx-basic-ng" }, ] @@ -427,18 +366,6 @@ version = "0.11.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/40/08/9e7f6b5d2b5bed6ad055cdd5925f192bb403a51280f86b56554d9d0699a2/librt-0.11.0.tar.gz", hash = "sha256:075dc3ef4458a278e0195cbf6ac9d38808d9b906c5a6c7f7f79c3888276a3fb1", size = 200139, upload-time = "2026-05-10T18:17:25.138Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/83/10/37fd9e9ba96cb0bd742dfb20fc3d082e54bdbec759d7300df927f360ef07/librt-0.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6e94ebfcfa2d5e9926d6c3b9aa4617ffc42a845b4321fb84021b872358c82a0f", size = 141706, upload-time = "2026-05-10T18:15:16.129Z" }, - { url = "https://files.pythonhosted.org/packages/cf/72/1b1466f358e4a0b728051f69bc27e67b432c6eaa2e05b88db49d3785ae0d/librt-0.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ae627397a2f351560440d872d6f7c8dbb4072e57868e7b2fc5b8b430fe489d45", size = 142605, upload-time = "2026-05-10T18:15:18.148Z" }, - { url = "https://files.pythonhosted.org/packages/ca/85/ed26dd2f6bc9a0baf48306433e579e8d354d70b2bcb78134ed950a5d0e1e/librt-0.11.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc329359321b67d24efdf4bc69012b0597001649544db662c001db5a0184794c", size = 476555, upload-time = "2026-05-10T18:15:19.569Z" }, - { url = "https://files.pythonhosted.org/packages/66/fe/11891191c0e0a3fd617724e891f6e67a71a7658974a892b9a9a97fdb2977/librt-0.11.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:7e82e642ab0f7608ce2fe53d76ca2280a9ee33a1b06556142c7c6fe80a86fc33", size = 468434, upload-time = "2026-05-10T18:15:20.87Z" }, - { url = "https://files.pythonhosted.org/packages/6f/50/5ec949d7f9ce1a07af903aa3e13abb98b717923bdead6e719b2f824ccc07/librt-0.11.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:88145c15c67731d54283d135b03244028c750cc9edc334a96a4f5950ebdb2884", size = 496918, upload-time = "2026-05-10T18:15:22.616Z" }, - { url = "https://files.pythonhosted.org/packages/ea/c4/177336c7524e34875a38bf668e88b193a6723a4eb4045d07f74df6e1506c/librt-0.11.0-cp310-cp310-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9d36a51b3d93320b686588e27123f4995804dbf1bce81df78c02fc3c6eea9280", size = 490334, upload-time = "2026-05-10T18:15:24.2Z" }, - { url = "https://files.pythonhosted.org/packages/13/1f/da3112f7569eda3b49f9a2629bae1fe059812b6085df16c885f6454dff49/librt-0.11.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d00f3ac06a2a8b246327f11e186a53a100a4d5c7ed52346367e5ec751d51586c", size = 511287, upload-time = "2026-05-10T18:15:26.226Z" }, - { url = "https://files.pythonhosted.org/packages/fa/94/03fec301522e172d105581431223be56b27594ff46440ebfbb658a3735d5/librt-0.11.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:461bbceede621f1ffb8839755f8663e886087ee7af16294cab7fb4d782c62eeb", size = 517202, upload-time = "2026-05-10T18:15:27.965Z" }, - { url = "https://files.pythonhosted.org/packages/b7/6e/339f6e5a7b413ce014f1917a756dae630fe59cc99f34153205b1cb540901/librt-0.11.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:0cad8a4d6a8ff03c9b76f9414caccd78e7cfbc8a2e12fa334d8e1d9932753783", size = 497517, upload-time = "2026-05-10T18:15:29.614Z" }, - { url = "https://files.pythonhosted.org/packages/cd/43/acdd5ce317cb46e8253ca9bfbdb8b12e68a24d745949336a7f3d5fb79ba0/librt-0.11.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f37aa505b3cf60701562eddb32df74b12a9e380c207fd8b06dd157a943ac7ea0", size = 538878, upload-time = "2026-05-10T18:15:30.928Z" }, - { url = "https://files.pythonhosted.org/packages/29/b5/7a25bb12e3172839f647f196b3e988318b7bb1ca7501732a225c4dce2ec0/librt-0.11.0-cp310-cp310-win32.whl", hash = "sha256:94663a21534637f0e787ec2a2a756022df6e5b7b2335a5cdd7d8e33d68a2af89", size = 100070, upload-time = "2026-05-10T18:15:32.551Z" }, - { url = "https://files.pythonhosted.org/packages/c6/0d/ebbcf4d77999c02c937b05d2b90ff4cd4dcc7e9a365ba132329ac1fe7a0f/librt-0.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:dec7db73758c2b54953fd8b7fe348c45188fe26b39ee18446196edd08453a5d4", size = 117918, upload-time = "2026-05-10T18:15:33.678Z" }, { url = "https://files.pythonhosted.org/packages/fe/87/2bf31fe17587b29e3f93ec31421e2b1e1c3e349b8bf6c7c313dbad1d5340/librt-0.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:93d95bd45b7d58343d8b90d904450a545144eec19a002511163426f8ab1fae29", size = 141092, upload-time = "2026-05-10T18:15:34.795Z" }, { url = "https://files.pythonhosted.org/packages/cf/08/5c5bf772920b7ebac6e32bc91a643e0ab3870199c0b542356d3baa83970a/librt-0.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ee278c769a713638cdacd4c0436d72156e75df3ebc0166ab2b9dc43acc386c9", size = 142035, upload-time = "2026-05-10T18:15:36.242Z" }, { url = "https://files.pythonhosted.org/packages/06/20/662a03d254e5b000d838e8b345d83303ddb768c080fd488e40634c0fa66b/librt-0.11.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f230cb1cbc9faaa616f9a678f530ebcf186e414b6bcbd88b960e4ba1b92428d5", size = 475022, upload-time = "2026-05-10T18:15:37.56Z" }, @@ -512,17 +439,6 @@ version = "3.0.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/4b/3541d44f3937ba468b75da9eebcae497dcf67adb65caa16760b0a6807ebb/markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559", size = 11631, upload-time = "2025-09-27T18:36:05.558Z" }, - { url = "https://files.pythonhosted.org/packages/98/1b/fbd8eed11021cabd9226c37342fa6ca4e8a98d8188a8d9b66740494960e4/markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419", size = 12057, upload-time = "2025-09-27T18:36:07.165Z" }, - { url = "https://files.pythonhosted.org/packages/40/01/e560d658dc0bb8ab762670ece35281dec7b6c1b33f5fbc09ebb57a185519/markupsafe-3.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695", size = 22050, upload-time = "2025-09-27T18:36:08.005Z" }, - { url = "https://files.pythonhosted.org/packages/af/cd/ce6e848bbf2c32314c9b237839119c5a564a59725b53157c856e90937b7a/markupsafe-3.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591", size = 20681, upload-time = "2025-09-27T18:36:08.881Z" }, - { url = "https://files.pythonhosted.org/packages/c9/2a/b5c12c809f1c3045c4d580b035a743d12fcde53cf685dbc44660826308da/markupsafe-3.0.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c", size = 20705, upload-time = "2025-09-27T18:36:10.131Z" }, - { url = "https://files.pythonhosted.org/packages/cf/e3/9427a68c82728d0a88c50f890d0fc072a1484de2f3ac1ad0bfc1a7214fd5/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f", size = 21524, upload-time = "2025-09-27T18:36:11.324Z" }, - { url = "https://files.pythonhosted.org/packages/bc/36/23578f29e9e582a4d0278e009b38081dbe363c5e7165113fad546918a232/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6", size = 20282, upload-time = "2025-09-27T18:36:12.573Z" }, - { url = "https://files.pythonhosted.org/packages/56/21/dca11354e756ebd03e036bd8ad58d6d7168c80ce1fe5e75218e4945cbab7/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1", size = 20745, upload-time = "2025-09-27T18:36:13.504Z" }, - { url = "https://files.pythonhosted.org/packages/87/99/faba9369a7ad6e4d10b6a5fbf71fa2a188fe4a593b15f0963b73859a1bbd/markupsafe-3.0.3-cp310-cp310-win32.whl", hash = "sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa", size = 14571, upload-time = "2025-09-27T18:36:14.779Z" }, - { url = "https://files.pythonhosted.org/packages/d6/25/55dc3ab959917602c96985cb1253efaa4ff42f71194bddeb61eb7278b8be/markupsafe-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8", size = 15056, upload-time = "2025-09-27T18:36:16.125Z" }, - { url = "https://files.pythonhosted.org/packages/d0/9e/0a02226640c255d1da0b8d12e24ac2aa6734da68bff14c05dd53b94a0fc3/markupsafe-3.0.3-cp310-cp310-win_arm64.whl", hash = "sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1", size = 13932, upload-time = "2025-09-27T18:36:17.311Z" }, { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, @@ -600,18 +516,10 @@ dependencies = [ { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, { name = "mypy-extensions" }, { name = "pathspec" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/82/15/cca9d88503549ed6fedeaa1d448cdddd542ee8a490232d732e278036fbf2/mypy-2.1.0.tar.gz", hash = "sha256:81e76ad12c2d804512e9b13240d1588316531bfba07558286078bfbce9613633", size = 3898359, upload-time = "2026-05-11T18:37:36.237Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/71/d351dca3e9b30da2328ee9d445c88b8388072808ebfbc49eb69d30b67749/mypy-2.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:11a6beb180257a805961aea9ec591bbd0bd17f1e18d35b8456d57aee5bedfedc", size = 14778792, upload-time = "2026-05-11T18:36:23.605Z" }, - { url = "https://files.pythonhosted.org/packages/2f/45/7d51594b644c17c0bcf74ed8cd5fc33b324276d708e8506f220b70dab9d9/mypy-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8ef78c1d306bbf9a8a12f526c44902c9c28dffd6c52c52bf6a72641ce18d3849", size = 13645739, upload-time = "2026-05-11T18:37:22.752Z" }, - { url = "https://files.pythonhosted.org/packages/65/01/455c31b170e9468265074840bf18863a8482a24103fdaabe4e199392aa5f/mypy-2.1.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c209a90853081ff01d01ee895cafe10f7db1474e0d95beaeef0f6c1db9119bbd", size = 14074199, upload-time = "2026-05-11T18:35:09.292Z" }, - { url = "https://files.pythonhosted.org/packages/41/5a/93093f0b29a9e982deafde698f740a2eb2e05886e79ccf0594c7fd5413a3/mypy-2.1.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47cebf61abde7c088a4e27718a8b13a81655686b2e9c251f5c0915a802248166", size = 14953128, upload-time = "2026-05-11T18:31:57.678Z" }, - { url = "https://files.pythonhosted.org/packages/7f/2f/a196f5331d96170ad3d28f144d2aba690d4b2911381f68d51e489c7ab82a/mypy-2.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d57a90ae5e872138a425ec328edbc9b235d1934c4377881a33ec05b341acc9a8", size = 15249378, upload-time = "2026-05-11T18:33:00.101Z" }, - { url = "https://files.pythonhosted.org/packages/54/de/94d321cc12da9f71341ac0c270efbed5c725750c7b4c334d957de9a087d9/mypy-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:aea7f7a8a55b459c34275fc468ada6ca7c173a5e43a68f5dbe588a563d8a06b8", size = 11060994, upload-time = "2026-05-11T18:33:18.848Z" }, - { url = "https://files.pythonhosted.org/packages/e1/62/0c27ca55219a7c764a7fb88c7bb2b7b2f9780ade8bbf16bc8ed8400eef6b/mypy-2.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:c989640253f0d76843e9c6c1bbf4bd48c5e85ada61bde4beb37cb3eca035685e", size = 9976743, upload-time = "2026-05-11T18:31:25.554Z" }, { url = "https://files.pythonhosted.org/packages/0a/a1/639f3024794a2a15899cb90707fe02e044c4412794c39c5769fd3df2e2ef/mypy-2.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a683016b16fe2f572dc04c72be7ee0504ac1605a265d0200f5cea695fb788f41", size = 14691685, upload-time = "2026-05-11T18:33:27.973Z" }, { url = "https://files.pythonhosted.org/packages/3b/08/9a585dea4325f20d8b80dc78623fa50d1fd2173b710f6237afd6ba6ab39b/mypy-2.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1a293c534adb55271fef24a26da04b855540a8c13cc07bc5917b9fd2c394f2ca", size = 13555165, upload-time = "2026-05-11T18:32:16.107Z" }, { url = "https://files.pythonhosted.org/packages/81/dc/7c42cc9c6cb01e8eb09961f1f738741d3e9c7e9d5c5b30ec69222625cd5f/mypy-2.1.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7406f4d048e71e576f5356d317e5b0a9e666dfd966bd99f9d14ca06e1a341538", size = 13994376, upload-time = "2026-05-11T18:32:39.256Z" }, @@ -662,9 +570,6 @@ wheels = [ [[package]] name = "nameparser" source = { editable = "." } -dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, -] [package.dev-dependencies] dev = [ @@ -675,8 +580,7 @@ dev = [ { name = "pytest-cov" }, { name = "pytest-timeout" }, { name = "ruff" }, - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] @@ -737,12 +641,10 @@ version = "9.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, { name = "pygments" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/84/0e/b5858858d74958632c49b72cb25a3976ff9f632397626715be71c89d3971/pytest-9.1.0.tar.gz", hash = "sha256:41dd9148c08072446394cefd3d79701701335a9f4cae69ba92e39f6c7f5c061c", size = 1634181, upload-time = "2026-06-13T18:52:45.983Z" } wheels = [ @@ -842,49 +744,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5e/f5/0c41cb68dcae6b7de4fac4188a3a9589e21fb31df21ea3a2e888db95e6c9/soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65", size = 37304, upload-time = "2026-05-24T13:55:55.406Z" }, ] -[[package]] -name = "sphinx" -version = "8.1.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11'", -] -dependencies = [ - { name = "alabaster" }, - { name = "babel" }, - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" } }, - { name = "imagesize" }, - { name = "jinja2" }, - { name = "packaging" }, - { name = "pygments" }, - { name = "requests" }, - { name = "snowballstemmer" }, - { name = "sphinxcontrib-applehelp" }, - { name = "sphinxcontrib-devhelp" }, - { name = "sphinxcontrib-htmlhelp" }, - { name = "sphinxcontrib-jsmath" }, - { name = "sphinxcontrib-qthelp" }, - { name = "sphinxcontrib-serializinghtml" }, - { name = "tomli" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/be0b61178fe2cdcb67e2a92fc9ebb488e3c51c4f74a36a7824c0adf23425/sphinx-8.1.3.tar.gz", hash = "sha256:43c1911eecb0d3e161ad78611bc905d1ad0e523e4ddc202a58a821773dc4c927", size = 8184611, upload-time = "2024-10-13T20:27:13.93Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/60/1ddff83a56d33aaf6f10ec8ce84b4c007d9368b21008876fceda7e7381ef/sphinx-8.1.3-py3-none-any.whl", hash = "sha256:09719015511837b76bf6e03e42eb7595ac8c2e41eeb9c29c5b755c6b677992a2", size = 3487125, upload-time = "2024-10-13T20:27:10.448Z" }, -] - [[package]] name = "sphinx" version = "9.0.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.11.*'", + "python_full_version < '3.12'", ] dependencies = [ { name = "alabaster" }, { name = "babel" }, { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "docutils", version = "0.22.4", source = { registry = "https://pypi.org/simple" } }, + { name = "docutils" }, { name = "imagesize" }, { name = "jinja2" }, { name = "packaging" }, @@ -916,7 +787,7 @@ dependencies = [ { name = "alabaster" }, { name = "babel" }, { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "docutils", version = "0.22.4", source = { registry = "https://pypi.org/simple" } }, + { name = "docutils" }, { name = "imagesize" }, { name = "jinja2" }, { name = "packaging" }, @@ -941,8 +812,7 @@ name = "sphinx-basic-ng" version = "1.0.0b2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/98/0b/a866924ded68efec7a1759587a4e478aec7559d8165fac8b2ad1c0e774d6/sphinx_basic_ng-1.0.0b2.tar.gz", hash = "sha256:9ec55a47c90c8c002b5960c57492ec3021f5193cb26cebc2dc4ea226848651c9", size = 20736, upload-time = "2023-07-08T18:40:54.166Z" } From 0a416ed8585da4bafdf320ab8366f5625e167e62 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 11 Jul 2026 23:44:22 -0700 Subject: [PATCH 003/275] Fail loud on malformed Token spans and non-string text Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 15 +++++++++++++-- tests/v2/test_types.py | 12 ++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/nameparser/_types.py b/nameparser/_types.py index cd759265..77b7b893 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -42,9 +42,20 @@ class Token: tags: frozenset[str] = frozenset() def __post_init__(self) -> None: - if not self.text: - raise ValueError("Token.text must be a non-empty string") + if not isinstance(self.text, str) or not self.text: + raise ValueError( + f"Token.text must be a non-empty string, got {self.text!r}" + ) if self.span is not None: + if not ( + isinstance(self.span, tuple) + and len(self.span) == 2 + and all(isinstance(v, int) for v in self.span) + ): + raise ValueError( + f"invalid span {self.span!r}: expected a (start, end) " + "pair of ints or None" + ) start, end = self.span if start < 0 or end < start: raise ValueError( diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 80db2bb6..e1ba6bfd 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -37,6 +37,18 @@ def test_token_rejects_negative_span(): Token("x", (-1, 1), Role.GIVEN) +def test_token_rejects_malformed_span_shapes(): + with pytest.raises(ValueError, match="expected a \\(start, end\\) pair"): + Token("x", (0, 4, 9), Role.GIVEN) # type: ignore[arg-type] + with pytest.raises(ValueError, match="expected a \\(start, end\\) pair"): + Token("x", 5, Role.GIVEN) # type: ignore[arg-type] + + +def test_token_rejects_non_string_text(): + with pytest.raises(ValueError, match="got None"): + Token(None, None, Role.GIVEN) # type: ignore[arg-type] + + def test_token_is_frozen_and_hashable(): t = Token("Juan", (0, 4), Role.GIVEN) with pytest.raises(AttributeError): From 3f3d4488a3136b5fc4040f2d4b7160cc2b5588fb Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 11 Jul 2026 23:47:16 -0700 Subject: [PATCH 004/275] Add v2 AmbiguityKind StrEnum and Ambiguity type Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 30 +++++++++++++++++++++++++++++- tests/v2/test_types.py | 20 ++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/nameparser/_types.py b/nameparser/_types.py index 77b7b893..43bc470d 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -6,7 +6,7 @@ from __future__ import annotations from dataclasses import dataclass -from enum import Enum +from enum import Enum, StrEnum from typing import NamedTuple @@ -63,3 +63,31 @@ def __post_init__(self) -> None: ) object.__setattr__(self, "span", Span(start, end)) object.__setattr__(self, "tags", frozenset(self.tags)) + + +class AmbiguityKind(StrEnum): + """Stable identifiers (API); members ARE their string values.""" + + ORDER = "order" + SUFFIX_OR_NICKNAME = "suffix-or-nickname" + PARTICLE_OR_GIVEN = "particle-or-given" + UNBALANCED_DELIMITER = "unbalanced-delimiter" + COMMA_STRUCTURE = "comma-structure" + + +@dataclass(frozen=True, slots=True) +class Ambiguity: + kind: AmbiguityKind + detail: str + tokens: tuple[Token, ...] + + def __post_init__(self) -> None: + if not isinstance(self.kind, AmbiguityKind): + try: + object.__setattr__(self, "kind", AmbiguityKind(self.kind)) + except ValueError: + valid = ", ".join(k.value for k in AmbiguityKind) + raise ValueError( + f"unknown AmbiguityKind {self.kind!r}; valid kinds: {valid}" + ) from None + object.__setattr__(self, "tokens", tuple(self.tokens)) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index e1ba6bfd..0389badb 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -54,3 +54,23 @@ def test_token_is_frozen_and_hashable(): with pytest.raises(AttributeError): t.text = "X" # type: ignore[misc] assert hash(t) == hash(Token("Juan", (0, 4), Role.GIVEN)) + + +from nameparser._types import Ambiguity, AmbiguityKind + + +def test_ambiguity_kind_members_are_their_string_values(): + assert AmbiguityKind.PARTICLE_OR_GIVEN == "particle-or-given" + assert AmbiguityKind("order") is AmbiguityKind.ORDER + + +def test_ambiguity_construction_coerces_kind_string(): + t = Token("Van", (0, 3), Role.GIVEN, frozenset({"particle"})) + a = Ambiguity("particle-or-given", "leading 'van' may be a particle", (t,)) + assert a.kind is AmbiguityKind.PARTICLE_OR_GIVEN + assert a.tokens == (t,) + + +def test_ambiguity_rejects_unknown_kind(): + with pytest.raises(ValueError, match="particle-or-given"): + Ambiguity("no-such-kind", "detail", ()) From de78ebf7f648354fd5d81cbbe9e718495fc9151e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 11 Jul 2026 23:50:10 -0700 Subject: [PATCH 005/275] Hoist v2 test imports to the top-of-file block Co-Authored-By: Claude Fable 5 --- tests/v2/test_types.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 0389badb..42424a9d 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -1,6 +1,6 @@ import pytest -from nameparser._types import Role, Span, Token +from nameparser._types import Ambiguity, AmbiguityKind, Role, Span, Token def test_role_declaration_order_is_canonical_field_order(): @@ -56,9 +56,6 @@ def test_token_is_frozen_and_hashable(): assert hash(t) == hash(Token("Juan", (0, 4), Role.GIVEN)) -from nameparser._types import Ambiguity, AmbiguityKind - - def test_ambiguity_kind_members_are_their_string_values(): assert AmbiguityKind.PARTICLE_OR_GIVEN == "particle-or-given" assert AmbiguityKind("order") is AmbiguityKind.ORDER From 072e2da1f50a05f152b00baaef2651c084501b03 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 11 Jul 2026 23:53:52 -0700 Subject: [PATCH 006/275] Validate Ambiguity detail and token element types Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 13 ++++++++++++- tests/v2/test_types.py | 10 ++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/nameparser/_types.py b/nameparser/_types.py index 43bc470d..7081069c 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -90,4 +90,15 @@ def __post_init__(self) -> None: raise ValueError( f"unknown AmbiguityKind {self.kind!r}; valid kinds: {valid}" ) from None - object.__setattr__(self, "tokens", tuple(self.tokens)) + if not isinstance(self.detail, str) or not self.detail: + raise ValueError( + f"Ambiguity.detail must be a non-empty string, got {self.detail!r}" + ) + toks = tuple(self.tokens) + for tok in toks: + if not isinstance(tok, Token): + raise ValueError( + f"Ambiguity.tokens must contain only Token instances, " + f"got {tok!r}" + ) + object.__setattr__(self, "tokens", toks) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 42424a9d..573e1c4f 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -71,3 +71,13 @@ def test_ambiguity_construction_coerces_kind_string(): def test_ambiguity_rejects_unknown_kind(): with pytest.raises(ValueError, match="particle-or-given"): Ambiguity("no-such-kind", "detail", ()) + + +def test_ambiguity_rejects_non_token_elements(): + with pytest.raises(ValueError, match="only Token instances"): + Ambiguity("order", "detail", ("not-a-token",)) # type: ignore[arg-type] + + +def test_ambiguity_rejects_empty_detail(): + with pytest.raises(ValueError, match="non-empty string"): + Ambiguity("order", "", ()) From ef6d2b833de95b13cc8790f235a43d71d6edd3c8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 11 Jul 2026 23:54:31 -0700 Subject: [PATCH 007/275] Add v2 ParsedName with constructor-enforced span invariants Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 44 +++++++++++++++++++++++++++ tests/v2/test_types.py | 67 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/nameparser/_types.py b/nameparser/_types.py index 7081069c..2d9bdbce 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -102,3 +102,47 @@ def __post_init__(self) -> None: f"got {tok!r}" ) object.__setattr__(self, "tokens", toks) + + +@dataclass(frozen=True, slots=True) +class ParsedName: + """Immutable result of a parse. Constructor-enforced invariants: + spans ascending, non-overlapping, in bounds of `original`; every + Ambiguity's tokens are a subset of `tokens`. Provenance semantics + (text == original[span] for parser-produced names) are documented, + not enforced -- transforms like replace() legitimately break them. + """ + + original: str + tokens: tuple[Token, ...] + ambiguities: tuple[Ambiguity, ...] = () + + def __post_init__(self) -> None: + object.__setattr__(self, "tokens", tuple(self.tokens)) + object.__setattr__(self, "ambiguities", tuple(self.ambiguities)) + prev_end = 0 + for tok in self.tokens: + if tok.span is None: + continue + if tok.span.end > len(self.original): + raise ValueError( + f"token {tok.text!r} span {tuple(tok.span)} is out of " + f"bounds for original of length {len(self.original)}" + ) + if tok.span.start < prev_end: + raise ValueError( + f"token spans must be ascending and non-overlapping; " + f"token {tok.text!r} at {tuple(tok.span)} begins before " + f"offset {prev_end}" + ) + prev_end = tok.span.end + for amb in self.ambiguities: + for tok in amb.tokens: + if tok not in self.tokens: + raise ValueError( + f"Ambiguity token {tok.text!r} is not a subset of " + f"this ParsedName's tokens" + ) + + def __bool__(self) -> bool: + return bool(self.tokens) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 573e1c4f..5dcd79e3 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -1,6 +1,6 @@ import pytest -from nameparser._types import Ambiguity, AmbiguityKind, Role, Span, Token +from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token def test_role_declaration_order_is_canonical_field_order(): @@ -81,3 +81,68 @@ def test_ambiguity_rejects_non_token_elements(): def test_ambiguity_rejects_empty_detail(): with pytest.raises(ValueError, match="non-empty string"): Ambiguity("order", "", ()) + + +def _pn(original, tokens, ambiguities=()): + return ParsedName(original=original, tokens=tuple(tokens), + ambiguities=tuple(ambiguities)) + + +def test_parsedname_accepts_valid_spans_and_is_truthy(): + pn = _pn("John Smith", [ + Token("John", (0, 4), Role.GIVEN), + Token("Smith", (5, 10), Role.FAMILY), + ]) + assert bool(pn) is True + + +def test_empty_parse_is_falsy(): + assert bool(_pn("", [])) is False + assert bool(_pn(" ", [])) is False + + +def test_parsedname_rejects_out_of_bounds_span(): + with pytest.raises(ValueError, match="out of bounds"): + _pn("John", [Token("Johnny", (0, 6), Role.GIVEN)]) + + +def test_parsedname_rejects_overlapping_spans(): + with pytest.raises(ValueError, match="ascending"): + _pn("John Smith", [ + Token("John", (0, 4), Role.GIVEN), + Token("ohn S", (1, 6), Role.FAMILY), + ]) + + +def test_parsedname_rejects_descending_spans(): + with pytest.raises(ValueError, match="ascending"): + _pn("John Smith", [ + Token("Smith", (5, 10), Role.FAMILY), + Token("John", (0, 4), Role.GIVEN), + ]) + + +def test_synthetic_tokens_skip_span_checks(): + pn = _pn("John Smith", [ + Token("John", (0, 4), Role.GIVEN), + Token("Qux", None, Role.MIDDLE), + Token("Smith", (5, 10), Role.FAMILY), + ]) + assert len(pn.tokens) == 3 + + +def test_ambiguity_tokens_must_be_subset_of_parse_tokens(): + inside = Token("Van", (0, 3), Role.GIVEN) + outside = Token("Zzz", None, Role.GIVEN) + with pytest.raises(ValueError, match="subset"): + _pn("Van Johnson", + [inside, Token("Johnson", (4, 11), Role.FAMILY)], + [Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "d", (outside,))]) + + +def test_parsedname_equality_is_strict_structural(): + a = _pn("John", [Token("John", (0, 4), Role.GIVEN)]) + b = _pn("John", [Token("John", (0, 4), Role.GIVEN)]) + c = _pn("John ", [Token("John", (0, 4), Role.GIVEN)]) + assert a == b and hash(a) == hash(b) + assert a != c # different original: not interchangeable From 4a79080a171d872e89808d3da441abde968a61b9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 11 Jul 2026 23:59:49 -0700 Subject: [PATCH 008/275] Validate ParsedName element types and original Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 16 ++++++++++++++++ tests/v2/test_types.py | 12 ++++++++++++ 2 files changed, 28 insertions(+) diff --git a/nameparser/_types.py b/nameparser/_types.py index 2d9bdbce..efede7e9 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -118,8 +118,24 @@ class ParsedName: ambiguities: tuple[Ambiguity, ...] = () def __post_init__(self) -> None: + if not isinstance(self.original, str): + raise ValueError( + f"ParsedName.original must be a str, got {self.original!r}" + ) object.__setattr__(self, "tokens", tuple(self.tokens)) object.__setattr__(self, "ambiguities", tuple(self.ambiguities)) + for tok in self.tokens: + if not isinstance(tok, Token): + raise ValueError( + f"ParsedName.tokens must contain only Token instances, " + f"got {tok!r}" + ) + for amb in self.ambiguities: + if not isinstance(amb, Ambiguity): + raise ValueError( + f"ParsedName.ambiguities must contain only Ambiguity " + f"instances, got {amb!r}" + ) prev_end = 0 for tok in self.tokens: if tok.span is None: diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 5dcd79e3..b14f9f90 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -146,3 +146,15 @@ def test_parsedname_equality_is_strict_structural(): c = _pn("John ", [Token("John", (0, 4), Role.GIVEN)]) assert a == b and hash(a) == hash(b) assert a != c # different original: not interchangeable + + +def test_parsedname_rejects_non_str_original(): + with pytest.raises(ValueError, match="must be a str"): + _pn(None, []) # type: ignore[arg-type] + + +def test_parsedname_rejects_non_token_and_non_ambiguity_elements(): + with pytest.raises(ValueError, match="only Token instances"): + _pn("x", ["not-a-token"]) # type: ignore[list-item] + with pytest.raises(ValueError, match="only Ambiguity instances"): + _pn("John", [Token("John", (0, 4), Role.GIVEN)], ["nope"]) # type: ignore[list-item] From 8e8160b870e892637ccf345cf4c928a116a0d960 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:00:21 -0700 Subject: [PATCH 009/275] Add v2 ParsedName string properties, derived views, and as_dict Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 74 ++++++++++++++++++++++++++++++++++++++++++ tests/v2/test_types.py | 58 +++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) diff --git a/nameparser/_types.py b/nameparser/_types.py index efede7e9..9d31a790 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -162,3 +162,77 @@ def __post_init__(self) -> None: def __bool__(self) -> bool: return bool(self.tokens) + + # -- string views (canonical order = Role declaration order) -------- + + def _text_for(self, *roles: Role, tag: str | None = None, + without_tag: str | None = None) -> str: + joiner = ", " if roles == (Role.SUFFIX,) else " " + parts = [] + for tok in self.tokens: + if tok.role not in roles: + continue + if tag is not None and tag not in tok.tags: + continue + if without_tag is not None and without_tag in tok.tags: + continue + parts.append(tok.text) + return joiner.join(parts) + + @property + def title(self) -> str: + return self._text_for(Role.TITLE) + + @property + def given(self) -> str: + return self._text_for(Role.GIVEN) + + @property + def middle(self) -> str: + return self._text_for(Role.MIDDLE) + + @property + def family(self) -> str: + return self._text_for(Role.FAMILY) + + @property + def suffix(self) -> str: + return self._text_for(Role.SUFFIX) + + @property + def nickname(self) -> str: + return self._text_for(Role.NICKNAME) + + @property + def maiden(self) -> str: + return self._text_for(Role.MAIDEN) + + # -- derived views (filters over roles + STABLE tags only) ---------- + + @property + def family_particles(self) -> str: + return self._text_for(Role.FAMILY, tag="particle") + + @property + def family_base(self) -> str: + return self._text_for(Role.FAMILY, without_tag="particle") + + @property + def surnames(self) -> str: + return self._text_for(Role.MIDDLE, Role.FAMILY) + + @property + def given_names(self) -> str: + return self._text_for(Role.GIVEN, Role.MIDDLE) + + # -- structured access ---------------------------------------------- + + def tokens_for(self, role: Role) -> tuple[Token, ...]: + return tuple(t for t in self.tokens if t.role is role) + + def as_dict(self, include_empty: bool = True) -> dict[str, str]: + # _text_for handles the suffix ", "-join (single-role SUFFIX call) + d = {role.value: self._text_for(role) for role in Role} + if not include_empty: + d = {k: v for k, v in d.items() if v} + return d diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index b14f9f90..ba5241b0 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -158,3 +158,61 @@ def test_parsedname_rejects_non_token_and_non_ambiguity_elements(): _pn("x", ["not-a-token"]) # type: ignore[list-item] with pytest.raises(ValueError, match="only Ambiguity instances"): _pn("John", [Token("John", (0, 4), Role.GIVEN)], ["nope"]) # type: ignore[list-item] + + +def _delavega(): + # "Dr. Juan de la Vega III" -- hand-built, spans verified by hand + # 0123456789012345678901234 + return _pn("Dr. Juan de la Vega III", [ + Token("Dr.", (0, 3), Role.TITLE), + Token("Juan", (4, 8), Role.GIVEN), + Token("de", (9, 11), Role.FAMILY, frozenset({"particle"})), + Token("la", (12, 14), Role.FAMILY, frozenset({"particle"})), + Token("Vega", (15, 19), Role.FAMILY), + Token("III", (20, 23), Role.SUFFIX), + ]) + + +def test_string_properties_join_by_role(): + pn = _delavega() + assert pn.title == "Dr." + assert pn.given == "Juan" + assert pn.middle == "" + assert pn.family == "de la Vega" + assert pn.suffix == "III" + assert pn.nickname == "" + assert pn.maiden == "" + + +def test_suffix_joins_with_comma_space(): + pn = _pn("John Smith PhD MD", [ + Token("John", (0, 4), Role.GIVEN), + Token("Smith", (5, 10), Role.FAMILY), + Token("PhD", (11, 14), Role.SUFFIX), + Token("MD", (15, 17), Role.SUFFIX), + ]) + assert pn.suffix == "PhD, MD" + + +def test_derived_views_filter_on_stable_particle_tag(): + pn = _delavega() + assert pn.family_particles == "de la" + assert pn.family_base == "Vega" + assert pn.surnames == "de la Vega" # middle + family + assert pn.given_names == "Juan" # given + middle + + +def test_tokens_for_preserves_order(): + pn = _delavega() + assert [t.text for t in pn.tokens_for(Role.FAMILY)] == ["de", "la", "Vega"] + assert pn.tokens_for(Role.NICKNAME) == () + + +def test_as_dict_canonical_order_and_empty_filtering(): + pn = _delavega() + d = pn.as_dict() + assert list(d) == ["title", "given", "middle", "family", + "suffix", "nickname", "maiden"] + assert d["family"] == "de la Vega" and d["middle"] == "" + d2 = pn.as_dict(include_empty=False) + assert list(d2) == ["title", "given", "family", "suffix"] From 3190abd3c34eb07ca3d9b9aead8146aa879f1737 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:05:29 -0700 Subject: [PATCH 010/275] Add v2 ParsedName.replace() with positional synthetic tokens Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 37 +++++++++++++++++++++++++++++++++++++ tests/v2/test_types.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/nameparser/_types.py b/nameparser/_types.py index 9d31a790..25fa4eff 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -236,3 +236,40 @@ def as_dict(self, include_empty: bool = True) -> dict[str, str]: if not include_empty: d = {k: v for k, v in d.items() if v} return d + + # -- editing ---------------------------------------------------------- + + def replace(self, **fields: str) -> "ParsedName": + """Return a new ParsedName with the named fields re-tokenized as + synthetic tokens (span=None). Whitespace-splits each value; an + empty value clears the field. original is unchanged (provenance). + """ + by_value = {role.value: role for role in Role} + for key in fields: + if key not in by_value: + raise TypeError( + f"unknown field {key!r}; expected one of " + f"{', '.join(by_value)}" + ) + + def synthetic(value: str, role: Role) -> list[Token]: + return [Token(word, None, role) for word in value.split()] + + replaced = {by_value[k]: v for k, v in fields.items()} + new_tokens: list[Token] = [] + emitted: set[Role] = set() + for tok in self.tokens: + if tok.role in replaced: + if tok.role not in emitted: + new_tokens.extend(synthetic(replaced[tok.role], tok.role)) + emitted.add(tok.role) + continue + new_tokens.append(tok) + for role, value in replaced.items(): + if role not in emitted: + new_tokens.extend(synthetic(value, role)) + kept = tuple( + amb for amb in self.ambiguities + if all(t in new_tokens for t in amb.tokens) + ) + return ParsedName(self.original, tuple(new_tokens), kept) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index ba5241b0..7889ead9 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -216,3 +216,44 @@ def test_as_dict_canonical_order_and_empty_filtering(): assert d["family"] == "de la Vega" and d["middle"] == "" d2 = pn.as_dict(include_empty=False) assert list(d2) == ["title", "given", "family", "suffix"] + + +def test_replace_swaps_field_with_synthetic_tokens_in_place(): + pn = _delavega() + pn2 = pn.replace(given="Jean Paul") + assert pn2.given == "Jean Paul" + assert pn2.family == "de la Vega" # untouched + assert pn2.original == pn.original # provenance unchanged + assert all(t.span is None for t in pn2.tokens_for(Role.GIVEN)) + assert pn.given == "Juan" # source object unchanged + # positional: synthetic given tokens sit where the old ones were + assert [t.role for t in pn2.tokens][:3] == [Role.TITLE, Role.GIVEN, Role.GIVEN] + + +def test_replace_adds_missing_field_at_end(): + pn = _pn("John Smith", [ + Token("John", (0, 4), Role.GIVEN), + Token("Smith", (5, 10), Role.FAMILY), + ]) + pn2 = pn.replace(suffix="Jr") + assert pn2.suffix == "Jr" + assert pn2.tokens[-1].role is Role.SUFFIX + + +def test_replace_with_empty_string_clears_field(): + pn = _delavega() + assert pn.replace(title="").title == "" + + +def test_replace_rejects_unknown_field(): + with pytest.raises(TypeError, match="given"): + _delavega().replace(firstname="X") + + +def test_replace_drops_ambiguities_referencing_removed_tokens(): + van = Token("Van", (0, 3), Role.GIVEN) + pn = _pn("Van Johnson", + [van, Token("Johnson", (4, 11), Role.FAMILY)], + [Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "d", (van,))]) + assert pn.replace(given="Bob").ambiguities == () + assert pn.replace(family="Smith").ambiguities != () From 06a7e221aa680c6399e90c7bcd0ab8f6f66555fd Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:10:32 -0700 Subject: [PATCH 011/275] Fail loud on non-str replace values; canonical append order - Validate that all replace() field values are str (user error if None) - Append missing-role synthetics in canonical Role order, not kwargs order - Unquote return type annotation (postponed annotations in effect) Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 14 +++++++++----- tests/v2/test_types.py | 11 +++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/nameparser/_types.py b/nameparser/_types.py index 25fa4eff..515eeb39 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -239,18 +239,22 @@ def as_dict(self, include_empty: bool = True) -> dict[str, str]: # -- editing ---------------------------------------------------------- - def replace(self, **fields: str) -> "ParsedName": + def replace(self, **fields: str) -> ParsedName: """Return a new ParsedName with the named fields re-tokenized as synthetic tokens (span=None). Whitespace-splits each value; an empty value clears the field. original is unchanged (provenance). """ by_value = {role.value: role for role in Role} - for key in fields: + for key, value in fields.items(): if key not in by_value: raise TypeError( f"unknown field {key!r}; expected one of " f"{', '.join(by_value)}" ) + if not isinstance(value, str): + raise TypeError( + f"field {key!r} must be a str, got {value!r}" + ) def synthetic(value: str, role: Role) -> list[Token]: return [Token(word, None, role) for word in value.split()] @@ -265,9 +269,9 @@ def synthetic(value: str, role: Role) -> list[Token]: emitted.add(tok.role) continue new_tokens.append(tok) - for role, value in replaced.items(): - if role not in emitted: - new_tokens.extend(synthetic(value, role)) + for role in Role: + if role in replaced and role not in emitted: + new_tokens.extend(synthetic(replaced[role], role)) kept = tuple( amb for amb in self.ambiguities if all(t in new_tokens for t in amb.tokens) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 7889ead9..3d620faf 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -257,3 +257,14 @@ def test_replace_drops_ambiguities_referencing_removed_tokens(): [Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "d", (van,))]) assert pn.replace(given="Bob").ambiguities == () assert pn.replace(family="Smith").ambiguities != () + + +def test_replace_rejects_non_str_value(): + with pytest.raises(TypeError, match="must be a str"): + _delavega().replace(given=None) # type: ignore[arg-type] + + +def test_replace_appends_missing_roles_in_canonical_order(): + pn = _pn("John", [Token("John", (0, 4), Role.GIVEN)]) + pn2 = pn.replace(maiden="X", suffix="Y") + assert [t.role for t in pn2.tokens] == [Role.GIVEN, Role.SUFFIX, Role.MAIDEN] From aeefe51156beb8becaaf1b8f7e7fdb82795b59d9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:11:02 -0700 Subject: [PATCH 012/275] Add v2 ParsedName.comparison_key() Casefolded seven-component tuple in canonical Role order for dedup, dict keys, and sorting. Semantic layer comparison; __eq__ remains strict. Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 8 ++++++++ tests/v2/test_types.py | 20 ++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/nameparser/_types.py b/nameparser/_types.py index 515eeb39..d0cc8022 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -277,3 +277,11 @@ def synthetic(value: str, role: Role) -> list[Token]: if all(t in new_tokens for t in amb.tokens) ) return ParsedName(self.original, tuple(new_tokens), kept) + + # -- comparison ------------------------------------------------------- + + def comparison_key(self) -> tuple[str, ...]: + """Casefolded seven components in canonical order, for dedup, + dict keys, and sorting. The semantic layer; __eq__ stays strict. + """ + return tuple(self._text_for(role).casefold() for role in Role) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 3d620faf..4ddfab08 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -268,3 +268,23 @@ def test_replace_appends_missing_roles_in_canonical_order(): pn = _pn("John", [Token("John", (0, 4), Role.GIVEN)]) pn2 = pn.replace(maiden="X", suffix="Y") assert [t.role for t in pn2.tokens] == [Role.GIVEN, Role.SUFFIX, Role.MAIDEN] + + +def test_comparison_key_is_casefolded_canonical_seven_tuple(): + pn = _delavega() + assert pn.comparison_key() == ( + "dr.", "juan", "", "de la vega", "iii", "", "", + ) + upper = _pn("JUAN DE LA VEGA", [ + Token("JUAN", (0, 4), Role.GIVEN), + Token("DE", (5, 7), Role.FAMILY, frozenset({"particle"})), + Token("LA", (8, 10), Role.FAMILY, frozenset({"particle"})), + Token("VEGA", (11, 15), Role.FAMILY), + ]) + lower = _pn("juan de la vega", [ + Token("juan", (0, 4), Role.GIVEN), + Token("de", (5, 7), Role.FAMILY, frozenset({"particle"})), + Token("la", (8, 10), Role.FAMILY, frozenset({"particle"})), + Token("vega", (11, 15), Role.FAMILY), + ]) + assert upper.comparison_key() == lower.comparison_key() From 9d781c79d605022639220de5a4eb799682521e0f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:16:54 -0700 Subject: [PATCH 013/275] Add v2 Lexicon with canonical storage and v1-sourced default Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 118 +++++++++++++++++++++++++++++++++++++++ tests/v2/test_lexicon.py | 48 ++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 nameparser/_lexicon.py create mode 100644 tests/v2/test_lexicon.py diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py new file mode 100644 index 00000000..545147ce --- /dev/null +++ b/nameparser/_lexicon.py @@ -0,0 +1,118 @@ +"""Immutable vocabulary configuration for the 2.0 API. + +Layering: may import nameparser.config DATA modules (titles, suffixes, +prefixes, conjunctions, capitalization, bound_first_names) as the single +source of vocabulary during 2.x -- never nameparser.config itself, never +nameparser.parser. Enforced by tests/v2/test_layering.py. +""" +from __future__ import annotations + +import functools +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from types import MappingProxyType + +#: Vocabulary set fields, in declaration order. add()/remove()/__or__ +#: (a later task) operate on exactly these; capitalization_exceptions is +#: deliberately excluded (its entries are pairs -- use dataclasses.replace). +_VOCAB_FIELDS = ( + "titles", "given_name_titles", "suffix_acronyms", "suffix_words", + "suffix_acronyms_ambiguous", "particles", "particles_ambiguous", + "conjunctions", "bound_given_names", "maiden_markers", +) + + +def _normalize(word: str) -> str: + """Casefold and strip periods -- v1's lc(). Membership tests never + re-normalize because construction already did.""" + return word.casefold().replace(".", "").strip() + + +def _normset(entries: Iterable[str]) -> frozenset[str]: + result = frozenset(_normalize(w) for w in entries) + return frozenset(w for w in result if w) + + +@dataclass(frozen=True, slots=True) +class Lexicon: + titles: frozenset[str] = frozenset() + given_name_titles: frozenset[str] = frozenset() + suffix_acronyms: frozenset[str] = frozenset() + suffix_words: frozenset[str] = frozenset() + suffix_acronyms_ambiguous: frozenset[str] = frozenset() + particles: frozenset[str] = frozenset() + particles_ambiguous: frozenset[str] = frozenset() + conjunctions: frozenset[str] = frozenset() + bound_given_names: frozenset[str] = frozenset() + maiden_markers: frozenset[str] = frozenset() + # Canonical storage: sorted tuple of (key, value) pairs. The + # constructor tolerates any Mapping (or pair iterable) at runtime and + # canonicalizes here; this closes the caller-aliasing hole and keeps + # Lexicon hashable. Read via capitalization_exceptions_map. + capitalization_exceptions: tuple[tuple[str, str], ...] = () + _cap_map: Mapping[str, str] = field( + init=False, repr=False, compare=False, hash=False, + default_factory=lambda: MappingProxyType({})) + + def __post_init__(self) -> None: + for name in _VOCAB_FIELDS: + object.__setattr__(self, name, _normset(getattr(self, name))) + raw = self.capitalization_exceptions + pairs = raw.items() if isinstance(raw, Mapping) else raw + canonical = tuple(sorted((_normalize(k), v) for k, v in pairs)) + object.__setattr__(self, "capitalization_exceptions", canonical) + object.__setattr__(self, "_cap_map", MappingProxyType(dict(canonical))) + if not self.particles_ambiguous <= self.particles: + extra = ", ".join(sorted(self.particles_ambiguous - self.particles)) + raise ValueError( + f"particles_ambiguous must be a subset of particles; " + f"not in particles: {extra}" + ) + + @property + def capitalization_exceptions_map(self) -> Mapping[str, str]: + return self._cap_map + + # -- constructors ---------------------------------------------------- + + @classmethod + def empty(cls) -> Lexicon: + return cls() + + @classmethod + def default(cls) -> Lexicon: + return _default_lexicon() + + +@functools.cache +def _default_lexicon() -> Lexicon: + # v1 data modules are the single source of vocabulary through 2.x. + from nameparser.config.bound_first_names import BOUND_FIRST_NAMES + from nameparser.config.capitalization import CAPITALIZATION_EXCEPTIONS + from nameparser.config.conjunctions import CONJUNCTIONS + from nameparser.config.prefixes import NON_FIRST_NAME_PREFIXES, PREFIXES + from nameparser.config.suffixes import ( + SUFFIX_ACRONYMS, SUFFIX_ACRONYMS_AMBIGUOUS, SUFFIX_NOT_ACRONYMS, + ) + from nameparser.config.titles import FIRST_NAME_TITLES, TITLES + + # v1 data modules export plain `set[str]`; wrap each at this call site + # so the strictly-typed frozenset[str] fields never see a bare set. + return Lexicon( + titles=frozenset(TITLES), + given_name_titles=frozenset(FIRST_NAME_TITLES), + suffix_acronyms=frozenset(SUFFIX_ACRONYMS), + suffix_words=frozenset(SUFFIX_NOT_ACRONYMS), + suffix_acronyms_ambiguous=frozenset(SUFFIX_ACRONYMS_AMBIGUOUS), + particles=frozenset(PREFIXES), + # FLIPPED from v1: v1 marks the never-given subset; v2 marks the + # may-be-given subset (migration: complement translation). + particles_ambiguous=frozenset(PREFIXES - NON_FIRST_NAME_PREFIXES), + conjunctions=frozenset(CONJUNCTIONS), + bound_given_names=frozenset(BOUND_FIRST_NAMES), + maiden_markers=frozenset({"née", "nee", "geb"}), + # pass canonical pair-tuples so this strictly-typed call site never + # feeds a Mapping to the tuple-annotated field; __post_init__ + # still tolerates a Mapping at runtime for interactive use + capitalization_exceptions=tuple(sorted(CAPITALIZATION_EXCEPTIONS.items())), + ) diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py new file mode 100644 index 00000000..74440cb5 --- /dev/null +++ b/tests/v2/test_lexicon.py @@ -0,0 +1,48 @@ +import dataclasses + +import pytest + +from nameparser._lexicon import Lexicon + + +def test_entries_are_normalized_at_construction(): + lex = Lexicon(titles={"Dr.", "MR"}) + assert lex.titles == frozenset({"dr", "mr"}) + + +def test_default_sources_v1_vocabulary(): + lex = Lexicon.default() + assert "dr" in lex.titles + assert "van" in lex.particles + assert "phd" in lex.suffix_acronyms + # flipped model: 'dos' is never-given in v1, so NOT ambiguous here + assert "dos" in lex.particles and "dos" not in lex.particles_ambiguous + assert "van" in lex.particles_ambiguous + # v1's CAPITALIZATION_EXCEPTIONS maps 'phd' -> 'Ph.D.' (verbatim, not + # normalized -- only keys are casefolded/period-stripped at + # construction, values pass through unchanged). + assert lex.capitalization_exceptions_map["phd"] == "Ph.D." + + +def test_default_is_cached_single_instance(): + assert Lexicon.default() is Lexicon.default() + + +def test_particles_ambiguous_must_be_subset_of_particles(): + with pytest.raises(ValueError, match="subset"): + Lexicon(particles_ambiguous={"van"}) + + +def test_capitalization_exceptions_canonical_and_no_aliasing(): + exceptions = {"phd": "PhD", "ii": "II"} + lex = Lexicon.empty() + lex2 = dataclasses.replace(lex, capitalization_exceptions=exceptions) + exceptions["iii"] = "III" # mutate caller's dict afterwards + assert "iii" not in lex2.capitalization_exceptions_map + # canonical: insertion order does not affect equality/hash + lex3 = dataclasses.replace(lex, capitalization_exceptions={"ii": "II", "phd": "PhD"}) + assert lex2 == lex3 and hash(lex2) == hash(lex3) + + +def test_lexicon_is_hashable(): + assert isinstance(hash(Lexicon.default()), int) From 61362bd2a50aa98fcb969d245ba58403188c1513 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:24:27 -0700 Subject: [PATCH 014/275] Fail loud on malformed Lexicon input; dedupe exception keys Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 34 ++++++++++++++++++++++++++++++---- tests/v2/test_lexicon.py | 22 ++++++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 545147ce..ffe75ed4 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -28,8 +28,22 @@ def _normalize(word: str) -> str: return word.casefold().replace(".", "").strip() -def _normset(entries: Iterable[str]) -> frozenset[str]: - result = frozenset(_normalize(w) for w in entries) +def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: + # Reject a bare str before iterating: iterating "dr" would silently + # yield the single characters {'d', 'r'} -- the set(str) footgun on + # the primary customization surface. + if isinstance(entries, str): + raise ValueError( + f"Lexicon.{field_name} must be an iterable of strings, " + f"not a bare string" + ) + items = tuple(entries) # materialize once; entries may be a generator + for w in items: + if not isinstance(w, str): + raise ValueError( + f"Lexicon.{field_name} entries must be strings, got {w!r}" + ) + result = frozenset(_normalize(w) for w in items) return frozenset(w for w in result if w) @@ -56,10 +70,22 @@ class Lexicon: def __post_init__(self) -> None: for name in _VOCAB_FIELDS: - object.__setattr__(self, name, _normset(getattr(self, name))) + object.__setattr__(self, name, _normset(getattr(self, name), name)) raw = self.capitalization_exceptions pairs = raw.items() if isinstance(raw, Mapping) else raw - canonical = tuple(sorted((_normalize(k), v) for k, v in pairs)) + # Dedupe on the NORMALIZED key before storing so the tuple and the + # map always agree ("Ph.D." and "phd" collide after normalization). + # Last occurrence wins, matching dict semantics and the right-bias + # rule used elsewhere. + deduped: dict[str, str] = {} + for k, v in pairs: + if not isinstance(k, str) or not isinstance(v, str): + raise ValueError( + f"capitalization_exceptions entries must be " + f"str -> str, got {k!r}: {v!r}" + ) + deduped[_normalize(k)] = v + canonical = tuple(sorted(deduped.items())) object.__setattr__(self, "capitalization_exceptions", canonical) object.__setattr__(self, "_cap_map", MappingProxyType(dict(canonical))) if not self.particles_ambiguous <= self.particles: diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 74440cb5..80723713 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -46,3 +46,25 @@ def test_capitalization_exceptions_canonical_and_no_aliasing(): def test_lexicon_is_hashable(): assert isinstance(hash(Lexicon.default()), int) + + +def test_lexicon_rejects_bare_string_vocab(): + with pytest.raises(ValueError, match="bare string"): + Lexicon(titles="dr") # type: ignore[arg-type] + + +def test_lexicon_rejects_non_str_vocab_entries(): + with pytest.raises(ValueError, match="entries must be strings"): + Lexicon(titles={"Dr.", 42}) # type: ignore[arg-type] + + +def test_colliding_exception_keys_dedupe_last_wins(): + lex = Lexicon(capitalization_exceptions={"Ph.D.": "A", "phd": "B"}) + assert lex.capitalization_exceptions == (("phd", "B"),) + rebuilt = Lexicon(capitalization_exceptions=lex.capitalization_exceptions_map) + assert rebuilt == lex and hash(rebuilt) == hash(lex) + + +def test_lexicon_rejects_non_str_exception_values(): + with pytest.raises(ValueError, match="str -> str"): + Lexicon(capitalization_exceptions={"phd": 42}) # type: ignore[dict-item] From f216e5fce24cd2e32452997b461702ea9faeae5b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:28:23 -0700 Subject: [PATCH 015/275] Add v2 Lexicon composition: add/remove and field-wise union Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 48 ++++++++++++++++++++++++++++++++++++++++ tests/v2/test_lexicon.py | 37 +++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index ffe75ed4..2499cc05 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -7,6 +7,7 @@ """ from __future__ import annotations +import dataclasses import functools from collections.abc import Iterable, Mapping from dataclasses import dataclass, field @@ -109,6 +110,53 @@ def empty(cls) -> Lexicon: def default(cls) -> Lexicon: return _default_lexicon() + # -- composition ------------------------------------------------------ + + def _edit(self, op: str, entries: Mapping[str, Iterable[str]]) -> Lexicon: + updates: dict[str, frozenset[str]] = {} + for name, words in entries.items(): + if name == "capitalization_exceptions": + raise TypeError( + "capitalization_exceptions holds key->value pairs; " + "use dataclasses.replace(lexicon, " + "capitalization_exceptions={...}) instead of " + f"{op}()" + ) + if name not in _VOCAB_FIELDS: + raise TypeError( + f"unknown Lexicon field {name!r}; valid fields: " + f"{', '.join(_VOCAB_FIELDS)}" + ) + current: frozenset[str] = getattr(self, name) + normalized = _normset(words, name) + updates[name] = (current | normalized if op == "add" + else current - normalized) + # mypy's dataclasses.replace() typing checks a **dict's single + # value type against every field's type (it can't see which keys + # are actually present behind the unpack), so a homogeneous + # frozenset[str] dict is flagged against the tuple/Mapping-typed + # capitalization_exceptions/_cap_map fields even though this dict + # never contains those keys (guarded above). + return dataclasses.replace(self, **updates) # type: ignore[arg-type] + + def add(self, **entries: Iterable[str]) -> Lexicon: + return self._edit("add", entries) + + def remove(self, **entries: Iterable[str]) -> Lexicon: + return self._edit("remove", entries) + + def __or__(self, other: Lexicon) -> Lexicon: + if not isinstance(other, Lexicon): + return NotImplemented + updates: dict[str, object] = { + name: getattr(self, name) | getattr(other, name) + for name in _VOCAB_FIELDS + } + # right-biased on key conflicts, mirroring later-wins for scalars + merged = dict(self._cap_map) | dict(other._cap_map) + updates["capitalization_exceptions"] = tuple(sorted(merged.items())) + return dataclasses.replace(self, **updates) # type: ignore[arg-type] + @functools.cache def _default_lexicon() -> Lexicon: diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 80723713..a952a922 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -68,3 +68,40 @@ def test_colliding_exception_keys_dedupe_last_wins(): def test_lexicon_rejects_non_str_exception_values(): with pytest.raises(ValueError, match="str -> str"): Lexicon(capitalization_exceptions={"phd": 42}) # type: ignore[dict-item] + + +def test_add_and_remove_return_new_lexicons(): + # "zqtitle" is a synthetic word absent from v1's TITLES data (unlike + # e.g. "dra", the feminine "dr." abbreviation, which is already there). + base = Lexicon.default() + lex = base.add(titles={"zqtitle"}).remove(suffix_words={"bishop"}) + assert "zqtitle" in lex.titles and "zqtitle" not in base.titles + assert "bishop" not in lex.suffix_words + + +def test_add_unknown_field_raises_with_valid_names(): + with pytest.raises(TypeError, match="prefixes"): + Lexicon.default().add(prefixes={"van"}) # v1 name: helpful error + + +def test_add_capitalization_exceptions_raises_pointing_at_replace(): + with pytest.raises(TypeError, match="dataclasses.replace"): + Lexicon.default().add(capitalization_exceptions={"x": "X"}) + + +def test_union_is_fieldwise_and_right_biased_for_exceptions(): + a = dataclasses.replace(Lexicon.empty(), + capitalization_exceptions={"phd": "PhD"}) + a = a.add(titles={"dr"}) + b = dataclasses.replace(Lexicon.empty(), + capitalization_exceptions={"phd": "Ph.D."}) + b = b.add(titles={"mr"}) + u = a | b + assert u.titles == frozenset({"dr", "mr"}) + assert u.capitalization_exceptions_map["phd"] == "Ph.D." # right wins + + +def test_remove_breaking_subset_invariant_raises(): + lex = Lexicon(particles={"van"}, particles_ambiguous={"van"}) + with pytest.raises(ValueError, match="subset"): + lex.remove(particles={"van"}) # would orphan particles_ambiguous From 0be9e5323aea265182676e050fe18cdd014a7242 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:33:31 -0700 Subject: [PATCH 016/275] Add v2 Policy with order-spec validation and PatronymicRule Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 72 +++++++++++++++++++++++++++++++++++++++++ tests/v2/test_policy.py | 49 ++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 nameparser/_policy.py create mode 100644 tests/v2/test_policy.py diff --git a/nameparser/_policy.py b/nameparser/_policy.py new file mode 100644 index 00000000..4067e30e --- /dev/null +++ b/nameparser/_policy.py @@ -0,0 +1,72 @@ +"""Immutable behavior configuration for the 2.0 API. + +Layering: imports nameparser._types only (tests/v2/test_layering.py, +added in a later task, enforces this). +""" +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum + +from nameparser._types import Role + + +class PatronymicRule(StrEnum): + """Stable rule names (API); implementations live in the pipeline.""" + + EAST_SLAVIC = "east-slavic" + TURKIC = "turkic" + + +# Order-spec constants (#270). Each reads as its contents because roles +# are named given/family, not first/last. +GIVEN_FIRST = (Role.GIVEN, Role.MIDDLE, Role.FAMILY) +FAMILY_FIRST = (Role.FAMILY, Role.GIVEN, Role.MIDDLE) +FAMILY_FIRST_GIVEN_LAST = (Role.FAMILY, Role.MIDDLE, Role.GIVEN) + +_NAME_ROLES = frozenset({Role.GIVEN, Role.MIDDLE, Role.FAMILY}) + + +@dataclass(frozen=True, slots=True) +class Policy: + name_order: tuple[Role, Role, Role] = GIVEN_FIRST + patronymic_rules: frozenset[PatronymicRule] = frozenset() + middle_as_family: bool = False # v1's middle_name_as_last + # v1 default delimiter set (#273) + nickname_delimiters: frozenset[tuple[str, str]] = frozenset( + {("'", "'"), ('"', '"'), ("(", ")")} + ) + # empty by default (v1 parity); route ("(", ")") here to send + # parenthesized content to maiden instead of nickname (#274) + maiden_delimiters: frozenset[tuple[str, str]] = frozenset() + extra_suffix_delimiters: frozenset[str] = frozenset() + lenient_comma_suffixes: bool = True + strip_emoji: bool = True + strip_bidi: bool = True # replaces v1's CONSTANTS.regexes.bidi = False + + def __post_init__(self) -> None: + order = tuple(self.name_order) + if len(order) != 3 or set(order) != _NAME_ROLES: + raise ValueError( + f"name_order must be a permutation of (Role.GIVEN, " + f"Role.MIDDLE, Role.FAMILY), got {order!r}; use " + f"GIVEN_FIRST, FAMILY_FIRST, or FAMILY_FIRST_GIVEN_LAST" + ) + object.__setattr__(self, "name_order", order) + try: + rules = frozenset(PatronymicRule(r) for r in self.patronymic_rules) + except ValueError: + valid = ", ".join(r.value for r in PatronymicRule) + raise ValueError( + f"unknown patronymic rule in {set(self.patronymic_rules)!r}; " + f"valid rules: {valid}" + ) from None + object.__setattr__(self, "patronymic_rules", rules) + for pairs_name in ("nickname_delimiters", "maiden_delimiters"): + for pair in getattr(self, pairs_name): + if (len(pair) != 2 or not all( + isinstance(s, str) and s for s in pair)): + raise ValueError( + f"{pairs_name} entries must be pairs of non-empty " + f"strings, got {pair!r}" + ) diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py new file mode 100644 index 00000000..430cfec2 --- /dev/null +++ b/tests/v2/test_policy.py @@ -0,0 +1,49 @@ +import dataclasses + +import pytest + +from nameparser._policy import ( + FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, GIVEN_FIRST, + PatronymicRule, Policy, +) +from nameparser._types import Role + + +def test_order_constants_read_as_their_contents(): + assert GIVEN_FIRST == (Role.GIVEN, Role.MIDDLE, Role.FAMILY) + assert FAMILY_FIRST == (Role.FAMILY, Role.GIVEN, Role.MIDDLE) + assert FAMILY_FIRST_GIVEN_LAST == (Role.FAMILY, Role.MIDDLE, Role.GIVEN) + + +def test_policy_defaults(): + p = Policy() + assert p.name_order == GIVEN_FIRST + assert p.patronymic_rules == frozenset() + assert ("(", ")") in p.nickname_delimiters + assert p.maiden_delimiters == frozenset() + assert p.strip_emoji and p.strip_bidi and p.lenient_comma_suffixes + + +def test_policy_is_hashable_and_replaceable(): + p = dataclasses.replace(Policy(), name_order=FAMILY_FIRST) + assert p.name_order == FAMILY_FIRST + assert isinstance(hash(p), int) + + +def test_name_order_must_be_permutation_and_error_names_constants(): + with pytest.raises(ValueError, match="FAMILY_FIRST_GIVEN_LAST"): + Policy(name_order=(Role.TITLE, Role.GIVEN, Role.FAMILY)) + with pytest.raises(ValueError, match="GIVEN_FIRST"): + Policy(name_order=(Role.GIVEN, Role.GIVEN, Role.FAMILY)) + + +def test_patronymic_rules_coerce_and_reject(): + p = Policy(patronymic_rules=frozenset({"east-slavic"})) + assert p.patronymic_rules == frozenset({PatronymicRule.EAST_SLAVIC}) + with pytest.raises(ValueError, match="east-slavic, turkic"): + Policy(patronymic_rules=frozenset({"klingon"})) + + +def test_delimiter_pairs_must_be_nonempty_string_pairs(): + with pytest.raises(ValueError, match="non-empty"): + Policy(nickname_delimiters=frozenset({("", ")")})) From 21c817b7f9405f989b7686f4c2b8816eebc6d5ec Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:38:56 -0700 Subject: [PATCH 017/275] Reject malformed Policy delimiter pairs and patronymic rules Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 17 +++++++++++------ tests/v2/test_policy.py | 12 ++++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 4067e30e..e8095b3b 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -53,20 +53,25 @@ def __post_init__(self) -> None: f"GIVEN_FIRST, FAMILY_FIRST, or FAMILY_FIRST_GIVEN_LAST" ) object.__setattr__(self, "name_order", order) + if isinstance(self.patronymic_rules, str): + raise ValueError( + f"patronymic_rules must be an iterable of rule names, " + f"not a bare string: {self.patronymic_rules!r}" + ) try: rules = frozenset(PatronymicRule(r) for r in self.patronymic_rules) - except ValueError: + except (TypeError, ValueError): valid = ", ".join(r.value for r in PatronymicRule) raise ValueError( - f"unknown patronymic rule in {set(self.patronymic_rules)!r}; " + f"unknown patronymic rule in {self.patronymic_rules!r}; " f"valid rules: {valid}" ) from None object.__setattr__(self, "patronymic_rules", rules) for pairs_name in ("nickname_delimiters", "maiden_delimiters"): for pair in getattr(self, pairs_name): - if (len(pair) != 2 or not all( - isinstance(s, str) and s for s in pair)): + if (not isinstance(pair, tuple) or len(pair) != 2 + or not all(isinstance(s, str) and s for s in pair)): raise ValueError( - f"{pairs_name} entries must be pairs of non-empty " - f"strings, got {pair!r}" + f"{pairs_name} entries must be (open, close) tuples " + f"of non-empty strings, got {pair!r}" ) diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 430cfec2..cef3f4f5 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -47,3 +47,15 @@ def test_patronymic_rules_coerce_and_reject(): def test_delimiter_pairs_must_be_nonempty_string_pairs(): with pytest.raises(ValueError, match="non-empty"): Policy(nickname_delimiters=frozenset({("", ")")})) + + +def test_delimiter_pair_rejects_two_char_string(): + with pytest.raises(ValueError, match="tuples"): + Policy(nickname_delimiters=frozenset({"()"})) # type: ignore[arg-type] + + +def test_patronymic_rules_rejects_bare_string_and_non_iterable(): + with pytest.raises(ValueError, match="bare string"): + Policy(patronymic_rules="east-slavic") # type: ignore[arg-type] + with pytest.raises(ValueError, match="valid rules"): + Policy(patronymic_rules=5) # type: ignore[arg-type] From 719fee6d418ebcb80c22ff693546081d39ee8805 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:42:39 -0700 Subject: [PATCH 018/275] Coerce and validate all Policy collection fields Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 19 ++++++++++++++++++- tests/v2/test_policy.py | 24 ++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index e8095b3b..1e6534b1 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -68,10 +68,27 @@ def __post_init__(self) -> None: ) from None object.__setattr__(self, "patronymic_rules", rules) for pairs_name in ("nickname_delimiters", "maiden_delimiters"): - for pair in getattr(self, pairs_name): + pairs = tuple(getattr(self, pairs_name)) + for pair in pairs: if (not isinstance(pair, tuple) or len(pair) != 2 or not all(isinstance(s, str) and s for s in pair)): raise ValueError( f"{pairs_name} entries must be (open, close) tuples " f"of non-empty strings, got {pair!r}" ) + object.__setattr__(self, pairs_name, frozenset(pairs)) + if isinstance(self.extra_suffix_delimiters, str): + raise ValueError( + f"extra_suffix_delimiters must be an iterable of strings, " + f"not a bare string: {self.extra_suffix_delimiters!r}" + ) + delimiters = tuple(self.extra_suffix_delimiters) + for d in delimiters: + if not isinstance(d, str) or not d: + raise ValueError( + f"extra_suffix_delimiters entries must be non-empty " + f"strings, got {d!r}" + ) + object.__setattr__( + self, "extra_suffix_delimiters", frozenset(delimiters) + ) diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index cef3f4f5..1a242e95 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -59,3 +59,27 @@ def test_patronymic_rules_rejects_bare_string_and_non_iterable(): Policy(patronymic_rules="east-slavic") # type: ignore[arg-type] with pytest.raises(ValueError, match="valid rules"): Policy(patronymic_rules=5) # type: ignore[arg-type] + + +def test_policy_delimiters_coerce_to_frozensets(): + p = Policy(nickname_delimiters=[("(", ")")]) # type: ignore[arg-type] + assert isinstance(p.nickname_delimiters, frozenset) + assert isinstance(hash(p), int) + assert p == Policy(nickname_delimiters=frozenset({("(", ")")})) + + +def test_policy_delimiters_do_not_alias_caller_containers(): + source = {("(", ")")} + p = Policy(nickname_delimiters=source) # type: ignore[arg-type] + source.add(("'", "'")) + assert ("'", "'") not in p.nickname_delimiters + + +def test_extra_suffix_delimiters_validated_and_coerced(): + with pytest.raises(ValueError, match="bare string"): + Policy(extra_suffix_delimiters="ab") # type: ignore[arg-type] + with pytest.raises(ValueError, match="non-empty strings"): + Policy(extra_suffix_delimiters={""}) + p = Policy(extra_suffix_delimiters=["-"]) # type: ignore[arg-type] + assert p.extra_suffix_delimiters == frozenset({"-"}) + assert isinstance(hash(p), int) From 0a47671a7692c6f3b4969db6e13fff9f618d1b25 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:45:38 -0700 Subject: [PATCH 019/275] Add v2 PolicyPatch with UNSET sentinel and declared composition Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 55 +++++++++++++++++++++++++++++++++++++++-- tests/v2/test_policy.py | 32 +++++++++++++++++++++++- 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 1e6534b1..adee223c 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -5,8 +5,9 @@ """ from __future__ import annotations -from dataclasses import dataclass -from enum import StrEnum +import dataclasses +from dataclasses import dataclass, field +from enum import Enum, StrEnum, auto from nameparser._types import Role @@ -92,3 +93,53 @@ def __post_init__(self) -> None: object.__setattr__( self, "extra_suffix_delimiters", frozenset(delimiters) ) + + +class _Unset(Enum): + UNSET = auto() + + +#: Sentinel for "this patch does not set this field" (picklable enum +#: member, distinguishable from every real value including None/False). +UNSET = _Unset.UNSET + +_UNION = {"compose": "union"} # field metadata: set-valued -> union + + +@dataclass(frozen=True, slots=True) +class PolicyPatch: + """A partial Policy: one field per Policy field, all defaulting to + UNSET. Composition per field is DECLARED via metadata -- set-valued + fields union, scalars override (later wins). Kept in lockstep with + Policy by the parity test in tests/v2/test_policy.py. + """ + + name_order: tuple[Role, Role, Role] | _Unset = UNSET + patronymic_rules: frozenset[PatronymicRule] | _Unset = field( + default=UNSET, metadata=_UNION) + middle_as_family: bool | _Unset = UNSET + nickname_delimiters: frozenset[tuple[str, str]] | _Unset = field( + default=UNSET, metadata=_UNION) + maiden_delimiters: frozenset[tuple[str, str]] | _Unset = field( + default=UNSET, metadata=_UNION) + extra_suffix_delimiters: frozenset[str] | _Unset = field( + default=UNSET, metadata=_UNION) + lenient_comma_suffixes: bool | _Unset = UNSET + strip_emoji: bool | _Unset = UNSET + strip_bidi: bool | _Unset = UNSET + + +def apply_patch(policy: Policy, patch: PolicyPatch) -> Policy: + """Fold a PolicyPatch onto a Policy. Policy.__post_init__ re-runs via + dataclasses.replace, so patched values are revalidated for free.""" + updates: dict[str, object] = {} + for f in dataclasses.fields(PolicyPatch): + value = getattr(patch, f.name) + if value is UNSET: + continue + if f.metadata.get("compose") == "union": + value = getattr(policy, f.name) | value + updates[f.name] = value + if not updates: + return policy + return dataclasses.replace(policy, **updates) # type: ignore[arg-type] diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 1a242e95..f52bf1ab 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -4,7 +4,7 @@ from nameparser._policy import ( FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, GIVEN_FIRST, - PatronymicRule, Policy, + PatronymicRule, Policy, PolicyPatch, UNSET, apply_patch, ) from nameparser._types import Role @@ -83,3 +83,33 @@ def test_extra_suffix_delimiters_validated_and_coerced(): p = Policy(extra_suffix_delimiters=["-"]) # type: ignore[arg-type] assert p.extra_suffix_delimiters == frozenset({"-"}) assert isinstance(hash(p), int) + + +def test_policy_patch_mirrors_policy_fields_and_types(): + policy_fields = {f.name for f in dataclasses.fields(Policy)} + patch_fields = {f.name for f in dataclasses.fields(PolicyPatch)} + assert policy_fields == patch_fields + + +def test_apply_patch_overrides_scalars_and_unions_sets(): + base = Policy(patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})) + patch = PolicyPatch( + name_order=FAMILY_FIRST, + patronymic_rules=frozenset({PatronymicRule.TURKIC}), + ) + out = apply_patch(base, patch) + assert out.name_order == FAMILY_FIRST # override + assert out.patronymic_rules == frozenset( # union + {PatronymicRule.EAST_SLAVIC, PatronymicRule.TURKIC}) + assert out.strip_emoji is True # untouched + + +def test_apply_patch_with_empty_patch_returns_same_policy(): + base = Policy() + assert apply_patch(base, PolicyPatch()) is base + + +def test_unset_fields_are_distinguishable_from_defaults(): + patch = PolicyPatch(strip_emoji=True) # explicitly set to the default value + assert patch.strip_emoji is True + assert PolicyPatch().strip_emoji is UNSET From cd46673aede740fe54c51941dbde43142dc18c9d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:48:45 -0700 Subject: [PATCH 020/275] Document the replace type-ignore; rename parity test Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 2 ++ tests/v2/test_policy.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index adee223c..629ced9b 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -142,4 +142,6 @@ def apply_patch(policy: Policy, patch: PolicyPatch) -> Policy: updates[f.name] = value if not updates: return policy + # Known mypy limitation with **dict-unpacked replace; see the full + # explanation at Lexicon._edit in _lexicon.py. return dataclasses.replace(policy, **updates) # type: ignore[arg-type] diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index f52bf1ab..f5572d1a 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -85,7 +85,7 @@ def test_extra_suffix_delimiters_validated_and_coerced(): assert isinstance(hash(p), int) -def test_policy_patch_mirrors_policy_fields_and_types(): +def test_policy_patch_mirrors_policy_field_names(): policy_fields = {f.name for f in dataclasses.fields(Policy)} patch_fields = {f.name for f in dataclasses.fields(PolicyPatch)} assert policy_fields == patch_fields From 29f603fe76991ad98e50b7925baa3965d4c84ad3 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:53:34 -0700 Subject: [PATCH 021/275] Canonicalize PolicyPatch union fields; lock field types in parity test Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 20 ++++++++++++++++++++ tests/v2/test_policy.py | 19 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 629ced9b..e6267d03 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -112,6 +112,9 @@ class PolicyPatch: UNSET. Composition per field is DECLARED via metadata -- set-valued fields union, scalars override (later wins). Kept in lockstep with Policy by the parity test in tests/v2/test_policy.py. + + Values are validated when the patch is applied (Policy's constructor + re-runs), not at patch construction. """ name_order: tuple[Role, Role, Role] | _Unset = UNSET @@ -128,6 +131,23 @@ class PolicyPatch: strip_emoji: bool | _Unset = UNSET strip_bidi: bool | _Unset = UNSET + def __post_init__(self) -> None: + # Canonicalize (but do NOT validate) union-composed fields so a + # patch built from a set/list literal is hashable and unions + # cleanly in apply_patch. + for f in dataclasses.fields(self): + if f.metadata.get("compose") != "union": + continue + value = getattr(self, f.name) + if value is UNSET: + continue + if isinstance(value, str): + raise ValueError( + f"{f.name} must be an iterable, " + f"not a bare string: {value!r}" + ) + object.__setattr__(self, f.name, frozenset(value)) + def apply_patch(policy: Policy, patch: PolicyPatch) -> Policy: """Fold a PolicyPatch onto a Policy. Policy.__post_init__ re-runs via diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index f5572d1a..ac0293d3 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -91,6 +91,25 @@ def test_policy_patch_mirrors_policy_field_names(): assert policy_fields == patch_fields +def test_policy_patch_mirrors_policy_field_types(): + for f in dataclasses.fields(Policy): + patch_annotation = PolicyPatch.__dataclass_fields__[f.name].type + assert patch_annotation == f"{f.type} | _Unset" + + +def test_policy_patch_canonicalizes_union_fields(): + p = PolicyPatch(extra_suffix_delimiters={"-"}) + assert isinstance(p.extra_suffix_delimiters, frozenset) + assert isinstance(hash(p), int) + out = apply_patch(Policy(), PolicyPatch(extra_suffix_delimiters=["-"])) # type: ignore[arg-type] + assert out.extra_suffix_delimiters == frozenset({"-"}) + + +def test_policy_patch_rejects_bare_string_union_fields(): + with pytest.raises(ValueError, match="bare string"): + PolicyPatch(extra_suffix_delimiters="ab") # type: ignore[arg-type] + + def test_apply_patch_overrides_scalars_and_unions_sets(): base = Policy(patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})) patch = PolicyPatch( From 2d3dd044f3426612cc0b2d04babb5760a9808e7d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:57:22 -0700 Subject: [PATCH 022/275] Add v2 Locale type Co-Authored-By: Claude Fable 5 --- nameparser/_locale.py | 40 ++++++++++++++++++++++++++++++++++++++++ tests/v2/test_locale.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 nameparser/_locale.py create mode 100644 tests/v2/test_locale.py diff --git a/nameparser/_locale.py b/nameparser/_locale.py new file mode 100644 index 00000000..8a6707d0 --- /dev/null +++ b/nameparser/_locale.py @@ -0,0 +1,40 @@ +"""The Locale type: a named delta over (Lexicon, Policy). + +A Locale dissolves at parser construction (parser_for, a later plan): +lexicon fragments union onto the base, the PolicyPatch folds via +apply_patch. Packs are pure data; they have no privileged capabilities. + +Layering: imports _lexicon and _policy only (tests/v2/test_layering.py, +added in a later task, enforces this). +""" +from __future__ import annotations + +from dataclasses import dataclass + +from nameparser._lexicon import Lexicon +from nameparser._policy import PolicyPatch + + +@dataclass(frozen=True, slots=True) +class Locale: + code: str + lexicon: Lexicon + policy: PolicyPatch = PolicyPatch() + + def __post_init__(self) -> None: + if not isinstance(self.code, str) or not self.code.strip(): + raise ValueError( + f"Locale.code must be a non-empty string, got {self.code!r}" + ) + if self.code != self.code.lower(): + raise ValueError( + f"Locale.code must be lowercase, got {self.code!r}" + ) + if not isinstance(self.lexicon, Lexicon): + raise ValueError( + f"Locale.lexicon must be a Lexicon, got {self.lexicon!r}" + ) + if not isinstance(self.policy, PolicyPatch): + raise ValueError( + f"Locale.policy must be a PolicyPatch, got {self.policy!r}" + ) diff --git a/tests/v2/test_locale.py b/tests/v2/test_locale.py new file mode 100644 index 00000000..6617708f --- /dev/null +++ b/tests/v2/test_locale.py @@ -0,0 +1,40 @@ +import pytest + +from nameparser._lexicon import Lexicon +from nameparser._locale import Locale +from nameparser._policy import PatronymicRule, PolicyPatch + + +def test_locale_holds_code_lexicon_fragment_and_patch(): + ru = Locale( + code="ru", + lexicon=Lexicon.empty(), + policy=PolicyPatch( + patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})), + ) + assert ru.code == "ru" + assert ru.policy.patronymic_rules == frozenset( + {PatronymicRule.EAST_SLAVIC}) + + +def test_locale_defaults_to_empty_patch(): + assert Locale(code="xx", lexicon=Lexicon.empty()).policy == PolicyPatch() + + +def test_locale_code_must_be_nonempty_lowercase(): + with pytest.raises(ValueError, match="lowercase"): + Locale(code="RU", lexicon=Lexicon.empty()) + with pytest.raises(ValueError, match="non-empty"): + Locale(code="", lexicon=Lexicon.empty()) + + +def test_locale_is_hashable(): + loc = Locale(code="ru", lexicon=Lexicon.empty()) + assert isinstance(hash(loc), int) + + +def test_locale_validates_component_types(): + with pytest.raises(ValueError, match="Lexicon"): + Locale(code="ru", lexicon={"titles": set()}) # type: ignore[arg-type] + with pytest.raises(ValueError, match="PolicyPatch"): + Locale(code="ru", lexicon=Lexicon.empty(), policy={"name_order": None}) # type: ignore[arg-type] From 91845fdfaa26a70397c1f2afa9d73d8a0cd8ebf1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 01:01:59 -0700 Subject: [PATCH 023/275] Reject whitespace in Locale codes Co-Authored-By: Claude Fable 5 --- nameparser/_locale.py | 4 ++++ tests/v2/test_locale.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/nameparser/_locale.py b/nameparser/_locale.py index 8a6707d0..d483d343 100644 --- a/nameparser/_locale.py +++ b/nameparser/_locale.py @@ -30,6 +30,10 @@ def __post_init__(self) -> None: raise ValueError( f"Locale.code must be lowercase, got {self.code!r}" ) + if any(c.isspace() for c in self.code): + raise ValueError( + f"Locale.code must not contain whitespace, got {self.code!r}" + ) if not isinstance(self.lexicon, Lexicon): raise ValueError( f"Locale.lexicon must be a Lexicon, got {self.lexicon!r}" diff --git a/tests/v2/test_locale.py b/tests/v2/test_locale.py index 6617708f..b3a8f541 100644 --- a/tests/v2/test_locale.py +++ b/tests/v2/test_locale.py @@ -28,6 +28,12 @@ def test_locale_code_must_be_nonempty_lowercase(): Locale(code="", lexicon=Lexicon.empty()) +def test_locale_code_rejects_whitespace(): + for bad in ("ru ", " ru", "ru\n", "r u"): + with pytest.raises(ValueError, match="whitespace"): + Locale(code=bad, lexicon=Lexicon.empty()) + + def test_locale_is_hashable(): loc = Locale(code="ru", lexicon=Lexicon.empty()) assert isinstance(hash(loc), int) From b88d3ac95934251184c843d792ded6587b46dd25 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 01:06:40 -0700 Subject: [PATCH 024/275] Drop exception keys that normalize to empty Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 5 ++++- tests/v2/test_lexicon.py | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 2499cc05..f7394b65 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -85,7 +85,10 @@ def __post_init__(self) -> None: f"capitalization_exceptions entries must be " f"str -> str, got {k!r}: {v!r}" ) - deduped[_normalize(k)] = v + normalized_key = _normalize(k) + if not normalized_key: + continue # mirror _normset's drop-empty rule + deduped[normalized_key] = v canonical = tuple(sorted(deduped.items())) object.__setattr__(self, "capitalization_exceptions", canonical) object.__setattr__(self, "_cap_map", MappingProxyType(dict(canonical))) diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index a952a922..944c4fb2 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -58,6 +58,11 @@ def test_lexicon_rejects_non_str_vocab_entries(): Lexicon(titles={"Dr.", 42}) # type: ignore[arg-type] +def test_exception_keys_normalizing_to_empty_are_dropped(): + lex = Lexicon(capitalization_exceptions={"...": "X", "phd": "PhD"}) + assert lex.capitalization_exceptions == (("phd", "PhD"),) + + def test_colliding_exception_keys_dedupe_last_wins(): lex = Lexicon(capitalization_exceptions={"Ph.D.": "A", "phd": "B"}) assert lex.capitalization_exceptions == (("phd", "B"),) From 0838e9878d158b31c0dbc590cf132bb75cb0038f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 01:08:36 -0700 Subject: [PATCH 025/275] Add bounded deviation-only reprs to all v2 types Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 19 +++++++++++++ nameparser/_locale.py | 12 ++++++++- nameparser/_policy.py | 24 +++++++++++++++++ nameparser/_types.py | 29 ++++++++++++++++++++ tests/v2/test_reprs.py | 61 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 tests/v2/test_reprs.py diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index f7394b65..9394899f 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -99,6 +99,25 @@ def __post_init__(self) -> None: f"not in particles: {extra}" ) + def __repr__(self) -> str: + # Bounded: renders only which fields deviate from default() and by + # how many entries -- never the entries themselves (design rule, + # see nameparser._types module docstring). + default = Lexicon.default() + if self == default: + return "Lexicon(default)" + deltas = [] + for name in _VOCAB_FIELDS + ("capitalization_exceptions",): + mine = set(getattr(self, name)) + theirs = set(getattr(default, name)) + added, removed = len(mine - theirs), len(theirs - mine) + if added or removed: + delta = "".join( + part for part, n in ((f"+{added}", added), (f"-{removed}", removed)) if n + ) + deltas.append(f"{name}: {delta}") + return f"Lexicon(default + {', '.join(deltas)})" + @property def capitalization_exceptions_map(self) -> Mapping[str, str]: return self._cap_map diff --git a/nameparser/_locale.py b/nameparser/_locale.py index d483d343..2494b3eb 100644 --- a/nameparser/_locale.py +++ b/nameparser/_locale.py @@ -9,10 +9,11 @@ """ from __future__ import annotations +import dataclasses from dataclasses import dataclass from nameparser._lexicon import Lexicon -from nameparser._policy import PolicyPatch +from nameparser._policy import UNSET, PolicyPatch @dataclass(frozen=True, slots=True) @@ -42,3 +43,12 @@ def __post_init__(self) -> None: raise ValueError( f"Locale.policy must be a PolicyPatch, got {self.policy!r}" ) + + def __repr__(self) -> str: + # Bounded: shows the code and which Policy fields the patch sets, + # never the Lexicon contents or the patched values themselves + # (design rule, see nameparser._types module docstring). + patched = [f.name for f in dataclasses.fields(self.policy) + if getattr(self.policy, f.name) is not UNSET] + suffix = f": {', '.join(patched)}" if patched else "" + return f"Locale({self.code!r}{suffix})" diff --git a/nameparser/_policy.py b/nameparser/_policy.py index e6267d03..7427f166 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -94,6 +94,30 @@ def __post_init__(self) -> None: self, "extra_suffix_delimiters", frozenset(delimiters) ) + def __repr__(self) -> str: + # Bounded: only fields that deviate from the default are shown + # (design rule, see nameparser._types module docstring). + constant_names = { + GIVEN_FIRST: "GIVEN_FIRST", + FAMILY_FIRST: "FAMILY_FIRST", + FAMILY_FIRST_GIVEN_LAST: "FAMILY_FIRST_GIVEN_LAST", + } + parts = [] + for f in dataclasses.fields(self): + value = getattr(self, f.name) + if value == f.default: + continue + if f.name == "name_order": + # Only 3 of the 6 possible role permutations have named + # constants; fall back to a compact role-name tuple for + # the rest so this can't KeyError on an unnamed order. + order_repr = constant_names.get( + value, "(" + ", ".join(r.name for r in value) + ")") + parts.append(f"name_order={order_repr}") + else: + parts.append(f"{f.name}={value!r}") + return f"Policy({', '.join(parts)})" + class _Unset(Enum): UNSET = auto() diff --git a/nameparser/_types.py b/nameparser/_types.py index d0cc8022..97dd941b 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -2,6 +2,11 @@ Layering (enforced by tests/v2/test_layering.py): this module imports nothing from nameparser -- it is the bottom of the dependency graph. + +Repr policy (applies to every v2 type's __repr__, across this module and +_lexicon.py/_policy.py/_locale.py): bounded output only. No repr may scale +with vocabulary size -- collections render as counts or deltas, never +contents. """ from __future__ import annotations @@ -64,6 +69,14 @@ def __post_init__(self) -> None: object.__setattr__(self, "span", Span(start, end)) object.__setattr__(self, "tags", frozenset(self.tags)) + def __repr__(self) -> str: + # Bounded output: a single token's text/span/role/tags, never + # scales with vocabulary size (design rule -- see module docstring). + where = (f"@{self.span.start}:{self.span.end}" + if self.span is not None else "@synthetic") + tags = f" {{{', '.join(sorted(self.tags))}}}" if self.tags else "" + return f"Token({self.text!r} {where} {self.role.name}{tags})" + class AmbiguityKind(StrEnum): """Stable identifiers (API); members ARE their string values.""" @@ -103,6 +116,10 @@ def __post_init__(self) -> None: ) object.__setattr__(self, "tokens", toks) + def __repr__(self) -> str: + texts = "/".join(repr(t.text) for t in self.tokens) + return f"Ambiguity({self.kind.value!r}: {texts})" + @dataclass(frozen=True, slots=True) class ParsedName: @@ -163,6 +180,18 @@ def __post_init__(self) -> None: def __bool__(self) -> bool: return bool(self.tokens) + def __repr__(self) -> str: + lines = [] + for role in Role: + text = self._text_for(role) + if text: + lines.append(f"\t{role.value}: {text!r}") + if self.ambiguities: + kinds = [a.kind.value for a in self.ambiguities] + lines.append(f"\tambiguities: {kinds!r}") + body = "\n".join(lines) + return f"" if lines else "" + # -- string views (canonical order = Role declaration order) -------- def _text_for(self, *roles: Role, tag: str | None = None, diff --git a/tests/v2/test_reprs.py b/tests/v2/test_reprs.py new file mode 100644 index 00000000..a5c421db --- /dev/null +++ b/tests/v2/test_reprs.py @@ -0,0 +1,61 @@ +import dataclasses + +from nameparser._lexicon import Lexicon +from nameparser._locale import Locale +from nameparser._policy import FAMILY_FIRST, PatronymicRule, Policy, PolicyPatch +from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Token + + +def test_token_repr_is_compact(): + t = Token("de", (9, 11), Role.FAMILY, frozenset({"particle"})) + assert repr(t) == "Token('de' @9:11 FAMILY {particle})" + assert repr(Token("Jane", None, Role.GIVEN)) == "Token('Jane' @synthetic GIVEN)" + + +def test_ambiguity_repr_shows_kind_and_token_texts(): + van = Token("Van", (0, 3), Role.GIVEN) + a = Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "detail", (van,)) + assert repr(a) == "Ambiguity('particle-or-given': 'Van')" + + +def test_parsedname_repr_lists_nonempty_fields_in_canonical_order(): + pn = ParsedName("John Smith", ( + Token("John", (0, 4), Role.GIVEN), + Token("Smith", (5, 10), Role.FAMILY), + )) + assert repr(pn) == ( + "" + ) + + +def test_parsedname_repr_includes_ambiguities_line_when_present(): + van = Token("Van", (0, 3), Role.GIVEN) + pn = ParsedName("Van Johnson", + (van, Token("Johnson", (4, 11), Role.FAMILY)), + (Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "d", (van,)),)) + assert "ambiguities: ['particle-or-given']" in repr(pn) + + +def test_empty_parsedname_repr(): + assert repr(ParsedName("", ())) == "" + + +def test_policy_repr_shows_only_nondefault_fields(): + assert repr(Policy()) == "Policy()" + p = Policy(name_order=FAMILY_FIRST, strip_bidi=False) + assert repr(p) == "Policy(name_order=FAMILY_FIRST, strip_bidi=False)" + + +def test_lexicon_repr_is_bounded(): + assert repr(Lexicon.default()) == "Lexicon(default)" + lex = Lexicon.default().add(titles={"zqx", "zqy"}) + assert repr(lex) == "Lexicon(default + titles: +2)" + assert "zqx" not in repr(lex) # never dump contents + + +def test_locale_repr_shows_code_and_patched_fields(): + ru = Locale("ru", Lexicon.empty(), + PolicyPatch(patronymic_rules=frozenset( + {PatronymicRule.EAST_SLAVIC}))) + assert repr(ru) == "Locale('ru': patronymic_rules)" + assert repr(Locale("xx", Lexicon.empty())) == "Locale('xx')" From af6a01ad787cc704d5e97cadb684866231a948c8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 01:20:54 -0700 Subject: [PATCH 026/275] Export v2 core types; enforce layering and mypy strictness Adds tests/v2/test_layering.py to mechanically enforce the conventions doc's import layering and the public v2 export surface. Appends the v2 core types (Span, Role, Token, Ambiguity, AmbiguityKind, ParsedName, Lexicon, Policy, PolicyPatch, PatronymicRule, UNSET, GIVEN_FIRST, FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, Locale) to nameparser/__init__.py alongside the existing v1 HumanName export. Turns on component-flag mypy strictness for the four v2 modules and check_untyped_defs for tests/v2, then fixes the resulting test-only type mismatches by using the precise constructed types (Span(...), frozenset({...}), tuple-of-pairs) where the literal type didn't matter to the test, and adding narrow # type: ignore[arg-type] comments only where a test deliberately exercises runtime coercion/validation of an intentionally mismatched static type (e.g. Token span/Ambiguity kind coercion tests, PatronymicRule string coercion, dict aliasing test). Co-Authored-By: Claude Fable 5 --- nameparser/__init__.py | 29 ++++++++++++ pyproject.toml | 17 +++++++ tests/v2/test_layering.py | 52 +++++++++++++++++++++ tests/v2/test_lexicon.py | 22 ++++----- tests/v2/test_policy.py | 8 ++-- tests/v2/test_reprs.py | 14 +++--- tests/v2/test_types.py | 96 +++++++++++++++++++-------------------- 7 files changed, 168 insertions(+), 70 deletions(-) create mode 100644 tests/v2/test_layering.py diff --git a/nameparser/__init__.py b/nameparser/__init__.py index c82d8002..5e3adcb2 100644 --- a/nameparser/__init__.py +++ b/nameparser/__init__.py @@ -5,3 +5,32 @@ __author_email__ = 'derek73@gmail.com' __license__ = "LGPL" __url__ = "https://github.com/derek73/python-nameparser" + +from nameparser._lexicon import Lexicon +from nameparser._locale import Locale +from nameparser._policy import ( + FAMILY_FIRST, + FAMILY_FIRST_GIVEN_LAST, + GIVEN_FIRST, + UNSET, + PatronymicRule, + Policy, + PolicyPatch, +) +from nameparser._types import ( + Ambiguity, + AmbiguityKind, + ParsedName, + Role, + Span, + Token, +) + +__all__ = [ + # v1 (compatibility layer) + "HumanName", + # v2 core + "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", + "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET", + "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "Locale", +] diff --git a/pyproject.toml b/pyproject.toml index 1e9ffef2..1beee8fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,23 @@ module = [ ] ignore_missing_imports = true +[[tool.mypy.overrides]] +module = [ + "nameparser._types", + "nameparser._lexicon", + "nameparser._policy", + "nameparser._locale", +] +disallow_untyped_defs = true +disallow_incomplete_defs = true +disallow_any_generics = true +warn_return_any = true +strict_equality = true + +[[tool.mypy.overrides]] +module = ["tests.v2.*"] +check_untyped_defs = true + [tool.pytest.ini_options] testpaths = ["tests", "nameparser"] addopts = "--doctest-modules" diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py new file mode 100644 index 00000000..99ae45c8 --- /dev/null +++ b/tests/v2/test_layering.py @@ -0,0 +1,52 @@ +"""Enforce the conventions doc's import layering mechanically.""" +import ast +import pathlib + +import nameparser + +PKG = pathlib.Path(nameparser.__file__).parent + +# module -> prefixes it may import from within nameparser +ALLOWED = { + "_types.py": (), + "_lexicon.py": ("nameparser.config.",), # DATA modules only, during 2.x + "_policy.py": ("nameparser._types",), + "_locale.py": ("nameparser._lexicon", "nameparser._policy"), +} + + +def _nameparser_imports(path: pathlib.Path) -> list[str]: + tree = ast.parse(path.read_text()) + found = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + found += [a.name for a in node.names] + elif isinstance(node, ast.ImportFrom) and node.module: + found.append(node.module) + return [m for m in found if m.startswith("nameparser")] + + +def test_layering_contract(): + for mod, allowed in ALLOWED.items(): + for imported in _nameparser_imports(PKG / mod): + assert imported.startswith(allowed), ( + f"{mod} imports {imported}, which the layering contract " + f"forbids (allowed prefixes: {allowed or 'none'})" + ) + + +def test_lexicon_never_imports_config_package_root_or_parser(): + for imported in _nameparser_imports(PKG / "_lexicon.py"): + assert imported != "nameparser.config" + assert not imported.startswith("nameparser.parser") + + +def test_public_exports(): + expected = { + "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", + "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET", + "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "Locale", + } + assert expected <= set(nameparser.__all__) + for name in expected: + assert getattr(nameparser, name) is not None diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 944c4fb2..4102a271 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -6,7 +6,7 @@ def test_entries_are_normalized_at_construction(): - lex = Lexicon(titles={"Dr.", "MR"}) + lex = Lexicon(titles=frozenset({"Dr.", "MR"})) assert lex.titles == frozenset({"dr", "mr"}) @@ -30,17 +30,17 @@ def test_default_is_cached_single_instance(): def test_particles_ambiguous_must_be_subset_of_particles(): with pytest.raises(ValueError, match="subset"): - Lexicon(particles_ambiguous={"van"}) + Lexicon(particles_ambiguous=frozenset({"van"})) def test_capitalization_exceptions_canonical_and_no_aliasing(): exceptions = {"phd": "PhD", "ii": "II"} lex = Lexicon.empty() - lex2 = dataclasses.replace(lex, capitalization_exceptions=exceptions) + lex2 = dataclasses.replace(lex, capitalization_exceptions=exceptions) # type: ignore[arg-type] exceptions["iii"] = "III" # mutate caller's dict afterwards assert "iii" not in lex2.capitalization_exceptions_map # canonical: insertion order does not affect equality/hash - lex3 = dataclasses.replace(lex, capitalization_exceptions={"ii": "II", "phd": "PhD"}) + lex3 = dataclasses.replace(lex, capitalization_exceptions=(("ii", "II"), ("phd", "PhD"))) assert lex2 == lex3 and hash(lex2) == hash(lex3) @@ -59,20 +59,20 @@ def test_lexicon_rejects_non_str_vocab_entries(): def test_exception_keys_normalizing_to_empty_are_dropped(): - lex = Lexicon(capitalization_exceptions={"...": "X", "phd": "PhD"}) + lex = Lexicon(capitalization_exceptions=(("...", "X"), ("phd", "PhD"))) assert lex.capitalization_exceptions == (("phd", "PhD"),) def test_colliding_exception_keys_dedupe_last_wins(): - lex = Lexicon(capitalization_exceptions={"Ph.D.": "A", "phd": "B"}) + lex = Lexicon(capitalization_exceptions=(("Ph.D.", "A"), ("phd", "B"))) assert lex.capitalization_exceptions == (("phd", "B"),) - rebuilt = Lexicon(capitalization_exceptions=lex.capitalization_exceptions_map) + rebuilt = Lexicon(capitalization_exceptions=lex.capitalization_exceptions_map) # type: ignore[arg-type] assert rebuilt == lex and hash(rebuilt) == hash(lex) def test_lexicon_rejects_non_str_exception_values(): with pytest.raises(ValueError, match="str -> str"): - Lexicon(capitalization_exceptions={"phd": 42}) # type: ignore[dict-item] + Lexicon(capitalization_exceptions={"phd": 42}) # type: ignore[dict-item, arg-type] def test_add_and_remove_return_new_lexicons(): @@ -96,10 +96,10 @@ def test_add_capitalization_exceptions_raises_pointing_at_replace(): def test_union_is_fieldwise_and_right_biased_for_exceptions(): a = dataclasses.replace(Lexicon.empty(), - capitalization_exceptions={"phd": "PhD"}) + capitalization_exceptions=(("phd", "PhD"),)) a = a.add(titles={"dr"}) b = dataclasses.replace(Lexicon.empty(), - capitalization_exceptions={"phd": "Ph.D."}) + capitalization_exceptions=(("phd", "Ph.D."),)) b = b.add(titles={"mr"}) u = a | b assert u.titles == frozenset({"dr", "mr"}) @@ -107,6 +107,6 @@ def test_union_is_fieldwise_and_right_biased_for_exceptions(): def test_remove_breaking_subset_invariant_raises(): - lex = Lexicon(particles={"van"}, particles_ambiguous={"van"}) + lex = Lexicon(particles=frozenset({"van"}), particles_ambiguous=frozenset({"van"})) with pytest.raises(ValueError, match="subset"): lex.remove(particles={"van"}) # would orphan particles_ambiguous diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index ac0293d3..83e33bf4 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -38,10 +38,10 @@ def test_name_order_must_be_permutation_and_error_names_constants(): def test_patronymic_rules_coerce_and_reject(): - p = Policy(patronymic_rules=frozenset({"east-slavic"})) + p = Policy(patronymic_rules=frozenset({"east-slavic"})) # type: ignore[arg-type] assert p.patronymic_rules == frozenset({PatronymicRule.EAST_SLAVIC}) with pytest.raises(ValueError, match="east-slavic, turkic"): - Policy(patronymic_rules=frozenset({"klingon"})) + Policy(patronymic_rules=frozenset({"klingon"})) # type: ignore[arg-type] def test_delimiter_pairs_must_be_nonempty_string_pairs(): @@ -79,7 +79,7 @@ def test_extra_suffix_delimiters_validated_and_coerced(): with pytest.raises(ValueError, match="bare string"): Policy(extra_suffix_delimiters="ab") # type: ignore[arg-type] with pytest.raises(ValueError, match="non-empty strings"): - Policy(extra_suffix_delimiters={""}) + Policy(extra_suffix_delimiters=frozenset({""})) p = Policy(extra_suffix_delimiters=["-"]) # type: ignore[arg-type] assert p.extra_suffix_delimiters == frozenset({"-"}) assert isinstance(hash(p), int) @@ -98,7 +98,7 @@ def test_policy_patch_mirrors_policy_field_types(): def test_policy_patch_canonicalizes_union_fields(): - p = PolicyPatch(extra_suffix_delimiters={"-"}) + p = PolicyPatch(extra_suffix_delimiters=frozenset({"-"})) assert isinstance(p.extra_suffix_delimiters, frozenset) assert isinstance(hash(p), int) out = apply_patch(Policy(), PolicyPatch(extra_suffix_delimiters=["-"])) # type: ignore[arg-type] diff --git a/tests/v2/test_reprs.py b/tests/v2/test_reprs.py index a5c421db..4f6672b8 100644 --- a/tests/v2/test_reprs.py +++ b/tests/v2/test_reprs.py @@ -3,25 +3,25 @@ from nameparser._lexicon import Lexicon from nameparser._locale import Locale from nameparser._policy import FAMILY_FIRST, PatronymicRule, Policy, PolicyPatch -from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Token +from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token def test_token_repr_is_compact(): - t = Token("de", (9, 11), Role.FAMILY, frozenset({"particle"})) + t = Token("de", Span(9, 11), Role.FAMILY, frozenset({"particle"})) assert repr(t) == "Token('de' @9:11 FAMILY {particle})" assert repr(Token("Jane", None, Role.GIVEN)) == "Token('Jane' @synthetic GIVEN)" def test_ambiguity_repr_shows_kind_and_token_texts(): - van = Token("Van", (0, 3), Role.GIVEN) + van = Token("Van", Span(0, 3), Role.GIVEN) a = Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "detail", (van,)) assert repr(a) == "Ambiguity('particle-or-given': 'Van')" def test_parsedname_repr_lists_nonempty_fields_in_canonical_order(): pn = ParsedName("John Smith", ( - Token("John", (0, 4), Role.GIVEN), - Token("Smith", (5, 10), Role.FAMILY), + Token("John", Span(0, 4), Role.GIVEN), + Token("Smith", Span(5, 10), Role.FAMILY), )) assert repr(pn) == ( "" @@ -29,9 +29,9 @@ def test_parsedname_repr_lists_nonempty_fields_in_canonical_order(): def test_parsedname_repr_includes_ambiguities_line_when_present(): - van = Token("Van", (0, 3), Role.GIVEN) + van = Token("Van", Span(0, 3), Role.GIVEN) pn = ParsedName("Van Johnson", - (van, Token("Johnson", (4, 11), Role.FAMILY)), + (van, Token("Johnson", Span(4, 11), Role.FAMILY)), (Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "d", (van,)),)) assert "ambiguities: ['particle-or-given']" in repr(pn) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 4ddfab08..21cedd8e 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -10,7 +10,7 @@ def test_role_declaration_order_is_canonical_field_order(): def test_token_construction_and_span_coercion(): - t = Token("Juan", (0, 4), Role.GIVEN) + t = Token("Juan", (0, 4), Role.GIVEN) # type: ignore[arg-type] assert t.span == Span(0, 4) assert isinstance(t.span, Span) assert t.span.start == 0 and t.span.end == 4 @@ -24,17 +24,17 @@ def test_synthetic_token_has_no_span(): def test_token_rejects_empty_text(): with pytest.raises(ValueError, match="non-empty"): - Token("", (0, 0), Role.GIVEN) + Token("", Span(0, 0), Role.GIVEN) def test_token_rejects_inverted_span(): with pytest.raises(ValueError, match="start <= end"): - Token("x", (5, 2), Role.GIVEN) + Token("x", Span(5, 2), Role.GIVEN) def test_token_rejects_negative_span(): with pytest.raises(ValueError, match="start <= end"): - Token("x", (-1, 1), Role.GIVEN) + Token("x", Span(-1, 1), Role.GIVEN) def test_token_rejects_malformed_span_shapes(): @@ -50,10 +50,10 @@ def test_token_rejects_non_string_text(): def test_token_is_frozen_and_hashable(): - t = Token("Juan", (0, 4), Role.GIVEN) + t = Token("Juan", Span(0, 4), Role.GIVEN) with pytest.raises(AttributeError): t.text = "X" # type: ignore[misc] - assert hash(t) == hash(Token("Juan", (0, 4), Role.GIVEN)) + assert hash(t) == hash(Token("Juan", Span(0, 4), Role.GIVEN)) def test_ambiguity_kind_members_are_their_string_values(): @@ -62,15 +62,15 @@ def test_ambiguity_kind_members_are_their_string_values(): def test_ambiguity_construction_coerces_kind_string(): - t = Token("Van", (0, 3), Role.GIVEN, frozenset({"particle"})) - a = Ambiguity("particle-or-given", "leading 'van' may be a particle", (t,)) + t = Token("Van", Span(0, 3), Role.GIVEN, frozenset({"particle"})) + a = Ambiguity("particle-or-given", "leading 'van' may be a particle", (t,)) # type: ignore[arg-type] assert a.kind is AmbiguityKind.PARTICLE_OR_GIVEN assert a.tokens == (t,) def test_ambiguity_rejects_unknown_kind(): with pytest.raises(ValueError, match="particle-or-given"): - Ambiguity("no-such-kind", "detail", ()) + Ambiguity("no-such-kind", "detail", ()) # type: ignore[arg-type] def test_ambiguity_rejects_non_token_elements(): @@ -80,7 +80,7 @@ def test_ambiguity_rejects_non_token_elements(): def test_ambiguity_rejects_empty_detail(): with pytest.raises(ValueError, match="non-empty string"): - Ambiguity("order", "", ()) + Ambiguity(AmbiguityKind.ORDER, "", ()) def _pn(original, tokens, ambiguities=()): @@ -90,8 +90,8 @@ def _pn(original, tokens, ambiguities=()): def test_parsedname_accepts_valid_spans_and_is_truthy(): pn = _pn("John Smith", [ - Token("John", (0, 4), Role.GIVEN), - Token("Smith", (5, 10), Role.FAMILY), + Token("John", Span(0, 4), Role.GIVEN), + Token("Smith", Span(5, 10), Role.FAMILY), ]) assert bool(pn) is True @@ -103,47 +103,47 @@ def test_empty_parse_is_falsy(): def test_parsedname_rejects_out_of_bounds_span(): with pytest.raises(ValueError, match="out of bounds"): - _pn("John", [Token("Johnny", (0, 6), Role.GIVEN)]) + _pn("John", [Token("Johnny", Span(0, 6), Role.GIVEN)]) def test_parsedname_rejects_overlapping_spans(): with pytest.raises(ValueError, match="ascending"): _pn("John Smith", [ - Token("John", (0, 4), Role.GIVEN), - Token("ohn S", (1, 6), Role.FAMILY), + Token("John", Span(0, 4), Role.GIVEN), + Token("ohn S", Span(1, 6), Role.FAMILY), ]) def test_parsedname_rejects_descending_spans(): with pytest.raises(ValueError, match="ascending"): _pn("John Smith", [ - Token("Smith", (5, 10), Role.FAMILY), - Token("John", (0, 4), Role.GIVEN), + Token("Smith", Span(5, 10), Role.FAMILY), + Token("John", Span(0, 4), Role.GIVEN), ]) def test_synthetic_tokens_skip_span_checks(): pn = _pn("John Smith", [ - Token("John", (0, 4), Role.GIVEN), + Token("John", Span(0, 4), Role.GIVEN), Token("Qux", None, Role.MIDDLE), - Token("Smith", (5, 10), Role.FAMILY), + Token("Smith", Span(5, 10), Role.FAMILY), ]) assert len(pn.tokens) == 3 def test_ambiguity_tokens_must_be_subset_of_parse_tokens(): - inside = Token("Van", (0, 3), Role.GIVEN) + inside = Token("Van", Span(0, 3), Role.GIVEN) outside = Token("Zzz", None, Role.GIVEN) with pytest.raises(ValueError, match="subset"): _pn("Van Johnson", - [inside, Token("Johnson", (4, 11), Role.FAMILY)], + [inside, Token("Johnson", Span(4, 11), Role.FAMILY)], [Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "d", (outside,))]) def test_parsedname_equality_is_strict_structural(): - a = _pn("John", [Token("John", (0, 4), Role.GIVEN)]) - b = _pn("John", [Token("John", (0, 4), Role.GIVEN)]) - c = _pn("John ", [Token("John", (0, 4), Role.GIVEN)]) + a = _pn("John", [Token("John", Span(0, 4), Role.GIVEN)]) + b = _pn("John", [Token("John", Span(0, 4), Role.GIVEN)]) + c = _pn("John ", [Token("John", Span(0, 4), Role.GIVEN)]) assert a == b and hash(a) == hash(b) assert a != c # different original: not interchangeable @@ -157,19 +157,19 @@ def test_parsedname_rejects_non_token_and_non_ambiguity_elements(): with pytest.raises(ValueError, match="only Token instances"): _pn("x", ["not-a-token"]) # type: ignore[list-item] with pytest.raises(ValueError, match="only Ambiguity instances"): - _pn("John", [Token("John", (0, 4), Role.GIVEN)], ["nope"]) # type: ignore[list-item] + _pn("John", [Token("John", Span(0, 4), Role.GIVEN)], ["nope"]) # type: ignore[list-item] def _delavega(): # "Dr. Juan de la Vega III" -- hand-built, spans verified by hand # 0123456789012345678901234 return _pn("Dr. Juan de la Vega III", [ - Token("Dr.", (0, 3), Role.TITLE), - Token("Juan", (4, 8), Role.GIVEN), - Token("de", (9, 11), Role.FAMILY, frozenset({"particle"})), - Token("la", (12, 14), Role.FAMILY, frozenset({"particle"})), - Token("Vega", (15, 19), Role.FAMILY), - Token("III", (20, 23), Role.SUFFIX), + Token("Dr.", Span(0, 3), Role.TITLE), + Token("Juan", Span(4, 8), Role.GIVEN), + Token("de", Span(9, 11), Role.FAMILY, frozenset({"particle"})), + Token("la", Span(12, 14), Role.FAMILY, frozenset({"particle"})), + Token("Vega", Span(15, 19), Role.FAMILY), + Token("III", Span(20, 23), Role.SUFFIX), ]) @@ -186,10 +186,10 @@ def test_string_properties_join_by_role(): def test_suffix_joins_with_comma_space(): pn = _pn("John Smith PhD MD", [ - Token("John", (0, 4), Role.GIVEN), - Token("Smith", (5, 10), Role.FAMILY), - Token("PhD", (11, 14), Role.SUFFIX), - Token("MD", (15, 17), Role.SUFFIX), + Token("John", Span(0, 4), Role.GIVEN), + Token("Smith", Span(5, 10), Role.FAMILY), + Token("PhD", Span(11, 14), Role.SUFFIX), + Token("MD", Span(15, 17), Role.SUFFIX), ]) assert pn.suffix == "PhD, MD" @@ -232,8 +232,8 @@ def test_replace_swaps_field_with_synthetic_tokens_in_place(): def test_replace_adds_missing_field_at_end(): pn = _pn("John Smith", [ - Token("John", (0, 4), Role.GIVEN), - Token("Smith", (5, 10), Role.FAMILY), + Token("John", Span(0, 4), Role.GIVEN), + Token("Smith", Span(5, 10), Role.FAMILY), ]) pn2 = pn.replace(suffix="Jr") assert pn2.suffix == "Jr" @@ -251,9 +251,9 @@ def test_replace_rejects_unknown_field(): def test_replace_drops_ambiguities_referencing_removed_tokens(): - van = Token("Van", (0, 3), Role.GIVEN) + van = Token("Van", Span(0, 3), Role.GIVEN) pn = _pn("Van Johnson", - [van, Token("Johnson", (4, 11), Role.FAMILY)], + [van, Token("Johnson", Span(4, 11), Role.FAMILY)], [Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "d", (van,))]) assert pn.replace(given="Bob").ambiguities == () assert pn.replace(family="Smith").ambiguities != () @@ -265,7 +265,7 @@ def test_replace_rejects_non_str_value(): def test_replace_appends_missing_roles_in_canonical_order(): - pn = _pn("John", [Token("John", (0, 4), Role.GIVEN)]) + pn = _pn("John", [Token("John", Span(0, 4), Role.GIVEN)]) pn2 = pn.replace(maiden="X", suffix="Y") assert [t.role for t in pn2.tokens] == [Role.GIVEN, Role.SUFFIX, Role.MAIDEN] @@ -276,15 +276,15 @@ def test_comparison_key_is_casefolded_canonical_seven_tuple(): "dr.", "juan", "", "de la vega", "iii", "", "", ) upper = _pn("JUAN DE LA VEGA", [ - Token("JUAN", (0, 4), Role.GIVEN), - Token("DE", (5, 7), Role.FAMILY, frozenset({"particle"})), - Token("LA", (8, 10), Role.FAMILY, frozenset({"particle"})), - Token("VEGA", (11, 15), Role.FAMILY), + Token("JUAN", Span(0, 4), Role.GIVEN), + Token("DE", Span(5, 7), Role.FAMILY, frozenset({"particle"})), + Token("LA", Span(8, 10), Role.FAMILY, frozenset({"particle"})), + Token("VEGA", Span(11, 15), Role.FAMILY), ]) lower = _pn("juan de la vega", [ - Token("juan", (0, 4), Role.GIVEN), - Token("de", (5, 7), Role.FAMILY, frozenset({"particle"})), - Token("la", (8, 10), Role.FAMILY, frozenset({"particle"})), - Token("vega", (11, 15), Role.FAMILY), + Token("juan", Span(0, 4), Role.GIVEN), + Token("de", Span(5, 7), Role.FAMILY, frozenset({"particle"})), + Token("la", Span(8, 10), Role.FAMILY, frozenset({"particle"})), + Token("vega", Span(11, 15), Role.FAMILY), ]) assert upper.comparison_key() == lower.comparison_key() From 30ef7630c1007e94a5bba666c3b924d1283170c3 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 04:54:37 -0700 Subject: [PATCH 027/275] Make Lexicon picklable despite its mappingproxy slot pickle.dumps(Lexicon.default()) raised TypeError because the default slots-dataclass pickle path serializes the _cap_map MappingProxyType. Ship every other slot and rebuild the proxy from the canonical capitalization_exceptions tuple on load. Parser (Plan 3) is picklable by construction per the core spec, and a Parser holds a Lexicon. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 13 +++++++++++++ tests/v2/test_lexicon.py | 15 +++++++++++++++ tests/v2/test_locale.py | 7 +++++++ 3 files changed, 35 insertions(+) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 9394899f..b9e20bab 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -118,6 +118,19 @@ def __repr__(self) -> str: deltas.append(f"{name}: {delta}") return f"Lexicon(default + {', '.join(deltas)})" + def __getstate__(self) -> dict[str, object]: + # _cap_map is a MappingProxyType, which pickle rejects; ship every + # other slot and rebuild the proxy from the canonical tuple on load. + return {f.name: getattr(self, f.name) + for f in dataclasses.fields(self) if f.name != "_cap_map"} + + def __setstate__(self, state: dict[str, object]) -> None: + for name, value in state.items(): + object.__setattr__(self, name, value) + object.__setattr__( + self, "_cap_map", + MappingProxyType(dict(self.capitalization_exceptions))) + @property def capitalization_exceptions_map(self) -> Mapping[str, str]: return self._cap_map diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 4102a271..5b1e2729 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -110,3 +110,18 @@ def test_remove_breaking_subset_invariant_raises(): lex = Lexicon(particles=frozenset({"van"}), particles_ambiguous=frozenset({"van"})) with pytest.raises(ValueError, match="subset"): lex.remove(particles={"van"}) # would orphan particles_ambiguous + + +def test_pickle_round_trip_preserves_equality_and_cap_map(): + # _cap_map holds a MappingProxyType, which pickle rejects; Lexicon + # must round-trip anyway because Parser is picklable by construction + # (spec: 2026-07-11-v2-core-api-design.md) and a Parser holds a Lexicon. + import pickle + + for lex in (Lexicon.default(), + Lexicon.empty().add(titles={"Dr."})): + loaded = pickle.loads(pickle.dumps(lex)) + assert loaded == lex + assert hash(loaded) == hash(lex) + assert (loaded.capitalization_exceptions_map + == lex.capitalization_exceptions_map) diff --git a/tests/v2/test_locale.py b/tests/v2/test_locale.py index b3a8f541..b5829ab8 100644 --- a/tests/v2/test_locale.py +++ b/tests/v2/test_locale.py @@ -44,3 +44,10 @@ def test_locale_validates_component_types(): Locale(code="ru", lexicon={"titles": set()}) # type: ignore[arg-type] with pytest.raises(ValueError, match="PolicyPatch"): Locale(code="ru", lexicon=Lexicon.empty(), policy={"name_order": None}) # type: ignore[arg-type] + + +def test_locale_with_lexicon_pickles_round_trip(): + import pickle + + loc = Locale(code="ru", lexicon=Lexicon.empty().add(titles={"Dr."})) + assert pickle.loads(pickle.dumps(loc)) == loc From 401535cae216bd1daf182c1b0a99ae7b3d21f976 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 04:55:25 -0700 Subject: [PATCH 028/275] Drop Python 3.10 from the CI matrix (#257, partial) requires-python is >=3.11 on this branch, so the 3.10 job can no longer install the package (uv sync either fails or silently substitutes a managed 3.11). The rest of #257 (typing_extensions, classifiers) stays tracked on the issue. Co-Authored-By: Claude Fable 5 --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 290ee0ae..072f44c0 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v7 From f2b42c8bb68946b32f5fc80351ebd8ea1d841de7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 04:58:10 -0700 Subject: [PATCH 029/275] Drop the pre-3.11 typing_extensions shim (#257) With requires-python at >=3.11, Self imports directly from typing and the conditional dependency can never activate; ruff (UP035/UP036) flags the dead version block once the floor is raised. Co-Authored-By: Claude Fable 5 --- nameparser/config/__init__.py | 7 +------ pyproject.toml | 4 +--- tests/v2/test_layering.py | 15 ++++++++++++++- tests/v2/test_types.py | 7 ++++++- uv.lock | 1 - 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/nameparser/config/__init__.py b/nameparser/config/__init__.py index 6f1e7984..a0bbdcea 100644 --- a/nameparser/config/__init__.py +++ b/nameparser/config/__init__.py @@ -45,12 +45,7 @@ import sys import warnings from collections.abc import Callable, Iterable, Iterator, Mapping, Set -from typing import Any, TypeVar, overload - -if sys.version_info >= (3, 11): - from typing import Self -else: - from typing_extensions import Self +from typing import Any, Self, TypeVar, overload from nameparser.util import lc from nameparser.config.prefixes import PREFIXES, NON_FIRST_NAME_PREFIXES diff --git a/pyproject.toml b/pyproject.toml index 1beee8fa..afbfecae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,9 +22,7 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Text Processing :: Linguistic", ] -dependencies = [ - "typing_extensions (>=4.5.0); python_version < '3.11'" -] +dependencies = [] [build-system] requires = ["setuptools (>=82.0.1)"] diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 99ae45c8..67f4a49f 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -26,10 +26,23 @@ def _nameparser_imports(path: pathlib.Path) -> list[str]: return [m for m in found if m.startswith("nameparser")] +def _permitted(imported: str, allowed: tuple[str, ...]) -> bool: + # An entry ending in "." is a pure prefix (subpackage contents only); + # any other entry means that exact module or its submodules -- a bare + # startswith would also admit siblings like nameparser._types_helpers. + for entry in allowed: + if entry.endswith("."): + if imported.startswith(entry): + return True + elif imported == entry or imported.startswith(entry + "."): + return True + return False + + def test_layering_contract(): for mod, allowed in ALLOWED.items(): for imported in _nameparser_imports(PKG / mod): - assert imported.startswith(allowed), ( + assert _permitted(imported, allowed), ( f"{mod} imports {imported}, which the layering contract " f"forbids (allowed prefixes: {allowed or 'none'})" ) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 21cedd8e..95bef688 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -1,6 +1,8 @@ import pytest -from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token +from nameparser._types import ( + STABLE_TAGS, Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token, +) def test_role_declaration_order_is_canonical_field_order(): @@ -195,6 +197,9 @@ def test_suffix_joins_with_comma_space(): def test_derived_views_filter_on_stable_particle_tag(): + # Pin the hard-coded "particle" string in _text_for to the published + # contract until Plan 3's tag-emission contract tests land. + assert "particle" in STABLE_TAGS pn = _delavega() assert pn.family_particles == "de la" assert pn.family_base == "Vega" diff --git a/uv.lock b/uv.lock index c9b8164c..fe847877 100644 --- a/uv.lock +++ b/uv.lock @@ -585,7 +585,6 @@ dev = [ ] [package.metadata] -requires-dist = [{ name = "typing-extensions", marker = "python_full_version < '3.11'", specifier = ">=4.5.0" }] [package.metadata.requires-dev] dev = [ From 4c171fc06a6bb0ee4b5b9b24ab8fc4a9972f4bc2 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 04:59:11 -0700 Subject: [PATCH 030/275] Annotate v2 test returns for ruff ANN compliance The v1 suite is fully annotated (since #250 put tests/ under mypy), so pyproject carries no per-file ANN ignores; the new tests/v2 modules must be annotated too or 'ruff check' fails in CI. Also drops an unused import ruff flagged (F401). Co-Authored-By: Claude Fable 5 --- tests/v2/test_lexicon.py | 34 +++++++++++++++++----------------- tests/v2/test_locale.py | 14 +++++++------- tests/v2/test_policy.py | 36 ++++++++++++++++++------------------ tests/v2/test_reprs.py | 17 ++++++++--------- 4 files changed, 50 insertions(+), 51 deletions(-) diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 5b1e2729..d2cfe5f5 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -5,12 +5,12 @@ from nameparser._lexicon import Lexicon -def test_entries_are_normalized_at_construction(): +def test_entries_are_normalized_at_construction() -> None: lex = Lexicon(titles=frozenset({"Dr.", "MR"})) assert lex.titles == frozenset({"dr", "mr"}) -def test_default_sources_v1_vocabulary(): +def test_default_sources_v1_vocabulary() -> None: lex = Lexicon.default() assert "dr" in lex.titles assert "van" in lex.particles @@ -24,16 +24,16 @@ def test_default_sources_v1_vocabulary(): assert lex.capitalization_exceptions_map["phd"] == "Ph.D." -def test_default_is_cached_single_instance(): +def test_default_is_cached_single_instance() -> None: assert Lexicon.default() is Lexicon.default() -def test_particles_ambiguous_must_be_subset_of_particles(): +def test_particles_ambiguous_must_be_subset_of_particles() -> None: with pytest.raises(ValueError, match="subset"): Lexicon(particles_ambiguous=frozenset({"van"})) -def test_capitalization_exceptions_canonical_and_no_aliasing(): +def test_capitalization_exceptions_canonical_and_no_aliasing() -> None: exceptions = {"phd": "PhD", "ii": "II"} lex = Lexicon.empty() lex2 = dataclasses.replace(lex, capitalization_exceptions=exceptions) # type: ignore[arg-type] @@ -44,38 +44,38 @@ def test_capitalization_exceptions_canonical_and_no_aliasing(): assert lex2 == lex3 and hash(lex2) == hash(lex3) -def test_lexicon_is_hashable(): +def test_lexicon_is_hashable() -> None: assert isinstance(hash(Lexicon.default()), int) -def test_lexicon_rejects_bare_string_vocab(): +def test_lexicon_rejects_bare_string_vocab() -> None: with pytest.raises(ValueError, match="bare string"): Lexicon(titles="dr") # type: ignore[arg-type] -def test_lexicon_rejects_non_str_vocab_entries(): +def test_lexicon_rejects_non_str_vocab_entries() -> None: with pytest.raises(ValueError, match="entries must be strings"): Lexicon(titles={"Dr.", 42}) # type: ignore[arg-type] -def test_exception_keys_normalizing_to_empty_are_dropped(): +def test_exception_keys_normalizing_to_empty_are_dropped() -> None: lex = Lexicon(capitalization_exceptions=(("...", "X"), ("phd", "PhD"))) assert lex.capitalization_exceptions == (("phd", "PhD"),) -def test_colliding_exception_keys_dedupe_last_wins(): +def test_colliding_exception_keys_dedupe_last_wins() -> None: lex = Lexicon(capitalization_exceptions=(("Ph.D.", "A"), ("phd", "B"))) assert lex.capitalization_exceptions == (("phd", "B"),) rebuilt = Lexicon(capitalization_exceptions=lex.capitalization_exceptions_map) # type: ignore[arg-type] assert rebuilt == lex and hash(rebuilt) == hash(lex) -def test_lexicon_rejects_non_str_exception_values(): +def test_lexicon_rejects_non_str_exception_values() -> None: with pytest.raises(ValueError, match="str -> str"): Lexicon(capitalization_exceptions={"phd": 42}) # type: ignore[dict-item, arg-type] -def test_add_and_remove_return_new_lexicons(): +def test_add_and_remove_return_new_lexicons() -> None: # "zqtitle" is a synthetic word absent from v1's TITLES data (unlike # e.g. "dra", the feminine "dr." abbreviation, which is already there). base = Lexicon.default() @@ -84,17 +84,17 @@ def test_add_and_remove_return_new_lexicons(): assert "bishop" not in lex.suffix_words -def test_add_unknown_field_raises_with_valid_names(): +def test_add_unknown_field_raises_with_valid_names() -> None: with pytest.raises(TypeError, match="prefixes"): Lexicon.default().add(prefixes={"van"}) # v1 name: helpful error -def test_add_capitalization_exceptions_raises_pointing_at_replace(): +def test_add_capitalization_exceptions_raises_pointing_at_replace() -> None: with pytest.raises(TypeError, match="dataclasses.replace"): Lexicon.default().add(capitalization_exceptions={"x": "X"}) -def test_union_is_fieldwise_and_right_biased_for_exceptions(): +def test_union_is_fieldwise_and_right_biased_for_exceptions() -> None: a = dataclasses.replace(Lexicon.empty(), capitalization_exceptions=(("phd", "PhD"),)) a = a.add(titles={"dr"}) @@ -106,13 +106,13 @@ def test_union_is_fieldwise_and_right_biased_for_exceptions(): assert u.capitalization_exceptions_map["phd"] == "Ph.D." # right wins -def test_remove_breaking_subset_invariant_raises(): +def test_remove_breaking_subset_invariant_raises() -> None: lex = Lexicon(particles=frozenset({"van"}), particles_ambiguous=frozenset({"van"})) with pytest.raises(ValueError, match="subset"): lex.remove(particles={"van"}) # would orphan particles_ambiguous -def test_pickle_round_trip_preserves_equality_and_cap_map(): +def test_pickle_round_trip_preserves_equality_and_cap_map() -> None: # _cap_map holds a MappingProxyType, which pickle rejects; Lexicon # must round-trip anyway because Parser is picklable by construction # (spec: 2026-07-11-v2-core-api-design.md) and a Parser holds a Lexicon. diff --git a/tests/v2/test_locale.py b/tests/v2/test_locale.py index b5829ab8..0af17fec 100644 --- a/tests/v2/test_locale.py +++ b/tests/v2/test_locale.py @@ -5,7 +5,7 @@ from nameparser._policy import PatronymicRule, PolicyPatch -def test_locale_holds_code_lexicon_fragment_and_patch(): +def test_locale_holds_code_lexicon_fragment_and_patch() -> None: ru = Locale( code="ru", lexicon=Lexicon.empty(), @@ -17,36 +17,36 @@ def test_locale_holds_code_lexicon_fragment_and_patch(): {PatronymicRule.EAST_SLAVIC}) -def test_locale_defaults_to_empty_patch(): +def test_locale_defaults_to_empty_patch() -> None: assert Locale(code="xx", lexicon=Lexicon.empty()).policy == PolicyPatch() -def test_locale_code_must_be_nonempty_lowercase(): +def test_locale_code_must_be_nonempty_lowercase() -> None: with pytest.raises(ValueError, match="lowercase"): Locale(code="RU", lexicon=Lexicon.empty()) with pytest.raises(ValueError, match="non-empty"): Locale(code="", lexicon=Lexicon.empty()) -def test_locale_code_rejects_whitespace(): +def test_locale_code_rejects_whitespace() -> None: for bad in ("ru ", " ru", "ru\n", "r u"): with pytest.raises(ValueError, match="whitespace"): Locale(code=bad, lexicon=Lexicon.empty()) -def test_locale_is_hashable(): +def test_locale_is_hashable() -> None: loc = Locale(code="ru", lexicon=Lexicon.empty()) assert isinstance(hash(loc), int) -def test_locale_validates_component_types(): +def test_locale_validates_component_types() -> None: with pytest.raises(ValueError, match="Lexicon"): Locale(code="ru", lexicon={"titles": set()}) # type: ignore[arg-type] with pytest.raises(ValueError, match="PolicyPatch"): Locale(code="ru", lexicon=Lexicon.empty(), policy={"name_order": None}) # type: ignore[arg-type] -def test_locale_with_lexicon_pickles_round_trip(): +def test_locale_with_lexicon_pickles_round_trip() -> None: import pickle loc = Locale(code="ru", lexicon=Lexicon.empty().add(titles={"Dr."})) diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 83e33bf4..0c9b6100 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -9,13 +9,13 @@ from nameparser._types import Role -def test_order_constants_read_as_their_contents(): +def test_order_constants_read_as_their_contents() -> None: assert GIVEN_FIRST == (Role.GIVEN, Role.MIDDLE, Role.FAMILY) assert FAMILY_FIRST == (Role.FAMILY, Role.GIVEN, Role.MIDDLE) assert FAMILY_FIRST_GIVEN_LAST == (Role.FAMILY, Role.MIDDLE, Role.GIVEN) -def test_policy_defaults(): +def test_policy_defaults() -> None: p = Policy() assert p.name_order == GIVEN_FIRST assert p.patronymic_rules == frozenset() @@ -24,58 +24,58 @@ def test_policy_defaults(): assert p.strip_emoji and p.strip_bidi and p.lenient_comma_suffixes -def test_policy_is_hashable_and_replaceable(): +def test_policy_is_hashable_and_replaceable() -> None: p = dataclasses.replace(Policy(), name_order=FAMILY_FIRST) assert p.name_order == FAMILY_FIRST assert isinstance(hash(p), int) -def test_name_order_must_be_permutation_and_error_names_constants(): +def test_name_order_must_be_permutation_and_error_names_constants() -> None: with pytest.raises(ValueError, match="FAMILY_FIRST_GIVEN_LAST"): Policy(name_order=(Role.TITLE, Role.GIVEN, Role.FAMILY)) with pytest.raises(ValueError, match="GIVEN_FIRST"): Policy(name_order=(Role.GIVEN, Role.GIVEN, Role.FAMILY)) -def test_patronymic_rules_coerce_and_reject(): +def test_patronymic_rules_coerce_and_reject() -> None: p = Policy(patronymic_rules=frozenset({"east-slavic"})) # type: ignore[arg-type] assert p.patronymic_rules == frozenset({PatronymicRule.EAST_SLAVIC}) with pytest.raises(ValueError, match="east-slavic, turkic"): Policy(patronymic_rules=frozenset({"klingon"})) # type: ignore[arg-type] -def test_delimiter_pairs_must_be_nonempty_string_pairs(): +def test_delimiter_pairs_must_be_nonempty_string_pairs() -> None: with pytest.raises(ValueError, match="non-empty"): Policy(nickname_delimiters=frozenset({("", ")")})) -def test_delimiter_pair_rejects_two_char_string(): +def test_delimiter_pair_rejects_two_char_string() -> None: with pytest.raises(ValueError, match="tuples"): Policy(nickname_delimiters=frozenset({"()"})) # type: ignore[arg-type] -def test_patronymic_rules_rejects_bare_string_and_non_iterable(): +def test_patronymic_rules_rejects_bare_string_and_non_iterable() -> None: with pytest.raises(ValueError, match="bare string"): Policy(patronymic_rules="east-slavic") # type: ignore[arg-type] with pytest.raises(ValueError, match="valid rules"): Policy(patronymic_rules=5) # type: ignore[arg-type] -def test_policy_delimiters_coerce_to_frozensets(): +def test_policy_delimiters_coerce_to_frozensets() -> None: p = Policy(nickname_delimiters=[("(", ")")]) # type: ignore[arg-type] assert isinstance(p.nickname_delimiters, frozenset) assert isinstance(hash(p), int) assert p == Policy(nickname_delimiters=frozenset({("(", ")")})) -def test_policy_delimiters_do_not_alias_caller_containers(): +def test_policy_delimiters_do_not_alias_caller_containers() -> None: source = {("(", ")")} p = Policy(nickname_delimiters=source) # type: ignore[arg-type] source.add(("'", "'")) assert ("'", "'") not in p.nickname_delimiters -def test_extra_suffix_delimiters_validated_and_coerced(): +def test_extra_suffix_delimiters_validated_and_coerced() -> None: with pytest.raises(ValueError, match="bare string"): Policy(extra_suffix_delimiters="ab") # type: ignore[arg-type] with pytest.raises(ValueError, match="non-empty strings"): @@ -85,19 +85,19 @@ def test_extra_suffix_delimiters_validated_and_coerced(): assert isinstance(hash(p), int) -def test_policy_patch_mirrors_policy_field_names(): +def test_policy_patch_mirrors_policy_field_names() -> None: policy_fields = {f.name for f in dataclasses.fields(Policy)} patch_fields = {f.name for f in dataclasses.fields(PolicyPatch)} assert policy_fields == patch_fields -def test_policy_patch_mirrors_policy_field_types(): +def test_policy_patch_mirrors_policy_field_types() -> None: for f in dataclasses.fields(Policy): patch_annotation = PolicyPatch.__dataclass_fields__[f.name].type assert patch_annotation == f"{f.type} | _Unset" -def test_policy_patch_canonicalizes_union_fields(): +def test_policy_patch_canonicalizes_union_fields() -> None: p = PolicyPatch(extra_suffix_delimiters=frozenset({"-"})) assert isinstance(p.extra_suffix_delimiters, frozenset) assert isinstance(hash(p), int) @@ -105,12 +105,12 @@ def test_policy_patch_canonicalizes_union_fields(): assert out.extra_suffix_delimiters == frozenset({"-"}) -def test_policy_patch_rejects_bare_string_union_fields(): +def test_policy_patch_rejects_bare_string_union_fields() -> None: with pytest.raises(ValueError, match="bare string"): PolicyPatch(extra_suffix_delimiters="ab") # type: ignore[arg-type] -def test_apply_patch_overrides_scalars_and_unions_sets(): +def test_apply_patch_overrides_scalars_and_unions_sets() -> None: base = Policy(patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})) patch = PolicyPatch( name_order=FAMILY_FIRST, @@ -123,12 +123,12 @@ def test_apply_patch_overrides_scalars_and_unions_sets(): assert out.strip_emoji is True # untouched -def test_apply_patch_with_empty_patch_returns_same_policy(): +def test_apply_patch_with_empty_patch_returns_same_policy() -> None: base = Policy() assert apply_patch(base, PolicyPatch()) is base -def test_unset_fields_are_distinguishable_from_defaults(): +def test_unset_fields_are_distinguishable_from_defaults() -> None: patch = PolicyPatch(strip_emoji=True) # explicitly set to the default value assert patch.strip_emoji is True assert PolicyPatch().strip_emoji is UNSET diff --git a/tests/v2/test_reprs.py b/tests/v2/test_reprs.py index 4f6672b8..cacfdbc6 100644 --- a/tests/v2/test_reprs.py +++ b/tests/v2/test_reprs.py @@ -1,4 +1,3 @@ -import dataclasses from nameparser._lexicon import Lexicon from nameparser._locale import Locale @@ -6,19 +5,19 @@ from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token -def test_token_repr_is_compact(): +def test_token_repr_is_compact() -> None: t = Token("de", Span(9, 11), Role.FAMILY, frozenset({"particle"})) assert repr(t) == "Token('de' @9:11 FAMILY {particle})" assert repr(Token("Jane", None, Role.GIVEN)) == "Token('Jane' @synthetic GIVEN)" -def test_ambiguity_repr_shows_kind_and_token_texts(): +def test_ambiguity_repr_shows_kind_and_token_texts() -> None: van = Token("Van", Span(0, 3), Role.GIVEN) a = Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "detail", (van,)) assert repr(a) == "Ambiguity('particle-or-given': 'Van')" -def test_parsedname_repr_lists_nonempty_fields_in_canonical_order(): +def test_parsedname_repr_lists_nonempty_fields_in_canonical_order() -> None: pn = ParsedName("John Smith", ( Token("John", Span(0, 4), Role.GIVEN), Token("Smith", Span(5, 10), Role.FAMILY), @@ -28,7 +27,7 @@ def test_parsedname_repr_lists_nonempty_fields_in_canonical_order(): ) -def test_parsedname_repr_includes_ambiguities_line_when_present(): +def test_parsedname_repr_includes_ambiguities_line_when_present() -> None: van = Token("Van", Span(0, 3), Role.GIVEN) pn = ParsedName("Van Johnson", (van, Token("Johnson", Span(4, 11), Role.FAMILY)), @@ -36,24 +35,24 @@ def test_parsedname_repr_includes_ambiguities_line_when_present(): assert "ambiguities: ['particle-or-given']" in repr(pn) -def test_empty_parsedname_repr(): +def test_empty_parsedname_repr() -> None: assert repr(ParsedName("", ())) == "" -def test_policy_repr_shows_only_nondefault_fields(): +def test_policy_repr_shows_only_nondefault_fields() -> None: assert repr(Policy()) == "Policy()" p = Policy(name_order=FAMILY_FIRST, strip_bidi=False) assert repr(p) == "Policy(name_order=FAMILY_FIRST, strip_bidi=False)" -def test_lexicon_repr_is_bounded(): +def test_lexicon_repr_is_bounded() -> None: assert repr(Lexicon.default()) == "Lexicon(default)" lex = Lexicon.default().add(titles={"zqx", "zqy"}) assert repr(lex) == "Lexicon(default + titles: +2)" assert "zqx" not in repr(lex) # never dump contents -def test_locale_repr_shows_code_and_patched_fields(): +def test_locale_repr_shows_code_and_patched_fields() -> None: ru = Locale("ru", Lexicon.empty(), PolicyPatch(patronymic_rules=frozenset( {PatronymicRule.EAST_SLAVIC}))) From 353d10aceb5cc038c5076c7de6bcb9fd136e83a8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 04:59:11 -0700 Subject: [PATCH 031/275] Tighten the layering test to exact-module matching A bare startswith('nameparser._types') would also admit a future sibling like nameparser._types_helpers. Entries ending in '.' stay pure prefixes (subpackage contents); anything else now means that exact module or its submodules. Also carries this file's ANN annotations. Co-Authored-By: Claude Fable 5 --- tests/v2/test_layering.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 67f4a49f..f9d9605b 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -39,7 +39,7 @@ def _permitted(imported: str, allowed: tuple[str, ...]) -> bool: return False -def test_layering_contract(): +def test_layering_contract() -> None: for mod, allowed in ALLOWED.items(): for imported in _nameparser_imports(PKG / mod): assert _permitted(imported, allowed), ( @@ -48,13 +48,13 @@ def test_layering_contract(): ) -def test_lexicon_never_imports_config_package_root_or_parser(): +def test_lexicon_never_imports_config_package_root_or_parser() -> None: for imported in _nameparser_imports(PKG / "_lexicon.py"): assert imported != "nameparser.config" assert not imported.startswith("nameparser.parser") -def test_public_exports(): +def test_public_exports() -> None: expected = { "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET", From 09cc85d6105d0b2d036bde78ad5e41df7761e16c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 04:59:11 -0700 Subject: [PATCH 032/275] Pin the derived views' particle tag to STABLE_TAGS Nothing tied the 'particle' string hard-coded in _text_for callers to the published STABLE_TAGS constant; assert the linkage in the derived-view test until Plan 3's tag-emission contract tests land. Also carries this file's ANN annotations. Co-Authored-By: Claude Fable 5 --- tests/v2/test_types.py | 81 ++++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 95bef688..7446cbde 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -1,3 +1,5 @@ +from collections.abc import Iterable + import pytest from nameparser._types import ( @@ -5,13 +7,13 @@ ) -def test_role_declaration_order_is_canonical_field_order(): +def test_role_declaration_order_is_canonical_field_order() -> None: assert [r.value for r in Role] == [ "title", "given", "middle", "family", "suffix", "nickname", "maiden", ] -def test_token_construction_and_span_coercion(): +def test_token_construction_and_span_coercion() -> None: t = Token("Juan", (0, 4), Role.GIVEN) # type: ignore[arg-type] assert t.span == Span(0, 4) assert isinstance(t.span, Span) @@ -19,78 +21,79 @@ def test_token_construction_and_span_coercion(): assert t.tags == frozenset() -def test_synthetic_token_has_no_span(): +def test_synthetic_token_has_no_span() -> None: t = Token("Jane", None, Role.GIVEN) assert t.span is None -def test_token_rejects_empty_text(): +def test_token_rejects_empty_text() -> None: with pytest.raises(ValueError, match="non-empty"): Token("", Span(0, 0), Role.GIVEN) -def test_token_rejects_inverted_span(): +def test_token_rejects_inverted_span() -> None: with pytest.raises(ValueError, match="start <= end"): Token("x", Span(5, 2), Role.GIVEN) -def test_token_rejects_negative_span(): +def test_token_rejects_negative_span() -> None: with pytest.raises(ValueError, match="start <= end"): Token("x", Span(-1, 1), Role.GIVEN) -def test_token_rejects_malformed_span_shapes(): +def test_token_rejects_malformed_span_shapes() -> None: with pytest.raises(ValueError, match="expected a \\(start, end\\) pair"): Token("x", (0, 4, 9), Role.GIVEN) # type: ignore[arg-type] with pytest.raises(ValueError, match="expected a \\(start, end\\) pair"): Token("x", 5, Role.GIVEN) # type: ignore[arg-type] -def test_token_rejects_non_string_text(): +def test_token_rejects_non_string_text() -> None: with pytest.raises(ValueError, match="got None"): Token(None, None, Role.GIVEN) # type: ignore[arg-type] -def test_token_is_frozen_and_hashable(): +def test_token_is_frozen_and_hashable() -> None: t = Token("Juan", Span(0, 4), Role.GIVEN) with pytest.raises(AttributeError): t.text = "X" # type: ignore[misc] assert hash(t) == hash(Token("Juan", Span(0, 4), Role.GIVEN)) -def test_ambiguity_kind_members_are_their_string_values(): +def test_ambiguity_kind_members_are_their_string_values() -> None: assert AmbiguityKind.PARTICLE_OR_GIVEN == "particle-or-given" assert AmbiguityKind("order") is AmbiguityKind.ORDER -def test_ambiguity_construction_coerces_kind_string(): +def test_ambiguity_construction_coerces_kind_string() -> None: t = Token("Van", Span(0, 3), Role.GIVEN, frozenset({"particle"})) a = Ambiguity("particle-or-given", "leading 'van' may be a particle", (t,)) # type: ignore[arg-type] assert a.kind is AmbiguityKind.PARTICLE_OR_GIVEN assert a.tokens == (t,) -def test_ambiguity_rejects_unknown_kind(): +def test_ambiguity_rejects_unknown_kind() -> None: with pytest.raises(ValueError, match="particle-or-given"): Ambiguity("no-such-kind", "detail", ()) # type: ignore[arg-type] -def test_ambiguity_rejects_non_token_elements(): +def test_ambiguity_rejects_non_token_elements() -> None: with pytest.raises(ValueError, match="only Token instances"): Ambiguity("order", "detail", ("not-a-token",)) # type: ignore[arg-type] -def test_ambiguity_rejects_empty_detail(): +def test_ambiguity_rejects_empty_detail() -> None: with pytest.raises(ValueError, match="non-empty string"): Ambiguity(AmbiguityKind.ORDER, "", ()) -def _pn(original, tokens, ambiguities=()): +def _pn(original: str, tokens: Iterable[Token], + ambiguities: Iterable[Ambiguity] = ()) -> ParsedName: return ParsedName(original=original, tokens=tuple(tokens), ambiguities=tuple(ambiguities)) -def test_parsedname_accepts_valid_spans_and_is_truthy(): +def test_parsedname_accepts_valid_spans_and_is_truthy() -> None: pn = _pn("John Smith", [ Token("John", Span(0, 4), Role.GIVEN), Token("Smith", Span(5, 10), Role.FAMILY), @@ -98,17 +101,17 @@ def test_parsedname_accepts_valid_spans_and_is_truthy(): assert bool(pn) is True -def test_empty_parse_is_falsy(): +def test_empty_parse_is_falsy() -> None: assert bool(_pn("", [])) is False assert bool(_pn(" ", [])) is False -def test_parsedname_rejects_out_of_bounds_span(): +def test_parsedname_rejects_out_of_bounds_span() -> None: with pytest.raises(ValueError, match="out of bounds"): _pn("John", [Token("Johnny", Span(0, 6), Role.GIVEN)]) -def test_parsedname_rejects_overlapping_spans(): +def test_parsedname_rejects_overlapping_spans() -> None: with pytest.raises(ValueError, match="ascending"): _pn("John Smith", [ Token("John", Span(0, 4), Role.GIVEN), @@ -116,7 +119,7 @@ def test_parsedname_rejects_overlapping_spans(): ]) -def test_parsedname_rejects_descending_spans(): +def test_parsedname_rejects_descending_spans() -> None: with pytest.raises(ValueError, match="ascending"): _pn("John Smith", [ Token("Smith", Span(5, 10), Role.FAMILY), @@ -124,7 +127,7 @@ def test_parsedname_rejects_descending_spans(): ]) -def test_synthetic_tokens_skip_span_checks(): +def test_synthetic_tokens_skip_span_checks() -> None: pn = _pn("John Smith", [ Token("John", Span(0, 4), Role.GIVEN), Token("Qux", None, Role.MIDDLE), @@ -133,7 +136,7 @@ def test_synthetic_tokens_skip_span_checks(): assert len(pn.tokens) == 3 -def test_ambiguity_tokens_must_be_subset_of_parse_tokens(): +def test_ambiguity_tokens_must_be_subset_of_parse_tokens() -> None: inside = Token("Van", Span(0, 3), Role.GIVEN) outside = Token("Zzz", None, Role.GIVEN) with pytest.raises(ValueError, match="subset"): @@ -142,7 +145,7 @@ def test_ambiguity_tokens_must_be_subset_of_parse_tokens(): [Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "d", (outside,))]) -def test_parsedname_equality_is_strict_structural(): +def test_parsedname_equality_is_strict_structural() -> None: a = _pn("John", [Token("John", Span(0, 4), Role.GIVEN)]) b = _pn("John", [Token("John", Span(0, 4), Role.GIVEN)]) c = _pn("John ", [Token("John", Span(0, 4), Role.GIVEN)]) @@ -150,19 +153,19 @@ def test_parsedname_equality_is_strict_structural(): assert a != c # different original: not interchangeable -def test_parsedname_rejects_non_str_original(): +def test_parsedname_rejects_non_str_original() -> None: with pytest.raises(ValueError, match="must be a str"): _pn(None, []) # type: ignore[arg-type] -def test_parsedname_rejects_non_token_and_non_ambiguity_elements(): +def test_parsedname_rejects_non_token_and_non_ambiguity_elements() -> None: with pytest.raises(ValueError, match="only Token instances"): _pn("x", ["not-a-token"]) # type: ignore[list-item] with pytest.raises(ValueError, match="only Ambiguity instances"): _pn("John", [Token("John", Span(0, 4), Role.GIVEN)], ["nope"]) # type: ignore[list-item] -def _delavega(): +def _delavega() -> ParsedName: # "Dr. Juan de la Vega III" -- hand-built, spans verified by hand # 0123456789012345678901234 return _pn("Dr. Juan de la Vega III", [ @@ -175,7 +178,7 @@ def _delavega(): ]) -def test_string_properties_join_by_role(): +def test_string_properties_join_by_role() -> None: pn = _delavega() assert pn.title == "Dr." assert pn.given == "Juan" @@ -186,7 +189,7 @@ def test_string_properties_join_by_role(): assert pn.maiden == "" -def test_suffix_joins_with_comma_space(): +def test_suffix_joins_with_comma_space() -> None: pn = _pn("John Smith PhD MD", [ Token("John", Span(0, 4), Role.GIVEN), Token("Smith", Span(5, 10), Role.FAMILY), @@ -196,7 +199,7 @@ def test_suffix_joins_with_comma_space(): assert pn.suffix == "PhD, MD" -def test_derived_views_filter_on_stable_particle_tag(): +def test_derived_views_filter_on_stable_particle_tag() -> None: # Pin the hard-coded "particle" string in _text_for to the published # contract until Plan 3's tag-emission contract tests land. assert "particle" in STABLE_TAGS @@ -207,13 +210,13 @@ def test_derived_views_filter_on_stable_particle_tag(): assert pn.given_names == "Juan" # given + middle -def test_tokens_for_preserves_order(): +def test_tokens_for_preserves_order() -> None: pn = _delavega() assert [t.text for t in pn.tokens_for(Role.FAMILY)] == ["de", "la", "Vega"] assert pn.tokens_for(Role.NICKNAME) == () -def test_as_dict_canonical_order_and_empty_filtering(): +def test_as_dict_canonical_order_and_empty_filtering() -> None: pn = _delavega() d = pn.as_dict() assert list(d) == ["title", "given", "middle", "family", @@ -223,7 +226,7 @@ def test_as_dict_canonical_order_and_empty_filtering(): assert list(d2) == ["title", "given", "family", "suffix"] -def test_replace_swaps_field_with_synthetic_tokens_in_place(): +def test_replace_swaps_field_with_synthetic_tokens_in_place() -> None: pn = _delavega() pn2 = pn.replace(given="Jean Paul") assert pn2.given == "Jean Paul" @@ -235,7 +238,7 @@ def test_replace_swaps_field_with_synthetic_tokens_in_place(): assert [t.role for t in pn2.tokens][:3] == [Role.TITLE, Role.GIVEN, Role.GIVEN] -def test_replace_adds_missing_field_at_end(): +def test_replace_adds_missing_field_at_end() -> None: pn = _pn("John Smith", [ Token("John", Span(0, 4), Role.GIVEN), Token("Smith", Span(5, 10), Role.FAMILY), @@ -245,17 +248,17 @@ def test_replace_adds_missing_field_at_end(): assert pn2.tokens[-1].role is Role.SUFFIX -def test_replace_with_empty_string_clears_field(): +def test_replace_with_empty_string_clears_field() -> None: pn = _delavega() assert pn.replace(title="").title == "" -def test_replace_rejects_unknown_field(): +def test_replace_rejects_unknown_field() -> None: with pytest.raises(TypeError, match="given"): _delavega().replace(firstname="X") -def test_replace_drops_ambiguities_referencing_removed_tokens(): +def test_replace_drops_ambiguities_referencing_removed_tokens() -> None: van = Token("Van", Span(0, 3), Role.GIVEN) pn = _pn("Van Johnson", [van, Token("Johnson", Span(4, 11), Role.FAMILY)], @@ -264,18 +267,18 @@ def test_replace_drops_ambiguities_referencing_removed_tokens(): assert pn.replace(family="Smith").ambiguities != () -def test_replace_rejects_non_str_value(): +def test_replace_rejects_non_str_value() -> None: with pytest.raises(TypeError, match="must be a str"): _delavega().replace(given=None) # type: ignore[arg-type] -def test_replace_appends_missing_roles_in_canonical_order(): +def test_replace_appends_missing_roles_in_canonical_order() -> None: pn = _pn("John", [Token("John", Span(0, 4), Role.GIVEN)]) pn2 = pn.replace(maiden="X", suffix="Y") assert [t.role for t in pn2.tokens] == [Role.GIVEN, Role.SUFFIX, Role.MAIDEN] -def test_comparison_key_is_casefolded_canonical_seven_tuple(): +def test_comparison_key_is_casefolded_canonical_seven_tuple() -> None: pn = _delavega() assert pn.comparison_key() == ( "dr.", "juan", "", "de la vega", "iii", "", "", From 473c0b96292109b8a713fe9aa504ec15b27bfa90 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 04:59:46 -0700 Subject: [PATCH 033/275] Reject mappings passed to plain Lexicon vocab fields Lexicon(titles={'Dr.': 'Doctor'}) silently kept only the keys -- the lone quiet coercion on an otherwise fail-loud surface, and a dict here almost always means the field was confused with capitalization_exceptions. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 8 ++++++++ tests/v2/test_lexicon.py | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index b9e20bab..16086961 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -38,6 +38,14 @@ def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: f"Lexicon.{field_name} must be an iterable of strings, " f"not a bare string" ) + # A Mapping would silently contribute only its keys; a dict here + # almost always means the caller confused this field with + # capitalization_exceptions. + if isinstance(entries, Mapping): + raise ValueError( + f"Lexicon.{field_name} must be an iterable of strings, not a " + f"mapping (only capitalization_exceptions holds key->value pairs)" + ) items = tuple(entries) # materialize once; entries may be a generator for w in items: if not isinstance(w, str): diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index d2cfe5f5..d200440a 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -125,3 +125,13 @@ def test_pickle_round_trip_preserves_equality_and_cap_map() -> None: assert hash(loaded) == hash(lex) assert (loaded.capitalization_exceptions_map == lex.capitalization_exceptions_map) + + +def test_lexicon_rejects_mapping_for_plain_vocab_field() -> None: + # A dict here almost always means the caller confused the field with + # capitalization_exceptions; silently keeping just the keys would be + # the lone quiet coercion on an otherwise fail-loud surface. + with pytest.raises(ValueError, match="mapping"): + Lexicon(titles={"Dr.": "Doctor"}) # type: ignore[arg-type] + with pytest.raises(ValueError, match="mapping"): + Lexicon.empty().add(titles={"Dr.": "Doctor"}) From ce7b36bc4ac9eddced6de3c4f5ee1c08fa8c7a72 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 05:00:32 -0700 Subject: [PATCH 034/275] Diff Lexicon reprs against the nearer named baseline repr(Lexicon.empty().add(titles={'zqx'})) rendered as default() plus ten '-N' deltas -- technically bounded, but the wrong story for the documented build-from-empty() power-user path. Compare against both named constructors and render the smaller diff. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 38 +++++++++++++++++++++++++------------- tests/v2/test_reprs.py | 8 ++++++++ 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 16086961..7087222a 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -107,24 +107,36 @@ def __post_init__(self) -> None: f"not in particles: {extra}" ) - def __repr__(self) -> str: - # Bounded: renders only which fields deviate from default() and by - # how many entries -- never the entries themselves (design rule, - # see nameparser._types module docstring). - default = Lexicon.default() - if self == default: - return "Lexicon(default)" + def _deltas_from(self, baseline: Lexicon) -> list[tuple[str, int, int]]: deltas = [] for name in _VOCAB_FIELDS + ("capitalization_exceptions",): mine = set(getattr(self, name)) - theirs = set(getattr(default, name)) + theirs = set(getattr(baseline, name)) added, removed = len(mine - theirs), len(theirs - mine) if added or removed: - delta = "".join( - part for part, n in ((f"+{added}", added), (f"-{removed}", removed)) if n - ) - deltas.append(f"{name}: {delta}") - return f"Lexicon(default + {', '.join(deltas)})" + deltas.append((name, added, removed)) + return deltas + + def __repr__(self) -> str: + # Bounded: renders only which fields deviate from the nearer of + # the two named constructors and by how many entries -- never the + # entries themselves (design rule, see nameparser._types module + # docstring). Diffing empty()-built lexicons against default() + # would tell the wrong story ("default minus ~700 entries"). + if self == Lexicon.default(): + return "Lexicon(default)" + if self == Lexicon.empty(): + return "Lexicon(empty)" + candidates = [(label, self._deltas_from(baseline)) + for label, baseline in (("default", Lexicon.default()), + ("empty", Lexicon.empty()))] + label, deltas = min( + candidates, key=lambda c: sum(a + r for _, a, r in c[1])) + rendered = ", ".join( + name + ": " + "".join( + part for part, n in ((f"+{a}", a), (f"-{r}", r)) if n) + for name, a, r in deltas) + return f"Lexicon({label} + {rendered})" def __getstate__(self) -> dict[str, object]: # _cap_map is a MappingProxyType, which pickle rejects; ship every diff --git a/tests/v2/test_reprs.py b/tests/v2/test_reprs.py index cacfdbc6..61c31e17 100644 --- a/tests/v2/test_reprs.py +++ b/tests/v2/test_reprs.py @@ -58,3 +58,11 @@ def test_locale_repr_shows_code_and_patched_fields() -> None: {PatronymicRule.EAST_SLAVIC}))) assert repr(ru) == "Locale('ru': patronymic_rules)" assert repr(Locale("xx", Lexicon.empty())) == "Locale('xx')" + + +def test_lexicon_repr_diffs_against_nearer_baseline() -> None: + # An empty()-built Lexicon must not render as default() minus ~700 + # entries -- diff against whichever named constructor is nearer. + assert repr(Lexicon.empty()) == "Lexicon(empty)" + lex = Lexicon.empty().add(titles={"zqx"}) + assert repr(lex) == "Lexicon(empty + titles: +1)" From 334c617ff954f6d7a482067e42dbf55902ae2c77 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 05:01:46 -0700 Subject: [PATCH 035/275] =?UTF-8?q?Order=20Lexicon=20sections=20per=20conv?= =?UTF-8?q?entions=20doc=20=C2=A74?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure move: constructors ahead of dunders, __or__ grouped with the dunders, then properties, then the editing methods (with _edit at the head of its section per the documented exception). No behavior change. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 50 +++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 7087222a..296c18e7 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -107,6 +107,18 @@ def __post_init__(self) -> None: f"not in particles: {extra}" ) + # -- constructors ---------------------------------------------------- + + @classmethod + def empty(cls) -> Lexicon: + return cls() + + @classmethod + def default(cls) -> Lexicon: + return _default_lexicon() + + # -- dunders ---------------------------------------------------------- + def _deltas_from(self, baseline: Lexicon) -> list[tuple[str, int, int]]: deltas = [] for name in _VOCAB_FIELDS + ("capitalization_exceptions",): @@ -151,21 +163,25 @@ def __setstate__(self, state: dict[str, object]) -> None: self, "_cap_map", MappingProxyType(dict(self.capitalization_exceptions))) + def __or__(self, other: Lexicon) -> Lexicon: + if not isinstance(other, Lexicon): + return NotImplemented + updates: dict[str, object] = { + name: getattr(self, name) | getattr(other, name) + for name in _VOCAB_FIELDS + } + # right-biased on key conflicts, mirroring later-wins for scalars + merged = dict(self._cap_map) | dict(other._cap_map) + updates["capitalization_exceptions"] = tuple(sorted(merged.items())) + return dataclasses.replace(self, **updates) # type: ignore[arg-type] + + # -- properties ------------------------------------------------------- + @property def capitalization_exceptions_map(self) -> Mapping[str, str]: return self._cap_map - # -- constructors ---------------------------------------------------- - - @classmethod - def empty(cls) -> Lexicon: - return cls() - - @classmethod - def default(cls) -> Lexicon: - return _default_lexicon() - - # -- composition ------------------------------------------------------ + # -- editing ---------------------------------------------------------- def _edit(self, op: str, entries: Mapping[str, Iterable[str]]) -> Lexicon: updates: dict[str, frozenset[str]] = {} @@ -200,18 +216,6 @@ def add(self, **entries: Iterable[str]) -> Lexicon: def remove(self, **entries: Iterable[str]) -> Lexicon: return self._edit("remove", entries) - def __or__(self, other: Lexicon) -> Lexicon: - if not isinstance(other, Lexicon): - return NotImplemented - updates: dict[str, object] = { - name: getattr(self, name) | getattr(other, name) - for name in _VOCAB_FIELDS - } - # right-biased on key conflicts, mirroring later-wins for scalars - merged = dict(self._cap_map) | dict(other._cap_map) - updates["capitalization_exceptions"] = tuple(sorted(merged.items())) - return dataclasses.replace(self, **updates) # type: ignore[arg-type] - @functools.cache def _default_lexicon() -> Lexicon: From 5a5bf801cf9a3e70683ffa9165bad38064087a6d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 11:23:50 -0700 Subject: [PATCH 036/275] Adopt the stdlib exception taxonomy across v2 validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wrong type (including wrong element type in a collection, bare str for an iterable-of-strings field, Mapping for a plain iterable) raises TypeError; well-typed but unacceptable values (empty required strings, inverted spans, unknown enum values, subset violations) raise ValueError. Matches the boundary the generated dataclass __init__ already draws -- unknown kwargs are TypeError natively -- so the constructors can never present an all-ValueError contract anyway. Mixed checks (Token.text, Ambiguity.detail, Locale.code, delimiter pairs, extra_suffix_delimiters entries) split into a type check and a value check. Enum lookups stay ValueError for any element, matching stdlib EnumType.__call__; non-iterable patronymic_rules now surfaces its natural TypeError instead of being converted. Rule recorded in the conventions doc §6. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 8 ++++---- nameparser/_locale.py | 10 +++++++--- nameparser/_policy.py | 31 +++++++++++++++++++++---------- nameparser/_types.py | 26 +++++++++++++++----------- tests/v2/test_lexicon.py | 10 +++++----- tests/v2/test_locale.py | 6 ++++-- tests/v2/test_policy.py | 12 +++++++----- tests/v2/test_types.py | 19 ++++++++++++------- 8 files changed, 75 insertions(+), 47 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 296c18e7..61e356dd 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -34,7 +34,7 @@ def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: # yield the single characters {'d', 'r'} -- the set(str) footgun on # the primary customization surface. if isinstance(entries, str): - raise ValueError( + raise TypeError( f"Lexicon.{field_name} must be an iterable of strings, " f"not a bare string" ) @@ -42,14 +42,14 @@ def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: # almost always means the caller confused this field with # capitalization_exceptions. if isinstance(entries, Mapping): - raise ValueError( + raise TypeError( f"Lexicon.{field_name} must be an iterable of strings, not a " f"mapping (only capitalization_exceptions holds key->value pairs)" ) items = tuple(entries) # materialize once; entries may be a generator for w in items: if not isinstance(w, str): - raise ValueError( + raise TypeError( f"Lexicon.{field_name} entries must be strings, got {w!r}" ) result = frozenset(_normalize(w) for w in items) @@ -89,7 +89,7 @@ def __post_init__(self) -> None: deduped: dict[str, str] = {} for k, v in pairs: if not isinstance(k, str) or not isinstance(v, str): - raise ValueError( + raise TypeError( f"capitalization_exceptions entries must be " f"str -> str, got {k!r}: {v!r}" ) diff --git a/nameparser/_locale.py b/nameparser/_locale.py index 2494b3eb..5e06d176 100644 --- a/nameparser/_locale.py +++ b/nameparser/_locale.py @@ -23,7 +23,11 @@ class Locale: policy: PolicyPatch = PolicyPatch() def __post_init__(self) -> None: - if not isinstance(self.code, str) or not self.code.strip(): + if not isinstance(self.code, str): + raise TypeError( + f"Locale.code must be a str, got {self.code!r}" + ) + if not self.code.strip(): raise ValueError( f"Locale.code must be a non-empty string, got {self.code!r}" ) @@ -36,11 +40,11 @@ def __post_init__(self) -> None: f"Locale.code must not contain whitespace, got {self.code!r}" ) if not isinstance(self.lexicon, Lexicon): - raise ValueError( + raise TypeError( f"Locale.lexicon must be a Lexicon, got {self.lexicon!r}" ) if not isinstance(self.policy, PolicyPatch): - raise ValueError( + raise TypeError( f"Locale.policy must be a PolicyPatch, got {self.policy!r}" ) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 7427f166..53ab123d 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -55,13 +55,15 @@ def __post_init__(self) -> None: ) object.__setattr__(self, "name_order", order) if isinstance(self.patronymic_rules, str): - raise ValueError( + raise TypeError( f"patronymic_rules must be an iterable of rule names, " f"not a bare string: {self.patronymic_rules!r}" ) + # A non-iterable raises its natural TypeError from the frozenset + # call; only failed enum lookups get the enriched message. try: rules = frozenset(PatronymicRule(r) for r in self.patronymic_rules) - except (TypeError, ValueError): + except ValueError: valid = ", ".join(r.value for r in PatronymicRule) raise ValueError( f"unknown patronymic rule in {self.patronymic_rules!r}; " @@ -72,23 +74,32 @@ def __post_init__(self) -> None: pairs = tuple(getattr(self, pairs_name)) for pair in pairs: if (not isinstance(pair, tuple) or len(pair) != 2 - or not all(isinstance(s, str) and s for s in pair)): - raise ValueError( + or not all(isinstance(s, str) for s in pair)): + raise TypeError( f"{pairs_name} entries must be (open, close) tuples " - f"of non-empty strings, got {pair!r}" + f"of strings, got {pair!r}" + ) + if not all(pair): + raise ValueError( + f"{pairs_name} entries must be pairs of non-empty " + f"strings, got {pair!r}" ) object.__setattr__(self, pairs_name, frozenset(pairs)) if isinstance(self.extra_suffix_delimiters, str): - raise ValueError( + raise TypeError( f"extra_suffix_delimiters must be an iterable of strings, " f"not a bare string: {self.extra_suffix_delimiters!r}" ) delimiters = tuple(self.extra_suffix_delimiters) for d in delimiters: - if not isinstance(d, str) or not d: + if not isinstance(d, str): + raise TypeError( + f"extra_suffix_delimiters entries must be strings, " + f"got {d!r}" + ) + if not d: raise ValueError( - f"extra_suffix_delimiters entries must be non-empty " - f"strings, got {d!r}" + "extra_suffix_delimiters entries must be non-empty strings" ) object.__setattr__( self, "extra_suffix_delimiters", frozenset(delimiters) @@ -166,7 +177,7 @@ def __post_init__(self) -> None: if value is UNSET: continue if isinstance(value, str): - raise ValueError( + raise TypeError( f"{f.name} must be an iterable, " f"not a bare string: {value!r}" ) diff --git a/nameparser/_types.py b/nameparser/_types.py index 97dd941b..5ec3f197 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -47,17 +47,19 @@ class Token: tags: frozenset[str] = frozenset() def __post_init__(self) -> None: - if not isinstance(self.text, str) or not self.text: - raise ValueError( - f"Token.text must be a non-empty string, got {self.text!r}" + if not isinstance(self.text, str): + raise TypeError( + f"Token.text must be a str, got {self.text!r}" ) + if not self.text: + raise ValueError("Token.text must be a non-empty string") if self.span is not None: if not ( isinstance(self.span, tuple) and len(self.span) == 2 and all(isinstance(v, int) for v in self.span) ): - raise ValueError( + raise TypeError( f"invalid span {self.span!r}: expected a (start, end) " "pair of ints or None" ) @@ -103,14 +105,16 @@ def __post_init__(self) -> None: raise ValueError( f"unknown AmbiguityKind {self.kind!r}; valid kinds: {valid}" ) from None - if not isinstance(self.detail, str) or not self.detail: - raise ValueError( - f"Ambiguity.detail must be a non-empty string, got {self.detail!r}" + if not isinstance(self.detail, str): + raise TypeError( + f"Ambiguity.detail must be a str, got {self.detail!r}" ) + if not self.detail: + raise ValueError("Ambiguity.detail must be a non-empty string") toks = tuple(self.tokens) for tok in toks: if not isinstance(tok, Token): - raise ValueError( + raise TypeError( f"Ambiguity.tokens must contain only Token instances, " f"got {tok!r}" ) @@ -136,20 +140,20 @@ class ParsedName: def __post_init__(self) -> None: if not isinstance(self.original, str): - raise ValueError( + raise TypeError( f"ParsedName.original must be a str, got {self.original!r}" ) object.__setattr__(self, "tokens", tuple(self.tokens)) object.__setattr__(self, "ambiguities", tuple(self.ambiguities)) for tok in self.tokens: if not isinstance(tok, Token): - raise ValueError( + raise TypeError( f"ParsedName.tokens must contain only Token instances, " f"got {tok!r}" ) for amb in self.ambiguities: if not isinstance(amb, Ambiguity): - raise ValueError( + raise TypeError( f"ParsedName.ambiguities must contain only Ambiguity " f"instances, got {amb!r}" ) diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index d200440a..210da29f 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -49,12 +49,12 @@ def test_lexicon_is_hashable() -> None: def test_lexicon_rejects_bare_string_vocab() -> None: - with pytest.raises(ValueError, match="bare string"): + with pytest.raises(TypeError, match="bare string"): Lexicon(titles="dr") # type: ignore[arg-type] def test_lexicon_rejects_non_str_vocab_entries() -> None: - with pytest.raises(ValueError, match="entries must be strings"): + with pytest.raises(TypeError, match="entries must be strings"): Lexicon(titles={"Dr.", 42}) # type: ignore[arg-type] @@ -71,7 +71,7 @@ def test_colliding_exception_keys_dedupe_last_wins() -> None: def test_lexicon_rejects_non_str_exception_values() -> None: - with pytest.raises(ValueError, match="str -> str"): + with pytest.raises(TypeError, match="str -> str"): Lexicon(capitalization_exceptions={"phd": 42}) # type: ignore[dict-item, arg-type] @@ -131,7 +131,7 @@ def test_lexicon_rejects_mapping_for_plain_vocab_field() -> None: # A dict here almost always means the caller confused the field with # capitalization_exceptions; silently keeping just the keys would be # the lone quiet coercion on an otherwise fail-loud surface. - with pytest.raises(ValueError, match="mapping"): + with pytest.raises(TypeError, match="mapping"): Lexicon(titles={"Dr.": "Doctor"}) # type: ignore[arg-type] - with pytest.raises(ValueError, match="mapping"): + with pytest.raises(TypeError, match="mapping"): Lexicon.empty().add(titles={"Dr.": "Doctor"}) diff --git a/tests/v2/test_locale.py b/tests/v2/test_locale.py index 0af17fec..bc25eda7 100644 --- a/tests/v2/test_locale.py +++ b/tests/v2/test_locale.py @@ -40,10 +40,12 @@ def test_locale_is_hashable() -> None: def test_locale_validates_component_types() -> None: - with pytest.raises(ValueError, match="Lexicon"): + with pytest.raises(TypeError, match="Lexicon"): Locale(code="ru", lexicon={"titles": set()}) # type: ignore[arg-type] - with pytest.raises(ValueError, match="PolicyPatch"): + with pytest.raises(TypeError, match="PolicyPatch"): Locale(code="ru", lexicon=Lexicon.empty(), policy={"name_order": None}) # type: ignore[arg-type] + with pytest.raises(TypeError, match="must be a str"): + Locale(code=5, lexicon=Lexicon.empty()) # type: ignore[arg-type] def test_locale_with_lexicon_pickles_round_trip() -> None: diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 0c9b6100..06cef02a 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -50,14 +50,14 @@ def test_delimiter_pairs_must_be_nonempty_string_pairs() -> None: def test_delimiter_pair_rejects_two_char_string() -> None: - with pytest.raises(ValueError, match="tuples"): + with pytest.raises(TypeError, match="tuples"): Policy(nickname_delimiters=frozenset({"()"})) # type: ignore[arg-type] def test_patronymic_rules_rejects_bare_string_and_non_iterable() -> None: - with pytest.raises(ValueError, match="bare string"): + with pytest.raises(TypeError, match="bare string"): Policy(patronymic_rules="east-slavic") # type: ignore[arg-type] - with pytest.raises(ValueError, match="valid rules"): + with pytest.raises(TypeError, match="iterable"): Policy(patronymic_rules=5) # type: ignore[arg-type] @@ -76,8 +76,10 @@ def test_policy_delimiters_do_not_alias_caller_containers() -> None: def test_extra_suffix_delimiters_validated_and_coerced() -> None: - with pytest.raises(ValueError, match="bare string"): + with pytest.raises(TypeError, match="bare string"): Policy(extra_suffix_delimiters="ab") # type: ignore[arg-type] + with pytest.raises(TypeError, match="must be strings"): + Policy(extra_suffix_delimiters=frozenset({5})) # type: ignore[arg-type] with pytest.raises(ValueError, match="non-empty strings"): Policy(extra_suffix_delimiters=frozenset({""})) p = Policy(extra_suffix_delimiters=["-"]) # type: ignore[arg-type] @@ -106,7 +108,7 @@ def test_policy_patch_canonicalizes_union_fields() -> None: def test_policy_patch_rejects_bare_string_union_fields() -> None: - with pytest.raises(ValueError, match="bare string"): + with pytest.raises(TypeError, match="bare string"): PolicyPatch(extra_suffix_delimiters="ab") # type: ignore[arg-type] diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 7446cbde..973021cf 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -42,14 +42,14 @@ def test_token_rejects_negative_span() -> None: def test_token_rejects_malformed_span_shapes() -> None: - with pytest.raises(ValueError, match="expected a \\(start, end\\) pair"): + with pytest.raises(TypeError, match="expected a \\(start, end\\) pair"): Token("x", (0, 4, 9), Role.GIVEN) # type: ignore[arg-type] - with pytest.raises(ValueError, match="expected a \\(start, end\\) pair"): + with pytest.raises(TypeError, match="expected a \\(start, end\\) pair"): Token("x", 5, Role.GIVEN) # type: ignore[arg-type] def test_token_rejects_non_string_text() -> None: - with pytest.raises(ValueError, match="got None"): + with pytest.raises(TypeError, match="got None"): Token(None, None, Role.GIVEN) # type: ignore[arg-type] @@ -78,7 +78,7 @@ def test_ambiguity_rejects_unknown_kind() -> None: def test_ambiguity_rejects_non_token_elements() -> None: - with pytest.raises(ValueError, match="only Token instances"): + with pytest.raises(TypeError, match="only Token instances"): Ambiguity("order", "detail", ("not-a-token",)) # type: ignore[arg-type] @@ -87,6 +87,11 @@ def test_ambiguity_rejects_empty_detail() -> None: Ambiguity(AmbiguityKind.ORDER, "", ()) +def test_ambiguity_rejects_non_str_detail() -> None: + with pytest.raises(TypeError, match="must be a str"): + Ambiguity(AmbiguityKind.ORDER, None, ()) # type: ignore[arg-type] + + def _pn(original: str, tokens: Iterable[Token], ambiguities: Iterable[Ambiguity] = ()) -> ParsedName: return ParsedName(original=original, tokens=tuple(tokens), @@ -154,14 +159,14 @@ def test_parsedname_equality_is_strict_structural() -> None: def test_parsedname_rejects_non_str_original() -> None: - with pytest.raises(ValueError, match="must be a str"): + with pytest.raises(TypeError, match="must be a str"): _pn(None, []) # type: ignore[arg-type] def test_parsedname_rejects_non_token_and_non_ambiguity_elements() -> None: - with pytest.raises(ValueError, match="only Token instances"): + with pytest.raises(TypeError, match="only Token instances"): _pn("x", ["not-a-token"]) # type: ignore[list-item] - with pytest.raises(ValueError, match="only Ambiguity instances"): + with pytest.raises(TypeError, match="only Ambiguity instances"): _pn("John", [Token("John", Span(0, 4), Role.GIVEN)], ["nope"]) # type: ignore[list-item] From 974aec2218f0f5fbf0867b480fceaeb0cea55300 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 12:14:55 -0700 Subject: [PATCH 037/275] Source maiden markers from a config data module (#274) maiden_markers was the only Lexicon.default() field fed by an inline literal instead of a nameparser/config data module. Add config/maiden_markers.py with the #274 candidate set: French, German, Dutch, Czech/Slovak, and Russian markers, both genders and both e/yo spellings (casefold does not fold them). Non-colliding Cyrillic entries belong in the default lexicon per the locales design's sorting rule. Deliberately absent: 'z domu' (two-token marker, pending the pipeline's multi-token matching decision) and Scandinavian 'f.' (collides with the initial F). The 1.x parser does not read the module. Co-Authored-By: Claude Fable 5 --- docs/modules.rst | 2 ++ nameparser/_lexicon.py | 3 ++- nameparser/config/maiden_markers.py | 30 +++++++++++++++++++++++++++++ tests/v2/test_lexicon.py | 6 ++++++ 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 nameparser/config/maiden_markers.py diff --git a/docs/modules.rst b/docs/modules.rst index bdc6b52d..8f3b1990 100644 --- a/docs/modules.rst +++ b/docs/modules.rst @@ -33,6 +33,8 @@ HumanName.config Defaults :members: .. automodule:: nameparser.config.conjunctions :members: +.. automodule:: nameparser.config.maiden_markers + :members: .. automodule:: nameparser.config.capitalization :members: .. automodule:: nameparser.config.regexes diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 61e356dd..a43e9e73 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -223,6 +223,7 @@ def _default_lexicon() -> Lexicon: from nameparser.config.bound_first_names import BOUND_FIRST_NAMES from nameparser.config.capitalization import CAPITALIZATION_EXCEPTIONS from nameparser.config.conjunctions import CONJUNCTIONS + from nameparser.config.maiden_markers import MAIDEN_MARKERS from nameparser.config.prefixes import NON_FIRST_NAME_PREFIXES, PREFIXES from nameparser.config.suffixes import ( SUFFIX_ACRONYMS, SUFFIX_ACRONYMS_AMBIGUOUS, SUFFIX_NOT_ACRONYMS, @@ -243,7 +244,7 @@ def _default_lexicon() -> Lexicon: particles_ambiguous=frozenset(PREFIXES - NON_FIRST_NAME_PREFIXES), conjunctions=frozenset(CONJUNCTIONS), bound_given_names=frozenset(BOUND_FIRST_NAMES), - maiden_markers=frozenset({"née", "nee", "geb"}), + maiden_markers=frozenset(MAIDEN_MARKERS), # pass canonical pair-tuples so this strictly-typed call site never # feeds a Mapping to the tuple-annotated field; __post_init__ # still tolerates a Mapping at runtime for interactive use diff --git a/nameparser/config/maiden_markers.py b/nameparser/config/maiden_markers.py new file mode 100644 index 00000000..47910bf0 --- /dev/null +++ b/nameparser/config/maiden_markers.py @@ -0,0 +1,30 @@ +MAIDEN_MARKERS = { + 'née', + 'né', + 'nee', + 'geb', + 'geborene', + 'geboren', + 'roz', + 'rozená', + 'урожд', + 'урождённая', + 'урожденная', + 'урождённый', + 'урожденный', +} +""" +Marker words that introduce a birth surname, e.g. "Jane Smith née Jones" +(#274). French née/né/nee, German geb./geborene, Dutch geboren, +Czech/Slovak roz./rozená, Russian урожд./урождённая (both ё and е +spellings — ``str.casefold()`` does not fold them, and running text +routinely writes е). Entries are stored normalized: lowercase, no +periods. + +Consumed by the 2.0 parser's default lexicon. The 1.x parser does not +read this module. + +Deliberately absent: Polish "z domu" (a two-token marker; pending the +2.0 pipeline's multi-token matching decision) and Scandinavian "f." +(født/född — collides with the initial "F."). +""" diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 210da29f..69456159 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -22,6 +22,12 @@ def test_default_sources_v1_vocabulary() -> None: # normalized -- only keys are casefolded/period-stripped at # construction, values pass through unchanged). assert lex.capitalization_exceptions_map["phd"] == "Ph.D." + # maiden markers source from the same data-module pattern (#274); + # non-colliding Cyrillic entries live in the default per the locales + # design's sorting rule, and both ё/е spellings are listed because + # casefold() does not fold them. + assert "geborene" in lex.maiden_markers + assert "урожденная" in lex.maiden_markers and "урождённая" in lex.maiden_markers def test_default_is_cached_single_instance() -> None: From 5d7893375f1ae2e94872b6b46a23ec8f030d7813 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 12:36:34 -0700 Subject: [PATCH 038/275] Add Scandinavian participle maiden markers (#274) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The issue's veto covered only the abbreviation 'f.' (collides with the initial F); the full participles født (da/nb), fødd (nn), and född (sv) cannot appear as name tokens and are the standard convention in running text. No ASCII variants -- dropping the diacritic is not standard practice in Scandinavian text, unlike nee in English. Co-Authored-By: Claude Fable 5 --- nameparser/config/maiden_markers.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/nameparser/config/maiden_markers.py b/nameparser/config/maiden_markers.py index 47910bf0..0c36c5ee 100644 --- a/nameparser/config/maiden_markers.py +++ b/nameparser/config/maiden_markers.py @@ -7,6 +7,9 @@ 'geboren', 'roz', 'rozená', + 'født', + 'fødd', + 'född', 'урожд', 'урождённая', 'урожденная', @@ -16,15 +19,16 @@ """ Marker words that introduce a birth surname, e.g. "Jane Smith née Jones" (#274). French née/né/nee, German geb./geborene, Dutch geboren, -Czech/Slovak roz./rozená, Russian урожд./урождённая (both ё and е -spellings — ``str.casefold()`` does not fold them, and running text -routinely writes е). Entries are stored normalized: lowercase, no -periods. +Czech/Slovak roz./rozená, Danish/Norwegian født (Nynorsk fødd), Swedish +född, Russian урожд./урождённая (both ё and е spellings — +``str.casefold()`` does not fold them, and running text routinely +writes е). Entries are stored normalized: lowercase, no periods. Consumed by the 2.0 parser's default lexicon. The 1.x parser does not read this module. Deliberately absent: Polish "z domu" (a two-token marker; pending the -2.0 pipeline's multi-token matching decision) and Scandinavian "f." -(født/född — collides with the initial "F."). +2.0 pipeline's multi-token matching decision) and the Scandinavian +abbreviation "f." (collides with the initial "F." — only the full +participles are safe). """ From a16364edf539ca92e6e6924f4a393939fb5b0edc Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 12:48:17 -0700 Subject: [PATCH 039/275] Distill 2.0 implementation conventions into AGENTS.md The enforceable subset of the (untracked) conventions doc -- module layout, layering, canonical field order, method organization, exception taxonomy, bounded reprs, typing/doctest posture, pickling, the one-global rule, and tests/v2 conventions -- now has a tracked home the v1 sections don't cover, including this week's two amendments (section-head helpers, TypeError/ValueError split). Establishes the same-commit rule: convention changes update this section in the commit that makes them. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 46115f44..6aefe8aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -107,6 +107,21 @@ Parse flow: Each named attribute (`title`, `first`, etc.) is a `@property` that joins its corresponding `_list`. Setters call `_set_list()` which runs the value through `parse_pieces()`, so assigning `hn.last = "de la Vega"` correctly re-parses prefix tokens. +## 2.0 API modules (`nameparser/_*.py`, `tests/v2/`) + +The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These conventions apply to all new-API code and are stricter than the v1 sections above. The full design record (rationale, settled-decision logs, dated amendments) lives in untracked `docs/superpowers/specs/`; this section is the enforceable subset. **A commit that establishes or amends one of these conventions must update this section in the same commit** — grep-driven staleness sweeps miss paraphrased prose (see Workflow above), so write-time maintenance is the mechanism, audits are the backstop. + +- **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`; later `_pipeline/`, `_parser.py`, `_render.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Old paths (`nameparser.parser`, `nameparser.config`) stay v1-shaped. +- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching): `_types` imports nothing internal; `_lexicon` and `_policy` sit above it independently; `_locale` on top. `_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x (`config/maiden_markers.py` is 2.0-only data that lives there for the same reason). Extend the test's `ALLOWED` table when adding a module. +- **Canonical field order** — `title, given, middle, family, suffix, nickname, maiden` — is defined once by `Role` enum declaration order and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. +- **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. +- **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). +- **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). +- **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; mypy strict via per-module overrides in pyproject. Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. +- **Pickling**: v2 types must round-trip (`Parser` will be picklable by construction, and it holds a `Lexicon`). `Lexicon` needs its custom `__getstate__`/`__setstate__` because of its `mappingproxy` slot; a new unpicklable slot type needs the same treatment plus a round-trip test. +- **One sanctioned global**: the (future) cached default `Parser`. Any second piece of module-level mutable state requires amending the conventions doc, on purpose, in review. +- **Tests**: all v2 tests live in the `tests/v2/` package (its `conftest.py` overrides the v1 dual-run fixture — v2 code never reads shared `CONSTANTS`), one test module per source module, names stating behavior. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. + ## Extension Patterns **Adding a scalar `Constants` attribute + `HumanName` kwarg** (e.g. `initials_separator`, `suffix_delimiter`): From 339a3edb53592a87c6c571081e6b1e359531c912 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:08:30 -0700 Subject: [PATCH 040/275] Guard Token.tags and coerce Token.role tags was the one collection input on the v2 surface without the bare-string/mapping/element-type guards: tags='particle' silently became its character set and every STABLE_TAGS-based derived view misclassified with no error. role was the one enum field without coercion; a raw 'given' string broke 'role is Role.GIVEN' identity checks silently. role now mirrors Ambiguity.kind: coerce the string form, enriched ValueError naming valid roles on any failed lookup. Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 29 ++++++++++++++++++++++++++++- tests/v2/test_types.py | 23 +++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/nameparser/_types.py b/nameparser/_types.py index 5ec3f197..c73d8c8b 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -10,6 +10,7 @@ """ from __future__ import annotations +from collections.abc import Mapping from dataclasses import dataclass from enum import Enum, StrEnum from typing import NamedTuple @@ -53,6 +54,14 @@ def __post_init__(self) -> None: ) if not self.text: raise ValueError("Token.text must be a non-empty string") + if not isinstance(self.role, Role): + try: + object.__setattr__(self, "role", Role(self.role)) + except ValueError: + valid = ", ".join(r.value for r in Role) + raise ValueError( + f"unknown Role {self.role!r}; valid roles: {valid}" + ) from None if self.span is not None: if not ( isinstance(self.span, tuple) @@ -69,7 +78,25 @@ def __post_init__(self) -> None: f"invalid span ({start}, {end}): need 0 <= start <= end" ) object.__setattr__(self, "span", Span(start, end)) - object.__setattr__(self, "tags", frozenset(self.tags)) + # The same guards _normset applies to Lexicon vocabulary: a bare + # string would become its character set, a mapping would silently + # contribute only its keys. + if isinstance(self.tags, str): + raise TypeError( + "Token.tags must be an iterable of strings, " + "not a bare string" + ) + if isinstance(self.tags, Mapping): + raise TypeError( + "Token.tags must be an iterable of strings, not a mapping" + ) + tags = frozenset(self.tags) + for tag in tags: + if not isinstance(tag, str): + raise TypeError( + f"Token.tags must contain only strings, got {tag!r}" + ) + object.__setattr__(self, "tags", tags) def __repr__(self) -> str: # Bounded output: a single token's text/span/role/tags, never diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 973021cf..921ff395 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -301,3 +301,26 @@ def test_comparison_key_is_casefolded_canonical_seven_tuple() -> None: Token("vega", Span(11, 15), Role.FAMILY), ]) assert upper.comparison_key() == lower.comparison_key() + + +def test_token_rejects_bare_string_and_mapping_tags() -> None: + # frozenset("particle") is the set(str) footgun: eight single chars. + with pytest.raises(TypeError, match="bare string"): + Token("Van", None, Role.GIVEN, tags="particle") # type: ignore[arg-type] + with pytest.raises(TypeError, match="mapping"): + Token("Van", None, Role.GIVEN, tags={"particle": 1}) # type: ignore[arg-type] + + +def test_token_rejects_non_str_tags() -> None: + with pytest.raises(TypeError, match="tags must contain only strings"): + Token("Van", None, Role.GIVEN, tags=frozenset({1})) # type: ignore[arg-type] + + +def test_token_coerces_role_string_and_rejects_unknown() -> None: + # mirror Ambiguity.kind: coerce the string form, ValueError for any + # failed enum lookup (stdlib EnumType precedent). + assert Token("Juan", None, "given").role is Role.GIVEN # type: ignore[arg-type] + with pytest.raises(ValueError, match="title, given"): + Token("Juan", None, "chief") # type: ignore[arg-type] + with pytest.raises(ValueError, match="title, given"): + Token("Juan", None, 5) # type: ignore[arg-type] From 9d7493c5fcb8db1e61866ddd3f7a7860f6978b4c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:09:26 -0700 Subject: [PATCH 041/275] Close the fail-loud gaps in Policy.__post_init__ The four bool flags were the only unvalidated Policy fields: truthy strings like 'no' stored fine and behaved as True, silently inverting the caller's intent. And the patronymic-rule try wrapped the whole iteration, so a ValueError raised inside a caller's generator was rewritten as 'unknown patronymic rule' with the real traceback erased by 'from None'. Materialize first (the _normset pattern), convert per-element, and name the offending entry in the error. Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 37 ++++++++++++++++++++++++++----------- tests/v2/test_policy.py | 25 +++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 11 deletions(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 53ab123d..80c6c69a 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -59,17 +59,22 @@ def __post_init__(self) -> None: f"patronymic_rules must be an iterable of rule names, " f"not a bare string: {self.patronymic_rules!r}" ) - # A non-iterable raises its natural TypeError from the frozenset - # call; only failed enum lookups get the enriched message. - try: - rules = frozenset(PatronymicRule(r) for r in self.patronymic_rules) - except ValueError: - valid = ", ".join(r.value for r in PatronymicRule) - raise ValueError( - f"unknown patronymic rule in {self.patronymic_rules!r}; " - f"valid rules: {valid}" - ) from None - object.__setattr__(self, "patronymic_rules", rules) + # Materialize before converting (the _normset pattern): a + # non-iterable raises its natural TypeError here, and an exception + # raised inside a caller's generator propagates untouched instead + # of being rewritten as an unknown-rule error. Only the enum + # lookup itself gets the enriched message, naming the offender. + items = tuple(self.patronymic_rules) + rules = set() + for r in items: + try: + rules.add(PatronymicRule(r)) + except ValueError: + valid = ", ".join(v.value for v in PatronymicRule) + raise ValueError( + f"unknown patronymic rule {r!r}; valid rules: {valid}" + ) from None + object.__setattr__(self, "patronymic_rules", frozenset(rules)) for pairs_name in ("nickname_delimiters", "maiden_delimiters"): pairs = tuple(getattr(self, pairs_name)) for pair in pairs: @@ -104,6 +109,16 @@ def __post_init__(self) -> None: object.__setattr__( self, "extra_suffix_delimiters", frozenset(delimiters) ) + # Truthy strings ("no", "false") would silently invert the + # caller's intent downstream; bools are the one field kind the + # coercing checks above can't cover. + for flag in ("middle_as_family", "lenient_comma_suffixes", + "strip_emoji", "strip_bidi"): + value = getattr(self, flag) + if not isinstance(value, bool): + raise TypeError( + f"{flag} must be a bool, got {value!r}" + ) def __repr__(self) -> str: # Bounded: only fields that deviate from the default are shown diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 06cef02a..a0674187 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -134,3 +134,28 @@ def test_unset_fields_are_distinguishable_from_defaults() -> None: patch = PolicyPatch(strip_emoji=True) # explicitly set to the default value assert patch.strip_emoji is True assert PolicyPatch().strip_emoji is UNSET + + +def test_policy_rejects_non_bool_flags() -> None: + # "no" and "false" are truthy: storing them would silently invert + # the caller's intent downstream. + for flag in ("middle_as_family", "lenient_comma_suffixes", + "strip_emoji", "strip_bidi"): + with pytest.raises(TypeError, match="must be a bool"): + Policy(**{flag: "no"}) # type: ignore[arg-type] + + +def test_patronymic_rules_generator_errors_propagate_untouched() -> None: + # A ValueError raised inside the caller's own generator must not be + # rewritten as "unknown patronymic rule" with the traceback erased. + def bad_loader(): # noqa: ANN202 + yield "east-slavic" + raise ValueError("config line 7: bad int") + + with pytest.raises(ValueError, match="config line 7"): + Policy(patronymic_rules=bad_loader()) # type: ignore[arg-type] + + +def test_unknown_patronymic_rule_error_names_the_offender() -> None: + with pytest.raises(ValueError, match="klingon"): + Policy(patronymic_rules=iter(["east-slavic", "klingon"])) # type: ignore[arg-type] From cff23fff154feb28dc716e948b16fbe6db8cc8f2 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:10:07 -0700 Subject: [PATCH 042/275] Canonicalize PolicyPatch's scalar name_order to a tuple Union fields were already canonicalized at patch construction, but a list name_order stored as-is -- making the patch and any Locale holding it unhashable, with the failure surfacing only when a locale became a dict key. Policy re-coerces at apply time, so the patch was the one container in the chain that could silently carry the list. Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 10 +++++++--- tests/v2/test_policy.py | 8 ++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 80c6c69a..df27793f 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -182,9 +182,13 @@ class PolicyPatch: strip_bidi: bool | _Unset = UNSET def __post_init__(self) -> None: - # Canonicalize (but do NOT validate) union-composed fields so a - # patch built from a set/list literal is hashable and unions - # cleanly in apply_patch. + # Canonicalize (but do NOT validate) collection fields so a patch + # built from a set/list literal is hashable and unions cleanly in + # apply_patch. name_order needs the same treatment: Policy would + # coerce a list at apply time, but the patch itself (and any + # Locale holding it) must already be hashable. + if self.name_order is not UNSET: + object.__setattr__(self, "name_order", tuple(self.name_order)) for f in dataclasses.fields(self): if f.metadata.get("compose") != "union": continue diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index a0674187..abf21af4 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -159,3 +159,11 @@ def bad_loader(): # noqa: ANN202 def test_unknown_patronymic_rule_error_names_the_offender() -> None: with pytest.raises(ValueError, match="klingon"): Policy(patronymic_rules=iter(["east-slavic", "klingon"])) # type: ignore[arg-type] + + +def test_policy_patch_canonicalizes_scalar_name_order() -> None: + # A list name_order stored as-is made the patch -- and any Locale + # holding it -- unhashable, failing far from the construction site. + p = PolicyPatch(name_order=[Role.FAMILY, Role.GIVEN, Role.MIDDLE]) # type: ignore[arg-type] + assert p.name_order == (Role.FAMILY, Role.GIVEN, Role.MIDDLE) + assert isinstance(hash(p), int) From d4cb1b234ab6cbc5155038a62c3925975efec991 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:11:03 -0700 Subject: [PATCH 043/275] Validate capitalization_exceptions entry shapes A bare string or a mis-shaped entry leaked raw unpack errors ('not enough values to unpack') naming neither the field nor the fix -- and a 2-char string entry like 'ab' unpacked silently into {'a': 'b'}. All three now raise the taxonomy's TypeError with the offending value and expected form. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 22 +++++++++++++++++++++- tests/v2/test_lexicon.py | 11 +++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index a43e9e73..620f2a8c 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -81,13 +81,33 @@ def __post_init__(self) -> None: for name in _VOCAB_FIELDS: object.__setattr__(self, name, _normset(getattr(self, name), name)) raw = self.capitalization_exceptions + if isinstance(raw, str): + raise TypeError( + "capitalization_exceptions must be a mapping or an " + "iterable of (key, value) pairs, not a bare string" + ) pairs = raw.items() if isinstance(raw, Mapping) else raw # Dedupe on the NORMALIZED key before storing so the tuple and the # map always agree ("Ph.D." and "phd" collide after normalization). # Last occurrence wins, matching dict semantics and the right-bias # rule used elsewhere. deduped: dict[str, str] = {} - for k, v in pairs: + for entry in pairs: + # A 2-char str entry would unpack "ab" into ("a", "b") + # silently, so reject str outright; other mis-shapes would + # otherwise surface as bare unpack errors. + if isinstance(entry, str): + raise TypeError( + f"capitalization_exceptions entries must be " + f"(key, value) pairs, got {entry!r}" + ) + try: + k, v = entry + except (TypeError, ValueError): + raise TypeError( + f"capitalization_exceptions entries must be " + f"(key, value) pairs, got {entry!r}" + ) from None if not isinstance(k, str) or not isinstance(v, str): raise TypeError( f"capitalization_exceptions entries must be " diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 69456159..ef8e1144 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -141,3 +141,14 @@ def test_lexicon_rejects_mapping_for_plain_vocab_field() -> None: Lexicon(titles={"Dr.": "Doctor"}) # type: ignore[arg-type] with pytest.raises(TypeError, match="mapping"): Lexicon.empty().add(titles={"Dr.": "Doctor"}) + + +def test_capitalization_exceptions_rejects_malformed_shapes() -> None: + with pytest.raises(TypeError, match="bare string"): + Lexicon(capitalization_exceptions="ab") # type: ignore[arg-type] + with pytest.raises(TypeError, match="pairs"): + Lexicon(capitalization_exceptions=(("a", "b", "c"),)) # type: ignore[arg-type] + with pytest.raises(TypeError, match="pairs"): + # a 2-char string entry would unpack into two chars and silently + # store {"a": "b"} -- reject str entries outright + Lexicon(capitalization_exceptions=("ab",)) # type: ignore[arg-type] From af3aad02bf9b3340a22e9e1922382095e5d51a8b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:11:37 -0700 Subject: [PATCH 044/275] Fail at unpickle time on Lexicon field-layout skew __setstate__ accepted any state dict: a pickle from an older or newer Lexicon (field added, renamed) loaded fine and failed at the first attribute read, far from the unpickle site with no hint the cause was a stale pickle. Check the field-name set at load and name the missing/unexpected fields. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 11 +++++++++++ tests/v2/test_lexicon.py | 16 ++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 620f2a8c..b4a53077 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -177,6 +177,17 @@ def __getstate__(self) -> dict[str, object]: for f in dataclasses.fields(self) if f.name != "_cap_map"} def __setstate__(self, state: dict[str, object]) -> None: + # Fail at the unpickle site if the state comes from a different + # Lexicon field layout (version skew) -- silently loading it + # would defer the failure to some distant attribute read. + expected = {f.name for f in dataclasses.fields(Lexicon)} - {"_cap_map"} + if set(state) != expected: + missing = ", ".join(sorted(expected - set(state))) or "none" + unexpected = ", ".join(sorted(set(state) - expected)) or "none" + raise ValueError( + f"incompatible Lexicon pickle: missing fields: {missing}; " + f"unexpected fields: {unexpected}" + ) for name, value in state.items(): object.__setattr__(self, name, value) object.__setattr__( diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index ef8e1144..ebdea70e 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -152,3 +152,19 @@ def test_capitalization_exceptions_rejects_malformed_shapes() -> None: # a 2-char string entry would unpack into two chars and silently # store {"a": "b"} -- reject str entries outright Lexicon(capitalization_exceptions=("ab",)) # type: ignore[arg-type] + + +def test_setstate_rejects_mismatched_field_layout() -> None: + # A pickle from a different Lexicon version (field added/renamed) + # must fail at load time with a message naming the mismatch, not at + # some later attribute read far from the unpickle site. + lex = Lexicon.empty() + good_state = lex.__getstate__() + missing = dict(good_state) + del missing["titles"] + with pytest.raises(ValueError, match="titles"): + Lexicon.__new__(Lexicon).__setstate__(missing) + extra = dict(good_state) + extra["zq_future_field"] = frozenset() + with pytest.raises(ValueError, match="zq_future_field"): + Lexicon.__new__(Lexicon).__setstate__(extra) From c5090d1f2c37c069ddd77ce47600d7e3b3b60e87 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:12:18 -0700 Subject: [PATCH 045/275] Fix vacuous removal assertion in the add/remove test 'bishop' is a v1 TITLE, not a suffix word, so removing it from suffix_words asserted nothing -- the test kept passing even if remove() were a no-op. Remove an entry that is actually present and assert the precondition first. Co-Authored-By: Claude Fable 5 --- tests/v2/test_lexicon.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index ebdea70e..9eb4d19a 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -85,9 +85,12 @@ def test_add_and_remove_return_new_lexicons() -> None: # "zqtitle" is a synthetic word absent from v1's TITLES data (unlike # e.g. "dra", the feminine "dr." abbreviation, which is already there). base = Lexicon.default() - lex = base.add(titles={"zqtitle"}).remove(suffix_words={"bishop"}) + lex = base.add(titles={"zqtitle"}).remove(suffix_words={"esquire"}) assert "zqtitle" in lex.titles and "zqtitle" not in base.titles - assert "bishop" not in lex.suffix_words + # precondition, or the removal assertion passes vacuously (this is a + # guard for the operation under test, not a vocabulary content pin) + assert "esquire" in base.suffix_words + assert "esquire" not in lex.suffix_words def test_add_unknown_field_raises_with_valid_names() -> None: From 2310fb1c790d0aedb21c85adceb509940c0336a7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:12:56 -0700 Subject: [PATCH 046/275] Pin four stated-but-untested v2 contracts - apply_patch revalidates deferred patch values (PolicyPatch documents lazy validation; nothing asserted the apply-time failure) - all four set-valued PolicyPatch fields declare union composition (apply_patch is metadata-driven; dropping one silently flips locale layering from union to override) - pickle round-trips for Token/Ambiguity/ParsedName/Policy/PolicyPatch, pinning that UNSET survives unpickling BY IDENTITY -- apply_patch gates on 'is UNSET', which only works because Enum members unpickle to the same object - _normalize is casefold + all-periods stripping, stricter than v1's lc() (lower + edge periods only) Co-Authored-By: Claude Fable 5 --- tests/v2/test_lexicon.py | 8 ++++++++ tests/v2/test_policy.py | 39 +++++++++++++++++++++++++++++++++++++++ tests/v2/test_types.py | 11 +++++++++++ 3 files changed, 58 insertions(+) diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 9eb4d19a..cdb8f24b 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -171,3 +171,11 @@ def test_setstate_rejects_mismatched_field_layout() -> None: extra["zq_future_field"] = frozenset() with pytest.raises(ValueError, match="zq_future_field"): Lexicon.__new__(Lexicon).__setstate__(extra) + + +def test_normalization_casefolds_and_strips_interior_periods() -> None: + # Stricter than v1's lc(), which lower()s and trims only EDGE + # periods: casefold handles ß, and interior periods are removed too. + # A "simplify to .lower()/.strip('.')" regression must fail here. + lex = Lexicon(titles=frozenset({"STRAßE", "Ph.D"})) + assert lex.titles == frozenset({"strasse", "phd"}) diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index abf21af4..045ac6e4 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -167,3 +167,42 @@ def test_policy_patch_canonicalizes_scalar_name_order() -> None: p = PolicyPatch(name_order=[Role.FAMILY, Role.GIVEN, Role.MIDDLE]) # type: ignore[arg-type] assert p.name_order == (Role.FAMILY, Role.GIVEN, Role.MIDDLE) assert isinstance(hash(p), int) + + +def test_apply_patch_revalidates_deferred_values() -> None: + # PolicyPatch documents lazy validation: invalid values sit latent in + # the patch and must fail when applied, not silently flow into Policy. + bad_order = PolicyPatch(name_order=(Role.TITLE, Role.GIVEN, Role.FAMILY)) + with pytest.raises(ValueError, match="permutation"): + apply_patch(Policy(), bad_order) + bad_rules = PolicyPatch(patronymic_rules=frozenset({"klingon"})) # type: ignore[arg-type] + with pytest.raises(ValueError, match="valid rules"): + apply_patch(Policy(), bad_rules) + + +def test_all_set_valued_patch_fields_declare_union_composition() -> None: + # apply_patch is driven by this metadata; dropping it from one field + # would silently flip locale layering from union to override. + union_fields = { + f.name for f in dataclasses.fields(PolicyPatch) + if f.metadata.get("compose") == "union" + } + assert union_fields == { + "patronymic_rules", "nickname_delimiters", + "maiden_delimiters", "extra_suffix_delimiters", + } + + +def test_policy_and_patch_pickle_round_trip_preserves_unset_identity() -> None: + import pickle + + p = Policy(patronymic_rules=frozenset({PatronymicRule.TURKIC})) + assert pickle.loads(pickle.dumps(p)) == p + patch = PolicyPatch(strip_emoji=False) + loaded = pickle.loads(pickle.dumps(patch)) + assert loaded == patch + # apply_patch gates on 'value is UNSET'; an unpickled patch is only + # correct because Enum members round-trip BY IDENTITY. A plain + # object() sentinel would break this silently. + assert loaded.name_order is UNSET + assert loaded.strip_emoji is False diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 921ff395..c956456c 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -324,3 +324,14 @@ def test_token_coerces_role_string_and_rejects_unknown() -> None: Token("Juan", None, "chief") # type: ignore[arg-type] with pytest.raises(ValueError, match="title, given"): Token("Juan", None, 5) # type: ignore[arg-type] + + +def test_types_pickle_round_trip() -> None: + import pickle + + pn = _delavega() + assert pickle.loads(pickle.dumps(pn)) == pn + amb = Ambiguity(AmbiguityKind.ORDER, "two-comma structure", ()) + assert pickle.loads(pickle.dumps(amb)) == amb + tok = Token("de", Span(9, 11), Role.FAMILY, frozenset({"particle"})) + assert pickle.loads(pickle.dumps(tok)) == tok From cbd8cdff837c2c43f5db520ef6eeafdf37a4f4fb Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:15:31 -0700 Subject: [PATCH 047/275] Fix stale and inaccurate comments found in review Three 'added in a later task' notes were false at HEAD; the _VOCAB_FIELDS comment claimed __or__ excludes capitalization_exceptions (it merges them right-biased); the _lexicon module docstring's data- module list omitted maiden_markers (now points at _default_lexicon's imports, which cannot rot); _normalize claimed equivalence with v1's lc() (lc lowers and trims only edge periods); the maiden_markers docstring omitted the masculine Russian forms it ships; replace() did not document that it drops ambiguities referencing replaced tokens; and AGENTS.md overstated 'mypy strict' (per-module strict is not valid mypy config) and the one-test-module-per-source rule. Also notes the layering table's config prefix enforces package granularity only. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 6 +++--- nameparser/_lexicon.py | 19 +++++++++++-------- nameparser/_locale.py | 4 ++-- nameparser/_policy.py | 6 +++--- nameparser/_types.py | 6 ++++-- nameparser/config/maiden_markers.py | 7 +++++-- tests/v2/test_layering.py | 4 +++- tests/v2/test_types.py | 2 +- 8 files changed, 32 insertions(+), 22 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6aefe8aa..4e528069 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -113,14 +113,14 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`; later `_pipeline/`, `_parser.py`, `_render.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Old paths (`nameparser.parser`, `nameparser.config`) stay v1-shaped. - **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching): `_types` imports nothing internal; `_lexicon` and `_policy` sit above it independently; `_locale` on top. `_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x (`config/maiden_markers.py` is 2.0-only data that lives there for the same reason). Extend the test's `ALLOWED` table when adding a module. -- **Canonical field order** — `title, given, middle, family, suffix, nickname, maiden` — is defined once by `Role` enum declaration order and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. +- **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). -- **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; mypy strict via per-module overrides in pyproject. Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. +- **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. - **Pickling**: v2 types must round-trip (`Parser` will be picklable by construction, and it holds a `Lexicon`). `Lexicon` needs its custom `__getstate__`/`__setstate__` because of its `mappingproxy` slot; a new unpicklable slot type needs the same treatment plus a round-trip test. - **One sanctioned global**: the (future) cached default `Parser`. Any second piece of module-level mutable state requires amending the conventions doc, on purpose, in review. -- **Tests**: all v2 tests live in the `tests/v2/` package (its `conftest.py` overrides the v1 dual-run fixture — v2 code never reads shared `CONSTANTS`), one test module per source module, names stating behavior. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. +- **Tests**: all v2 tests live in the `tests/v2/` package (its `conftest.py` overrides the v1 dual-run fixture — v2 code never reads shared `CONSTANTS`), one test module per source module plus cross-cutting `test_reprs.py` and `test_layering.py`, names stating behavior. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. ## Extension Patterns diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index b4a53077..e1d07a45 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -1,8 +1,8 @@ """Immutable vocabulary configuration for the 2.0 API. -Layering: may import nameparser.config DATA modules (titles, suffixes, -prefixes, conjunctions, capitalization, bound_first_names) as the single -source of vocabulary during 2.x -- never nameparser.config itself, never +Layering: may import nameparser.config DATA modules (the imports in +_default_lexicon() are the authoritative list) as the single source of +vocabulary during 2.x -- never nameparser.config itself, never nameparser.parser. Enforced by tests/v2/test_layering.py. """ from __future__ import annotations @@ -13,9 +13,10 @@ from dataclasses import dataclass, field from types import MappingProxyType -#: Vocabulary set fields, in declaration order. add()/remove()/__or__ -#: (a later task) operate on exactly these; capitalization_exceptions is -#: deliberately excluded (its entries are pairs -- use dataclasses.replace). +#: Vocabulary set fields, in declaration order. add()/remove() operate +#: on exactly these and reject capitalization_exceptions (its entries +#: are pairs -- use dataclasses.replace); __or__ unions these AND merges +#: capitalization_exceptions right-biased. _VOCAB_FIELDS = ( "titles", "given_name_titles", "suffix_acronyms", "suffix_words", "suffix_acronyms_ambiguous", "particles", "particles_ambiguous", @@ -24,8 +25,10 @@ def _normalize(word: str) -> str: - """Casefold and strip periods -- v1's lc(). Membership tests never - re-normalize because construction already did.""" + """Casefold, remove ALL periods, strip whitespace -- v2's stricter + analogue of v1's lc(), which lower()s and trims only edge periods. + Membership tests never re-normalize because construction already + did.""" return word.casefold().replace(".", "").strip() diff --git a/nameparser/_locale.py b/nameparser/_locale.py index 5e06d176..346fb0dd 100644 --- a/nameparser/_locale.py +++ b/nameparser/_locale.py @@ -4,8 +4,8 @@ lexicon fragments union onto the base, the PolicyPatch folds via apply_patch. Packs are pure data; they have no privileged capabilities. -Layering: imports _lexicon and _policy only (tests/v2/test_layering.py, -added in a later task, enforces this). +Layering: imports _lexicon and _policy only (enforced by +tests/v2/test_layering.py). """ from __future__ import annotations diff --git a/nameparser/_policy.py b/nameparser/_policy.py index df27793f..9f12fad8 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -1,7 +1,7 @@ """Immutable behavior configuration for the 2.0 API. -Layering: imports nameparser._types only (tests/v2/test_layering.py, -added in a later task, enforces this). +Layering: imports nameparser._types only (enforced by +tests/v2/test_layering.py). """ from __future__ import annotations @@ -43,7 +43,7 @@ class Policy: extra_suffix_delimiters: frozenset[str] = frozenset() lenient_comma_suffixes: bool = True strip_emoji: bool = True - strip_bidi: bool = True # replaces v1's CONSTANTS.regexes.bidi = False + strip_bidi: bool = True # =False replaces v1's opt-out CONSTANTS.regexes.bidi = False def __post_init__(self) -> None: order = tuple(self.name_order) diff --git a/nameparser/_types.py b/nameparser/_types.py index c73d8c8b..8420f3f4 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -303,6 +303,7 @@ def replace(self, **fields: str) -> ParsedName: """Return a new ParsedName with the named fields re-tokenized as synthetic tokens (span=None). Whitespace-splits each value; an empty value clears the field. original is unchanged (provenance). + Ambiguities referencing replaced tokens are dropped. """ by_value = {role.value: role for role in Role} for key, value in fields.items(): @@ -341,7 +342,8 @@ def synthetic(value: str, role: Role) -> list[Token]: # -- comparison ------------------------------------------------------- def comparison_key(self) -> tuple[str, ...]: - """Casefolded seven components in canonical order, for dedup, - dict keys, and sorting. The semantic layer; __eq__ stays strict. + """One casefolded component per Role, in canonical order, for + dedup, dict keys, and sorting. The semantic layer; __eq__ stays + strict. """ return tuple(self._text_for(role).casefold() for role in Role) diff --git a/nameparser/config/maiden_markers.py b/nameparser/config/maiden_markers.py index 0c36c5ee..22399e37 100644 --- a/nameparser/config/maiden_markers.py +++ b/nameparser/config/maiden_markers.py @@ -20,9 +20,12 @@ Marker words that introduce a birth surname, e.g. "Jane Smith née Jones" (#274). French née/né/nee, German geb./geborene, Dutch geboren, Czech/Slovak roz./rozená, Danish/Norwegian født (Nynorsk fødd), Swedish -född, Russian урожд./урождённая (both ё and е spellings — +född, Russian урожд./урождённая/урождённый (both ё and е spellings — ``str.casefold()`` does not fold them, and running text routinely -writes е). Entries are stored normalized: lowercase, no periods. +writes е). Both grammatical genders are listed where #274 or review +attested them (née/né, урождённая/урождённый); Czech masculine rozený +awaits the same vetting. Entries are stored normalized: lowercase, no +periods. Consumed by the 2.0 parser's default lexicon. The 1.x parser does not read this module. diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index f9d9605b..cfbd63b7 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -9,7 +9,9 @@ # module -> prefixes it may import from within nameparser ALLOWED = { "_types.py": (), - "_lexicon.py": ("nameparser.config.",), # DATA modules only, during 2.x + # intent: DATA modules only, during 2.x -- mechanically this admits + # any config submodule; "data only" holds by convention/review + "_lexicon.py": ("nameparser.config.",), "_policy.py": ("nameparser._types",), "_locale.py": ("nameparser._lexicon", "nameparser._policy"), } diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index c956456c..339e4e0c 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -206,7 +206,7 @@ def test_suffix_joins_with_comma_space() -> None: def test_derived_views_filter_on_stable_particle_tag() -> None: # Pin the hard-coded "particle" string in _text_for to the published - # contract until Plan 3's tag-emission contract tests land. + # contract until parser tag-emission contract tests land. assert "particle" in STABLE_TAGS pn = _delavega() assert pn.family_particles == "de la" From 358ce8074e6bf54b2fca4fb900301963f2b336d4 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:49:00 -0700 Subject: [PATCH 048/275] Raise on entries that normalize to empty '.', '', or whitespace is a data bug (stray split artifact, empty CSV cell) on the primary customization surface; silently dropping it also meant a future data-module typo would vanish instead of failing CI. One rule for vocab fields AND capitalization-exception keys -- the previous behavior dropped both silently (keys deliberately, vocab undocumented). Raising now and relaxing later is compatible; the reverse is a break. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 20 +++++++++++++++++--- tests/v2/test_lexicon.py | 13 ++++++++++--- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index e1d07a45..dea001a9 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -50,13 +50,23 @@ def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: f"mapping (only capitalization_exceptions holds key->value pairs)" ) items = tuple(entries) # materialize once; entries may be a generator + normalized = set() for w in items: if not isinstance(w, str): raise TypeError( f"Lexicon.{field_name} entries must be strings, got {w!r}" ) - result = frozenset(_normalize(w) for w in items) - return frozenset(w for w in result if w) + n = _normalize(w) + # "." or "" is a data bug (stray split artifact, empty CSV + # cell); dropping it silently would also let a data-module typo + # vanish instead of failing CI. + if not n: + raise ValueError( + f"Lexicon.{field_name} entry {w!r} normalizes to empty " + f"(casefold + strip periods/whitespace leaves nothing)" + ) + normalized.add(n) + return frozenset(normalized) @dataclass(frozen=True, slots=True) @@ -118,7 +128,11 @@ def __post_init__(self) -> None: ) normalized_key = _normalize(k) if not normalized_key: - continue # mirror _normset's drop-empty rule + raise ValueError( + f"capitalization_exceptions key {k!r} normalizes to " + f"empty (casefold + strip periods/whitespace leaves " + f"nothing)" + ) deduped[normalized_key] = v canonical = tuple(sorted(deduped.items())) object.__setattr__(self, "capitalization_exceptions", canonical) diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index cdb8f24b..99bf0950 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -64,9 +64,16 @@ def test_lexicon_rejects_non_str_vocab_entries() -> None: Lexicon(titles={"Dr.", 42}) # type: ignore[arg-type] -def test_exception_keys_normalizing_to_empty_are_dropped() -> None: - lex = Lexicon(capitalization_exceptions=(("...", "X"), ("phd", "PhD"))) - assert lex.capitalization_exceptions == (("phd", "PhD"),) +def test_entries_normalizing_to_empty_raise() -> None: + # "." or "" is a data bug (stray split artifact, empty CSV cell); + # dropping it silently would also let a future data-module typo + # vanish instead of failing CI. One rule for vocab AND exception keys. + with pytest.raises(ValueError, match="normalizes to empty"): + Lexicon(titles=frozenset({"Dr.", "."})) + with pytest.raises(ValueError, match="normalizes to empty"): + Lexicon.empty().add(titles={" "}) + with pytest.raises(ValueError, match="normalizes to empty"): + Lexicon(capitalization_exceptions=(("...", "X"), ("phd", "PhD"))) def test_colliding_exception_keys_dedupe_last_wins() -> None: From 22fe2eaf478438c538546d53ea7463bc20dc6c5e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:50:04 -0700 Subject: [PATCH 049/275] Block Span concatenation and bool span coordinates Span(0,2) + Span(3,4) inherited tuple concatenation, producing a 4-tuple -- the natural but wrong spelling of 'covering span' that the pipeline's join stage will tempt. A real cover() operation ships with that consumer; blocking + now is compatible, blocking it after 2.0.0 would be a break. Also reject bools in span coordinates (bool is an int subclass; (False, True) is a leaked comparison result, not a span). Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 17 +++++++++++++++-- tests/v2/test_types.py | 15 +++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/nameparser/_types.py b/nameparser/_types.py index 8420f3f4..497d7d58 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -13,7 +13,7 @@ from collections.abc import Mapping from dataclasses import dataclass from enum import Enum, StrEnum -from typing import NamedTuple +from typing import NamedTuple, NoReturn class Role(Enum): @@ -34,6 +34,16 @@ class Span(NamedTuple): start: int end: int + def __add__(self, other: object) -> NoReturn: # type: ignore[override] + # Inherited tuple + would concatenate two spans into a 4-tuple -- + # the natural but wrong spelling of "covering span". The real + # covering operation ships with its consumer, the pipeline's + # join stage. + raise TypeError( + "Span does not support +; tuple concatenation is not a " + "covering span" + ) + #: Stable, documented tag vocabulary (API). All other tags are #: namespaced ("vocab:...", "patronymic:...") and unstable. @@ -66,7 +76,10 @@ def __post_init__(self) -> None: if not ( isinstance(self.span, tuple) and len(self.span) == 2 - and all(isinstance(v, int) for v in self.span) + # bool is an int subclass: (False, True) is a comparison + # result leaking into a coordinate slot, not a span + and all(isinstance(v, int) and not isinstance(v, bool) + for v in self.span) ): raise TypeError( f"invalid span {self.span!r}: expected a (start, end) " diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 339e4e0c..5211730b 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -335,3 +335,18 @@ def test_types_pickle_round_trip() -> None: assert pickle.loads(pickle.dumps(amb)) == amb tok = Token("de", Span(9, 11), Role.FAMILY, frozenset({"particle"})) assert pickle.loads(pickle.dumps(tok)) == tok + + +def test_span_add_is_blocked() -> None: + # NamedTuple + would concatenate into a 4-tuple, the natural but + # wrong spelling of "covering span" (a real cover() arrives with the + # pipeline's join stage, its consumer). + with pytest.raises(TypeError, match="covering span"): + Span(0, 2) + Span(3, 4) # type: ignore[operator] + + +def test_token_rejects_bool_span_coordinates() -> None: + # bool is an int subclass; (False, True) is a comparison result + # leaking into a coordinate slot, not a span. + with pytest.raises(TypeError, match="pair of ints"): + Token("x", (False, True), Role.GIVEN) # type: ignore[arg-type] From f07082b369b94d2644770356a7834f006ffece61 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:50:57 -0700 Subject: [PATCH 050/275] Pin Locale.code to the registry-key charset [a-z0-9_]+ Codes become registry keys the moment parser_for and third-party packs exist; every character accepted today is supported forever. The pin matches the shipped ru/tr_az convention and deliberately allows one separator only -- accepting '-' as well would make tr-az and tr_az distinct keys. Relaxing later is compatible; tightening later breaks published packs. Co-Authored-By: Claude Fable 5 --- nameparser/_locale.py | 9 +++++++++ tests/v2/test_locale.py | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/nameparser/_locale.py b/nameparser/_locale.py index 346fb0dd..14784c3d 100644 --- a/nameparser/_locale.py +++ b/nameparser/_locale.py @@ -10,6 +10,7 @@ from __future__ import annotations import dataclasses +import re from dataclasses import dataclass from nameparser._lexicon import Lexicon @@ -39,6 +40,14 @@ def __post_init__(self) -> None: raise ValueError( f"Locale.code must not contain whitespace, got {self.code!r}" ) + # Codes are registry keys (parser_for, third-party packs): every + # accepted character is supported forever, so pin the charset + # while relaxing later is still compatible. One separator only -- + # allowing '-' too would make tr-az and tr_az distinct keys. + if not re.fullmatch(r"[a-z0-9_]+", self.code): + raise ValueError( + f"Locale.code must match [a-z0-9_]+, got {self.code!r}" + ) if not isinstance(self.lexicon, Lexicon): raise TypeError( f"Locale.lexicon must be a Lexicon, got {self.lexicon!r}" diff --git a/tests/v2/test_locale.py b/tests/v2/test_locale.py index bc25eda7..f84728f7 100644 --- a/tests/v2/test_locale.py +++ b/tests/v2/test_locale.py @@ -53,3 +53,14 @@ def test_locale_with_lexicon_pickles_round_trip() -> None: loc = Locale(code="ru", lexicon=Lexicon.empty().add(titles={"Dr."})) assert pickle.loads(pickle.dumps(loc)) == loc + + +def test_locale_code_is_pinned_to_registry_charset() -> None: + # Codes become registry keys the moment parser_for and third-party + # packs exist: every accepted character is supported forever, so pin + # [a-z0-9_]+ now (matches ru/tr_az). One separator only -- allowing + # both '-' and '_' would make tr-az and tr_az distinct keys. + for bad in ("ru!", "tr-az", "тр", "zh/tw"): + with pytest.raises(ValueError, match="a-z0-9_"): + Locale(code=bad, lexicon=Lexicon.empty()) + assert Locale(code="tr_az", lexicon=Lexicon.empty()).code == "tr_az" From 22ea503dc27efda001406eec39eab652ab81a878 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 14:05:06 -0700 Subject: [PATCH 051/275] Extract _coerce_enum for the duplicated enum coercion blocks Token.role and Ambiguity.kind carried byte-parallel coerce-or-raise blocks that had to be kept in sync by hand; both message shapes are pinned by tests. One parametrized module-private helper, two one-line call sites, identical messages. Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/nameparser/_types.py b/nameparser/_types.py index 497d7d58..f147909a 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -13,7 +13,7 @@ from collections.abc import Mapping from dataclasses import dataclass from enum import Enum, StrEnum -from typing import NamedTuple, NoReturn +from typing import NamedTuple, NoReturn, TypeVar class Role(Enum): @@ -49,6 +49,23 @@ def __add__(self, other: object) -> NoReturn: # type: ignore[override] #: namespaced ("vocab:...", "patronymic:...") and unstable. STABLE_TAGS = frozenset({"particle", "conjunction", "initial"}) +_E = TypeVar("_E", bound=Enum) + + +def _coerce_enum(value: object, enum_cls: type[_E], noun: str, plural: str) -> _E: + """Coerce value to enum_cls, or raise the enriched ValueError listing + every valid member (enum lookups stay ValueError for any input -- + stdlib EnumType precedent, see AGENTS.md's taxonomy rule).""" + if isinstance(value, enum_cls): + return value + try: + return enum_cls(value) + except ValueError: + valid = ", ".join(str(m.value) for m in enum_cls) + raise ValueError( + f"unknown {noun} {value!r}; valid {plural}: {valid}" + ) from None + @dataclass(frozen=True, slots=True) class Token: @@ -64,14 +81,8 @@ def __post_init__(self) -> None: ) if not self.text: raise ValueError("Token.text must be a non-empty string") - if not isinstance(self.role, Role): - try: - object.__setattr__(self, "role", Role(self.role)) - except ValueError: - valid = ", ".join(r.value for r in Role) - raise ValueError( - f"unknown Role {self.role!r}; valid roles: {valid}" - ) from None + object.__setattr__( + self, "role", _coerce_enum(self.role, Role, "Role", "roles")) if self.span is not None: if not ( isinstance(self.span, tuple) @@ -137,14 +148,9 @@ class Ambiguity: tokens: tuple[Token, ...] def __post_init__(self) -> None: - if not isinstance(self.kind, AmbiguityKind): - try: - object.__setattr__(self, "kind", AmbiguityKind(self.kind)) - except ValueError: - valid = ", ".join(k.value for k in AmbiguityKind) - raise ValueError( - f"unknown AmbiguityKind {self.kind!r}; valid kinds: {valid}" - ) from None + object.__setattr__( + self, "kind", + _coerce_enum(self.kind, AmbiguityKind, "AmbiguityKind", "kinds")) if not isinstance(self.detail, str): raise TypeError( f"Ambiguity.detail must be a str, got {self.detail!r}" From eaba92f260adfdd8d3fa4657c291db83d3f7ab19 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 14:05:47 -0700 Subject: [PATCH 052/275] Extract _normpairs as _normset's sibling The module's convention is that collection normalization lives in a module-level _norm* helper the constructor calls; capitalization exceptions were the one field whose ~40-line validation body sat inlined in __post_init__, mixing orchestration with one field's rules. Pure move -- messages and behavior unchanged. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 90 ++++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index dea001a9..e4f08e91 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -69,6 +69,53 @@ def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: return frozenset(normalized) +def _normpairs( + raw: Mapping[str, str] | Iterable[tuple[str, str]], +) -> tuple[tuple[str, str], ...]: + """Canonicalize capitalization_exceptions input: _normset's sibling + for the one pair-valued field. Dedupes on the NORMALIZED key so the + tuple and the derived map always agree ("Ph.D." and "phd" collide + after normalization); last occurrence wins, matching dict semantics + and the right-bias rule used elsewhere.""" + if isinstance(raw, str): + raise TypeError( + "capitalization_exceptions must be a mapping or an " + "iterable of (key, value) pairs, not a bare string" + ) + pairs = raw.items() if isinstance(raw, Mapping) else raw + deduped: dict[str, str] = {} + for entry in pairs: + # A 2-char str entry would unpack "ab" into ("a", "b") + # silently, so reject str outright; other mis-shapes would + # otherwise surface as bare unpack errors. + if isinstance(entry, str): + raise TypeError( + f"capitalization_exceptions entries must be " + f"(key, value) pairs, got {entry!r}" + ) + try: + k, v = entry + except (TypeError, ValueError): + raise TypeError( + f"capitalization_exceptions entries must be " + f"(key, value) pairs, got {entry!r}" + ) from None + if not isinstance(k, str) or not isinstance(v, str): + raise TypeError( + f"capitalization_exceptions entries must be " + f"str -> str, got {k!r}: {v!r}" + ) + normalized_key = _normalize(k) + if not normalized_key: + raise ValueError( + f"capitalization_exceptions key {k!r} normalizes to " + f"empty (casefold + strip periods/whitespace leaves " + f"nothing)" + ) + deduped[normalized_key] = v + return tuple(sorted(deduped.items())) + + @dataclass(frozen=True, slots=True) class Lexicon: titles: frozenset[str] = frozenset() @@ -93,48 +140,7 @@ class Lexicon: def __post_init__(self) -> None: for name in _VOCAB_FIELDS: object.__setattr__(self, name, _normset(getattr(self, name), name)) - raw = self.capitalization_exceptions - if isinstance(raw, str): - raise TypeError( - "capitalization_exceptions must be a mapping or an " - "iterable of (key, value) pairs, not a bare string" - ) - pairs = raw.items() if isinstance(raw, Mapping) else raw - # Dedupe on the NORMALIZED key before storing so the tuple and the - # map always agree ("Ph.D." and "phd" collide after normalization). - # Last occurrence wins, matching dict semantics and the right-bias - # rule used elsewhere. - deduped: dict[str, str] = {} - for entry in pairs: - # A 2-char str entry would unpack "ab" into ("a", "b") - # silently, so reject str outright; other mis-shapes would - # otherwise surface as bare unpack errors. - if isinstance(entry, str): - raise TypeError( - f"capitalization_exceptions entries must be " - f"(key, value) pairs, got {entry!r}" - ) - try: - k, v = entry - except (TypeError, ValueError): - raise TypeError( - f"capitalization_exceptions entries must be " - f"(key, value) pairs, got {entry!r}" - ) from None - if not isinstance(k, str) or not isinstance(v, str): - raise TypeError( - f"capitalization_exceptions entries must be " - f"str -> str, got {k!r}: {v!r}" - ) - normalized_key = _normalize(k) - if not normalized_key: - raise ValueError( - f"capitalization_exceptions key {k!r} normalizes to " - f"empty (casefold + strip periods/whitespace leaves " - f"nothing)" - ) - deduped[normalized_key] = v - canonical = tuple(sorted(deduped.items())) + canonical = _normpairs(self.capitalization_exceptions) object.__setattr__(self, "capitalization_exceptions", canonical) object.__setattr__(self, "_cap_map", MappingProxyType(dict(canonical))) if not self.particles_ambiguous <= self.particles: From cac42311899e79282f964e0ffe74a7e1f01d417e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 14:06:14 -0700 Subject: [PATCH 053/275] Drop the whitespace check the charset pin subsumed The [a-z0-9_]+ fullmatch landed after the whitespace check and fully absorbs it: all-whitespace input already fails the strip() check, and interior whitespace fails the charset with an accurate message. The empty and lowercase pre-checks stay -- they guard the common mistakes with materially friendlier messages. Co-Authored-By: Claude Fable 5 --- nameparser/_locale.py | 4 ---- tests/v2/test_locale.py | 4 +++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/nameparser/_locale.py b/nameparser/_locale.py index 14784c3d..640a6a67 100644 --- a/nameparser/_locale.py +++ b/nameparser/_locale.py @@ -36,10 +36,6 @@ def __post_init__(self) -> None: raise ValueError( f"Locale.code must be lowercase, got {self.code!r}" ) - if any(c.isspace() for c in self.code): - raise ValueError( - f"Locale.code must not contain whitespace, got {self.code!r}" - ) # Codes are registry keys (parser_for, third-party packs): every # accepted character is supported forever, so pin the charset # while relaxing later is still compatible. One separator only -- diff --git a/tests/v2/test_locale.py b/tests/v2/test_locale.py index f84728f7..c900ff5f 100644 --- a/tests/v2/test_locale.py +++ b/tests/v2/test_locale.py @@ -29,8 +29,10 @@ def test_locale_code_must_be_nonempty_lowercase() -> None: def test_locale_code_rejects_whitespace() -> None: + # caught by the [a-z0-9_]+ charset pin (a dedicated whitespace check + # would be pure redundancy; the charset message is accurate) for bad in ("ru ", " ru", "ru\n", "r u"): - with pytest.raises(ValueError, match="whitespace"): + with pytest.raises(ValueError, match="a-z0-9_"): Locale(code=bad, lexicon=Lexicon.empty()) From a2d88eaa61fbf558c8e0863095bb1171a37b1cc7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 14:28:09 -0700 Subject: [PATCH 054/275] Test the real matrix interpreter, and add 3.15 pre-release (#259) Every matrix job was silently testing CPython 3.11: uv sync honors .python-version (pinned 3.11 by the floor bump) over setup-python's interpreter, so 'build (3.14)' ran the suite on 3.11.15 (verified in the PR #288 run logs). UV_PYTHON makes the matrix real. With that fixed, add 3.15 (final 2026-10-01) as a non-blocking pre-release job so interpreter breakage surfaces while 2.0 is being built; flip it to blocking when 3.15 goes final. Locally green today on 3.15.0b3: pytest, mypy, ruff. Co-Authored-By: Claude Fable 5 --- .github/workflows/python-package.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 072f44c0..aacd72a7 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -20,7 +20,14 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12", "3.13", "3.14"] + python-version: ["3.11", "3.12", "3.13", "3.14", "3.15"] + # 3.15 is pre-release until 2026-10-01 (#259): surface breakage + # without blocking the branch; flip to blocking when it goes final. + continue-on-error: ${{ matrix.python-version == '3.15' }} + env: + # Without this, uv sync honors .python-version (3.11) over the + # matrix interpreter and every job silently tests 3.11. + UV_PYTHON: ${{ matrix.python-version }} steps: - uses: actions/checkout@v7 @@ -28,6 +35,7 @@ jobs: uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - name: Install uv uses: astral-sh/setup-uv@v6 with: From 73e816eed8dae48085f9e83d055a99770a886013 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 15:37:10 -0700 Subject: [PATCH 055/275] Add _render with the #254-normative collapse; extend layering for it Co-Authored-By: Claude Fable 5 --- AGENTS.md | 2 +- nameparser/_render.py | 30 +++++++++++++++++++++++ pyproject.toml | 1 + tests/v2/test_layering.py | 50 +++++++++++++++++++++++++++++++++++---- tests/v2/test_render.py | 20 ++++++++++++++++ 5 files changed, 97 insertions(+), 6 deletions(-) create mode 100644 nameparser/_render.py create mode 100644 tests/v2/test_render.py diff --git a/AGENTS.md b/AGENTS.md index 4e528069..0166c0e2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -112,7 +112,7 @@ Each named attribute (`title`, `first`, etc.) is a `@property` that joins its co The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These conventions apply to all new-API code and are stricter than the v1 sections above. The full design record (rationale, settled-decision logs, dated amendments) lives in untracked `docs/superpowers/specs/`; this section is the enforceable subset. **A commit that establishes or amends one of these conventions must update this section in the same commit** — grep-driven staleness sweeps miss paraphrased prose (see Workflow above), so write-time maintenance is the mechanism, audits are the backstop. - **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`; later `_pipeline/`, `_parser.py`, `_render.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Old paths (`nameparser.parser`, `nameparser.config`) stay v1-shaped. -- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching): `_types` imports nothing internal; `_lexicon` and `_policy` sit above it independently; `_locale` on top. `_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x (`config/maiden_markers.py` is 2.0-only data that lives there for the same reason). Extend the test's `ALLOWED` table when adding a module. +- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy`; `_render` imports `_types` and `_lexicon` (for `Lexicon.default()`). Extend the test's `ALLOWED` table when adding a module. - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). diff --git a/nameparser/_render.py b/nameparser/_render.py new file mode 100644 index 00000000..fa673031 --- /dev/null +++ b/nameparser/_render.py @@ -0,0 +1,30 @@ +"""Rendering for the 2.0 API: ParsedName -> display strings. + +Layering: imports nameparser._types, and nameparser._lexicon only for +Lexicon.default() when capitalized() receives lexicon=None (enforced by +tests/v2/test_layering.py). Parsing code never imports this module; +ParsedName's rendering methods delegate here via call-time imports. +""" +from __future__ import annotations + +import re + +_SPACES = re.compile(r"\s+") +_SPACE_BEFORE_COMMA = re.compile(r"\s+,") +_COMMA_CHAR = re.compile(r"[,،，]") # ASCII, Arabic, fullwidth + + +def _collapse(rendered: str) -> str: + """The #254 collapse, normative (core spec §5b): empty fields + substitute '' and every artifact of that is removed -- dangling + empty-nickname wrappers, space runs, space-before-comma, one + trailing comma character (any script), leading/trailing ', ' + debris.""" + rendered = (rendered.replace(" ()", "") + .replace(" ''", "") + .replace(' ""', "")) + rendered = _SPACE_BEFORE_COMMA.sub(",", rendered) + rendered = _SPACES.sub(" ", rendered.strip()) + if rendered and _COMMA_CHAR.fullmatch(rendered[-1]): + rendered = rendered[:-1] + return rendered.strip(", ") diff --git a/pyproject.toml b/pyproject.toml index afbfecae..bba07a32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,7 @@ module = [ "nameparser._types", "nameparser._lexicon", "nameparser._policy", + "nameparser._render", "nameparser._locale", ] disallow_untyped_defs = true diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index cfbd63b7..cb7c7c3a 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -8,23 +8,47 @@ # module -> prefixes it may import from within nameparser ALLOWED = { - "_types.py": (), + # call-time imports only (inside the rendering-delegate method + # bodies); module level stays import-free. TYPE_CHECKING imports + # are skipped by _nameparser_imports and need no entry. + "_types.py": ("nameparser._render",), # intent: DATA modules only, during 2.x -- mechanically this admits # any config submodule; "data only" holds by convention/review "_lexicon.py": ("nameparser.config.",), "_policy.py": ("nameparser._types",), "_locale.py": ("nameparser._lexicon", "nameparser._policy"), + # _lexicon is needed at runtime: capitalized(lexicon=None) resolves + # to Lexicon.default() + "_render.py": ("nameparser._types", "nameparser._lexicon"), } +def _is_type_checking(test: ast.expr) -> bool: + return (isinstance(test, ast.Name) and test.id == "TYPE_CHECKING") or ( + isinstance(test, ast.Attribute) and test.attr == "TYPE_CHECKING") + + def _nameparser_imports(path: pathlib.Path) -> list[str]: - tree = ast.parse(path.read_text()) - found = [] - for node in ast.walk(tree): + """All nameparser-internal imports in the module, at any nesting + depth, EXCEPT those under `if TYPE_CHECKING:` -- annotation-only + imports are not runtime dependencies.""" + found: list[str] = [] + + def _record(node: ast.AST) -> None: + # guard checked on EVERY node uniformly, so `elif TYPE_CHECKING:` + # (a nested If in orelse) is skipped exactly like the plain form + if isinstance(node, ast.If) and _is_type_checking(node.test): + for stmt in node.orelse: + _record(stmt) + return if isinstance(node, ast.Import): - found += [a.name for a in node.names] + found.extend(a.name for a in node.names) elif isinstance(node, ast.ImportFrom) and node.module: found.append(node.module) + for child in ast.iter_child_nodes(node): + _record(child) + + _record(ast.parse(path.read_text())) return [m for m in found if m.startswith("nameparser")] @@ -65,3 +89,19 @@ def test_public_exports() -> None: assert expected <= set(nameparser.__all__) for name in expected: assert getattr(nameparser, name) is not None + + +def test_type_checking_imports_do_not_count(tmp_path: pathlib.Path) -> None: + mod = tmp_path / "snippet.py" + mod.write_text( + "from typing import TYPE_CHECKING\n" + "if TYPE_CHECKING:\n" + " from nameparser._lexicon import Lexicon\n" + "elif TYPE_CHECKING:\n" + " import nameparser.never_runtime\n" + "else:\n" + " import nameparser.util\n" + "def f():\n" + " from nameparser import _render\n" + ) + assert _nameparser_imports(mod) == ["nameparser.util", "nameparser"] diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py new file mode 100644 index 00000000..7cfa87a8 --- /dev/null +++ b/tests/v2/test_render.py @@ -0,0 +1,20 @@ +# NOTE: the import block grows task by task -- importing names before +# their task lands would fail ruff F401. Task 1 needs only _collapse. +from nameparser._render import _collapse + + +def test_collapse_is_the_254_algorithm() -> None: + # normative: leading/trailing whitespace, doubled spaces, + # space-before-comma, one trailing comma char (incl. Arabic/CJK), + # leading/trailing ', ' debris, and empty-wrapper artifacts from + # empty fields are removed + assert _collapse(" John Smith ") == "John Smith" + assert _collapse("Smith , John") == "Smith, John" + assert _collapse("John Smith ,") == "John Smith" + assert _collapse("John Smith،") == "John Smith" # Arabic comma + assert _collapse("John Smith，") == "John Smith" # fullwidth comma + assert _collapse(", John Smith, ") == "John Smith" + assert _collapse("John Smith ()") == "John Smith" + assert _collapse("John Smith ''") == "John Smith" + assert _collapse('John Smith ""') == "John Smith" + assert _collapse("") == "" From a6750dc3a95378cedc989216ecea943ff5efcf26 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 15:57:59 -0700 Subject: [PATCH 056/275] Add render(): spec formatting over role fields and derived views Co-Authored-By: Claude Fable 5 --- nameparser/_render.py | 22 ++++++++++++++++ tests/v2/test_render.py | 57 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/nameparser/_render.py b/nameparser/_render.py index fa673031..14b9b684 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -9,10 +9,17 @@ import re +from nameparser._types import ParsedName, Role + _SPACES = re.compile(r"\s+") _SPACE_BEFORE_COMMA = re.compile(r"\s+,") _COMMA_CHAR = re.compile(r"[,،，]") # ASCII, Arabic, fullwidth +#: str.format keys render() accepts: the seven role fields in canonical +#: order (derived from Role -- never restated) plus the derived views. +_DERIVED_VIEWS = ("family_base", "family_particles", "surnames", "given_names") +_RENDER_KEYS = tuple(r.value for r in Role) + _DERIVED_VIEWS + def _collapse(rendered: str) -> str: """The #254 collapse, normative (core spec §5b): empty fields @@ -28,3 +35,18 @@ def _collapse(rendered: str) -> str: if rendered and _COMMA_CHAR.fullmatch(rendered[-1]): rendered = rendered[:-1] return rendered.strip(", ") + + +def render(name: ParsedName, spec: str) -> str: + """Fill the str.format spec from the seven role fields and the + derived views (empty fields substitute ''), then apply the #254 + collapse. Unknown keys raise KeyError naming the valid fields.""" + values = {key: getattr(name, key) for key in _RENDER_KEYS} + try: + rendered = spec.format(**values) + except KeyError as exc: + raise KeyError( + f"unknown render field {exc.args[0]!r}; valid fields: " + f"{', '.join(_RENDER_KEYS)}" + ) from None + return _collapse(rendered) diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 7cfa87a8..142707b9 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -1,6 +1,7 @@ -# NOTE: the import block grows task by task -- importing names before -# their task lands would fail ruff F401. Task 1 needs only _collapse. +import pytest + from nameparser._render import _collapse +from nameparser._types import ParsedName, Role, Span, Token def test_collapse_is_the_254_algorithm() -> None: @@ -18,3 +19,55 @@ def test_collapse_is_the_254_algorithm() -> None: assert _collapse("John Smith ''") == "John Smith" assert _collapse('John Smith ""') == "John Smith" assert _collapse("") == "" + + +def _pn(original: str, tokens: list[Token]) -> ParsedName: + return ParsedName(original=original, tokens=tuple(tokens)) + + +def _delavega() -> ParsedName: + # "Dr. Juan de la Vega III" -- spans verified by hand + # 0123456789012345678901234 + return _pn("Dr. Juan de la Vega III", [ + Token("Dr.", Span(0, 3), Role.TITLE), + Token("Juan", Span(4, 8), Role.GIVEN), + Token("de", Span(9, 11), Role.FAMILY, frozenset({"particle"})), + Token("la", Span(12, 14), Role.FAMILY, frozenset({"particle"})), + Token("Vega", Span(15, 19), Role.FAMILY), + Token("III", Span(20, 23), Role.SUFFIX), + ]) + + +def test_render_fills_fields_and_collapses() -> None: + from nameparser._render import render + pn = _delavega() + assert render(pn, "{title} {given} {middle} {family} {suffix}") \ + == "Dr. Juan de la Vega III" + # empty middle collapses; comma survives correctly + assert render(pn, "{family}, {given} {middle}") == "de la Vega, Juan" + + +def test_render_accepts_derived_view_keys() -> None: + from nameparser._render import render + assert render(_delavega(), "{family_base}, {given} {family_particles}") \ + == "Vega, Juan de la" + assert render(_delavega(), "{surnames}") == "de la Vega" + assert render(_delavega(), "{given_names}") == "Juan" + + +def test_render_every_role_key_is_valid() -> None: + from nameparser._render import render + pn = _delavega() + for role in Role: + render(pn, f"{{{role.value}}}") # must not raise + + +def test_render_unknown_key_raises_enriched_keyerror() -> None: + from nameparser._render import render + with pytest.raises(KeyError, match="valid fields"): + render(_delavega(), "{first}") # v1 spelling: redirected loudly + + +def test_render_empty_parse_is_empty_string() -> None: + from nameparser._render import render + assert render(_pn("", []), "{title} {given} {middle} {family} {suffix}") == "" From bbca3653209747dac1dc29ba8dc2c964fb6ad0c9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 16:03:58 -0700 Subject: [PATCH 057/275] Add ParsedName.render() and __str__ via call-time _render delegation Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 15 +++++++++++++++ tests/v2/test_render.py | 8 ++++++++ 2 files changed, 23 insertions(+) diff --git a/nameparser/_types.py b/nameparser/_types.py index f147909a..7d6410eb 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -230,6 +230,9 @@ def __post_init__(self) -> None: def __bool__(self) -> bool: return bool(self.tokens) + def __str__(self) -> str: + return self.render() + def __repr__(self) -> str: lines = [] for role in Role: @@ -366,3 +369,15 @@ def comparison_key(self) -> tuple[str, ...]: strict. """ return tuple(self._text_for(role).casefold() for role in Role) + + # -- rendering delegates ---------------------------------------------- + # One-line delegation to nameparser._render (core spec §5b): parsing + # code physically cannot import formatting logic, so these import at + # call time -- module level stays internal-import-free. + + def render(self, spec: str = "{title} {given} {middle} {family} {suffix}") -> str: + """Fill the str.format spec from the seven role fields and the + derived views; empty fields collapse (#254). Unknown keys raise + KeyError naming the valid fields.""" + import nameparser._render as _render + return _render.render(self, spec) diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 142707b9..064f74ca 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -71,3 +71,11 @@ def test_render_unknown_key_raises_enriched_keyerror() -> None: def test_render_empty_parse_is_empty_string() -> None: from nameparser._render import render assert render(_pn("", []), "{title} {given} {middle} {family} {suffix}") == "" + + +def test_parsedname_render_and_str_delegate() -> None: + pn = _delavega() + assert pn.render() == "Dr. Juan de la Vega III" + assert pn.render("{family}, {given}") == "de la Vega, Juan" + assert str(pn) == pn.render() + assert str(_pn("", [])) == "" From 445571d138a4007e60775574026d35677fa8c4b8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 16:07:50 -0700 Subject: [PATCH 058/275] Add initials() with call-site delimiter/separator arguments Co-Authored-By: Claude Fable 5 --- nameparser/_render.py | 34 ++++++++++++++++++++++ nameparser/_types.py | 8 ++++++ tests/v2/test_render.py | 62 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+) diff --git a/nameparser/_render.py b/nameparser/_render.py index 14b9b684..2f8ae5aa 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -20,6 +20,13 @@ _DERIVED_VIEWS = ("family_base", "family_particles", "surnames", "given_names") _RENDER_KEYS = tuple(r.value for r in Role) + _DERIVED_VIEWS +#: str.format keys initials() accepts: the three name-bearing roles. +_INITIALS_KEYS = (Role.GIVEN.value, Role.MIDDLE.value, Role.FAMILY.value) + +#: Tags whose tokens contribute no initial outside the given group. +#: Not STABLE_TAGS -- that also contains "initial", which must contribute. +_SKIP_TAGS = frozenset({"particle", "conjunction"}) + def _collapse(rendered: str) -> str: """The #254 collapse, normative (core spec §5b): empty fields @@ -50,3 +57,30 @@ def render(name: ParsedName, spec: str) -> str: f"{', '.join(_RENDER_KEYS)}" ) from None return _collapse(rendered) + + +def initials(name: ParsedName, spec: str, delimiter: str, separator: str) -> str: + """First letter of each contributing token per group, v1 semantics: + delimiter follows each initial, separator sits between initials + within a group. Tokens tagged particle/conjunction contribute no + initial in middle/family (given-name tokens always contribute); + tags come from the pipeline -- hand-built untagged tokens all + contribute. Valid spec keys: given, middle, family.""" + values: dict[str, str] = {} + for key in _INITIALS_KEYS: + role = Role(key) + tokens = name.tokens_for(role) + if role is not Role.GIVEN: + tokens = tuple(t for t in tokens + if not (_SKIP_TAGS & t.tags)) + letters = [t.text[0] for t in tokens] + values[key] = ((delimiter + separator).join(letters) + delimiter + if letters else "") + try: + rendered = spec.format(**values) + except KeyError as exc: + raise KeyError( + f"unknown initials field {exc.args[0]!r}; valid fields: " + f"{', '.join(_INITIALS_KEYS)}" + ) from None + return _collapse(rendered) diff --git a/nameparser/_types.py b/nameparser/_types.py index 7d6410eb..a700bd1a 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -381,3 +381,11 @@ def render(self, spec: str = "{title} {given} {middle} {family} {suffix}") -> st KeyError naming the valid fields.""" import nameparser._render as _render return _render.render(self, spec) + + def initials(self, spec: str = "{given} {middle} {family}", + delimiter: str = ".", separator: str = " ") -> str: + """Initials per group; v1's initials_format/_delimiter/_separator + become call-site arguments (core spec §5b). Valid spec keys: + given, middle, family.""" + import nameparser._render as _render + return _render.initials(self, spec, delimiter, separator) diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 064f74ca..15e08611 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -79,3 +79,65 @@ def test_parsedname_render_and_str_delegate() -> None: assert pn.render("{family}, {given}") == "de la Vega, Juan" assert str(pn) == pn.render() assert str(_pn("", [])) == "" + + +def _bobdole() -> ParsedName: + # "Sir Bob Andrew Dole" + # 01234567890123456789 + return _pn("Sir Bob Andrew Dole", [ + Token("Sir", Span(0, 3), Role.TITLE), + Token("Bob", Span(4, 7), Role.GIVEN), + Token("Andrew", Span(8, 14), Role.MIDDLE), + Token("Dole", Span(15, 19), Role.FAMILY), + ]) + + +def test_initials_default_spec() -> None: + assert _bobdole().initials() == "B. A. D." + + +def test_initials_skips_tagged_particles_outside_given() -> None: + # family "de la Vega" with particle tags -> only V contributes; + # a given-name token always contributes even if tagged + assert _delavega().initials() == "J. V." + # conjunction tag skips too + pn = _pn("Mr. and Mrs. Smith", [ + Token("Mr.", Span(0, 3), Role.TITLE), + Token("and", Span(4, 7), Role.FAMILY, frozenset({"conjunction"})), + Token("Smith", Span(13, 18), Role.FAMILY), + ]) + assert pn.initials("{family}") == "S." + + +def test_initials_custom_delimiter_and_separator() -> None: + assert _bobdole().initials(delimiter="", separator="") == "B A D" + + +def test_initials_multiword_group_joins_within_group() -> None: + pn = _pn("Mary Jane Watson", [ + Token("Mary", Span(0, 4), Role.GIVEN), + Token("Jane", Span(5, 9), Role.GIVEN), + Token("Watson", Span(10, 16), Role.FAMILY), + ]) + assert pn.initials() == "M. J. W." + + +def test_initials_custom_spec_and_unknown_key() -> None: + assert _bobdole().initials("{given} {middle}") == "B. A." + with pytest.raises(KeyError, match="valid fields"): + _bobdole().initials("{title}") + + +def test_initials_already_initial_words() -> None: + pn = _pn("J. Doe", [ + Token("J.", Span(0, 2), Role.GIVEN), + Token("Doe", Span(3, 6), Role.FAMILY), + ]) + assert pn.initials() == "J. D." + + +def test_initials_empty_group_renders_empty() -> None: + # v2 returns "" for an empty result -- no v1-style + # empty_attribute_default fallback + assert _bobdole().initials("{middle}") == "A." + assert _pn("Cher", [Token("Cher", Span(0, 4), Role.GIVEN)]).initials("{middle}") == "" From da94eb5f18c18d90a8d6e8414a826c8fd91f335c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 16:14:53 -0700 Subject: [PATCH 059/275] Add capitalized(): lexicon-driven case fixing, same spans Co-Authored-By: Claude Fable 5 --- nameparser/_render.py | 56 +++++++++++++++++++++++++++- nameparser/_types.py | 14 ++++++- tests/v2/test_render.py | 82 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 149 insertions(+), 3 deletions(-) diff --git a/nameparser/_render.py b/nameparser/_render.py index 2f8ae5aa..6a4dd0aa 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -9,11 +9,14 @@ import re -from nameparser._types import ParsedName, Role +from nameparser._lexicon import Lexicon, _normalize +from nameparser._types import Ambiguity, ParsedName, Role, Token _SPACES = re.compile(r"\s+") _SPACE_BEFORE_COMMA = re.compile(r"\s+,") _COMMA_CHAR = re.compile(r"[,،，]") # ASCII, Arabic, fullwidth +_MAC = re.compile(r"^(ma?c)(\w{2,})", re.IGNORECASE) +_WORD = re.compile(r"(\w|\.)+") #: str.format keys render() accepts: the seven role fields in canonical #: order (derived from Role -- never restated) plus the derived views. @@ -84,3 +87,54 @@ def initials(name: ParsedName, spec: str, delimiter: str, separator: str) -> str f"{', '.join(_INITIALS_KEYS)}" ) from None return _collapse(rendered) + + +def _cap_word(word: str, role: Role, lex: Lexicon) -> str: + # v1 cap_word order: particle/conjunction rule first, then the + # exceptions map, then Mac/Mc, then str.capitalize + normalized = _normalize(word) + if (normalized in lex.particles + and role in (Role.MIDDLE, Role.FAMILY)) \ + or normalized in lex.conjunctions: + return word.lower() + exception = lex.capitalization_exceptions_map.get(normalized) + if exception is not None: + return exception + if _MAC.match(word): + return _MAC.sub( + lambda m: m.group(1).capitalize() + m.group(2).capitalize(), + word) + return word.capitalize() + + +def _cap_text(text: str, role: Role, lex: Lexicon) -> str: + # word-by-word within the token text: hyphenated names capitalize + # both sides ("macdole-eisenhower" -> "MacDole-Eisenhower") + return _WORD.sub(lambda m: _cap_word(m.group(0), role, lex), text) + + +def capitalized(name: ParsedName, lexicon: Lexicon | None, *, + force: bool) -> ParsedName: + """Case-fixing transform -> new ParsedName, same spans, new token + texts (core spec §5b). Gate (v1 parity): only single-case input is + touched unless force=True; the gate reads the joined token texts. + Idempotent: without force, a capitalized result is mixed-case and + the gate returns it unchanged; with force, every _cap_word rule is + a fixpoint on its own output.""" + lex = Lexicon.default() if lexicon is None else lexicon + joined = " ".join(t.text for t in name.tokens) + if not force and joined not in (joined.upper(), joined.lower()): + return name + new_tokens = tuple( + Token(_cap_text(t.text, t.role, lex), t.span, t.role, t.tags) + for t in name.tokens) + # equal tokens (possible only for synthetic span=None duplicates) + # collapse to one mapping entry -- benign: the rebuilt ambiguity + # references an equal token, so the subset invariant still holds + replacement = dict(zip(name.tokens, new_tokens)) + new_ambiguities = tuple( + Ambiguity(a.kind, a.detail, + tuple(replacement[t] for t in a.tokens)) + for a in name.ambiguities) + return ParsedName(original=name.original, tokens=new_tokens, + ambiguities=new_ambiguities) diff --git a/nameparser/_types.py b/nameparser/_types.py index a700bd1a..fee45e46 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -13,7 +13,10 @@ from collections.abc import Mapping from dataclasses import dataclass from enum import Enum, StrEnum -from typing import NamedTuple, NoReturn, TypeVar +from typing import TYPE_CHECKING, NamedTuple, NoReturn, TypeVar + +if TYPE_CHECKING: + from nameparser._lexicon import Lexicon class Role(Enum): @@ -389,3 +392,12 @@ def initials(self, spec: str = "{given} {middle} {family}", given, middle, family.""" import nameparser._render as _render return _render.initials(self, spec, delimiter, separator) + + def capitalized(self, lexicon: Lexicon | None = None, *, + force: bool = False) -> ParsedName: + """Case-fixing transform -> new ParsedName, same spans, new + token texts. Needs a lexicon for capitalization_exceptions and + particle rules; None uses the default lexicon. force=False + preserves mixed-case input (v1 parity). Idempotent.""" + import nameparser._render as _render + return _render.capitalized(self, lexicon, force=force) diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 15e08611..b79db4f6 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -1,7 +1,8 @@ import pytest +from nameparser._lexicon import Lexicon from nameparser._render import _collapse -from nameparser._types import ParsedName, Role, Span, Token +from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token def test_collapse_is_the_254_algorithm() -> None: @@ -141,3 +142,82 @@ def test_initials_empty_group_renders_empty() -> None: # empty_attribute_default fallback assert _bobdole().initials("{middle}") == "A." assert _pn("Cher", [Token("Cher", Span(0, 4), Role.GIVEN)]).initials("{middle}") == "" + + +def _lowercase_mac() -> ParsedName: + # v1 capitalize() doctest input: 'bob v. de la macdole-eisenhower phd' + # 0123456789012345678901234567890123456 + return _pn("bob v. de la macdole-eisenhower phd", [ + Token("bob", Span(0, 3), Role.GIVEN), + Token("v.", Span(4, 6), Role.MIDDLE), + Token("de", Span(7, 9), Role.FAMILY), + Token("la", Span(10, 12), Role.FAMILY), + Token("macdole-eisenhower", Span(13, 31), Role.FAMILY), + Token("phd", Span(32, 35), Role.SUFFIX), + ]) + + +def test_capitalized_all_lower_input_v1_parity() -> None: + out = _lowercase_mac().capitalized() + assert out.given == "Bob" + assert out.middle == "V." + assert out.family == "de la MacDole-Eisenhower" # particles stay lower + assert out.suffix == "Ph.D." # exceptions map, verbatim + # same spans, new texts (provenance is a documented non-invariant) + assert [t.span for t in out.tokens] == [t.span for t in _lowercase_mac().tokens] + + +def test_capitalized_all_upper_input() -> None: + pn = _pn("JOHN SMITH", [ + Token("JOHN", Span(0, 4), Role.GIVEN), + Token("SMITH", Span(5, 10), Role.FAMILY), + ]) + assert str(pn.capitalized()) == "John Smith" + + +def test_capitalized_preserves_mixed_case_unless_forced() -> None: + pn = _pn("Shirley Maclaine", [ + Token("Shirley", Span(0, 7), Role.GIVEN), + Token("Maclaine", Span(8, 16), Role.FAMILY), + ]) + assert pn.capitalized() == pn # untouched + assert pn.capitalized(force=True).family == "MacLaine" + + +def test_capitalized_is_idempotent() -> None: + once = _lowercase_mac().capitalized() + assert once.capitalized() == once + assert once.capitalized(force=True) == once + + +def test_capitalized_with_explicit_lexicon() -> None: + # empty lexicon: no particle rule, no exceptions -> plain capitalize + out = _lowercase_mac().capitalized(Lexicon.empty()) + assert out.family == "De La MacDole-Eisenhower" + assert out.suffix == "Phd" + + +def test_capitalized_lowers_conjunctions() -> None: + # 01234567890123456789 + pn = _pn("juan ortega Y gasset", [ + Token("juan", Span(0, 4), Role.GIVEN), + Token("ortega", Span(5, 11), Role.FAMILY), + Token("Y", Span(12, 13), Role.FAMILY, frozenset({"conjunction"})), + Token("gasset", Span(14, 20), Role.FAMILY), + ]) + out = pn.capitalized(force=True) + assert out.family == "Ortega y Gasset" + + +def test_capitalized_rebuilds_ambiguity_tokens() -> None: + tok = Token("van", Span(0, 3), Role.GIVEN, frozenset({"particle"})) + pn = ParsedName( + original="van johnson", + tokens=(tok, Token("johnson", Span(4, 11), Role.FAMILY)), + ambiguities=(Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, + "leading 'van' may be a particle", (tok,)),), + ) + out = pn.capitalized() + # the ambiguity references the NEW capitalized token (subset invariant) + assert out.ambiguities[0].tokens[0] is out.tokens[0] + assert out.ambiguities[0].tokens[0].text == "Van" From 936d3476b29a3d8513d193510a67fc6895c5a431 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 16:27:28 -0700 Subject: [PATCH 060/275] True up module docstrings to the rendering delegation _types still claimed to import nothing from nameparser -- now false in the unqualified form (call-time _render imports, TYPE_CHECKING Lexicon) and contradicting the layering ALLOWED entry; AGENTS.md had the qualified wording but the docstring a reader actually opens did not. _render's docstring understated its _lexicon dependency (_normalize is used unconditionally) and now notes that only unknown KEYS get the enriched KeyError. Co-Authored-By: Claude Fable 5 --- nameparser/_render.py | 13 +++++++++---- nameparser/_types.py | 5 ++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/nameparser/_render.py b/nameparser/_render.py index 6a4dd0aa..d3c7ea76 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -1,9 +1,14 @@ """Rendering for the 2.0 API: ParsedName -> display strings. -Layering: imports nameparser._types, and nameparser._lexicon only for -Lexicon.default() when capitalized() receives lexicon=None (enforced by -tests/v2/test_layering.py). Parsing code never imports this module; -ParsedName's rendering methods delegate here via call-time imports. +Layering: imports nameparser._types, and nameparser._lexicon for +Lexicon.default() (capitalized() with lexicon=None) and _normalize +(enforced by tests/v2/test_layering.py). Parsing code never imports +this module; ParsedName's rendering methods delegate here via +call-time imports. + +Malformed str.format specs beyond unknown keys (positional fields, +bad conversions) surface the raw str.format error; only unknown KEYS +get the enriched KeyError. """ from __future__ import annotations diff --git a/nameparser/_types.py b/nameparser/_types.py index fee45e46..195c7e6f 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -1,7 +1,10 @@ """Core value types for the 2.0 API. Layering (enforced by tests/v2/test_layering.py): this module imports -nothing from nameparser -- it is the bottom of the dependency graph. +nothing from nameparser at module level -- it is the bottom of the +module-import dependency graph. The rendering delegates import _render +at call time, and a TYPE_CHECKING-only _lexicon import supplies the +Lexicon annotation. Repr policy (applies to every v2 type's __repr__, across this module and _lexicon.py/_policy.py/_locale.py): bounded output only. No repr may scale From 27616dcdc1240e3086cda99b52322037cbb78129 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 16:59:36 -0700 Subject: [PATCH 061/275] Guard all frozen types against pickle layout skew Lexicon failed at the load site on field-layout mismatch; its five frozen siblings (Token, Ambiguity, ParsedName, Policy, PolicyPatch, Locale) restored slots silently and failed at a distant attribute read. Shared _guarded_getstate/_guarded_setstate functions in _types are ASSIGNED IN EACH CLASS BODY -- @dataclass(slots=True) regenerates the class and installs its own pickle methods unless the names are in the class's own __dict__, which is why a mixin cannot work. Lexicon keeps its own copy (layering forbids _lexicon importing _types) plus its mappingproxy rebuild. Policy, stated in code and AGENTS.md: unpickle validates field LAYOUT, not values -- pickle is not a security boundary, and canonical state comes only from a validated instance. _locale's ALLOWED entry gains the downward _types import. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 4 ++-- nameparser/_locale.py | 5 +++++ nameparser/_policy.py | 10 ++++++++- nameparser/_types.py | 46 +++++++++++++++++++++++++++++++++++++++ tests/v2/test_layering.py | 5 ++++- tests/v2/test_locale.py | 8 +++++++ tests/v2/test_policy.py | 7 ++++++ tests/v2/test_types.py | 18 +++++++++++++++ 8 files changed, 99 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0166c0e2..8b7ed4e3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -112,13 +112,13 @@ Each named attribute (`title`, `first`, etc.) is a `@property` that joins its co The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These conventions apply to all new-API code and are stricter than the v1 sections above. The full design record (rationale, settled-decision logs, dated amendments) lives in untracked `docs/superpowers/specs/`; this section is the enforceable subset. **A commit that establishes or amends one of these conventions must update this section in the same commit** — grep-driven staleness sweeps miss paraphrased prose (see Workflow above), so write-time maintenance is the mechanism, audits are the backstop. - **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`; later `_pipeline/`, `_parser.py`, `_render.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Old paths (`nameparser.parser`, `nameparser.config`) stay v1-shaped. -- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy`; `_render` imports `_types` and `_lexicon` (for `Lexicon.default()`). Extend the test's `ALLOWED` table when adding a module. +- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`). Extend the test's `ALLOWED` table when adding a module. - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). - **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. -- **Pickling**: v2 types must round-trip (`Parser` will be picklable by construction, and it holds a `Lexicon`). `Lexicon` needs its custom `__getstate__`/`__setstate__` because of its `mappingproxy` slot; a new unpicklable slot type needs the same treatment plus a round-trip test. +- **Pickling**: v2 types must round-trip (`Parser` will be picklable by construction, and it holds a `Lexicon`). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, and values are deliberately NOT re-validated (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` keeps its own copy of the guard (layering) plus the `mappingproxy` slot rebuild; a new unpicklable slot type needs the same treatment plus a round-trip test. - **One sanctioned global**: the (future) cached default `Parser`. Any second piece of module-level mutable state requires amending the conventions doc, on purpose, in review. - **Tests**: all v2 tests live in the `tests/v2/` package (its `conftest.py` overrides the v1 dual-run fixture — v2 code never reads shared `CONSTANTS`), one test module per source module plus cross-cutting `test_reprs.py` and `test_layering.py`, names stating behavior. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. diff --git a/nameparser/_locale.py b/nameparser/_locale.py index 640a6a67..d50cd6ed 100644 --- a/nameparser/_locale.py +++ b/nameparser/_locale.py @@ -15,6 +15,7 @@ from nameparser._lexicon import Lexicon from nameparser._policy import UNSET, PolicyPatch +from nameparser._types import _guarded_getstate, _guarded_setstate @dataclass(frozen=True, slots=True) @@ -23,6 +24,10 @@ class Locale: lexicon: Lexicon policy: PolicyPatch = PolicyPatch() + # in the class body so @dataclass(slots=True) keeps them + __getstate__ = _guarded_getstate + __setstate__ = _guarded_setstate + def __post_init__(self) -> None: if not isinstance(self.code, str): raise TypeError( diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 9f12fad8..f0c375aa 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -9,7 +9,7 @@ from dataclasses import dataclass, field from enum import Enum, StrEnum, auto -from nameparser._types import Role +from nameparser._types import Role, _guarded_getstate, _guarded_setstate class PatronymicRule(StrEnum): @@ -45,6 +45,10 @@ class Policy: strip_emoji: bool = True strip_bidi: bool = True # =False replaces v1's opt-out CONSTANTS.regexes.bidi = False + # in the class body so @dataclass(slots=True) keeps them + __getstate__ = _guarded_getstate + __setstate__ = _guarded_setstate + def __post_init__(self) -> None: order = tuple(self.name_order) if len(order) != 3 or set(order) != _NAME_ROLES: @@ -181,6 +185,10 @@ class PolicyPatch: strip_emoji: bool | _Unset = UNSET strip_bidi: bool | _Unset = UNSET + # in the class body so @dataclass(slots=True) keeps them + __getstate__ = _guarded_getstate + __setstate__ = _guarded_setstate + def __post_init__(self) -> None: # Canonicalize (but do NOT validate) collection fields so a patch # built from a set/list literal is hashable and unions cleanly in diff --git a/nameparser/_types.py b/nameparser/_types.py index 195c7e6f..58829195 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -13,6 +13,7 @@ """ from __future__ import annotations +import dataclasses from collections.abc import Mapping from dataclasses import dataclass from enum import Enum, StrEnum @@ -73,6 +74,39 @@ def _coerce_enum(value: object, enum_cls: type[_E], noun: str, plural: str) -> _ ) from None +# Pickle support shared by the frozen slots dataclasses: fail at the +# LOAD site when a pickle's field layout does not match this version of +# the class (version skew) -- silently loading would defer the failure +# to a distant attribute read. Values are deliberately NOT re-validated: +# pickle is not a security boundary (arbitrary pickles can execute code +# anyway), and canonical state only comes from a validated instance. +# These are ASSIGNED IN EACH CLASS BODY (not inherited from a mixin): +# @dataclass(slots=True) regenerates the class and installs its own +# pickle methods unless __getstate__/__setstate__ are in the class's +# own __dict__. Lexicon duplicates this logic by design (its slots also +# carry a rebuilt mappingproxy) -- layering keeps _lexicon import-free +# of _types. + + +def _guarded_getstate(self: object) -> dict[str, object]: + fields = dataclasses.fields(self) # type: ignore[arg-type] + return {f.name: getattr(self, f.name) for f in fields} + + +def _guarded_setstate(self: object, state: dict[str, object]) -> None: + fields = dataclasses.fields(self) # type: ignore[arg-type] + expected = {f.name for f in fields} + if set(state) != expected: + missing = ", ".join(sorted(expected - set(state))) or "none" + unexpected = ", ".join(sorted(set(state) - expected)) or "none" + raise ValueError( + f"incompatible {type(self).__name__} pickle: missing " + f"fields: {missing}; unexpected fields: {unexpected}" + ) + for name, value in state.items(): + object.__setattr__(self, name, value) + + @dataclass(frozen=True, slots=True) class Token: text: str @@ -80,6 +114,10 @@ class Token: role: Role tags: frozenset[str] = frozenset() + # in the class body so @dataclass(slots=True) keeps them + __getstate__ = _guarded_getstate + __setstate__ = _guarded_setstate + def __post_init__(self) -> None: if not isinstance(self.text, str): raise TypeError( @@ -153,6 +191,10 @@ class Ambiguity: detail: str tokens: tuple[Token, ...] + # in the class body so @dataclass(slots=True) keeps them + __getstate__ = _guarded_getstate + __setstate__ = _guarded_setstate + def __post_init__(self) -> None: object.__setattr__( self, "kind", @@ -190,6 +232,10 @@ class ParsedName: tokens: tuple[Token, ...] ambiguities: tuple[Ambiguity, ...] = () + # in the class body so @dataclass(slots=True) keeps them + __getstate__ = _guarded_getstate + __setstate__ = _guarded_setstate + def __post_init__(self) -> None: if not isinstance(self.original, str): raise TypeError( diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index cb7c7c3a..63ec1225 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -16,7 +16,10 @@ # any config submodule; "data only" holds by convention/review "_lexicon.py": ("nameparser.config.",), "_policy.py": ("nameparser._types",), - "_locale.py": ("nameparser._lexicon", "nameparser._policy"), + # _types is a downward import (bottom of the graph): Locale shares + # the _guarded_getstate/_guarded_setstate pickle functions + "_locale.py": ("nameparser._types", "nameparser._lexicon", + "nameparser._policy"), # _lexicon is needed at runtime: capitalized(lexicon=None) resolves # to Lexicon.default() "_render.py": ("nameparser._types", "nameparser._lexicon"), diff --git a/tests/v2/test_locale.py b/tests/v2/test_locale.py index c900ff5f..924a7ce2 100644 --- a/tests/v2/test_locale.py +++ b/tests/v2/test_locale.py @@ -66,3 +66,11 @@ def test_locale_code_is_pinned_to_registry_charset() -> None: with pytest.raises(ValueError, match="a-z0-9_"): Locale(code=bad, lexicon=Lexicon.empty()) assert Locale(code="tr_az", lexicon=Lexicon.empty()).code == "tr_az" + + +def test_setstate_rejects_layout_skew() -> None: + loc = Locale(code="ru", lexicon=Lexicon.empty()) + state = dict(loc.__getstate__()) + del state["code"] + with pytest.raises(ValueError, match="code"): + Locale.__new__(Locale).__setstate__(state) diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 045ac6e4..497cf9a0 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -206,3 +206,10 @@ def test_policy_and_patch_pickle_round_trip_preserves_unset_identity() -> None: # object() sentinel would break this silently. assert loaded.name_order is UNSET assert loaded.strip_emoji is False + + +def test_setstate_rejects_layout_skew() -> None: + state = dict(Policy().__getstate__()) + del state["name_order"] + with pytest.raises(ValueError, match="name_order"): + Policy.__new__(Policy).__setstate__(state) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 5211730b..84952eb3 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -350,3 +350,21 @@ def test_token_rejects_bool_span_coordinates() -> None: # leaking into a coordinate slot, not a span. with pytest.raises(TypeError, match="pair of ints"): Token("x", (False, True), Role.GIVEN) # type: ignore[arg-type] + + +def test_setstate_rejects_layout_skew_on_frozen_types() -> None: + # version-skewed pickles must fail at the LOAD site, naming the + # mismatch -- not at a distant attribute read (same policy as + # Lexicon; values are deliberately NOT re-validated: pickle is not + # a security boundary) + tok = Token("Juan", Span(0, 4), Role.GIVEN) + state = tok.__getstate__() + bad = dict(state) + del bad["tags"] + with pytest.raises(ValueError, match="tags"): + Token.__new__(Token).__setstate__(bad) + pn = _pn("Juan", [tok]) + bad_pn = dict(pn.__getstate__()) + bad_pn["zq_future"] = () + with pytest.raises(ValueError, match="zq_future"): + ParsedName.__new__(ParsedName).__setstate__(bad_pn) From 35e1836bf4349d3028b1f208aa454d184ca1a043 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 17:02:20 -0700 Subject: [PATCH 062/275] Validate rendering arguments eagerly capitalized() with a non-Lexicon argument was a silent no-op on mixed-case input (the gate returned before touching the argument) and a deep AttributeError on single-case input; render()/initials() with non-str spec/delimiter/separator failed inside str.format. All three now raise the taxonomy's eager TypeError naming the argument, matching every constructor on the branch. Co-Authored-By: Claude Fable 5 --- nameparser/_render.py | 10 ++++++++++ tests/v2/test_render.py | 22 ++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/nameparser/_render.py b/nameparser/_render.py index d3c7ea76..d10a56f3 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -56,6 +56,8 @@ def render(name: ParsedName, spec: str) -> str: """Fill the str.format spec from the seven role fields and the derived views (empty fields substitute ''), then apply the #254 collapse. Unknown keys raise KeyError naming the valid fields.""" + if not isinstance(spec, str): + raise TypeError(f"spec must be a str, got {spec!r}") values = {key: getattr(name, key) for key in _RENDER_KEYS} try: rendered = spec.format(**values) @@ -74,6 +76,10 @@ def initials(name: ParsedName, spec: str, delimiter: str, separator: str) -> str initial in middle/family (given-name tokens always contribute); tags come from the pipeline -- hand-built untagged tokens all contribute. Valid spec keys: given, middle, family.""" + for arg_name, arg in (("spec", spec), ("delimiter", delimiter), + ("separator", separator)): + if not isinstance(arg, str): + raise TypeError(f"{arg_name} must be a str, got {arg!r}") values: dict[str, str] = {} for key in _INITIALS_KEYS: role = Role(key) @@ -126,6 +132,10 @@ def capitalized(name: ParsedName, lexicon: Lexicon | None, *, Idempotent: without force, a capitalized result is mixed-case and the gate returns it unchanged; with force, every _cap_word rule is a fixpoint on its own output.""" + if lexicon is not None and not isinstance(lexicon, Lexicon): + # eager, before the gate: a garbage argument must not become a + # silent no-op on mixed-case input or a deep AttributeError + raise TypeError(f"lexicon must be a Lexicon or None, got {lexicon!r}") lex = Lexicon.default() if lexicon is None else lexicon joined = " ".join(t.text for t in name.tokens) if not force and joined not in (joined.upper(), joined.lower()): diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index b79db4f6..0d14af7d 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -221,3 +221,25 @@ def test_capitalized_rebuilds_ambiguity_tokens() -> None: # the ambiguity references the NEW capitalized token (subset invariant) assert out.ambiguities[0].tokens[0] is out.tokens[0] assert out.ambiguities[0].tokens[0].text == "Van" + + +def test_render_and_initials_reject_non_str_arguments() -> None: + # eager, like every constructor: not an AttributeError frames deep + pn = _delavega() + with pytest.raises(TypeError, match="spec must be a str"): + pn.render(7) # type: ignore[arg-type] + with pytest.raises(TypeError, match="spec must be a str"): + pn.initials(7) # type: ignore[arg-type] + with pytest.raises(TypeError, match="delimiter must be a str"): + pn.initials(delimiter=None) # type: ignore[arg-type] + with pytest.raises(TypeError, match="separator must be a str"): + pn.initials(separator=0) # type: ignore[arg-type] + + +def test_capitalized_rejects_non_lexicon_argument() -> None: + # previously a silent no-op on mixed-case input and a deep + # AttributeError on single-case input + with pytest.raises(TypeError, match="must be a Lexicon"): + _delavega().capitalized("garbage") # type: ignore[arg-type] + with pytest.raises(TypeError, match="must be a Lexicon"): + _lowercase_mac().capitalized({"titles": set()}) # type: ignore[arg-type] From 7e8b6fb703d83449387e6f39c37484c0504c52e8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 17:02:51 -0700 Subject: [PATCH 063/275] Reject bare-string name_order with the taxonomy TypeError tuple('gmf') is ('g', 'm', 'f'), so a bare string fell through to the permutation ValueError instead of the bare-string TypeError every other iterable-valued field raises. Guarded in both Policy and PolicyPatch (whose canonicalization would otherwise store the shredded tuple until apply time). Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 12 ++++++++++++ tests/v2/test_policy.py | 10 ++++++++++ 2 files changed, 22 insertions(+) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index f0c375aa..624202bf 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -50,6 +50,13 @@ class Policy: __setstate__ = _guarded_setstate def __post_init__(self) -> None: + # tuple("gmf") would be ("g", "m", "f") -- catch the bare string + # with the same TypeError every other iterable field raises + if isinstance(self.name_order, str): + raise TypeError( + f"name_order must be an iterable of three Roles, " + f"not a bare string: {self.name_order!r}" + ) order = tuple(self.name_order) if len(order) != 3 or set(order) != _NAME_ROLES: raise ValueError( @@ -196,6 +203,11 @@ def __post_init__(self) -> None: # coerce a list at apply time, but the patch itself (and any # Locale holding it) must already be hashable. if self.name_order is not UNSET: + if isinstance(self.name_order, str): + raise TypeError( + f"name_order must be an iterable of three Roles, " + f"not a bare string: {self.name_order!r}" + ) object.__setattr__(self, "name_order", tuple(self.name_order)) for f in dataclasses.fields(self): if f.metadata.get("compose") != "union": diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 497cf9a0..b84ecac5 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -213,3 +213,13 @@ def test_setstate_rejects_layout_skew() -> None: del state["name_order"] with pytest.raises(ValueError, match="name_order"): Policy.__new__(Policy).__setstate__(state) + + +def test_name_order_rejects_bare_string() -> None: + # tuple("gmf") is ("g","m","f"): without the guard a bare string + # fell through to the permutation ValueError instead of the + # taxonomy's bare-string TypeError every other iterable field raises + with pytest.raises(TypeError, match="bare string"): + Policy(name_order="gmf") # type: ignore[arg-type] + with pytest.raises(TypeError, match="bare string"): + PolicyPatch(name_order="gmf") # type: ignore[arg-type] From 4db87b7f7bad4c9e00a0f8cf2d75b9e687970eda Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 17:03:11 -0700 Subject: [PATCH 064/275] Pin two documented rendering contracts The given-group tag immunity in initials() (a tagged given token still contributes -- the PARTICLE_OR_GIVEN reading) and the raw-error contract for malformed render specs (positional fields, bad conversions) were documented but untested. Co-Authored-By: Claude Fable 5 --- tests/v2/test_render.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 0d14af7d..35ec6f5c 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -243,3 +243,23 @@ def test_capitalized_rejects_non_lexicon_argument() -> None: _delavega().capitalized("garbage") # type: ignore[arg-type] with pytest.raises(TypeError, match="must be a Lexicon"): _lowercase_mac().capitalized({"titles": set()}) # type: ignore[arg-type] + + +def test_initials_given_tokens_ignore_skip_tags() -> None: + # documented: a given-name token contributes even when tagged (the + # PARTICLE_OR_GIVEN case -- 'van' read as a given name) + pn = _pn("van Johnson", [ + Token("van", Span(0, 3), Role.GIVEN, frozenset({"particle"})), + Token("Johnson", Span(4, 11), Role.FAMILY), + ]) + assert pn.initials() == "v. J." + + +def test_render_malformed_specs_surface_raw_format_errors() -> None: + # documented contract: only unknown KEYS get the enriched KeyError; + # positional fields and bad conversions raise str.format's own error + pn = _delavega() + with pytest.raises(IndexError): + pn.render("{}") + with pytest.raises(ValueError): + pn.render("{given!q}") From 96b9166e3c11455fe7098ffe4b430ad5fd5bb711 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 17:03:42 -0700 Subject: [PATCH 065/275] Fix three stale doc claims from the PR review AGENTS.md listed _render under 'later' modules one line above the bullet describing its shipped layering; the one-sanctioned-global rule did not acknowledge Lexicon.default()'s functools.cache (a lazily cached FROZEN singleton is a constant, not state); and the Lexicon repr comment cited '~700 entries' when the default vocabulary is ~1460 -- de-numbered so it cannot rot again. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 4 ++-- nameparser/_lexicon.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8b7ed4e3..acedeaf5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -111,7 +111,7 @@ Each named attribute (`title`, `first`, etc.) is a `@property` that joins its co The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These conventions apply to all new-API code and are stricter than the v1 sections above. The full design record (rationale, settled-decision logs, dated amendments) lives in untracked `docs/superpowers/specs/`; this section is the enforceable subset. **A commit that establishes or amends one of these conventions must update this section in the same commit** — grep-driven staleness sweeps miss paraphrased prose (see Workflow above), so write-time maintenance is the mechanism, audits are the backstop. -- **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`; later `_pipeline/`, `_parser.py`, `_render.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Old paths (`nameparser.parser`, `nameparser.config`) stay v1-shaped. +- **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`; later `_pipeline/`, `_parser.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Old paths (`nameparser.parser`, `nameparser.config`) stay v1-shaped. - **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`). Extend the test's `ALLOWED` table when adding a module. - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. @@ -119,7 +119,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). - **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. - **Pickling**: v2 types must round-trip (`Parser` will be picklable by construction, and it holds a `Lexicon`). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, and values are deliberately NOT re-validated (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` keeps its own copy of the guard (layering) plus the `mappingproxy` slot rebuild; a new unpicklable slot type needs the same treatment plus a round-trip test. -- **One sanctioned global**: the (future) cached default `Parser`. Any second piece of module-level mutable state requires amending the conventions doc, on purpose, in review. +- **One sanctioned global**: the (future) cached default `Parser`. Lazily cached FROZEN singletons (`Lexicon.default()`'s `functools.cache`, the future default parser) are constants, not state; any second piece of module-level MUTABLE state requires amending the conventions doc, on purpose, in review. - **Tests**: all v2 tests live in the `tests/v2/` package (its `conftest.py` overrides the v1 dual-run fixture — v2 code never reads shared `CONSTANTS`), one test module per source module plus cross-cutting `test_reprs.py` and `test_layering.py`, names stating behavior. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. ## Extension Patterns diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index e4f08e91..87ed5429 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -177,7 +177,8 @@ def __repr__(self) -> str: # the two named constructors and by how many entries -- never the # entries themselves (design rule, see nameparser._types module # docstring). Diffing empty()-built lexicons against default() - # would tell the wrong story ("default minus ~700 entries"). + # would tell the wrong story ("default minus the entire + # default vocabulary"). if self == Lexicon.default(): return "Lexicon(default)" if self == Lexicon.empty(): From 79277de25e5d1cad8e7362e9454d0c11abd3b1f5 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 17:28:51 -0700 Subject: [PATCH 066/275] Simplify the rendering module Extract _format_spec for the identical fill/enrich/collapse tail of render() and initials() (the two error templates could drift independently); replace initials()'s guarded join-plus-append with separator.join(letter + delimiter ...), which makes the empty-group conditional unnecessary; drop a backslash continuation for the house paren style; record WHY capitalized()'s gate reads joined token texts rather than render() output. Co-Authored-By: Claude Fable 5 --- nameparser/_render.py | 52 +++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/nameparser/_render.py b/nameparser/_render.py index d10a56f3..3ce93c66 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -52,23 +52,30 @@ def _collapse(rendered: str) -> str: return rendered.strip(", ") -def render(name: ParsedName, spec: str) -> str: - """Fill the str.format spec from the seven role fields and the - derived views (empty fields substitute ''), then apply the #254 - collapse. Unknown keys raise KeyError naming the valid fields.""" +def _format_spec(spec: str, values: dict[str, str], noun: str, + keys: tuple[str, ...]) -> str: + """Shared tail of render()/initials(): fill the spec, enrich + unknown-KEY errors with the valid key list, collapse.""" if not isinstance(spec, str): raise TypeError(f"spec must be a str, got {spec!r}") - values = {key: getattr(name, key) for key in _RENDER_KEYS} try: rendered = spec.format(**values) except KeyError as exc: raise KeyError( - f"unknown render field {exc.args[0]!r}; valid fields: " - f"{', '.join(_RENDER_KEYS)}" + f"unknown {noun} field {exc.args[0]!r}; valid fields: " + f"{', '.join(keys)}" ) from None return _collapse(rendered) +def render(name: ParsedName, spec: str) -> str: + """Fill the str.format spec from the seven role fields and the + derived views (empty fields substitute ''), then apply the #254 + collapse. Unknown keys raise KeyError naming the valid fields.""" + values = {key: getattr(name, key) for key in _RENDER_KEYS} + return _format_spec(spec, values, "render", _RENDER_KEYS) + + def initials(name: ParsedName, spec: str, delimiter: str, separator: str) -> str: """First letter of each contributing token per group, v1 semantics: delimiter follows each initial, separator sits between initials @@ -76,10 +83,10 @@ def initials(name: ParsedName, spec: str, delimiter: str, separator: str) -> str initial in middle/family (given-name tokens always contribute); tags come from the pipeline -- hand-built untagged tokens all contribute. Valid spec keys: given, middle, family.""" - for arg_name, arg in (("spec", spec), ("delimiter", delimiter), - ("separator", separator)): - if not isinstance(arg, str): - raise TypeError(f"{arg_name} must be a str, got {arg!r}") + if not isinstance(delimiter, str): + raise TypeError(f"delimiter must be a str, got {delimiter!r}") + if not isinstance(separator, str): + raise TypeError(f"separator must be a str, got {separator!r}") values: dict[str, str] = {} for key in _INITIALS_KEYS: role = Role(key) @@ -87,26 +94,17 @@ def initials(name: ParsedName, spec: str, delimiter: str, separator: str) -> str if role is not Role.GIVEN: tokens = tuple(t for t in tokens if not (_SKIP_TAGS & t.tags)) - letters = [t.text[0] for t in tokens] - values[key] = ((delimiter + separator).join(letters) + delimiter - if letters else "") - try: - rendered = spec.format(**values) - except KeyError as exc: - raise KeyError( - f"unknown initials field {exc.args[0]!r}; valid fields: " - f"{', '.join(_INITIALS_KEYS)}" - ) from None - return _collapse(rendered) + values[key] = separator.join( + t.text[0] + delimiter for t in tokens) + return _format_spec(spec, values, "initials", _INITIALS_KEYS) def _cap_word(word: str, role: Role, lex: Lexicon) -> str: # v1 cap_word order: particle/conjunction rule first, then the # exceptions map, then Mac/Mc, then str.capitalize normalized = _normalize(word) - if (normalized in lex.particles - and role in (Role.MIDDLE, Role.FAMILY)) \ - or normalized in lex.conjunctions: + if ((normalized in lex.particles and role in (Role.MIDDLE, Role.FAMILY)) + or normalized in lex.conjunctions): return word.lower() exception = lex.capitalization_exceptions_map.get(normalized) if exception is not None: @@ -128,7 +126,9 @@ def capitalized(name: ParsedName, lexicon: Lexicon | None, *, force: bool) -> ParsedName: """Case-fixing transform -> new ParsedName, same spans, new token texts (core spec §5b). Gate (v1 parity): only single-case input is - touched unless force=True; the gate reads the joined token texts. + touched unless force=True; the gate reads the joined token texts + (not render() output -- the case gate stays decoupled from spec + formatting and the #254 collapse). Idempotent: without force, a capitalized result is mixed-case and the gate returns it unchanged; with force, every _cap_word rule is a fixpoint on its own output.""" From 60802f12e93ac4231116b01a94fc2ba690cae507 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 17:28:51 -0700 Subject: [PATCH 067/275] Single-source the bare-string name_order guard The identical TypeError lived in both Policy and PolicyPatch __post_init__ and could drift; extracted to a module helper. Also add the missing back-reference on Lexicon.__setstate__ pointing at its deliberately-duplicated twin in _types (the sync note was previously one-directional). Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 2 ++ nameparser/_policy.py | 25 +++++++++++++------------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 87ed5429..a1a08fda 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -204,6 +204,8 @@ def __setstate__(self, state: dict[str, object]) -> None: # Fail at the unpickle site if the state comes from a different # Lexicon field layout (version skew) -- silently loading it # would defer the failure to some distant attribute read. + # Message kept in sync with _types._guarded_setstate by design + # (layering keeps this module import-free of _types). expected = {f.name for f in dataclasses.fields(Lexicon)} - {"_cap_map"} if set(state) != expected: missing = ", ".join(sorted(expected - set(state))) or "none" diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 624202bf..3ef42336 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -28,6 +28,17 @@ class PatronymicRule(StrEnum): _NAME_ROLES = frozenset({Role.GIVEN, Role.MIDDLE, Role.FAMILY}) +def _reject_bare_string_order(value: object) -> None: + # tuple("gmf") would be ("g", "m", "f") -- catch the bare string + # with the same TypeError every other iterable field raises. + # Single-sourced: called from Policy AND PolicyPatch __post_init__. + if isinstance(value, str): + raise TypeError( + f"name_order must be an iterable of three Roles, " + f"not a bare string: {value!r}" + ) + + @dataclass(frozen=True, slots=True) class Policy: name_order: tuple[Role, Role, Role] = GIVEN_FIRST @@ -50,13 +61,7 @@ class Policy: __setstate__ = _guarded_setstate def __post_init__(self) -> None: - # tuple("gmf") would be ("g", "m", "f") -- catch the bare string - # with the same TypeError every other iterable field raises - if isinstance(self.name_order, str): - raise TypeError( - f"name_order must be an iterable of three Roles, " - f"not a bare string: {self.name_order!r}" - ) + _reject_bare_string_order(self.name_order) order = tuple(self.name_order) if len(order) != 3 or set(order) != _NAME_ROLES: raise ValueError( @@ -203,11 +208,7 @@ def __post_init__(self) -> None: # coerce a list at apply time, but the patch itself (and any # Locale holding it) must already be hashable. if self.name_order is not UNSET: - if isinstance(self.name_order, str): - raise TypeError( - f"name_order must be an iterable of three Roles, " - f"not a bare string: {self.name_order!r}" - ) + _reject_bare_string_order(self.name_order) object.__setattr__(self, "name_order", tuple(self.name_order)) for f in dataclasses.fields(self): if f.metadata.get("compose") != "union": From bc56bd2a5d8cf24c00aa40fbc39c0e5fd427c742 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 17:28:51 -0700 Subject: [PATCH 068/275] Enforce the pickle guards mechanically; flatten test imports Forgetting the two class-body guard assignments on a future frozen type is silent -- @dataclass(slots=True) installs its own working pickle methods without skew detection. A conventions test now enumerates every frozen dataclass in the package and asserts the guards are present (Lexicon allow-listed for its private copy), turning a per-class convention into a CI gate. Also folds five vestigial function-local render imports into the top import block. Co-Authored-By: Claude Fable 5 --- tests/v2/test_layering.py | 33 ++++++++++++++++++++++++++++++++- tests/v2/test_render.py | 7 +------ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 63ec1225..3968665c 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -1,4 +1,5 @@ -"""Enforce the conventions doc's import layering mechanically.""" +"""Enforce the conventions doc's contracts mechanically: import +layering, public exports, and the pickle layout guards.""" import ast import pathlib @@ -108,3 +109,33 @@ def test_type_checking_imports_do_not_count(tmp_path: pathlib.Path) -> None: " from nameparser import _render\n" ) assert _nameparser_imports(mod) == ["nameparser.util", "nameparser"] + + +def test_every_frozen_dataclass_carries_the_pickle_guards() -> None: + # Forgetting the two class-body assignments is SILENT -- + # @dataclass(slots=True) installs its own working pickle methods + # without skew detection. Lexicon keeps a private copy of the guard + # (layering keeps _lexicon import-free of _types). + import dataclasses + import inspect + + import nameparser._lexicon + import nameparser._locale + import nameparser._policy + import nameparser._types + from nameparser._types import _guarded_getstate, _guarded_setstate + + modules = (nameparser._types, nameparser._lexicon, + nameparser._policy, nameparser._locale) + for module in modules: + for _, cls in inspect.getmembers(module, inspect.isclass): + if cls.__module__ != module.__name__: + continue + if not dataclasses.is_dataclass(cls): + continue + if cls.__name__ == "Lexicon": + assert "__getstate__" in cls.__dict__ + assert "__setstate__" in cls.__dict__ + continue + assert cls.__dict__.get("__getstate__") is _guarded_getstate, cls + assert cls.__dict__.get("__setstate__") is _guarded_setstate, cls diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 35ec6f5c..249e3d47 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -1,7 +1,7 @@ import pytest from nameparser._lexicon import Lexicon -from nameparser._render import _collapse +from nameparser._render import _collapse, render from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token @@ -40,7 +40,6 @@ def _delavega() -> ParsedName: def test_render_fills_fields_and_collapses() -> None: - from nameparser._render import render pn = _delavega() assert render(pn, "{title} {given} {middle} {family} {suffix}") \ == "Dr. Juan de la Vega III" @@ -49,7 +48,6 @@ def test_render_fills_fields_and_collapses() -> None: def test_render_accepts_derived_view_keys() -> None: - from nameparser._render import render assert render(_delavega(), "{family_base}, {given} {family_particles}") \ == "Vega, Juan de la" assert render(_delavega(), "{surnames}") == "de la Vega" @@ -57,20 +55,17 @@ def test_render_accepts_derived_view_keys() -> None: def test_render_every_role_key_is_valid() -> None: - from nameparser._render import render pn = _delavega() for role in Role: render(pn, f"{{{role.value}}}") # must not raise def test_render_unknown_key_raises_enriched_keyerror() -> None: - from nameparser._render import render with pytest.raises(KeyError, match="valid fields"): render(_delavega(), "{first}") # v1 spelling: redirected loudly def test_render_empty_parse_is_empty_string() -> None: - from nameparser._render import render assert render(_pn("", []), "{title} {given} {middle} {family} {suffix}") == "" From bf6814796a2dbfdd1158e4eae9d70c9f3907b199 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 20:02:15 -0700 Subject: [PATCH 069/275] Add the pipeline package skeleton: ParseState and layering wiring Co-Authored-By: Claude Fable 5 --- AGENTS.md | 4 +- nameparser/_pipeline/__init__.py | 25 +++++++++++ nameparser/_pipeline/_state.py | 74 ++++++++++++++++++++++++++++++++ pyproject.toml | 2 + tests/v2/pipeline/__init__.py | 0 tests/v2/pipeline/test_state.py | 31 +++++++++++++ tests/v2/test_layering.py | 39 ++++++++++++++++- 7 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 nameparser/_pipeline/__init__.py create mode 100644 nameparser/_pipeline/_state.py create mode 100644 tests/v2/pipeline/__init__.py create mode 100644 tests/v2/pipeline/test_state.py diff --git a/AGENTS.md b/AGENTS.md index acedeaf5..5d62952e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -111,8 +111,8 @@ Each named attribute (`title`, `first`, etc.) is a `@property` that joins its co The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These conventions apply to all new-API code and are stricter than the v1 sections above. The full design record (rationale, settled-decision logs, dated amendments) lives in untracked `docs/superpowers/specs/`; this section is the enforceable subset. **A commit that establishes or amends one of these conventions must update this section in the same commit** — grep-driven staleness sweeps miss paraphrased prose (see Workflow above), so write-time maintenance is the mechanism, audits are the backstop. -- **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`; later `_pipeline/`, `_parser.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Old paths (`nameparser.parser`, `nameparser.config`) stay v1-shaped. -- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`). Extend the test's `ALLOWED` table when adding a module. +- **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`, `_pipeline/`, `_parser.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Old paths (`nameparser.parser`, `nameparser.config`) stay v1-shaped. +- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers; `_parser` sits on everything except `_render`. Extend the test's `ALLOWED` table when adding a module. - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). diff --git a/nameparser/_pipeline/__init__.py b/nameparser/_pipeline/__init__.py new file mode 100644 index 00000000..67fae774 --- /dev/null +++ b/nameparser/_pipeline/__init__.py @@ -0,0 +1,25 @@ +"""The 2.0 parse pipeline: pure stages folded over ParseState. + +Stage names and ParseState are NOT public API (core spec §6). The stage +list is data; runners other than the plain fold (explain(), parse_all()) +arrive in 2.x minors without changing any stage signature. + +Layering: imports only nameparser._pipeline.* (enforced by +tests/v2/test_layering.py). +""" +from __future__ import annotations + +from collections.abc import Callable + +from nameparser._pipeline._state import ParseState + +#: Filled in by later tasks as stages land; the fold is total either way. +STAGES: tuple[Callable[[ParseState], ParseState], ...] = () + + +def run(state: ParseState) -> ParseState: + """Fold the stages over the initial state. Pure: each stage returns + a new ParseState; content never raises (totality, spec §5a).""" + for stage in STAGES: + state = stage(state) + return state diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py new file mode 100644 index 00000000..d2a8e2e0 --- /dev/null +++ b/nameparser/_pipeline/_state.py @@ -0,0 +1,74 @@ +"""Internal pipeline state: WorkToken and ParseState. + +WorkTokens are pipeline-internal (no validation -- the tokenizer is the +only producer) and are addressed BY INDEX in every stage: pieces and +segments are runs of token indices, never joined strings, so value-based +lookup (v1's #100 family) is structurally impossible. + +Layering: imports _types, _lexicon, _policy only (enforced by +tests/v2/test_layering.py). +""" +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum, auto + +from nameparser._lexicon import Lexicon +from nameparser._policy import Policy +from nameparser._types import AmbiguityKind, Role, Span + + +@dataclass(frozen=True, slots=True) +class WorkToken: + """One tokenized word. role stays None until assign; extracted + nickname/maiden tokens arrive with their role pre-set.""" + + text: str + span: Span + tags: frozenset[str] = frozenset() + role: Role | None = None + + +class Structure(Enum): + """segment's comma-structure decision.""" + + NO_COMMA = auto() + FAMILY_COMMA = auto() # "Family, Given ..." (v1 lastname-comma) + SUFFIX_COMMA = auto() # "Given Family, Suffix ..." + + +@dataclass(frozen=True, slots=True) +class PendingAmbiguity: + """An ambiguity recorded mid-pipeline by token INDEX; assemble + materializes real Ambiguity objects over the final tokens.""" + + kind: AmbiguityKind + detail: str + indices: tuple[int, ...] = () + + +@dataclass(frozen=True, slots=True) +class ParseState: + """Carried through the stage fold. Frozen; stages return copies via + dataclasses.replace. Fields are filled progressively: + extract_delimited -> extracted/masked; tokenize -> tokens (span- + sorted)/comma_offsets; segment -> segments/structure; classify -> + token tags; group -> pieces/dropped; assign/post_rules -> token + roles; ambiguities accumulate anywhere.""" + + original: str + lexicon: Lexicon + policy: Policy + extracted: tuple[tuple[Role, Span], ...] = () + masked: tuple[Span, ...] = () + tokens: tuple[WorkToken, ...] = () + comma_offsets: tuple[int, ...] = () + segments: tuple[tuple[int, ...], ...] = () + structure: Structure = Structure.NO_COMMA + # pieces[s][p] = run of token indices: piece p of segment s. + # piece_tags[s][p] = derived flags for that piece ("title", "prefix", + # "suffix", "conjunction") set by group's joins. + pieces: tuple[tuple[tuple[int, ...], ...], ...] = () + piece_tags: tuple[tuple[frozenset[str], ...], ...] = () + dropped: tuple[int, ...] = () # structural tokens (maiden markers) + ambiguities: tuple[PendingAmbiguity, ...] = () diff --git a/pyproject.toml b/pyproject.toml index bba07a32..c56d715a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,8 @@ module = [ "nameparser._policy", "nameparser._render", "nameparser._locale", + "nameparser._pipeline.*", + "nameparser._parser", ] disallow_untyped_defs = true disallow_incomplete_defs = true diff --git a/tests/v2/pipeline/__init__.py b/tests/v2/pipeline/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/v2/pipeline/test_state.py b/tests/v2/pipeline/test_state.py new file mode 100644 index 00000000..bbda440c --- /dev/null +++ b/tests/v2/pipeline/test_state.py @@ -0,0 +1,31 @@ +import dataclasses + +from nameparser._lexicon import Lexicon +from nameparser._pipeline._state import ParseState, Structure, WorkToken +from nameparser._policy import Policy +from nameparser._types import Role, Span + + +def _state(text: str) -> ParseState: + return ParseState(original=text, lexicon=Lexicon.empty(), policy=Policy()) + + +def test_state_defaults_are_empty() -> None: + s = _state("John Smith") + assert s.tokens == () and s.segments == () and s.pieces == () + assert s.structure is Structure.NO_COMMA + assert s.ambiguities == () and s.extracted == () and s.masked == () + assert s.comma_offsets == () and s.dropped == () and s.piece_tags == () + + +def test_state_is_frozen_and_replace_works() -> None: + s = _state("x") + tok = WorkToken("x", Span(0, 1)) + s2 = dataclasses.replace(s, tokens=(tok,)) + assert s.tokens == () and s2.tokens == (tok,) + assert s2.tokens[0].role is None and s2.tokens[0].tags == frozenset() + + +def test_worktoken_carries_optional_role() -> None: + t = WorkToken("Jack", Span(6, 10), role=Role.NICKNAME) + assert t.role is Role.NICKNAME diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 3968665c..709a441d 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -7,12 +7,26 @@ PKG = pathlib.Path(nameparser.__file__).parent +# ALLOWED keys whose module must exist on disk -- the exists() skip in +# test_layering_contract is for entries that precede their module within +# a plan, and without this list a renamed existing module would silently +# drop out of enforcement. Later tasks add each stage file as it lands. +_MUST_EXIST = {"_types.py", "_lexicon.py", "_policy.py", "_locale.py", + "_render.py", "_pipeline/_state.py", "_pipeline/__init__.py"} + +_PIPELINE_STAGE_ALLOWED = ( + "nameparser._types", "nameparser._lexicon", "nameparser._policy", + # stages share _state plus in-package helpers (_vocab, _group's + # piece predicates); the prefix still forbids _render/_locale/_parser + "nameparser._pipeline.", +) + # module -> prefixes it may import from within nameparser ALLOWED = { # call-time imports only (inside the rendering-delegate method # bodies); module level stays import-free. TYPE_CHECKING imports # are skipped by _nameparser_imports and need no entry. - "_types.py": ("nameparser._render",), + "_types.py": ("nameparser._render", "nameparser._parser"), # intent: DATA modules only, during 2.x -- mechanically this admits # any config submodule; "data only" holds by convention/review "_lexicon.py": ("nameparser.config.",), @@ -24,6 +38,22 @@ # _lexicon is needed at runtime: capitalized(lexicon=None) resolves # to Lexicon.default() "_render.py": ("nameparser._types", "nameparser._lexicon"), + # every stage module: state + the three config/type modules + "_pipeline/_state.py": ("nameparser._types", "nameparser._lexicon", + "nameparser._policy"), + "_pipeline/__init__.py": ("nameparser._pipeline.",), + "_pipeline/_extract.py": _PIPELINE_STAGE_ALLOWED, + "_pipeline/_tokenize.py": _PIPELINE_STAGE_ALLOWED, + "_pipeline/_segment.py": _PIPELINE_STAGE_ALLOWED, + "_pipeline/_classify.py": _PIPELINE_STAGE_ALLOWED, + "_pipeline/_group.py": _PIPELINE_STAGE_ALLOWED, + "_pipeline/_assign.py": _PIPELINE_STAGE_ALLOWED, + "_pipeline/_post_rules.py": _PIPELINE_STAGE_ALLOWED, + "_pipeline/_assemble.py": _PIPELINE_STAGE_ALLOWED, + # Parser sits on everything except _render and the facade + "_parser.py": ("nameparser._types", "nameparser._lexicon", + "nameparser._policy", "nameparser._locale", + "nameparser._pipeline"), } @@ -71,7 +101,12 @@ def _permitted(imported: str, allowed: tuple[str, ...]) -> bool: def test_layering_contract() -> None: for mod, allowed in ALLOWED.items(): - for imported in _nameparser_imports(PKG / mod): + path = PKG / mod + if not path.exists(): + assert mod not in _MUST_EXIST, ( + f"{mod} keyed in ALLOWED but file is missing") + continue # entries may precede their module within a plan + for imported in _nameparser_imports(path): assert _permitted(imported, allowed), ( f"{mod} imports {imported}, which the layering contract " f"forbids (allowed prefixes: {allowed or 'none'})" From 24c5191b8bf70fa7b62b76c839ff52a7fd239579 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 20:11:07 -0700 Subject: [PATCH 070/275] Add extract_delimited: delimiter pairs to nickname/maiden spans Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/__init__.py | 3 +- nameparser/_pipeline/_extract.py | 99 +++++++++++++++++++++++++++++++ tests/v2/pipeline/test_extract.py | 90 ++++++++++++++++++++++++++++ tests/v2/test_layering.py | 3 +- 4 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 nameparser/_pipeline/_extract.py create mode 100644 tests/v2/pipeline/test_extract.py diff --git a/nameparser/_pipeline/__init__.py b/nameparser/_pipeline/__init__.py index 67fae774..7540f5b7 100644 --- a/nameparser/_pipeline/__init__.py +++ b/nameparser/_pipeline/__init__.py @@ -11,10 +11,11 @@ from collections.abc import Callable +from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._state import ParseState #: Filled in by later tasks as stages land; the fold is total either way. -STAGES: tuple[Callable[[ParseState], ParseState], ...] = () +STAGES: tuple[Callable[[ParseState], ParseState], ...] = (extract_delimited,) def run(state: ParseState) -> ParseState: diff --git a/nameparser/_pipeline/_extract.py b/nameparser/_pipeline/_extract.py new file mode 100644 index 00000000..1d0560f3 --- /dev/null +++ b/nameparser/_pipeline/_extract.py @@ -0,0 +1,99 @@ +"""Stage: extract_delimited. + +Consumes: ParseState.original. +Produces: extracted (role + inner span per delimited region), masked +(full regions incl. delimiter chars, skipped by tokenize), +UNBALANCED_DELIMITER ambiguities for opens with no close. +Reads: Policy.nickname_delimiters, Policy.maiden_delimiters. + +Matching rules (the #273 mechanism): pairs scan the original text left +to right, no nesting. For pairs whose open == close (quotes), the open +must sit at a word boundary (start of text or after whitespace) and the +close before one (end, whitespace, or a comma char) -- this is what +keeps the apostrophe in O'Connor literal. maiden_delimiters are scanned +before nickname_delimiters, so routing a pair to maiden (#274) wins if +a pair appears in both sets. Empty enclosures are masked (removed from +the token stream) but extract nothing. Overlapping candidate regions +resolve by scan order (maiden pairs, then nickname pairs, sorted within +each): the first match masks the region, and any delimiter chars inside +an extracted span stay literal. +""" +from __future__ import annotations + +import dataclasses + +from nameparser._pipeline._state import ParseState, PendingAmbiguity +from nameparser._types import AmbiguityKind, Role, Span + +_CLOSE_BOUNDARY_EXTRAS = {",", "،", "，"} # comma chars end a close + + +def _open_ok(text: str, i: int) -> bool: + return i == 0 or text[i - 1].isspace() + + +def _close_ok(text: str, j: int, width: int) -> bool: + k = j + width + return k >= len(text) or text[k].isspace() or text[k] in _CLOSE_BOUNDARY_EXTRAS + + +def _overlaps(span: Span, taken: list[Span]) -> bool: + return any(span.start < t.end and t.start < span.end for t in taken) + + +def extract_delimited(state: ParseState) -> ParseState: + text = state.original + extracted: list[tuple[Role, Span]] = [] + masked: list[Span] = [] + ambiguities: list[PendingAmbiguity] = [] + for role, pairs in ( + (Role.MAIDEN, state.policy.maiden_delimiters), + (Role.NICKNAME, state.policy.nickname_delimiters), + ): + for open_, close in sorted(pairs): + pos = 0 + while (i := text.find(open_, pos)) != -1: + if open_ == close and not _open_ok(text, i): + pos = i + 1 + continue + j = text.find(close, i + len(open_)) + while (open_ == close and j != -1 + and not _close_ok(text, j, len(close))): + j = text.find(close, j + 1) + if j == -1: + ambiguities.append(PendingAmbiguity( + AmbiguityKind.UNBALANCED_DELIMITER, + f"unmatched {open_!r} at offset {i}; treated as " + f"literal text", + )) + if open_ == close: + # _close_ok is open-independent, so a failed + # close-walk means NO boundary-valid close + # exists anywhere to the right: every remaining + # boundary-valid open is unmatched too. Record + # each in one forward pass without re-walking + # closes (keeps adversarial input linear). + scan = i + len(open_) + while (k := text.find(open_, scan)) != -1: + if _open_ok(text, k): + ambiguities.append(PendingAmbiguity( + AmbiguityKind.UNBALANCED_DELIMITER, + f"unmatched {open_!r} at offset {k}; " + f"treated as literal text", + )) + scan = k + 1 + break + pos = i + len(open_) + continue + full = Span(i, j + len(close)) + if not _overlaps(full, masked): + inner = Span(i + len(open_), j) + if inner.start < inner.end: + extracted.append((role, inner)) + masked.append(full) + pos = j + len(close) + extracted.sort(key=lambda pair: tuple(pair[1])) + masked.sort(key=tuple) + return dataclasses.replace( + state, extracted=tuple(extracted), masked=tuple(masked), + ambiguities=state.ambiguities + tuple(ambiguities)) diff --git a/tests/v2/pipeline/test_extract.py b/tests/v2/pipeline/test_extract.py new file mode 100644 index 00000000..0ee365d7 --- /dev/null +++ b/tests/v2/pipeline/test_extract.py @@ -0,0 +1,90 @@ +import dataclasses + +from nameparser._lexicon import Lexicon +from nameparser._pipeline._extract import extract_delimited +from nameparser._pipeline._state import ParseState +from nameparser._policy import Policy +from nameparser._types import AmbiguityKind, Role, Span + + +def _state(text: str, policy: Policy | None = None) -> ParseState: + return ParseState(original=text, lexicon=Lexicon.empty(), + policy=policy or Policy()) + + +def test_double_quoted_nickname_extracted() -> None: + # 0123456789012345678 + out = extract_delimited(_state('John "Jack" Kennedy')) + assert out.extracted == ((Role.NICKNAME, Span(6, 10)),) + assert out.masked == (Span(5, 11),) + + +def test_parenthesized_nickname_extracted() -> None: + out = extract_delimited(_state("John (Jack) Kennedy")) + assert out.extracted == ((Role.NICKNAME, Span(6, 10)),) + + +def test_same_char_delimiter_needs_boundaries() -> None: + # the apostrophe in O'Connor is not an opening quote + out = extract_delimited(_state("Sean O'Connor")) + assert out.extracted == () and out.masked == () + + +def test_single_quoted_nickname_at_boundaries() -> None: + # 01234567890123456789 + out = extract_delimited(_state("John 'Jack' Kennedy")) + assert out.extracted == ((Role.NICKNAME, Span(6, 10)),) + + +def test_unbalanced_delimiter_left_literal_with_ambiguity() -> None: + out = extract_delimited(_state('Jon "Nick Smith')) + assert out.extracted == () and out.masked == () + assert len(out.ambiguities) == 1 + assert out.ambiguities[0].kind is AmbiguityKind.UNBALANCED_DELIMITER + + +def test_maiden_delimiters_route_to_maiden() -> None: + policy = dataclasses.replace( + Policy(), + nickname_delimiters=frozenset({("'", "'"), ('"', '"')}), + maiden_delimiters=frozenset({("(", ")")}), + ) + out = extract_delimited(_state("Jane Smith (Jones)", policy)) + assert out.extracted == ((Role.MAIDEN, Span(12, 17)),) + + +def test_empty_enclosure_is_masked_but_not_extracted() -> None: + out = extract_delimited(_state("John () Kennedy")) + assert out.extracted == () + assert out.masked == (Span(5, 7),) + + +def test_multiple_extracts_and_no_overlap() -> None: + # 0123456789012345678901234 + out = extract_delimited(_state('"Jack" John (Jonny) Kim')) + roles = {span: role for role, span in out.extracted} + assert roles == {Span(1, 5): Role.NICKNAME, Span(13, 18): Role.NICKNAME} + + +def test_adjacent_pairs_extract_separately() -> None: + out = extract_delimited(_state('John "A" "B" Kim')) + assert len(out.extracted) == 2 + + +def test_close_at_end_of_string() -> None: + out = extract_delimited(_state('John "Jack"')) + assert out.extracted == ((Role.NICKNAME, Span(6, 10)),) + + +def test_unmatched_open_at_last_char() -> None: + out = extract_delimited(_state('John "')) + assert out.extracted == () + assert len(out.ambiguities) == 1 + + +def test_nested_delimiters_inner_scan_order_pins() -> None: + # parens win over quotes when the quote chars sit flush against the + # parens (their word-boundary test fails); the inner quote chars + # flow into the extracted span verbatim -- v1 parity, deliberate + out = extract_delimited(_state('John ("Jack") Kim')) + assert out.extracted == ((Role.NICKNAME, Span(6, 12)),) diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 709a441d..5c633c6d 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -12,7 +12,8 @@ # a plan, and without this list a renamed existing module would silently # drop out of enforcement. Later tasks add each stage file as it lands. _MUST_EXIST = {"_types.py", "_lexicon.py", "_policy.py", "_locale.py", - "_render.py", "_pipeline/_state.py", "_pipeline/__init__.py"} + "_render.py", "_pipeline/_state.py", "_pipeline/__init__.py", + "_pipeline/_extract.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", From 7e341a955e04ab2366be67ab343e5d75aac2feea Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 21:39:04 -0700 Subject: [PATCH 071/275] Add tokenize: char-classification tokenizer with exact spans Ports the emoji/bidi character classes verbatim from config/regexes.py per the layering contract's duplication-by-design note; whitespace, emoji, bidi marks, and comma chars all act as separators without ever entering a token, so token spans always index original exactly. Wires tokenize into STAGES and registers the new module in test_layering's _MUST_EXIST. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/__init__.py | 4 +- nameparser/_pipeline/_tokenize.py | 86 ++++++++++++++++++++++++++++++ tests/v2/pipeline/test_tokenize.py | 72 +++++++++++++++++++++++++ tests/v2/test_layering.py | 2 +- 4 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 nameparser/_pipeline/_tokenize.py create mode 100644 tests/v2/pipeline/test_tokenize.py diff --git a/nameparser/_pipeline/__init__.py b/nameparser/_pipeline/__init__.py index 7540f5b7..961efcda 100644 --- a/nameparser/_pipeline/__init__.py +++ b/nameparser/_pipeline/__init__.py @@ -13,9 +13,11 @@ from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._state import ParseState +from nameparser._pipeline._tokenize import tokenize #: Filled in by later tasks as stages land; the fold is total either way. -STAGES: tuple[Callable[[ParseState], ParseState], ...] = (extract_delimited,) +STAGES: tuple[Callable[[ParseState], ParseState], ...] = ( + extract_delimited, tokenize) def run(state: ParseState) -> ParseState: diff --git a/nameparser/_pipeline/_tokenize.py b/nameparser/_pipeline/_tokenize.py new file mode 100644 index 00000000..712e0943 --- /dev/null +++ b/nameparser/_pipeline/_tokenize.py @@ -0,0 +1,86 @@ +"""Stage: tokenize. + +Consumes: original, masked (regions to skip), extracted (regions that +tokenize with a pre-set role). +Produces: tokens (span-sorted WorkTokens; text always == original +slice), comma_offsets (segmentation points; never tokens). +Reads: Policy.strip_emoji, Policy.strip_bidi. + +There is NO text-rewriting normalize stage (core spec §6): whitespace +collapsing and emoji/bidi stripping are character-classification rules +here -- ignorable characters act as separators and never enter a token, +so spans always index the original exactly as given. + +v1's squash_emoji/squash_bidi REMOVED the char and joined neighbors +('A\U0001f600B' -> 'AB'); here an ignorable char is a SEPARATOR +('A\U0001f600B' -> 'A', 'B') -- the unavoidable consequence of spans +indexing the original exactly. +""" +from __future__ import annotations + +import dataclasses +import re + +from nameparser._pipeline._state import ParseState, WorkToken +from nameparser._types import Role, Span + +_COMMA_CHARS = {",", "،", "，"} # ASCII, Arabic, fullwidth + +# Ported verbatim from v1 (nameparser/config/regexes.py, "emoji" and +# "bidi") -- layering forbids importing the config package here, so the +# patterns are duplicated by design with this provenance note. When +# editing, keep both copies in sync. +_EMOJI = re.compile('[' # lgtm[py/overly-large-range] + '\U0001F300-\U0001F64F' + '\U0001F680-\U0001F6FF' + '\u2600-\u26FF\u2700-\u27BF]+') +_BIDI = re.compile('[\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+') + + +def _ignorable(ch: str, state: ParseState) -> bool: + if ch.isspace(): + return True + if state.policy.strip_bidi and _BIDI.match(ch): + return True + return bool(state.policy.strip_emoji and _EMOJI.match(ch)) + + +def _tokenize_region(state: ParseState, start: int, end: int, + role: Role | None, record_commas: bool, + tokens: list[WorkToken], commas: list[int]) -> None: + text = state.original + tok_start: int | None = None + for i in range(start, end): + ch = text[i] + if ch in _COMMA_CHARS or _ignorable(ch, state): + if tok_start is not None: + tokens.append(WorkToken(text[tok_start:i], + Span(tok_start, i), role=role)) + tok_start = None + if ch in _COMMA_CHARS and record_commas: + commas.append(i) + continue + if tok_start is None: + tok_start = i + if tok_start is not None: + tokens.append(WorkToken(text[tok_start:end], + Span(tok_start, end), role=role)) + + +def tokenize(state: ParseState) -> ParseState: + tokens: list[WorkToken] = [] + commas: list[int] = [] + # main stream: everything outside masked regions + boundaries = [0] + for m in state.masked: + boundaries.extend((m.start, m.end)) + boundaries.append(len(state.original)) + for start, end in zip(boundaries[::2], boundaries[1::2]): + _tokenize_region(state, start, end, None, True, tokens, commas) + # extracted regions: pre-set role, commas are mere separators + for role, inner in state.extracted: + _tokenize_region(state, inner.start, inner.end, role, False, + tokens, commas) + tokens.sort(key=lambda t: tuple(t.span)) + return dataclasses.replace(state, tokens=tuple(tokens), + comma_offsets=tuple(sorted(commas))) diff --git a/tests/v2/pipeline/test_tokenize.py b/tests/v2/pipeline/test_tokenize.py new file mode 100644 index 00000000..3e94e184 --- /dev/null +++ b/tests/v2/pipeline/test_tokenize.py @@ -0,0 +1,72 @@ +import dataclasses + +from nameparser._lexicon import Lexicon +from nameparser._pipeline._extract import extract_delimited +from nameparser._pipeline._state import ParseState +from nameparser._pipeline._tokenize import tokenize +from nameparser._policy import Policy +from nameparser._types import Role + + +def _tokenized(text: str, policy: Policy | None = None) -> ParseState: + state = ParseState(original=text, lexicon=Lexicon.empty(), + policy=policy or Policy()) + return tokenize(extract_delimited(state)) + + +def test_whitespace_split_with_spans() -> None: + out = _tokenized("John Smith") + assert [(t.text, tuple(t.span)) for t in out.tokens] == [ + ("John", (0, 4)), ("Smith", (6, 11)), + ] + assert all(t.role is None for t in out.tokens) + + +def test_provenance_text_equals_original_slice() -> None: + out = _tokenized(" Dr. Juan de la Vega ") + for t in out.tokens: + assert t.text == out.original[t.span.start:t.span.end] + + +def test_commas_are_separators_and_recorded() -> None: + out = _tokenized("Smith, John") + assert [t.text for t in out.tokens] == ["Smith", "John"] + assert out.comma_offsets == (5,) + + +def test_fullwidth_and_arabic_commas_segment() -> None: + out = _tokenized("سميث، جون") + assert out.comma_offsets == (4,) + out2 = _tokenized("山田，太郎") + assert out2.comma_offsets == (2,) + + +def test_extracted_regions_are_skipped_and_tokenized_with_role() -> None: + out = _tokenized('John "Jack Jr" Kennedy') + main = [(t.text, t.role) for t in out.tokens if t.role is None] + nick = [(t.text, t.role) for t in out.tokens if t.role is Role.NICKNAME] + assert main == [("John", None), ("Kennedy", None)] + assert nick == [("Jack", Role.NICKNAME), ("Jr", Role.NICKNAME)] + # tokens are span-sorted overall + starts = [t.span.start for t in out.tokens] + assert starts == sorted(starts) + + +def test_comma_inside_extracted_region_is_not_an_offset() -> None: + out = _tokenized('John "Jack, Jr" Kim') + assert out.comma_offsets == () + nick = [t.text for t in out.tokens if t.role is Role.NICKNAME] + assert nick == ["Jack", "Jr"] + + +def test_emoji_and_bidi_are_ignorable_by_policy() -> None: + out = _tokenized("John‏ \U0001f600Smith") + assert [t.text for t in out.tokens] == ["John", "Smith"] + keep = dataclasses.replace(Policy(), strip_emoji=False, strip_bidi=False) + out2 = _tokenized("John \U0001f600Smith", keep) + assert [t.text for t in out2.tokens] == ["John", "\U0001f600Smith"] + + +def test_empty_and_whitespace_yield_no_tokens() -> None: + assert _tokenized("").tokens == () + assert _tokenized(" ").tokens == () diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 5c633c6d..fe4758a9 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -13,7 +13,7 @@ # drop out of enforcement. Later tasks add each stage file as it lands. _MUST_EXIST = {"_types.py", "_lexicon.py", "_policy.py", "_locale.py", "_render.py", "_pipeline/_state.py", "_pipeline/__init__.py", - "_pipeline/_extract.py"} + "_pipeline/_extract.py", "_pipeline/_tokenize.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", From 29ceb1880f34f6c78dae5d52e575f0f1774a959d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 21:48:31 -0700 Subject: [PATCH 072/275] Add segment: comma-structure decision over token index runs Adds _vocab.py (is_initial/is_suffix_strict/is_suffix_lenient, shared across stages) and _segment.py (the family-comma/suffix-comma/no-comma decision). segment reads Lexicon suffix vocabulary directly rather than waiting for classify (recorded plan deviation #3: the suffix-comma vs family-comma call is definitionally vocabulary-dependent, mirroring v1's are_suffixes_after_comma). Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/__init__.py | 3 +- nameparser/_pipeline/_segment.py | 66 +++++++++++++++++++++++++++ nameparser/_pipeline/_vocab.py | 44 ++++++++++++++++++ tests/v2/pipeline/test_segment.py | 76 +++++++++++++++++++++++++++++++ tests/v2/pipeline/test_vocab.py | 34 ++++++++++++++ tests/v2/test_layering.py | 4 +- 6 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 nameparser/_pipeline/_segment.py create mode 100644 nameparser/_pipeline/_vocab.py create mode 100644 tests/v2/pipeline/test_segment.py create mode 100644 tests/v2/pipeline/test_vocab.py diff --git a/nameparser/_pipeline/__init__.py b/nameparser/_pipeline/__init__.py index 961efcda..863bd8fb 100644 --- a/nameparser/_pipeline/__init__.py +++ b/nameparser/_pipeline/__init__.py @@ -12,12 +12,13 @@ from collections.abc import Callable from nameparser._pipeline._extract import extract_delimited +from nameparser._pipeline._segment import segment from nameparser._pipeline._state import ParseState from nameparser._pipeline._tokenize import tokenize #: Filled in by later tasks as stages land; the fold is total either way. STAGES: tuple[Callable[[ParseState], ParseState], ...] = ( - extract_delimited, tokenize) + extract_delimited, tokenize, segment) def run(state: ParseState) -> ParseState: diff --git a/nameparser/_pipeline/_segment.py b/nameparser/_pipeline/_segment.py new file mode 100644 index 00000000..961f720e --- /dev/null +++ b/nameparser/_pipeline/_segment.py @@ -0,0 +1,66 @@ +"""Stage: segment. + +Consumes: tokens (role-None main stream), comma_offsets. +Produces: segments (runs of main-token indices), structure, +COMMA_STRUCTURE ambiguities for unrecognized extra segments. +Reads: Lexicon suffix vocabulary (via _vocab.is_suffix_lenient) -- +the suffix-comma decision is definitionally vocabulary-dependent +(recorded plan deviation #3); Policy is not consulted here. + +Decision (v1 parity): >=1 comma and every post-first segment entirely +lenient-suffix AND >1 word before the first comma -> SUFFIX_COMMA; +otherwise FAMILY_COMMA ("Family, Given ..."), with segments beyond the +second that are not lenient-suffix flagged COMMA_STRUCTURE (they are +still best-effort consumed as suffixes by assign, spec §5a). +""" +from __future__ import annotations + +import bisect +import dataclasses + +from nameparser._pipeline._state import ParseState, PendingAmbiguity, Structure +from nameparser._pipeline._vocab import is_suffix_lenient +from nameparser._types import AmbiguityKind + + +def segment(state: ParseState) -> ParseState: + main = [i for i, t in enumerate(state.tokens) if t.role is None] + if not main: + return dataclasses.replace(state, segments=(), + structure=Structure.NO_COMMA) + if not state.comma_offsets: + return dataclasses.replace(state, segments=(tuple(main),), + structure=Structure.NO_COMMA) + buckets: list[list[int]] = [[] for _ in range(len(state.comma_offsets) + 1)] + for i in main: + # comma_offsets is sorted and no offset ever equals a token + # start, so bisect_left counts the commas before this token + start = state.tokens[i].span.start + bucket = bisect.bisect_left(state.comma_offsets, start) + buckets[bucket].append(i) + groups = [tuple(b) for b in buckets if b] + if len(groups) <= 1: + segs = tuple(groups) if groups else (tuple(main),) + return dataclasses.replace(state, segments=segs, + structure=Structure.NO_COMMA) + + def suffixy(seg: tuple[int, ...]) -> bool: + return all(is_suffix_lenient(state.tokens[i].text, state.lexicon) + for i in seg) + + rest = groups[1:] + if all(suffixy(s) for s in rest) and len(groups[0]) > 1: + return dataclasses.replace(state, segments=tuple(groups), + structure=Structure.SUFFIX_COMMA) + ambiguities = list(state.ambiguities) + for seg in groups[2:]: + if not suffixy(seg): + texts = " ".join(state.tokens[i].text for i in seg) + ambiguities.append(PendingAmbiguity( + AmbiguityKind.COMMA_STRUCTURE, + f"segment {texts!r} beyond the recognized comma " + f"structures; consumed as suffix best-effort", + tuple(seg))) + return dataclasses.replace(state, segments=tuple(groups), + structure=Structure.FAMILY_COMMA, + ambiguities=tuple(ambiguities)) diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py new file mode 100644 index 00000000..48475fb2 --- /dev/null +++ b/nameparser/_pipeline/_vocab.py @@ -0,0 +1,44 @@ +"""Shared vocabulary predicates for pipeline stages. + +Text-level tests used by more than one stage; token/piece-level +predicates live with their stage. All take normalized-or-raw text +explicitly -- no state. + +Layering: imports _lexicon and _types only. +""" +from __future__ import annotations + +import re + +from nameparser._lexicon import Lexicon, _normalize + +# Ported verbatim from v1 (nameparser/config/regexes.py "initial") minus +# its empty-string alternative -- WorkToken text is never empty. Kept in +# sync by hand; layering forbids importing the config package here. +_INITIAL = re.compile(r"^(\w\.|[A-Z])$") + + +def is_initial(text: str) -> bool: + """'A.' / 'j.' / bare capital -- v1's is_an_initial.""" + return bool(_INITIAL.fullmatch(text)) + + +def is_suffix_strict(text: str, lexicon: Lexicon) -> bool: + """v1's is_suffix: suffix vocabulary with the initial veto ('V.' in + 'John V. Smith' is a middle initial, not roman five); ambiguous + acronyms count only when written with periods ('M.A.' yes, 'Ma' no). + """ + n = _normalize(text) + if "." in text and n in lexicon.suffix_acronyms_ambiguous: + return True + if is_initial(text): + return False + return n in lexicon.suffix_acronyms or n in lexicon.suffix_words + + +def is_suffix_lenient(text: str, lexicon: Lexicon) -> bool: + """v1's is_suffix_lenient: suffix_words accepted unconditionally, + bypassing the initial veto -- only safe in unambiguous positions + (after a comma).""" + return _normalize(text) in lexicon.suffix_words or \ + is_suffix_strict(text, lexicon) diff --git a/tests/v2/pipeline/test_segment.py b/tests/v2/pipeline/test_segment.py new file mode 100644 index 00000000..aad05671 --- /dev/null +++ b/tests/v2/pipeline/test_segment.py @@ -0,0 +1,76 @@ +from nameparser._lexicon import Lexicon +from nameparser._pipeline._extract import extract_delimited +from nameparser._pipeline._segment import segment +from nameparser._pipeline._state import ParseState, Structure +from nameparser._pipeline._tokenize import tokenize +from nameparser._policy import Policy +from nameparser._types import AmbiguityKind + +# synthetic vocabulary: behavior given a lexicon, never default() contents +_LEX = Lexicon( + suffix_acronyms=frozenset({"phd"}), + suffix_words=frozenset({"jr", "v"}), +) + + +def _segmented(text: str) -> ParseState: + state = ParseState(original=text, lexicon=_LEX, policy=Policy()) + return segment(tokenize(extract_delimited(state))) + + +def _texts(state: ParseState, seg: tuple[int, ...]) -> list[str]: + return [state.tokens[i].text for i in seg] + + +def test_no_comma() -> None: + out = _segmented("John Smith") + assert out.structure is Structure.NO_COMMA + assert [_texts(out, s) for s in out.segments] == [["John", "Smith"]] + + +def test_family_comma() -> None: + out = _segmented("Smith, John") + assert out.structure is Structure.FAMILY_COMMA + assert [_texts(out, s) for s in out.segments] == [["Smith"], ["John"]] + + +def test_suffix_comma_when_all_rest_groups_are_suffixes() -> None: + out = _segmented("John Smith, PhD") + assert out.structure is Structure.SUFFIX_COMMA + assert [_texts(out, s) for s in out.segments] == [["John", "Smith"], ["PhD"]] + + +def test_suffix_comma_lenient_accepts_initial_shaped_suffix_word() -> None: + # "V" is initial-shaped; the strict test vetoes it, the post-comma + # lenient test accepts suffix_words unconditionally (v1 parity) + out = _segmented("John Ingram, V") + assert out.structure is Structure.SUFFIX_COMMA + + +def test_family_comma_with_trailing_suffix_segment() -> None: + out = _segmented("Smith, John, Jr.") + assert out.structure is Structure.FAMILY_COMMA + assert [_texts(out, s) for s in out.segments] == [["Smith"], ["John"], ["Jr."]] + + +def test_single_pre_comma_word_never_suffix_comma() -> None: + # v1: suffix-comma requires >1 word before the comma + out = _segmented("Johnson, Jr.") + assert out.structure is Structure.FAMILY_COMMA + + +def test_excess_non_suffix_segment_flags_comma_structure() -> None: + out = _segmented("Smith, John, Extra, Jr.") + assert out.structure is Structure.FAMILY_COMMA + kinds = [a.kind for a in out.ambiguities] + assert AmbiguityKind.COMMA_STRUCTURE in kinds + + +def test_empty_input_yields_no_segments() -> None: + assert _segmented("").segments == () + + +def test_comma_only_input_is_no_comma_structure() -> None: + out = _segmented(",,,") + assert out.structure is Structure.NO_COMMA + assert out.segments == () diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py new file mode 100644 index 00000000..54d4cfc6 --- /dev/null +++ b/tests/v2/pipeline/test_vocab.py @@ -0,0 +1,34 @@ +from nameparser._lexicon import Lexicon +from nameparser._pipeline._vocab import is_initial, is_suffix_lenient, is_suffix_strict + +_LEX = Lexicon( + suffix_acronyms=frozenset({"phd"}), + suffix_words=frozenset({"jr", "v"}), + suffix_acronyms_ambiguous=frozenset({"ma"}), +) + + +def test_is_initial() -> None: + assert is_initial("A.") + assert is_initial("j.") + assert is_initial("B") + assert not is_initial("Jo") + assert not is_initial("b") # bare lowercase letter is not an initial + + +def test_strict_suffix_initial_veto() -> None: + assert is_suffix_strict("PhD", _LEX) + assert not is_suffix_strict("V.", _LEX) # initial veto + assert not is_suffix_strict("V", _LEX) # initial veto + assert is_suffix_strict("Jr", _LEX) + + +def test_ambiguous_acronym_needs_periods_and_beats_the_veto() -> None: + assert is_suffix_strict("M.A.", _LEX) + assert not is_suffix_strict("Ma", _LEX) + + +def test_lenient_accepts_suffix_words_unconditionally() -> None: + assert is_suffix_lenient("V", _LEX) + assert is_suffix_lenient("V.", _LEX) + assert not is_suffix_lenient("Ma", _LEX) diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index fe4758a9..1fd7ee82 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -13,7 +13,8 @@ # drop out of enforcement. Later tasks add each stage file as it lands. _MUST_EXIST = {"_types.py", "_lexicon.py", "_policy.py", "_locale.py", "_render.py", "_pipeline/_state.py", "_pipeline/__init__.py", - "_pipeline/_extract.py", "_pipeline/_tokenize.py"} + "_pipeline/_extract.py", "_pipeline/_tokenize.py", + "_pipeline/_vocab.py", "_pipeline/_segment.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", @@ -43,6 +44,7 @@ "_pipeline/_state.py": ("nameparser._types", "nameparser._lexicon", "nameparser._policy"), "_pipeline/__init__.py": ("nameparser._pipeline.",), + "_pipeline/_vocab.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_extract.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_tokenize.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_segment.py": _PIPELINE_STAGE_ALLOWED, From a91e46562794fb3ca7d51bb3087afb33f44ff65d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 21:58:49 -0700 Subject: [PATCH 073/275] Add classify: vocabulary tags from the lexicon Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/__init__.py | 3 +- nameparser/_pipeline/_classify.py | 61 +++++++++++++++++++++++++++ tests/v2/pipeline/test_classify.py | 68 ++++++++++++++++++++++++++++++ tests/v2/test_layering.py | 3 +- 4 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 nameparser/_pipeline/_classify.py create mode 100644 tests/v2/pipeline/test_classify.py diff --git a/nameparser/_pipeline/__init__.py b/nameparser/_pipeline/__init__.py index 863bd8fb..b964120c 100644 --- a/nameparser/_pipeline/__init__.py +++ b/nameparser/_pipeline/__init__.py @@ -11,6 +11,7 @@ from collections.abc import Callable +from nameparser._pipeline._classify import classify from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._segment import segment from nameparser._pipeline._state import ParseState @@ -18,7 +19,7 @@ #: Filled in by later tasks as stages land; the fold is total either way. STAGES: tuple[Callable[[ParseState], ParseState], ...] = ( - extract_delimited, tokenize, segment) + extract_delimited, tokenize, segment, classify) def run(state: ParseState) -> ParseState: diff --git a/nameparser/_pipeline/_classify.py b/nameparser/_pipeline/_classify.py new file mode 100644 index 00000000..60c35068 --- /dev/null +++ b/nameparser/_pipeline/_classify.py @@ -0,0 +1,61 @@ +"""Stage: classify. + +Consumes: tokens. +Produces: tokens with vocabulary tags added (text/span/role unchanged). +Reads: every Lexicon vocabulary field; Policy is not consulted. + +Tags emitted -- stable (API): "particle", "conjunction", "initial"; +namespaced (unstable): "vocab:title", "vocab:given-title", +"vocab:suffix", "vocab:suffix-word", "vocab:suffix-ambiguous", +"vocab:particle-ambiguous", "vocab:bound-given", "vocab:maiden-marker". +"vocab:suffix" means "counts as a suffix as written": unambiguous +suffix vocabulary, or an ambiguous acronym written with periods +('M.A.' yes, 'Ma' no -- 'Ma' gets only "vocab:suffix-ambiguous"). +The initial veto is assign's job, not classify's: 'V' carries both +"vocab:suffix" and "initial". +""" +from __future__ import annotations + +import dataclasses + +from nameparser._lexicon import _normalize +from nameparser._pipeline._state import ParseState, WorkToken +from nameparser._pipeline._vocab import is_initial + + +def _tags_for(token: WorkToken, state: ParseState) -> frozenset[str]: + lex = state.lexicon + n = _normalize(token.text) + tags = set(token.tags) + if n in lex.titles: + tags.add("vocab:title") + if n in lex.given_name_titles: + tags.add("vocab:given-title") + if n in lex.suffix_acronyms or n in lex.suffix_words: + tags.add("vocab:suffix") + if n in lex.suffix_words: + tags.add("vocab:suffix-word") + if n in lex.suffix_acronyms_ambiguous: + tags.add("vocab:suffix-ambiguous") + if "." in token.text: + tags.add("vocab:suffix") + if n in lex.particles: + tags.add("particle") + if n in lex.particles_ambiguous: + tags.add("vocab:particle-ambiguous") + if n in lex.conjunctions: + tags.add("conjunction") + if n in lex.bound_given_names: + tags.add("vocab:bound-given") + if n in lex.maiden_markers: + tags.add("vocab:maiden-marker") + if is_initial(token.text): + tags.add("initial") + return frozenset(tags) + + +def classify(state: ParseState) -> ParseState: + tokens = tuple( + dataclasses.replace(t, tags=_tags_for(t, state)) + for t in state.tokens) + return dataclasses.replace(state, tokens=tokens) diff --git a/tests/v2/pipeline/test_classify.py b/tests/v2/pipeline/test_classify.py new file mode 100644 index 00000000..1c4f0a4f --- /dev/null +++ b/tests/v2/pipeline/test_classify.py @@ -0,0 +1,68 @@ +from nameparser._lexicon import Lexicon +from nameparser._pipeline._classify import classify +from nameparser._pipeline._extract import extract_delimited +from nameparser._pipeline._segment import segment +from nameparser._pipeline._state import ParseState +from nameparser._pipeline._tokenize import tokenize +from nameparser._policy import Policy + +_LEX = Lexicon( + titles=frozenset({"dr", "sir"}), + given_name_titles=frozenset({"sir"}), + suffix_acronyms=frozenset({"phd"}), + suffix_words=frozenset({"jr", "v"}), + suffix_acronyms_ambiguous=frozenset({"ma"}), + particles=frozenset({"de", "la", "van"}), + particles_ambiguous=frozenset({"van"}), + conjunctions=frozenset({"and", "y"}), + bound_given_names=frozenset({"abdul"}), + maiden_markers=frozenset({"née"}), +) + + +def _classified(text: str) -> ParseState: + state = ParseState(original=text, lexicon=_LEX, policy=Policy()) + return classify(segment(tokenize(extract_delimited(state)))) + + +def _tags(state: ParseState, text: str) -> frozenset[str]: + return next(t.tags for t in state.tokens if t.text == text) + + +def test_vocabulary_tags() -> None: + out = _classified("Dr. van de la Smith and abdul née PhD Jr") + assert "vocab:title" in _tags(out, "Dr.") + assert {"particle", "vocab:particle-ambiguous"} <= _tags(out, "van") + assert "particle" in _tags(out, "de") + assert "vocab:particle-ambiguous" not in _tags(out, "de") + assert "conjunction" in _tags(out, "and") + assert "vocab:bound-given" in _tags(out, "abdul") + assert "vocab:maiden-marker" in _tags(out, "née") + assert "vocab:suffix" in _tags(out, "PhD") + assert {"vocab:suffix", "vocab:suffix-word"} <= _tags(out, "Jr") + assert _tags(out, "Smith") == frozenset() + + +def test_given_title_tag() -> None: + out = _classified("Sir John") + assert {"vocab:title", "vocab:given-title"} <= _tags(out, "Sir") + + +def test_initial_tag() -> None: + out = _classified("John A. B Smith") + assert "initial" in _tags(out, "A.") + assert "initial" in _tags(out, "B") + assert "initial" not in _tags(out, "John") + + +def test_ambiguous_suffix_acronym_needs_periods() -> None: + out = _classified("M.A. Ma") + assert "vocab:suffix" in _tags(out, "M.A.") + assert "vocab:suffix" not in _tags(out, "Ma") + assert "vocab:suffix-ambiguous" in _tags(out, "Ma") + + +def test_v_is_suffix_word_and_initial() -> None: + # both tags present; assign applies the veto, not classify + out = _classified("John V Smith") + assert {"vocab:suffix", "vocab:suffix-word", "initial"} <= _tags(out, "V") diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 1fd7ee82..98112eda 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -14,7 +14,8 @@ _MUST_EXIST = {"_types.py", "_lexicon.py", "_policy.py", "_locale.py", "_render.py", "_pipeline/_state.py", "_pipeline/__init__.py", "_pipeline/_extract.py", "_pipeline/_tokenize.py", - "_pipeline/_vocab.py", "_pipeline/_segment.py"} + "_pipeline/_vocab.py", "_pipeline/_segment.py", + "_pipeline/_classify.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", From 20e6bf644813b36d25647c5b36d34651612dd2e1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 22:06:18 -0700 Subject: [PATCH 074/275] Add group: piece boundaries by index; joins ported from v1 Ports v1's join_on_conjunctions, prefix chains, and _join_bound_first_name as pure index operations over WorkToken runs (the anti-#100 invariant: no value-based lookup, no string joins). Adds two v2 additions from the plan: the "Ph. D."-split merge (plan deviation #1 -- lands in group since assign needs the merged piece to exist first) and the maiden-marker consuming rule (#274). Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/__init__.py | 3 +- nameparser/_pipeline/_group.py | 211 +++++++++++++++++++++++++++++++ tests/v2/pipeline/test_group.py | 129 +++++++++++++++++++ tests/v2/test_layering.py | 2 +- 4 files changed, 343 insertions(+), 2 deletions(-) create mode 100644 nameparser/_pipeline/_group.py create mode 100644 tests/v2/pipeline/test_group.py diff --git a/nameparser/_pipeline/__init__.py b/nameparser/_pipeline/__init__.py index b964120c..d4f3fb94 100644 --- a/nameparser/_pipeline/__init__.py +++ b/nameparser/_pipeline/__init__.py @@ -13,13 +13,14 @@ from nameparser._pipeline._classify import classify from nameparser._pipeline._extract import extract_delimited +from nameparser._pipeline._group import group from nameparser._pipeline._segment import segment from nameparser._pipeline._state import ParseState from nameparser._pipeline._tokenize import tokenize #: Filled in by later tasks as stages land; the fold is total either way. STAGES: tuple[Callable[[ParseState], ParseState], ...] = ( - extract_delimited, tokenize, segment, classify) + extract_delimited, tokenize, segment, classify, group) def run(state: ParseState) -> ParseState: diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py new file mode 100644 index 00000000..4023c7e6 --- /dev/null +++ b/nameparser/_pipeline/_group.py @@ -0,0 +1,211 @@ +"""Stage: group. + +Consumes: tokens (classified), segments, structure. +Produces: pieces + piece_tags per segment (runs of token indices -- +tokens are NEVER joined into strings: the anti-#100 invariant); maiden +tail tokens get role=MAIDEN; marker tokens land in dropped. +Reads: token tags (from classify); Policy is not consulted. The v1 +"derived titles/prefixes" registration becomes piece_tags entries -- +per-parse state that dissolves with the state (v1 kept per-parse sets +for the same reason). + +Ports v1's join_on_conjunctions + prefix chains + _join_bound_first_name +plus two additions: the "Ph. D."-split merge (v1 fix_phd, recorded plan +deviation #1) and the maiden-marker consuming rule (#274: marker plus +following pieces until a suffix become maiden; the marker itself is +structural, like a delimiter char, and is dropped from assembly). +""" +from __future__ import annotations + +import dataclasses +import re + +from nameparser._pipeline._state import ParseState, Structure, WorkToken +from nameparser._types import Role + +_PH = re.compile(r"^ph\.?$", re.IGNORECASE) +_D = re.compile(r"^d\.?$", re.IGNORECASE) + +Piece = list[int] + + +def _is_title_piece(piece: Piece, ptags: set[str], + tokens: tuple[WorkToken, ...]) -> bool: + if "title" in ptags: + return True + return len(piece) == 1 and "vocab:title" in tokens[piece[0]].tags + + +def _is_prefix_piece(piece: Piece, ptags: set[str], + tokens: tuple[WorkToken, ...]) -> bool: + if "prefix" in ptags: + return True + return len(piece) == 1 and "particle" in tokens[piece[0]].tags + + +def _is_suffix_piece(piece: Piece, ptags: set[str], + tokens: tuple[WorkToken, ...]) -> bool: + if "suffix" in ptags: + return True + if len(piece) != 1: + return False + tags = tokens[piece[0]].tags + return "vocab:suffix" in tags and "initial" not in tags + + +def _is_conj_piece(piece: Piece, ptags: set[str], + tokens: tuple[WorkToken, ...]) -> bool: + if "conjunction" in ptags: + return True + return len(piece) == 1 and "conjunction" in tokens[piece[0]].tags + + +def _is_rootname(piece: Piece, ptags: set[str], + tokens: tuple[WorkToken, ...]) -> bool: + if len(piece) == 1 and "initial" in tokens[piece[0]].tags: + return False + return not (_is_title_piece(piece, ptags, tokens) + or _is_prefix_piece(piece, ptags, tokens) + or _is_suffix_piece(piece, ptags, tokens)) + + +def _group_segment(seg: tuple[int, ...], additional: int, + tokens: tuple[WorkToken, ...], + ) -> tuple[list[Piece], list[set[str]]]: + pieces: list[Piece] = [[i] for i in seg] + ptags: list[set[str]] = [set() for _ in seg] + + def title(k: int) -> bool: + return _is_title_piece(pieces[k], ptags[k], tokens) + + def prefix(k: int) -> bool: + return _is_prefix_piece(pieces[k], ptags[k], tokens) + + def suffix(k: int) -> bool: + return _is_suffix_piece(pieces[k], ptags[k], tokens) + + def conj(k: int) -> bool: + return _is_conj_piece(pieces[k], ptags[k], tokens) + + # ph-d merge first: "Ph." "D." adjacent -> one suffix piece (plan + # deviation #1; v1 fix_phd did this by regex on the raw string) + k = 0 + while k < len(pieces) - 1: + a, b = pieces[k], pieces[k + 1] + if (len(a) == 1 and len(b) == 1 + and _PH.fullmatch(tokens[a[0]].text) + and _D.fullmatch(tokens[b[0]].text)): + pieces[k:k + 2] = [a + b] + merged = ptags[k] | ptags[k + 1] | {"suffix"} + ptags[k:k + 2] = [merged] + else: + k += 1 + + if len(pieces) + additional >= 3: + total = sum(_is_rootname(p, t, tokens) + for p, t in zip(pieces, ptags)) + additional + # contiguous conjunction runs merge first (v1: "of the") + k = 0 + while k < len(pieces) - 1: + if conj(k) and conj(k + 1): + pieces[k:k + 2] = [pieces[k] + pieces[k + 1]] + ptags[k:k + 2] = [ptags[k] | ptags[k + 1] | {"conjunction"}] + else: + k += 1 + # each conjunction joins its neighbors (v1 issue #11 carve-out: + # a single-letter alphabetic conjunction in a short name is more + # likely an initial) + k = 0 + while k < len(pieces): + if not conj(k): + k += 1 + continue + text = " ".join(tokens[i].text for i in pieces[k]) + if len(text) == 1 and total < 4 and text.isalpha(): + k += 1 + continue + start = max(0, k - 1) + end = min(len(pieces), k + 2) + neighbor = start if start < k else end - 1 + new_tags = set().union(*ptags[start:end]) + if title(neighbor): + new_tags.add("title") + if prefix(neighbor): + new_tags.add("prefix") + merged_piece = [i for p in pieces[start:end] for i in p] + pieces[start:end] = [merged_piece] + ptags[start:end] = [new_tags] + k = start + 1 + # prefix chains: a non-leading prefix run absorbs everything to + # the next prefix or suffix (v1's leading_first_name rule keeps + # the first piece a name: "Van Johnson") + k = 0 + while k < len(pieces): + if k == 0 or not prefix(k): + k += 1 + continue + j = k + 1 + while j < len(pieces) and prefix(j): + j += 1 + while j < len(pieces) and not prefix(j) and not suffix(j): + j += 1 + merged_piece = [i for p in pieces[k:j] for i in p] + merged_tags = set().union(*ptags[k:j]) - {"prefix"} + pieces[k:j] = [merged_piece] + ptags[k:j] = [merged_tags] + k += 1 + # bound given names: first non-title piece joins the next when + # enough rootname pieces remain (v1 reserve_last) + first_name_k = next( + (k for k in range(len(pieces)) if not title(k)), None) + if (first_name_k is not None + and first_name_k + 1 < len(pieces) + and len(pieces[first_name_k]) == 1 + and "vocab:bound-given" + in tokens[pieces[first_name_k][0]].tags): + non_suffix = sum(1 for k in range(len(pieces)) + if not title(k) and not suffix(k)) + if non_suffix >= 3: + bg = first_name_k + pieces[bg:bg + 2] = [pieces[bg] + pieces[bg + 1]] + ptags[bg:bg + 2] = [ptags[bg] | ptags[bg + 1]] + return pieces, ptags + + +def group(state: ParseState) -> ParseState: + tokens = list(state.tokens) + dropped = list(state.dropped) + all_pieces: list[tuple[tuple[int, ...], ...]] = [] + all_ptags: list[tuple[frozenset[str], ...]] = [] + # v1 parity: additional_parts_count=1 applies only to FAMILY_COMMA + # parts (parser.py:1333); the SUFFIX_COMMA pre-comma segment gets 0. + additional = 1 if state.structure is Structure.FAMILY_COMMA else 0 + for seg in state.segments: + pieces, ptags = _group_segment(seg, additional, tuple(tokens)) + # maiden markers: a non-leading marker piece consumes following + # pieces until a suffix; consumed tokens become MAIDEN, the + # marker is dropped (#274) + m = next( + (k for k in range(1, len(pieces)) + if len(pieces[k]) == 1 + and "vocab:maiden-marker" in tokens[pieces[k][0]].tags), + None) + if m is not None: + j = m + 1 + consumed: list[int] = [] + while j < len(pieces) and not _is_suffix_piece( + pieces[j], ptags[j], tuple(tokens)): + consumed.extend(pieces[j]) + j += 1 + if consumed: + dropped.extend(pieces[m]) + for i in consumed: + tokens[i] = dataclasses.replace( + tokens[i], role=Role.MAIDEN) + pieces[m:j] = [] + ptags[m:j] = [] + all_pieces.append(tuple(tuple(p) for p in pieces)) + all_ptags.append(tuple(frozenset(t) for t in ptags)) + return dataclasses.replace( + state, tokens=tuple(tokens), pieces=tuple(all_pieces), + piece_tags=tuple(all_ptags), dropped=tuple(dropped)) diff --git a/tests/v2/pipeline/test_group.py b/tests/v2/pipeline/test_group.py new file mode 100644 index 00000000..f5fecc4e --- /dev/null +++ b/tests/v2/pipeline/test_group.py @@ -0,0 +1,129 @@ +from nameparser._lexicon import Lexicon +from nameparser._pipeline._classify import classify +from nameparser._pipeline._extract import extract_delimited +from nameparser._pipeline._group import group +from nameparser._pipeline._segment import segment +from nameparser._pipeline._state import ParseState +from nameparser._pipeline._tokenize import tokenize +from nameparser._policy import Policy +from nameparser._types import Role + +_LEX = Lexicon( + titles=frozenset({"mr", "mrs", "secretary", "the", "state"}), + suffix_acronyms=frozenset({"phd"}), + suffix_words=frozenset({"jr"}), + particles=frozenset({"de", "la", "van", "von", "und", "zu"}), + particles_ambiguous=frozenset({"van"}), + conjunctions=frozenset({"and", "of", "the", "y"}), + bound_given_names=frozenset({"abdul"}), + maiden_markers=frozenset({"née", "geb"}), +) +# NOTE: "the" sits in both titles and conjunctions here, mirroring v1's +# real vocabulary overlap; the subset rule only constrains particles. + + +def _grouped(text: str) -> ParseState: + state = ParseState(original=text, lexicon=_LEX, policy=Policy()) + return group(classify(segment(tokenize(extract_delimited(state))))) + + +def _piece_texts(state: ParseState) -> list[list[str]]: + return [[" ".join(state.tokens[i].text for i in piece) + for piece in seg] for seg in state.pieces] + + +def test_no_joins_pass_through() -> None: + out = _grouped("John Smith") + assert _piece_texts(out) == [["John", "Smith"]] + + +def test_conjunction_joins_neighbors() -> None: + out = _grouped("Mr. and Mrs. John Smith") + assert _piece_texts(out) == [["Mr. and Mrs.", "John", "Smith"]] + # joined-to-a-title piece is flagged a derived title + assert "title" in out.piece_tags[0][0] + + +def test_contiguous_conjunctions_join_first() -> None: + out = _grouped("The Secretary of State Hillary Clinton") + assert _piece_texts(out) == [["The Secretary of State", "Hillary", "Clinton"]] + assert "title" in out.piece_tags[0][0] + + +def test_single_letter_conjunction_prefers_initial_when_short() -> None: + # v1 issue #11: 3 rootname parts, single-letter conjunction "y" + out = _grouped("John y Smith") + assert _piece_texts(out) == [["John", "y", "Smith"]] + + +def test_prefix_chain_joins_to_following() -> None: + out = _grouped("Juan de la Vega") + assert _piece_texts(out) == [["Juan", "de la Vega"]] + + +def test_prefix_chain_absorbs_through_to_next_suffix() -> None: + out = _grouped("Juan de la Vega Martinez PhD") + assert _piece_texts(out) == [["Juan", "de la Vega Martinez", "PhD"]] + + +def test_leading_prefix_is_never_chained() -> None: + # "Van Johnson": the leading piece is a first name, not a particle + out = _grouped("Van Johnson") + assert _piece_texts(out) == [["Van", "Johnson"]] + + +def test_von_und_zu_bridges() -> None: + # conjunction "und" joins two prefixes; the joined piece is a derived + # prefix and still chains onto the following name (v1 PR #191) + out = _grouped("Otto von und zu Habsburg") + assert _piece_texts(out) == [["Otto", "von und zu Habsburg"]] + + +def test_bound_given_joins_with_three_rootnames() -> None: + assert _piece_texts(_grouped("abdul rahman al-said")) == \ + [["abdul rahman", "al-said"]] + # only two rootname pieces: no join (v1 reserve_last) + assert _piece_texts(_grouped("abdul rahman")) == [["abdul", "rahman"]] + + +def test_phd_split_across_tokens_merges_as_suffix() -> None: + out = _grouped("John Smith Ph. D.") + assert _piece_texts(out) == [["John", "Smith", "Ph. D."]] + assert "suffix" in out.piece_tags[0][2] + + +def test_maiden_marker_consumes_tail() -> None: + out = _grouped("Jane Smith née Jones") + assert _piece_texts(out) == [["Jane", "Smith"]] + maiden = [t.text for t in out.tokens if t.role is Role.MAIDEN] + assert maiden == ["Jones"] + # the marker token itself is structural: dropped from assembly + née_idx = next(i for i, t in enumerate(out.tokens) if t.text == "née") + assert née_idx in out.dropped + + +def test_maiden_marker_stops_at_suffix() -> None: + out = _grouped("Jane Smith née Jones PhD") + maiden = [t.text for t in out.tokens if t.role is Role.MAIDEN] + assert maiden == ["Jones"] + assert _piece_texts(out)[0][-1] == "PhD" + + +def test_leading_marker_is_not_consumed() -> None: + # "née Jones" alone: marker at piece 0 has no name before it + out = _grouped("née Jones") + assert _piece_texts(out) == [["née", "Jones"]] + + +def test_initials_do_not_count_as_rootnames_for_conjunction_carveout() -> None: + # v1 parity: 'J.' is an initial, so total rootnames stay under 4 and + # the single-letter conjunction 'y' is treated as an initial, not joined + out = _grouped("J. Ruiz y Gomez") + assert _piece_texts(out) == [["J.", "Ruiz", "y", "Gomez"]] + + +def test_suffix_comma_name_segment_gets_no_additional_count() -> None: + # v1 parity: additional_parts_count applies to FAMILY_COMMA parts only; + # ', PhD' must not tip the single-letter-conjunction carve-out + out = _grouped("John y Smith, PhD") + assert _piece_texts(out) == [["John", "y", "Smith"], ["PhD"]] diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 98112eda..ad6f777f 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -15,7 +15,7 @@ "_render.py", "_pipeline/_state.py", "_pipeline/__init__.py", "_pipeline/_extract.py", "_pipeline/_tokenize.py", "_pipeline/_vocab.py", "_pipeline/_segment.py", - "_pipeline/_classify.py"} + "_pipeline/_classify.py", "_pipeline/_group.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", From 5e22e78262013ace77f4da43d76b24f648fc151e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 22:25:59 -0700 Subject: [PATCH 075/275] Add assign: positional roles per name_order over pieces Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/__init__.py | 3 +- nameparser/_pipeline/_assign.py | 183 +++++++++++++++++++++++++++++++ tests/v2/pipeline/test_assign.py | 158 ++++++++++++++++++++++++++ tests/v2/test_layering.py | 3 +- 4 files changed, 345 insertions(+), 2 deletions(-) create mode 100644 nameparser/_pipeline/_assign.py create mode 100644 tests/v2/pipeline/test_assign.py diff --git a/nameparser/_pipeline/__init__.py b/nameparser/_pipeline/__init__.py index d4f3fb94..25f8149f 100644 --- a/nameparser/_pipeline/__init__.py +++ b/nameparser/_pipeline/__init__.py @@ -11,6 +11,7 @@ from collections.abc import Callable +from nameparser._pipeline._assign import assign from nameparser._pipeline._classify import classify from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._group import group @@ -20,7 +21,7 @@ #: Filled in by later tasks as stages land; the fold is total either way. STAGES: tuple[Callable[[ParseState], ParseState], ...] = ( - extract_delimited, tokenize, segment, classify, group) + extract_delimited, tokenize, segment, classify, group, assign) def run(state: ParseState) -> ParseState: diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py new file mode 100644 index 00000000..706294d6 --- /dev/null +++ b/nameparser/_pipeline/_assign.py @@ -0,0 +1,183 @@ +"""Stage: assign. + +Consumes: pieces + piece_tags (grouped), segments, structure, tokens. +Produces: tokens with roles set on every main-stream token. +Reads: Policy.name_order (#270); token/piece tags; Lexicon only through +tags already applied by classify (plus the leading-title period rule). + +Ports v1's assignment loops. NO_COMMA (per name_order): +leading title pieces chain while no given-position name has been seen +(a title needs a following piece, unless the whole name is one title); +then positional assignment per name_order with the trailing-suffix +rule: the piece from which everything after is a strict suffix is the +last name-position piece, the rest are suffixes. The v1 single-name+ +nickname rule lives here (plan deviation #2): one non-title piece plus +a nonempty nickname puts that piece in FAMILY. +FAMILY_COMMA: segment 0 wholly FAMILY (v1 parity); segment 1 gets +leading titles, then given, then middles with strict-suffix pieces to +suffix; segments 2+ are suffixes (lenient -- segment already flagged +non-suffixy ones COMMA_STRUCTURE). +SUFFIX_COMMA: segment 0 as NO_COMMA; segments 1+ wholly SUFFIX. +Emits PARTICLE_OR_GIVEN when the given position consumed a leading +particles_ambiguous token with more pieces following ("Van Johnson"). +""" +from __future__ import annotations + +import dataclasses +import re + +from nameparser._pipeline._group import ( + _is_suffix_piece, _is_title_piece, +) +from nameparser._pipeline._state import ( + ParseState, PendingAmbiguity, Structure, WorkToken, +) +from nameparser._types import AmbiguityKind, Role + +# Ported verbatim from v1 (nameparser/config/regexes.py +# "period_abbreviation" and "roman_numeral") -- layering forbids the +# config import; keep in sync by hand. +_PERIOD_ABBREV = re.compile(r'^[^\W\d_]{2,}\.$') +_ROMAN = re.compile(r'^(X|IX|IV|V?I{0,3})$', re.I) + + +def _set_roles(tokens: list[WorkToken], piece: tuple[int, ...], + role: Role) -> None: + for i in piece: + tokens[i] = dataclasses.replace(tokens[i], role=role) + + +def _is_leading_title(piece: tuple[int, ...], ptags: frozenset[str], + tokens: list[WorkToken]) -> bool: + if _is_title_piece(list(piece), set(ptags), tuple(tokens)): + return True + return (len(piece) == 1 + and bool(_PERIOD_ABBREV.match(tokens[piece[0]].text))) + + +def _name_positions(order: tuple[Role, Role, Role], + count: int) -> list[Role]: + """Roles for `count` name pieces (titles/suffixes already peeled), + per name_order. GIVEN_FIRST: given, middles..., family. + FAMILY_FIRST: family, given, middles... FAMILY_FIRST_GIVEN_LAST: + family, middles..., given. One piece takes order[0]'s role + (spec §5a); two pieces take order[0] and the other primary.""" + first, second = order[0], order[1] + if count == 1: + return [first] + if first is Role.GIVEN: # GIVEN_FIRST + return ([Role.GIVEN] + [Role.MIDDLE] * (count - 2) + + [Role.FAMILY]) + if second is Role.GIVEN: # FAMILY_FIRST + return ([Role.FAMILY, Role.GIVEN] + + [Role.MIDDLE] * (count - 2)) + return ([Role.FAMILY] + [Role.MIDDLE] * (count - 2) # F_F_GIVEN_LAST + + [Role.GIVEN]) + + +def _assign_main(seg_idx: int, state: ParseState, + tokens: list[WorkToken], + ambiguities: list[PendingAmbiguity]) -> None: + pieces = state.pieces[seg_idx] + ptags = state.piece_tags[seg_idx] + has_nickname = any(t.role is Role.NICKNAME for t in tokens) + # peel leading titles + n = 0 + while n < len(pieces): + has_next = n + 1 < len(pieces) + if ((has_next or len(pieces) == 1) + and _is_leading_title(pieces[n], ptags[n], tokens)): + _set_roles(tokens, pieces[n], Role.TITLE) + n += 1 + continue + break + rest = list(range(n, len(pieces))) + if not rest: + return + # v1 nickname rule (plan deviation #2) + if len(rest) == 1 and has_nickname: + _set_roles(tokens, pieces[rest[0]], Role.FAMILY) + return + # peel the trailing suffix run: k = first index in rest from which + # every piece is a strict suffix (v1's are_suffixes tail rule, with + # the roman-numeral special: a final roman numeral after a + # non-initial piece is a suffix) + k = len(rest) + while k > 0: + piece = pieces[rest[k - 1]] + tags = ptags[rest[k - 1]] + if _is_suffix_piece(list(piece), set(tags), tuple(tokens)): + k -= 1 + continue + if (k == len(rest) and k >= 2 and len(piece) == 1 + and _ROMAN.match(tokens[piece[0]].text) + and "initial" not in tokens[pieces[rest[k - 2]][0]].tags): + k -= 1 + continue + break + name_pieces, suffix_pieces = rest[:k], rest[k:] + if not name_pieces and suffix_pieces: + # everything suffix-shaped after titles: first one is the name + name_pieces, suffix_pieces = suffix_pieces[:1], suffix_pieces[1:] + roles = _name_positions(state.policy.name_order, len(name_pieces)) + for pos, piece_idx in enumerate(name_pieces): + _set_roles(tokens, pieces[piece_idx], roles[pos]) + for piece_idx in suffix_pieces: + _set_roles(tokens, pieces[piece_idx], Role.SUFFIX) + # leading ambiguous particle read as a name (#121 surfaced) + if name_pieces: + head = pieces[name_pieces[0]] + if (len(head) == 1 and len(name_pieces) > 1 + and "vocab:particle-ambiguous" in tokens[head[0]].tags): + ambiguities.append(PendingAmbiguity( + AmbiguityKind.PARTICLE_OR_GIVEN, + f"leading {tokens[head[0]].text!r} may be a family-name " + f"particle; read as a given name", + tuple(head))) + + +def assign(state: ParseState) -> ParseState: + tokens = list(state.tokens) + ambiguities = list(state.ambiguities) + if not state.segments: + return state + if state.structure is Structure.NO_COMMA: + _assign_main(0, state, tokens, ambiguities) + elif state.structure is Structure.SUFFIX_COMMA: + _assign_main(0, state, tokens, ambiguities) + for seg_idx in range(1, len(state.segments)): + for piece in state.pieces[seg_idx]: + _set_roles(tokens, piece, Role.SUFFIX) + else: # FAMILY_COMMA + # PARTICLE_OR_GIVEN is deliberately not emitted here: after a + # comma the family is already fixed, so a leading given-position + # particle is not meaningfully ambiguous. + for piece in state.pieces[0]: + _set_roles(tokens, piece, Role.FAMILY) + if len(state.segments) > 1: + pieces = state.pieces[1] + ptags = state.piece_tags[1] + given_done = False + n = 0 + while n < len(pieces): + if (not given_done + and _is_leading_title(pieces[n], ptags[n], tokens) + and (n + 1 < len(pieces) or len(pieces) == 1)): + _set_roles(tokens, pieces[n], Role.TITLE) + n += 1 + continue + break + for m in range(n, len(pieces)): + if _is_suffix_piece(list(pieces[m]), set(ptags[m]), + tuple(tokens)): + _set_roles(tokens, pieces[m], Role.SUFFIX) + elif not given_done: + _set_roles(tokens, pieces[m], Role.GIVEN) + given_done = True + else: + _set_roles(tokens, pieces[m], Role.MIDDLE) + for seg_idx in range(2, len(state.segments)): + for piece in state.pieces[seg_idx]: + _set_roles(tokens, piece, Role.SUFFIX) + return dataclasses.replace(state, tokens=tuple(tokens), + ambiguities=tuple(ambiguities)) diff --git a/tests/v2/pipeline/test_assign.py b/tests/v2/pipeline/test_assign.py new file mode 100644 index 00000000..88bb4352 --- /dev/null +++ b/tests/v2/pipeline/test_assign.py @@ -0,0 +1,158 @@ +# tests/v2/pipeline/test_assign.py +from nameparser._lexicon import Lexicon +from nameparser._pipeline._assign import assign +from nameparser._pipeline._classify import classify +from nameparser._pipeline._extract import extract_delimited +from nameparser._pipeline._group import group +from nameparser._pipeline._segment import segment +from nameparser._pipeline._state import ParseState +from nameparser._pipeline._tokenize import tokenize +from nameparser._policy import FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, Policy +from nameparser._types import AmbiguityKind, Role + +_LEX = Lexicon( + titles=frozenset({"dr", "mr", "mrs", "sir"}), + given_name_titles=frozenset({"sir"}), + suffix_acronyms=frozenset({"phd", "md"}), + suffix_words=frozenset({"jr", "iii", "v"}), + particles=frozenset({"de", "la", "van"}), + particles_ambiguous=frozenset({"van"}), + conjunctions=frozenset({"and"}), + maiden_markers=frozenset({"née"}), +) + + +def _assigned(text: str, policy: Policy | None = None) -> ParseState: + state = ParseState(original=text, lexicon=_LEX, + policy=policy or Policy()) + return assign(group(classify(segment(tokenize( + extract_delimited(state)))))) + + +def _by_role(state: ParseState, role: Role) -> str: + return " ".join(t.text for t in state.tokens if t.role is role) + + +def test_given_first_positional() -> None: + out = _assigned("Dr. Juan de la Vega III") + assert _by_role(out, Role.TITLE) == "Dr." + assert _by_role(out, Role.GIVEN) == "Juan" + assert _by_role(out, Role.FAMILY) == "de la Vega" + assert _by_role(out, Role.SUFFIX) == "III" + + +def test_middles() -> None: + out = _assigned("John Quincy Adams Smith") + assert _by_role(out, Role.GIVEN) == "John" + assert _by_role(out, Role.MIDDLE) == "Quincy Adams" + assert _by_role(out, Role.FAMILY) == "Smith" + + +def test_single_token_takes_name_order_head() -> None: + assert _by_role(_assigned("John"), Role.GIVEN) == "John" + out = _assigned("John", Policy(name_order=FAMILY_FIRST)) + assert _by_role(out, Role.FAMILY) == "John" + + +def test_title_only() -> None: + out = _assigned("Dr.") + assert _by_role(out, Role.TITLE) == "Dr." + assert not _by_role(out, Role.GIVEN) + + +def test_leading_ambiguous_particle_reads_as_given_with_ambiguity() -> None: + out = _assigned("Van Johnson") + assert _by_role(out, Role.GIVEN) == "Van" + assert _by_role(out, Role.FAMILY) == "Johnson" + assert any(a.kind is AmbiguityKind.PARTICLE_OR_GIVEN + for a in out.ambiguities) + # unambiguous input: no ambiguity recorded + assert not _assigned("John Smith").ambiguities + + +def test_family_comma() -> None: + out = _assigned("de la Vega, Juan") + assert _by_role(out, Role.FAMILY) == "de la Vega" + assert _by_role(out, Role.GIVEN) == "Juan" + + +def test_family_comma_with_title_and_middle() -> None: + out = _assigned("Smith, Dr. John A.") + assert _by_role(out, Role.TITLE) == "Dr." + assert _by_role(out, Role.GIVEN) == "John" + assert _by_role(out, Role.MIDDLE) == "A." + assert _by_role(out, Role.FAMILY) == "Smith" + + +def test_family_comma_lone_post_comma_title() -> None: + # v1 parity: a lone post-comma title is a TITLE ('Smith, Dr.'), + # not a given name or suffix; post_rules later applies the + # title+lone-family handling + out = _assigned("Smith, Dr.") + assert _by_role(out, Role.TITLE) == "Dr." + assert _by_role(out, Role.FAMILY) == "Smith" + assert not _by_role(out, Role.GIVEN) + + +def test_suffix_comma_and_extra_segments() -> None: + out = _assigned("Smith, John, Jr.") + assert _by_role(out, Role.GIVEN) == "John" + assert _by_role(out, Role.FAMILY) == "Smith" + assert _by_role(out, Role.SUFFIX) == "Jr." + + +def test_trailing_suffix_run_no_comma() -> None: + out = _assigned("John Jack Kennedy PhD MD") + assert _by_role(out, Role.MIDDLE) == "Jack" + assert _by_role(out, Role.FAMILY) == "Kennedy" + assert _by_role(out, Role.SUFFIX) == "PhD MD" + + +def test_initial_veto_keeps_v_in_middle() -> None: + out = _assigned("John V. Smith") + assert _by_role(out, Role.MIDDLE) == "V." + assert _by_role(out, Role.FAMILY) == "Smith" + + +def test_nickname_only_leaves_name_fields_empty() -> None: + out = _assigned("(Jack)") + assert _by_role(out, Role.NICKNAME) == "Jack" + assert not _by_role(out, Role.GIVEN) and not _by_role(out, Role.FAMILY) + + +def test_single_name_with_nickname_goes_to_family() -> None: + out = _assigned("John (Jack)") + assert _by_role(out, Role.FAMILY) == "John" + assert not _by_role(out, Role.GIVEN) + + +def test_family_first_order() -> None: + out = _assigned("Yamada Taro", Policy(name_order=FAMILY_FIRST)) + assert _by_role(out, Role.FAMILY) == "Yamada" + assert _by_role(out, Role.GIVEN) == "Taro" + out2 = _assigned("Yamada Hanako Taro", + Policy(name_order=FAMILY_FIRST)) + assert _by_role(out2, Role.FAMILY) == "Yamada" + assert _by_role(out2, Role.GIVEN) == "Hanako" + assert _by_role(out2, Role.MIDDLE) == "Taro" + + +def test_family_first_given_last_order() -> None: + # Pinned against the actual built pipeline (2026-07-12), not guessed: + # 'Van' is particles_ambiguous and non-leading, so group's prefix + # chain (which is name_order-agnostic -- it lands upstream of assign + # and doesn't consult Policy) absorbs 'Van Anh Thu' into ONE piece + # before assign ever sees it. With only two name pieces left, + # FAMILY_FIRST_GIVEN_LAST's positional rule (family, then given for + # count==2) correctly assigns Nguyen -> FAMILY and the whole merged + # piece -> GIVEN; there is no MIDDLE piece to assign. This is a + # sensible consequence of the current, order-agnostic group stage -- + # not a contract violation of assign -- and is the exact kind of gap + # the Vietnamese-aware locale pack (#270/#272 follow-ups) is meant to + # close by teaching group to suppress the particle chain under this + # name_order. Do not "fix" this by changing assign. + out = _assigned("Nguyen Van Anh Thu", + Policy(name_order=FAMILY_FIRST_GIVEN_LAST)) + assert _by_role(out, Role.FAMILY) == "Nguyen" + assert _by_role(out, Role.GIVEN) == "Van Anh Thu" + assert _by_role(out, Role.MIDDLE) == "" diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index ad6f777f..cb9749f7 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -15,7 +15,8 @@ "_render.py", "_pipeline/_state.py", "_pipeline/__init__.py", "_pipeline/_extract.py", "_pipeline/_tokenize.py", "_pipeline/_vocab.py", "_pipeline/_segment.py", - "_pipeline/_classify.py", "_pipeline/_group.py"} + "_pipeline/_classify.py", "_pipeline/_group.py", + "_pipeline/_assign.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", From 4f3a70766fd05db1b059c7d675e0e5749bff9a13 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 22:43:00 -0700 Subject: [PATCH 076/275] Add post_rules: firstname swap and opt-in patronymic rotations Completes the seven-stage STAGES tuple. Patronymic regexes verified byte-for-byte against nameparser/config/regexes.py's east_slavic/turkic entries -- no mismatch found. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/__init__.py | 7 +- nameparser/_pipeline/_post_rules.py | 98 ++++++++++++++++++++++++++++ tests/v2/pipeline/test_post_rules.py | 83 +++++++++++++++++++++++ tests/v2/test_layering.py | 2 +- 4 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 nameparser/_pipeline/_post_rules.py create mode 100644 tests/v2/pipeline/test_post_rules.py diff --git a/nameparser/_pipeline/__init__.py b/nameparser/_pipeline/__init__.py index 25f8149f..d9a3bfeb 100644 --- a/nameparser/_pipeline/__init__.py +++ b/nameparser/_pipeline/__init__.py @@ -15,13 +15,16 @@ from nameparser._pipeline._classify import classify from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._group import group +from nameparser._pipeline._post_rules import post_rules from nameparser._pipeline._segment import segment from nameparser._pipeline._state import ParseState from nameparser._pipeline._tokenize import tokenize -#: Filled in by later tasks as stages land; the fold is total either way. +#: The full seven-stage fold (spec §6). STAGES: tuple[Callable[[ParseState], ParseState], ...] = ( - extract_delimited, tokenize, segment, classify, group, assign) + extract_delimited, tokenize, segment, classify, group, assign, + post_rules, +) def run(state: ParseState) -> ParseState: diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py new file mode 100644 index 00000000..4edfb393 --- /dev/null +++ b/nameparser/_pipeline/_post_rules.py @@ -0,0 +1,98 @@ +"""Stage: post_rules. + +Consumes: tokens (roles assigned). +Produces: tokens with roles adjusted by the post rules. +Reads: Policy.patronymic_rules; Lexicon.given_name_titles. + +Rules (each a small pure function over the role-bearing tokens): +1. v1 handle_firstnames: when the parse is exactly a title plus ONE + given token (no other roles), and the title is not a given-name + title ('Sir'), that token is a family name -- "Mr. Johnson". +2. EAST_SLAVIC (opt-in): positional GIVEN/MIDDLE/FAMILY each exactly + one token, the FAMILY-position token carries an East Slavic + patronymic ending, and the MIDDLE-position token does NOT (given + + patronymic + patronymic-derived surname like Abramovich must not + rotate) -> rotate: given<-old MIDDLE, middle<-old FAMILY (the + patronymic), family<-old GIVEN (v1 parity, pinned live). +3. TURKIC (opt-in): exactly 1 GIVEN + 2 MIDDLE + 1 FAMILY tokens and + the FAMILY-position token is a standalone Turkic marker -> + given<-first MIDDLE, middle<-(second MIDDLE, marker), family<-old + GIVEN. + +Both rotations fire only on Structure.NO_COMMA (v1 gates them on +`not self._had_comma`): a comma already established the family. +""" +from __future__ import annotations + +import dataclasses +import re + +from nameparser._lexicon import _normalize +from nameparser._pipeline._state import ParseState, Structure, WorkToken +from nameparser._policy import PatronymicRule +from nameparser._types import Role + +# Ported verbatim from v1 (nameparser/config/regexes.py) -- layering +# forbids the config import; keep in sync by hand. +_EAST_SLAVIC = re.compile( + r"(ovich|ovna|evich|evna|ichna|ilyich|kuzmich|lukich|fomich|fokich)$", + re.I) +_EAST_SLAVIC_CYR = re.compile( + r"(ович|овна|евич|евна|ична|ильич|кузьмич|лукич|фомич|фокич)$", + re.I) +_TURKIC = re.compile( + r"^(oglu|oğlu|ogly|ogli|o['’ʻ]g['’ʻ]li" + r"|qizi|qızı|kizi|kyzy|gyzy|uly|uulu)$", re.I) +_TURKIC_CYR = re.compile( + r"^(оглу|оглы|оғлу|ўғли|угли|кызы|гызы|қызы|қизи|улы|ұлы|уулу)$", re.I) + + +def _idx(tokens: list[WorkToken], role: Role) -> list[int]: + return [i for i, t in enumerate(tokens) if t.role is role] + + +def _retag(tokens: list[WorkToken], i: int, role: Role) -> None: + tokens[i] = dataclasses.replace(tokens[i], role=role) + + +def post_rules(state: ParseState) -> ParseState: + tokens = list(state.tokens) + titles = _idx(tokens, Role.TITLE) + givens = _idx(tokens, Role.GIVEN) + middles = _idx(tokens, Role.MIDDLE) + families = _idx(tokens, Role.FAMILY) + others = [t for t in tokens + if t.role in (Role.SUFFIX, Role.NICKNAME, Role.MAIDEN)] + + # rule 1: title + lone given -> family (v1 handle_firstnames) + if titles and givens and not middles and not families and not others: + joined = " ".join(_normalize(tokens[i].text) for i in titles) + if joined not in state.lexicon.given_name_titles: + for i in givens: + _retag(tokens, i, Role.FAMILY) + + # v1 gates both rotations on `not self._had_comma` + if state.structure is not Structure.NO_COMMA: + return dataclasses.replace(state, tokens=tuple(tokens)) + rules = state.policy.patronymic_rules + if PatronymicRule.EAST_SLAVIC in rules and \ + len(givens) == 1 and len(middles) == 1 and len(families) == 1: + tail = tokens[families[0]].text + mid = tokens[middles[0]].text + if (_EAST_SLAVIC.search(tail) or _EAST_SLAVIC_CYR.search(tail)) \ + and not (_EAST_SLAVIC.search(mid) + or _EAST_SLAVIC_CYR.search(mid)): + g, m, f = givens[0], middles[0], families[0] + _retag(tokens, m, Role.GIVEN) + _retag(tokens, f, Role.MIDDLE) + _retag(tokens, g, Role.FAMILY) + if PatronymicRule.TURKIC in rules and \ + len(givens) == 1 and len(middles) == 2 and len(families) == 1: + tail = tokens[families[0]].text + if _TURKIC.match(tail) or _TURKIC_CYR.match(tail): + g, m1, m2, f = givens[0], middles[0], middles[1], families[0] + _retag(tokens, m1, Role.GIVEN) + _retag(tokens, m2, Role.MIDDLE) + _retag(tokens, f, Role.MIDDLE) + _retag(tokens, g, Role.FAMILY) + return dataclasses.replace(state, tokens=tuple(tokens)) diff --git a/tests/v2/pipeline/test_post_rules.py b/tests/v2/pipeline/test_post_rules.py new file mode 100644 index 00000000..12634ca5 --- /dev/null +++ b/tests/v2/pipeline/test_post_rules.py @@ -0,0 +1,83 @@ +from nameparser._lexicon import Lexicon +from nameparser._pipeline import run +from nameparser._pipeline._state import ParseState +from nameparser._policy import PatronymicRule, Policy +from nameparser._types import Role + +_LEX = Lexicon( + titles=frozenset({"mr", "sir"}), + given_name_titles=frozenset({"sir"}), +) + + +def _parsed(text: str, policy: Policy | None = None) -> ParseState: + return run(ParseState(original=text, lexicon=_LEX, + policy=policy or Policy())) + + +def _by_role(state: ParseState, role: Role) -> str: + return " ".join(t.text for t in state.tokens if t.role is role) + + +def test_plain_title_with_single_name_swaps_to_family() -> None: + out = _parsed("Mr. Johnson") + assert _by_role(out, Role.FAMILY) == "Johnson" + assert not _by_role(out, Role.GIVEN) + + +def test_given_name_title_keeps_given() -> None: + out = _parsed("Sir Bob") + assert _by_role(out, Role.GIVEN) == "Bob" + assert not _by_role(out, Role.FAMILY) + + +def test_no_swap_when_more_fields_present() -> None: + out = _parsed("Mr. John Johnson") + assert _by_role(out, Role.GIVEN) == "John" + assert _by_role(out, Role.FAMILY) == "Johnson" + + +_ES = Policy(patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})) +_TK = Policy(patronymic_rules=frozenset({PatronymicRule.TURKIC})) + + +def test_east_slavic_rotation() -> None: + out = _parsed("Сидоров Иван Петрович", _ES) + assert _by_role(out, Role.GIVEN) == "Иван" + assert _by_role(out, Role.MIDDLE) == "Петрович" + assert _by_role(out, Role.FAMILY) == "Сидоров" + + +def test_east_slavic_needs_one_one_one() -> None: + # four tokens: left unchanged (v1 parity) + out = _parsed("Anna Maria Petrova Ivanovna", _ES) + assert _by_role(out, Role.GIVEN) == "Anna" + + +def test_east_slavic_skips_comma_forms() -> None: + # v1 parity: patronymic reorder never fires on comma input -- + # the comma already established the family + out = _parsed("Abramovich, Roman Petrovich", _ES) + assert _by_role(out, Role.FAMILY) == "Abramovich" + assert _by_role(out, Role.GIVEN) == "Roman" + + +def test_east_slavic_skips_when_middle_is_also_patronymic() -> None: + # v1 parity: given + patronymic + patronymic-derived surname + # (Abramovich) must not rotate + out = _parsed("Roman Petrovich Abramovich", _ES) + assert _by_role(out, Role.GIVEN) == "Roman" + assert _by_role(out, Role.MIDDLE) == "Petrovich" + assert _by_role(out, Role.FAMILY) == "Abramovich" + + +def test_east_slavic_off_by_default() -> None: + out = _parsed("Сидоров Иван Петрович") + assert _by_role(out, Role.GIVEN) == "Сидоров" + + +def test_turkic_rotation() -> None: + out = _parsed("Mammadova Aygun Ali kizi", _TK) + assert _by_role(out, Role.GIVEN) == "Aygun" + assert _by_role(out, Role.MIDDLE) == "Ali kizi" + assert _by_role(out, Role.FAMILY) == "Mammadova" diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index cb9749f7..f16838be 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -16,7 +16,7 @@ "_pipeline/_extract.py", "_pipeline/_tokenize.py", "_pipeline/_vocab.py", "_pipeline/_segment.py", "_pipeline/_classify.py", "_pipeline/_group.py", - "_pipeline/_assign.py"} + "_pipeline/_assign.py", "_pipeline/_post_rules.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", From bb80c8d0ba7a5678b2c22a73c320fffe5254628d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 22:57:56 -0700 Subject: [PATCH 077/275] Add assemble: final ParseState to validated ParsedName Not a pipeline stage but the runner's tail: converts the seven-stage fold's ParseState into a public ParsedName, dropping structural marker tokens (e.g. maiden markers) and materializing PendingAmbiguity indices into real Ambiguity objects over the final token tuple. Also adds _pipeline/_assemble.py to test_layering's _MUST_EXIST now that the file exists. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_assemble.py | 33 +++++++++++++++++++ tests/v2/pipeline/test_assemble.py | 52 ++++++++++++++++++++++++++++++ tests/v2/test_layering.py | 3 +- 3 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 nameparser/_pipeline/_assemble.py create mode 100644 tests/v2/pipeline/test_assemble.py diff --git a/nameparser/_pipeline/_assemble.py b/nameparser/_pipeline/_assemble.py new file mode 100644 index 00000000..0eb48975 --- /dev/null +++ b/nameparser/_pipeline/_assemble.py @@ -0,0 +1,33 @@ +"""Not a stage: converts the final ParseState into a public ParsedName. + +Consumes: tokens (all roles set), dropped, ambiguities (by index). +Produces: a validated ParsedName -- the constructor re-checks every +invariant (span order/bounds, ambiguity subset), so a pipeline bug +that would produce an invalid result dies HERE, not in a renderer +three layers away. + +Structural tokens (dropped maiden markers) are omitted, like delimiter +characters. A main-stream token that somehow reaches here with no role +takes Role.GIVEN -- parse is total over str (spec §5a) and must not +raise on content; the fallback is deliberately boring. +""" +from __future__ import annotations + +from nameparser._pipeline._state import ParseState +from nameparser._types import Ambiguity, ParsedName, Role, Token + + +def assemble(state: ParseState) -> ParsedName: + dropped = set(state.dropped) + final: dict[int, Token] = {} + for i, t in enumerate(state.tokens): + if i in dropped: + continue + final[i] = Token(t.text, t.span, t.role or Role.GIVEN, t.tags) + ambiguities = tuple( + Ambiguity(p.kind, p.detail, + tuple(final[i] for i in p.indices if i in final)) + for p in state.ambiguities) + return ParsedName(original=state.original, + tokens=tuple(final.values()), + ambiguities=ambiguities) diff --git a/tests/v2/pipeline/test_assemble.py b/tests/v2/pipeline/test_assemble.py new file mode 100644 index 00000000..d1aaae9a --- /dev/null +++ b/tests/v2/pipeline/test_assemble.py @@ -0,0 +1,52 @@ +from nameparser._lexicon import Lexicon +from nameparser._pipeline import run +from nameparser._pipeline._assemble import assemble +from nameparser._pipeline._state import ParseState +from nameparser._policy import Policy +from nameparser._types import AmbiguityKind, ParsedName + +_LEX = Lexicon( + titles=frozenset({"dr"}), + particles=frozenset({"de", "la", "van"}), + particles_ambiguous=frozenset({"van"}), + suffix_words=frozenset({"iii"}), + maiden_markers=frozenset({"née"}), +) + + +def _parse(text: str) -> ParsedName: + return assemble(run(ParseState(original=text, lexicon=_LEX, + policy=Policy()))) + + +def test_assemble_produces_validated_parsedname() -> None: + pn = _parse("Dr. Juan de la Vega III") + assert pn.title == "Dr." + assert pn.given == "Juan" + assert pn.family == "de la Vega" + assert pn.family_base == "Vega" # particle tags carried over + assert pn.family_particles == "de la" + assert pn.suffix == "III" + assert pn.original == "Dr. Juan de la Vega III" + for t in pn.tokens: + assert t.span is not None + assert t.text == pn.original[t.span.start:t.span.end] + + +def test_assemble_materializes_ambiguities_on_final_tokens() -> None: + pn = _parse("Van Johnson") + assert len(pn.ambiguities) == 1 + amb = pn.ambiguities[0] + assert amb.kind is AmbiguityKind.PARTICLE_OR_GIVEN + assert amb.tokens[0] is pn.tokens[0] + + +def test_assemble_drops_structural_marker_tokens() -> None: + pn = _parse("Jane Smith née Jones") + assert [t.text for t in pn.tokens] == ["Jane", "Smith", "Jones"] + assert pn.maiden == "Jones" + + +def test_empty_parse_is_falsy() -> None: + assert not _parse("") + assert not _parse(" ") diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index f16838be..5c8ed722 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -16,7 +16,8 @@ "_pipeline/_extract.py", "_pipeline/_tokenize.py", "_pipeline/_vocab.py", "_pipeline/_segment.py", "_pipeline/_classify.py", "_pipeline/_group.py", - "_pipeline/_assign.py", "_pipeline/_post_rules.py"} + "_pipeline/_assign.py", "_pipeline/_post_rules.py", + "_pipeline/_assemble.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", From 9bcd0073fda0554d78b817a011c4a036f21a2549 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 23:07:24 -0700 Subject: [PATCH 078/275] Add Parser and parse(): the 2.0 pipeline goes end to end Co-Authored-By: Claude Fable 5 --- nameparser/__init__.py | 2 + nameparser/_parser.py | 73 +++++++++++++++++++++++++++++++++++ tests/v2/test_layering.py | 8 +++- tests/v2/test_parser.py | 80 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 nameparser/_parser.py create mode 100644 tests/v2/test_parser.py diff --git a/nameparser/__init__.py b/nameparser/__init__.py index 5e3adcb2..1b49624e 100644 --- a/nameparser/__init__.py +++ b/nameparser/__init__.py @@ -8,6 +8,7 @@ from nameparser._lexicon import Lexicon from nameparser._locale import Locale +from nameparser._parser import Parser, parse from nameparser._policy import ( FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, @@ -33,4 +34,5 @@ "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET", "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "Locale", + "Parser", "parse", ] diff --git a/nameparser/_parser.py b/nameparser/_parser.py new file mode 100644 index 00000000..f6562b00 --- /dev/null +++ b/nameparser/_parser.py @@ -0,0 +1,73 @@ +"""Parser and the module-level parse() for the 2.0 API. + +Layering: sits on _types/_lexicon/_policy/_locale/_pipeline; never +imports _render or the v1 facade (enforced by tests/v2/test_layering.py). + +_default_parser is THE one sanctioned module-level global (conventions +§8): a functools.cache'd frozen Parser over default config. +""" +from __future__ import annotations + +import functools +from dataclasses import dataclass + +from nameparser._lexicon import Lexicon +from nameparser._pipeline import run +from nameparser._pipeline._assemble import assemble +from nameparser._pipeline._state import ParseState +from nameparser._policy import Policy +from nameparser._types import ParsedName, _guarded_getstate, _guarded_setstate + + +@dataclass(frozen=True, slots=True) +class Parser: + """Immutable, thread-safe, picklable by construction (spec §4): all + validity checking happens at construction; a Parser that constructs + successfully cannot fail at parse time on any str content.""" + + lexicon: Lexicon = None # type: ignore[assignment] # None -> default() + policy: Policy = None # type: ignore[assignment] # None -> Policy() + + # in the class body so @dataclass(slots=True) keeps them + __getstate__ = _guarded_getstate + __setstate__ = _guarded_setstate + + def __post_init__(self) -> None: + if self.lexicon is None: + object.__setattr__(self, "lexicon", Lexicon.default()) + elif not isinstance(self.lexicon, Lexicon): + raise TypeError( + f"lexicon must be a Lexicon or None, got {self.lexicon!r}") + if self.policy is None: + object.__setattr__(self, "policy", Policy()) + elif not isinstance(self.policy, Policy): + raise TypeError( + f"policy must be a Policy or None, got {self.policy!r}") + + def __repr__(self) -> str: + # composes the two bounded component reprs (spec §2 reprs) + return f"Parser({self.lexicon!r}, {self.policy!r})" + + def parse(self, text: str) -> ParsedName: + """Total over str (spec §5a): content never raises; non-str + raises TypeError eagerly, with a decode hint for bytes (#245: + bytes support ended with 1.x).""" + if isinstance(text, bytes): + raise TypeError( + "parse() takes str, not bytes -- decode first, e.g. " + "raw.decode('utf-8')") + if not isinstance(text, str): + raise TypeError(f"parse() takes str, got {text!r}") + state = ParseState(original=text, lexicon=self.lexicon, + policy=self.policy) + return assemble(run(state)) + + +@functools.cache +def _default_parser() -> Parser: + return Parser() + + +def parse(text: str) -> ParsedName: + """Module-level convenience over a lazily created default Parser.""" + return _default_parser().parse(text) diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 5c8ed722..9d847585 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -17,7 +17,7 @@ "_pipeline/_vocab.py", "_pipeline/_segment.py", "_pipeline/_classify.py", "_pipeline/_group.py", "_pipeline/_assign.py", "_pipeline/_post_rules.py", - "_pipeline/_assemble.py"} + "_pipeline/_assemble.py", "_parser.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", @@ -130,6 +130,7 @@ def test_public_exports() -> None: "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET", "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "Locale", + "Parser", "parse", } assert expected <= set(nameparser.__all__) for name in expected: @@ -162,12 +163,15 @@ def test_every_frozen_dataclass_carries_the_pickle_guards() -> None: import nameparser._lexicon import nameparser._locale + import nameparser._parser import nameparser._policy import nameparser._types from nameparser._types import _guarded_getstate, _guarded_setstate + # _pipeline internals (ParseState, WorkToken, ...) are exempt: they + # are never pickled and are not public API. modules = (nameparser._types, nameparser._lexicon, - nameparser._policy, nameparser._locale) + nameparser._policy, nameparser._locale, nameparser._parser) for module in modules: for _, cls in inspect.getmembers(module, inspect.isclass): if cls.__module__ != module.__name__: diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py new file mode 100644 index 00000000..dc31655c --- /dev/null +++ b/tests/v2/test_parser.py @@ -0,0 +1,80 @@ +import pickle + +import pytest + +from nameparser import Lexicon, Parser, Policy, parse +from nameparser._policy import FAMILY_FIRST +from nameparser._types import AmbiguityKind + + +def test_parser_defaults_and_properties() -> None: + p = Parser() + assert p.lexicon == Lexicon.default() + assert p.policy == Policy() + + +def test_parser_rejects_wrong_types_eagerly() -> None: + with pytest.raises(TypeError, match="lexicon"): + Parser(lexicon={"titles": set()}) # type: ignore[arg-type] + with pytest.raises(TypeError, match="policy"): + Parser(policy="strict") # type: ignore[arg-type] + + +def test_parse_end_to_end_with_default_vocabulary() -> None: + pn = parse("Dr. Juan de la Vega III") + assert pn.title == "Dr." + assert pn.given == "Juan" + assert pn.family == "de la Vega" + assert pn.suffix == "III" + assert str(pn) == "Dr. Juan de la Vega III" + + +def test_parse_rejects_non_str_with_decode_hint() -> None: + with pytest.raises(TypeError, match="decode"): + parse(b"John Smith") # type: ignore[arg-type] + with pytest.raises(TypeError, match="str"): + parse(None) # type: ignore[arg-type] + + +def test_degenerate_inputs_are_total() -> None: + # spec §5a table + assert not parse("") + assert not parse(" ") + assert parse("").original == "" + single = parse("John") + assert single.given == "John" + family_first = Parser(policy=Policy(name_order=FAMILY_FIRST)) + assert family_first.parse("Yamada").family == "Yamada" + title_only = parse("Dr.") + assert title_only.title == "Dr." and not title_only.given + unbalanced = parse('Jon "Nick Smith') + kinds = {a.kind for a in unbalanced.ambiguities} + assert AmbiguityKind.UNBALANCED_DELIMITER in kinds + assert '"Nick' in [t.text for t in unbalanced.tokens] # literal + + +def test_parser_is_picklable_and_frozen() -> None: + p = Parser(policy=Policy(name_order=FAMILY_FIRST)) + loaded = pickle.loads(pickle.dumps(p)) + assert loaded == p + assert loaded.parse("Yamada Taro").family == "Yamada" + with pytest.raises(AttributeError): + p.policy = Policy() # type: ignore[misc] + + +def test_parser_repr_composes_component_reprs() -> None: + assert repr(Parser()) == "Parser(Lexicon(default), Policy())" + p = Parser(policy=Policy(name_order=FAMILY_FIRST)) + assert repr(p) == "Parser(Lexicon(default), Policy(name_order=FAMILY_FIRST))" + + +def test_parsedname_repr_includes_ambiguities_line() -> None: + pn = parse("Van Johnson") + r = repr(pn) + assert "given: 'Van'" in r + assert "ambiguities:" in r and "particle-or-given" in r + + +def test_module_parse_reuses_the_default_parser() -> None: + import nameparser._parser as parser_mod + assert parser_mod._default_parser() is parser_mod._default_parser() From 8ad0247fc3a02d0a03c55f030015163f73d0e438 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 23:16:34 -0700 Subject: [PATCH 079/275] Add parser_for() with locale-identified errors and matches() Co-Authored-By: Claude Fable 5 --- nameparser/__init__.py | 4 +-- nameparser/_parser.py | 46 ++++++++++++++++++++++++++++++- nameparser/_types.py | 15 +++++++++++ tests/v2/test_layering.py | 2 +- tests/v2/test_parser.py | 57 +++++++++++++++++++++++++++++++++++++-- 5 files changed, 118 insertions(+), 6 deletions(-) diff --git a/nameparser/__init__.py b/nameparser/__init__.py index 1b49624e..fb9575fc 100644 --- a/nameparser/__init__.py +++ b/nameparser/__init__.py @@ -8,7 +8,7 @@ from nameparser._lexicon import Lexicon from nameparser._locale import Locale -from nameparser._parser import Parser, parse +from nameparser._parser import Parser, parse, parser_for from nameparser._policy import ( FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, @@ -34,5 +34,5 @@ "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET", "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "Locale", - "Parser", "parse", + "Parser", "parse", "parser_for", ] diff --git a/nameparser/_parser.py b/nameparser/_parser.py index f6562b00..b94039ca 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -8,14 +8,17 @@ """ from __future__ import annotations +import dataclasses import functools +import warnings from dataclasses import dataclass from nameparser._lexicon import Lexicon +from nameparser._locale import Locale from nameparser._pipeline import run from nameparser._pipeline._assemble import assemble from nameparser._pipeline._state import ParseState -from nameparser._policy import Policy +from nameparser._policy import UNSET, Policy, PolicyPatch, apply_patch from nameparser._types import ParsedName, _guarded_getstate, _guarded_setstate @@ -71,3 +74,44 @@ def _default_parser() -> Parser: def parse(text: str) -> ParsedName: """Module-level convenience over a lazily created default Parser.""" return _default_parser().parse(text) + + +def parser_for(*locales: Locale, base: Parser | None = None) -> Parser: + """Lexicon fragments unioned left-to-right onto base's; policy + patches applied left-to-right (later wins; set-valued fields union + per the patch metadata). Validation errors raised while applying a + pack are wrapped with that pack's identity (spec §4 amendment) -- + PolicyPatch validates lazily, so with stacked packs the raw error + would otherwise point at nothing. Two packs setting the same SCALAR + field is a declared conflict: UserWarning, later wins.""" + if base is not None and not isinstance(base, Parser): + raise TypeError(f"base must be a Parser or None, got {base!r}") + for loc in locales: + if not isinstance(loc, Locale): + raise TypeError(f"parser_for() takes Locale packs, got {loc!r}") + lexicon = base.lexicon if base is not None else Lexicon.default() + policy = base.policy if base is not None else Policy() + scalar_setters: dict[str, str] = {} + for loc in locales: + for f in dataclasses.fields(PolicyPatch): + if f.metadata.get("compose") == "union": + continue + if getattr(loc.policy, f.name) is UNSET: + continue + if f.name in scalar_setters and scalar_setters[f.name] != loc.code: + warnings.warn( + f"locale {loc.code!r} overrides scalar policy field " + f"{f.name!r} already set by locale " + f"{scalar_setters[f.name]!r}; later wins", + UserWarning, stacklevel=2) + scalar_setters[f.name] = loc.code + try: + lexicon = lexicon | loc.lexicon + policy = apply_patch(policy, loc.policy) + except (TypeError, ValueError) as exc: + # safe: every raise in the apply path (Policy.__post_init__, + # Lexicon.__or__, apply_patch) is a PLAIN TypeError/ValueError -- + # a subclass with extra mandatory args would break this rewrap + raise type(exc)( + f"while applying locale {loc.code!r}: {exc}") from exc + return Parser(lexicon=lexicon, policy=policy) diff --git a/nameparser/_types.py b/nameparser/_types.py index 58829195..af6789a3 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -21,6 +21,7 @@ if TYPE_CHECKING: from nameparser._lexicon import Lexicon + from nameparser._parser import Parser class Role(Enum): @@ -422,6 +423,20 @@ def comparison_key(self) -> tuple[str, ...]: """ return tuple(self._text_for(role).casefold() for role in Role) + def matches(self, other: str | ParsedName, *, + parser: Parser | None = None) -> bool: + """Component-wise case-insensitive comparison (the semantic + layer; __eq__ stays strict). A str argument is parsed with + `parser`, or the default parser when None.""" + if isinstance(other, str): + import nameparser._parser as _parser + active = parser if parser is not None else _parser._default_parser() + other = active.parse(other) + if not isinstance(other, ParsedName): + raise TypeError( + f"matches() takes a str or ParsedName, got {other!r}") + return self.comparison_key() == other.comparison_key() + # -- rendering delegates ---------------------------------------------- # One-line delegation to nameparser._render (core spec §5b): parsing # code physically cannot import formatting logic, so these import at diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 9d847585..5b852188 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -130,7 +130,7 @@ def test_public_exports() -> None: "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET", "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "Locale", - "Parser", "parse", + "Parser", "parse", "parser_for", } assert expected <= set(nameparser.__all__) for name in expected: diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index dc31655c..d82aa674 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -2,8 +2,8 @@ import pytest -from nameparser import Lexicon, Parser, Policy, parse -from nameparser._policy import FAMILY_FIRST +from nameparser import Lexicon, Locale, Parser, Policy, PolicyPatch, parse, parser_for +from nameparser._policy import FAMILY_FIRST, PatronymicRule from nameparser._types import AmbiguityKind @@ -78,3 +78,56 @@ def test_parsedname_repr_includes_ambiguities_line() -> None: def test_module_parse_reuses_the_default_parser() -> None: import nameparser._parser as parser_mod assert parser_mod._default_parser() is parser_mod._default_parser() + + +def test_parser_for_stacks_locales() -> None: + ru = Locale(code="ru", + lexicon=Lexicon.empty().add(titles={"г-н"}), + policy=PolicyPatch(patronymic_rules=frozenset( + {PatronymicRule.EAST_SLAVIC}))) + p = parser_for(ru) + assert PatronymicRule.EAST_SLAVIC in p.policy.patronymic_rules + pn = p.parse("г-н Сидоров Иван Петрович") + assert pn.title == "г-н" + assert pn.given == "Иван" + assert pn.family == "Сидоров" + + +def test_parser_for_rejects_non_locales() -> None: + with pytest.raises(TypeError, match="Locale"): + parser_for("ru") # type: ignore[arg-type] + + +def test_parser_for_wraps_pack_errors_with_identity() -> None: + # PolicyPatch validates lazily (by design), so an invalid value sits + # latent in a perfectly constructible Locale until apply time + bad = Locale(code="xx", lexicon=Lexicon.empty(), + policy=PolicyPatch(name_order=(1, 2, 3))) # type: ignore[arg-type] + with pytest.raises(ValueError, match="while applying locale 'xx'"): + parser_for(bad) + + +def test_parser_for_warns_on_scalar_conflict() -> None: + a = Locale(code="aa", lexicon=Lexicon.empty(), + policy=PolicyPatch(strip_emoji=False)) + b = Locale(code="bb", lexicon=Lexicon.empty(), + policy=PolicyPatch(strip_emoji=True)) + with pytest.warns(UserWarning, match="strip_emoji"): + p = parser_for(a, b) + assert p.policy.strip_emoji is True # later wins + + +def test_matches_component_wise_case_insensitive() -> None: + pn = parse("John Smith") + assert pn.matches("JOHN SMITH") + assert pn.matches(parse("john smith")) + assert not pn.matches("John Smythe") + with pytest.raises(TypeError, match="str or ParsedName"): + pn.matches(42) # type: ignore[arg-type] + + +def test_matches_accepts_explicit_parser() -> None: + family_first = Parser(policy=Policy(name_order=FAMILY_FIRST)) + pn = family_first.parse("Yamada Taro") + assert pn.matches("Yamada Taro", parser=family_first) + assert not pn.matches("Yamada Taro") # default parser reads given-first From c7e3fb011b151a3515da393ac73448daaa2395a8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 23:25:39 -0700 Subject: [PATCH 080/275] Fix the shared case-table format and seed it with the pinned corpus Adds tests/v2/cases.py (Case dataclass + CASES tuple) and tests/v2/test_cases.py (the core runner), seeded with the v1 parity battery pinned in the plan header plus three carried-forward rows verified live against v1.4.0: - suffix_stays_suffix / suffix_stays_suffix_title: v1 routes a lone trailing recognized suffix (PhD) to the family/last field; v2 keeps it in suffix. Classified fix(suffix-routing). - family_comma_lone_title: v1 puts the pre-comma piece in first when it's followed only by a title; v2 treats pre-comma as definitionally family. Classified fix(comma-family). All 35 case rows pass against the pipeline as built through Task 11 with no corrections needed to the plan's predicted values; the East Slavic/Turkic policy rows were cross-checked against v1's equivalent patronymic_name_order opt-in and confirmed as true parity. Co-Authored-By: Claude Fable 5 --- tests/v2/cases.py | 125 +++++++++++++++++++++++++++++++++++++++++ tests/v2/test_cases.py | 21 +++++++ 2 files changed, 146 insertions(+) create mode 100644 tests/v2/cases.py create mode 100644 tests/v2/test_cases.py diff --git a/tests/v2/cases.py b/tests/v2/cases.py new file mode 100644 index 00000000..01579313 --- /dev/null +++ b/tests/v2/cases.py @@ -0,0 +1,125 @@ +"""THE shared behavior case table (core spec §7.2). + +Format is fixed here, in the first pipeline PR, and never per-PR: +one Case per input, expected values for exactly the non-empty fields, +optional Policy/Locale context, and a mandatory classification -- +"parity" (matches v1.4.0, pinned live 2026-07-12) or "fix(#N)" / +"fix()" (an intentional 2.0 behavior change, annotated with its +issue or a design-decision slug). No silent expectation edits: +changing a row means changing its classification. + +The v1 suite's full corpus is extracted into this table by the +migration plan (facade runner consumes the same rows); this file seeds +it with the pinned battery. +""" +from __future__ import annotations + +from dataclasses import dataclass + +from nameparser import Policy +from nameparser._policy import PatronymicRule + + +@dataclass(frozen=True) +class Case: + id: str + text: str + expect: dict[str, str] # field -> value; absent fields == "" + policy: Policy | None = None + classification: str = "parity" + ambiguities: tuple[str, ...] = () # expected AmbiguityKind values + notes: str = "" + + +_ES = Policy(patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})) +_TK = Policy(patronymic_rules=frozenset({PatronymicRule.TURKIC})) + +CASES: tuple[Case, ...] = ( + Case("plain", "John Smith", {"given": "John", "family": "Smith"}), + Case("family_comma", "Smith, John", + {"given": "John", "family": "Smith"}), + Case("suffix_comma", "John Smith, PhD", + {"given": "John", "family": "Smith", "suffix": "PhD"}), + Case("delavega", "Dr. Juan de la Vega III", + {"title": "Dr.", "given": "Juan", "family": "de la Vega", + "suffix": "III"}), + Case("prefix_chain_to_end", "Juan de la Vega Martinez", + {"given": "Juan", "family": "de la Vega Martinez"}), + Case("van_johnson", "Van Johnson", + {"given": "Van", "family": "Johnson"}, + ambiguities=("particle-or-given",), + notes="v2 surfaces #121's irreducible ambiguity"), + Case("family_comma_particles", "de la Vega, Juan", + {"given": "Juan", "family": "de la Vega"}), + Case("nickname_quotes", 'John "Jack" Kennedy', + {"given": "John", "family": "Kennedy", "nickname": "Jack"}), + Case("nickname_parens", "John (Jack) Kennedy", + {"given": "John", "family": "Kennedy", "nickname": "Jack"}), + Case("sir_bob", "Sir Bob Andrew Dole", + {"title": "Sir", "given": "Bob", "middle": "Andrew", + "family": "Dole"}), + Case("long_title", "President of the United States Barack Obama", + {"title": "President of the United States", + "given": "Barack", "family": "Obama"}), + Case("secretary", "The Secretary of State Hillary Clinton", + {"title": "The Secretary of State", "given": "Hillary", + "family": "Clinton"}), + Case("comma_middle_initial", "Doe, John A.", + {"given": "John", "middle": "A.", "family": "Doe"}), + Case("single", "John", {"given": "John"}), + Case("title_only", "Dr.", {"title": "Dr."}), + Case("double_comma_suffix", "Smith, John, Jr.", + {"given": "John", "family": "Smith", "suffix": "Jr."}), + Case("bound_given_two", "abdul rahman", + {"given": "abdul", "family": "rahman"}), + Case("bound_given_three", "abdul rahman al-said", + {"given": "abdul rahman", "family": "al-said"}), + Case("mr_and_mrs", "Mr. and Mrs. John Smith", + {"title": "Mr. and Mrs.", "given": "John", "family": "Smith"}), + Case("roman_suffix", "John Smith V", + {"given": "John", "family": "Smith", "suffix": "V"}), + Case("initial_not_suffix", "John V. Smith", + {"given": "John", "middle": "V.", "family": "Smith"}), + Case("lenient_after_comma", "John Ingram, V", + {"given": "John", "family": "Ingram", "suffix": "V"}), + Case("comma_then_title", "Smith, Dr. John", + {"title": "Dr.", "given": "John", "family": "Smith"}), + Case("nickname_single_name", "John (Jack)", + {"family": "John", "nickname": "Jack"}), + Case("nickname_only", "(Jack)", {"nickname": "Jack"}), + Case("suffix_run", "John Jack Kennedy PhD MD", + {"given": "John", "middle": "Jack", "family": "Kennedy", + "suffix": "PhD, MD"}), + Case("maiden_marker", "Jane Smith née Jones", + {"given": "Jane", "family": "Smith", "maiden": "Jones"}, + classification="fix(#274)", + notes="v1 mangles to middle='Smith née'"), + Case("east_slavic", "Сидоров Иван Петрович", + {"given": "Иван", "middle": "Петрович", "family": "Сидоров"}, + policy=_ES), + Case("turkic", "Mammadova Aygun Ali kizi", + {"given": "Aygun", "middle": "Ali kizi", "family": "Mammadova"}, + policy=_TK), + Case("empty", "", {}), + Case("whitespace", " ", {}), + Case("unbalanced_quote", 'Jon "Nick Smith', + {"given": "Jon", "middle": '"Nick', "family": "Smith"}, + ambiguities=("unbalanced-delimiter",), + notes="quote char stays literal (spec §5a)"), + Case("suffix_stays_suffix", "Johnson PhD", + {"given": "Johnson", "suffix": "PhD"}, + classification="fix(suffix-routing)", + notes="v1 routes a lone trailing suffix to family " + "(first=Johnson last=PhD); v2 keeps recognized " + "suffixes in suffix"), + Case("suffix_stays_suffix_title", "Mr. Johnson PhD", + {"title": "Mr.", "given": "Johnson", "suffix": "PhD"}, + classification="fix(suffix-routing)", + notes="v1 routes a lone trailing suffix to family " + "(title=Mr. first=Johnson last=PhD); v2 keeps " + "recognized suffixes in suffix"), + Case("family_comma_lone_title", "Smith, Dr.", + {"title": "Dr.", "family": "Smith"}, + classification="fix(comma-family)", + notes="pre-comma is definitionally family; v1 put it in first"), +) diff --git a/tests/v2/test_cases.py b/tests/v2/test_cases.py new file mode 100644 index 00000000..aea0e19a --- /dev/null +++ b/tests/v2/test_cases.py @@ -0,0 +1,21 @@ +"""Core runner over the shared case table (spec §7.2). The facade +runner (migration plan) consumes the same CASES.""" +import pytest + +from nameparser import Parser + +from .cases import CASES, Case + +_FIELDS = ("title", "given", "middle", "family", "suffix", "nickname", + "maiden") + + +@pytest.mark.parametrize("case", CASES, ids=lambda c: c.id) +def test_case(case: Case) -> None: + parser = Parser(policy=case.policy) if case.policy else Parser() + pn = parser.parse(case.text) + actual = {f: getattr(pn, f) for f in _FIELDS if getattr(pn, f)} + assert actual == case.expect, f"{case.text!r} ({case.classification})" + kinds = sorted(a.kind.value for a in pn.ambiguities) + assert kinds == sorted(case.ambiguities), \ + f"{case.text!r} ({case.classification})" From 93478485a6ffdef8897c46a0fc039749c5460ba8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 23:33:48 -0700 Subject: [PATCH 081/275] Add property, contract, and benchmark test layers Co-Authored-By: Claude Fable 5 --- pyproject.toml | 1 + tests/v2/test_benchmark.py | 15 ++++++++ tests/v2/test_contracts.py | 67 ++++++++++++++++++++++++++++++++++ tests/v2/test_properties.py | 52 +++++++++++++++++++++++++++ uv.lock | 72 +++++++++++++++++++++++++++++++++++++ 5 files changed, 207 insertions(+) create mode 100644 tests/v2/test_benchmark.py create mode 100644 tests/v2/test_contracts.py create mode 100644 tests/v2/test_properties.py diff --git a/pyproject.toml b/pyproject.toml index c56d715a..6a86e4b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ dev = [ "ruff (>=0.15)", "pytest-timeout>=2.4.0", "pytest-cov>=6", + "hypothesis", ] [tool.mypy] diff --git a/tests/v2/test_benchmark.py b/tests/v2/test_benchmark.py new file mode 100644 index 00000000..f3716991 --- /dev/null +++ b/tests/v2/test_benchmark.py @@ -0,0 +1,15 @@ +"""Perf smoke (core spec §7 tail): parse cost stays v1-comparable +(microseconds per name). Deliberately generous bound -- guards against +order-of-magnitude regressions, does not gate normal variance.""" +import time + +from nameparser import parse + + +def test_parse_thousand_names_under_a_second() -> None: + parse("warm up the default parser cache") + start = time.perf_counter() + for i in range(1000): + parse(f"Dr. Juan{i} de la Vega III") + elapsed = time.perf_counter() - start + assert elapsed < 1.0, f"1000 parses took {elapsed:.2f}s" diff --git a/tests/v2/test_contracts.py b/tests/v2/test_contracts.py new file mode 100644 index 00000000..78f284d5 --- /dev/null +++ b/tests/v2/test_contracts.py @@ -0,0 +1,67 @@ +"""Stable-string contract tests (core spec §7.4): every enum member and +stable tag has a canonical triggering input, parametrized by iterating +the registries -- a new member without an entry here fails loudly.""" +import pytest + +from nameparser import Parser, Policy, parse +from nameparser._policy import PatronymicRule +from nameparser._types import STABLE_TAGS, AmbiguityKind + +_AMBIGUITY_TRIGGERS: dict[AmbiguityKind, str | None] = { + AmbiguityKind.PARTICLE_OR_GIVEN: "Van Johnson", + AmbiguityKind.UNBALANCED_DELIMITER: 'Jon "Nick Smith', + AmbiguityKind.COMMA_STRUCTURE: "Smith, John, Extra, Jr.", + # no emitter yet -- arrives with locale-pack order detection (2.x) + AmbiguityKind.ORDER: None, + # no emitter yet -- arrives with suffix/nickname refinement (2.x) + AmbiguityKind.SUFFIX_OR_NICKNAME: None, +} + + +@pytest.mark.parametrize("kind", [ + pytest.param(k, marks=pytest.mark.xfail( + strict=True, reason=f"{k.value}: emitter not yet implemented")) + if k in _AMBIGUITY_TRIGGERS and _AMBIGUITY_TRIGGERS[k] is None else k + for k in AmbiguityKind +]) +def test_every_ambiguity_kind_has_a_registered_trigger( + kind: AmbiguityKind) -> None: + assert kind in _AMBIGUITY_TRIGGERS, ( + f"new AmbiguityKind {kind.value!r} needs a canonical trigger " + f"(or an explicit None with its planned emitter)") + trigger = _AMBIGUITY_TRIGGERS[kind] + assert trigger is not None # None triggers are strict-xfail marked + kinds = {a.kind for a in parse(trigger).ambiguities} + assert kind in kinds + + +_PATRONYMIC_TRIGGERS: dict[PatronymicRule, tuple[str, str]] = { + # rule -> (input, expected given) + PatronymicRule.EAST_SLAVIC: ("Сидоров Иван Петрович", "Иван"), + PatronymicRule.TURKIC: ("Mammadova Aygun Ali kizi", "Aygun"), +} + + +@pytest.mark.parametrize("rule", list(PatronymicRule)) +def test_every_patronymic_rule_has_a_trigger(rule: PatronymicRule) -> None: + assert rule in _PATRONYMIC_TRIGGERS + text, expected_given = _PATRONYMIC_TRIGGERS[rule] + p = Parser(policy=Policy(patronymic_rules=frozenset({rule}))) + assert p.parse(text).given == expected_given + + +_TAG_TRIGGERS: dict[str, tuple[str, str]] = { + # tag -> (input, token text carrying the tag) + "particle": ("Juan de la Vega", "de"), + "conjunction": ("Mr. and Mrs. John Smith", "and"), + "initial": ("John A. Smith", "A."), +} + + +@pytest.mark.parametrize("tag", sorted(STABLE_TAGS)) +def test_every_stable_tag_has_a_trigger(tag: str) -> None: + assert tag in _TAG_TRIGGERS + text, token_text = _TAG_TRIGGERS[tag] + pn = parse(text) + tagged = next(t for t in pn.tokens if t.text == token_text) + assert tag in tagged.tags diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py new file mode 100644 index 00000000..157116d1 --- /dev/null +++ b/tests/v2/test_properties.py @@ -0,0 +1,52 @@ +"""Property layer (core spec §7.3). Hypothesis is a dev dependency only. + +The alphabet is punctuation-heavy on purpose: plain st.text() spreads +over all of Unicode, so commas, quotes, and delimiters almost never +appear and the interesting planes go unexercised. derandomize=True +keeps runs reproducible on shared CI runners -- this layer guards +against regressions; exploratory fuzzing happened during review. +""" +from hypothesis import given, settings +from hypothesis import strategies as st + +from nameparser import parse + +_ALPHABET = st.sampled_from( + 'abcdefgh ABC .,،，\'"()«»‏‏\U0001f600éñßЖ-') + + +@given(st.text(alphabet=_ALPHABET, max_size=200)) +@settings(max_examples=300, deadline=None, derandomize=True) +def test_parse_never_raises_on_str(text: str) -> None: + parse(text) + + +@given(st.text(alphabet=_ALPHABET, max_size=200)) +@settings(max_examples=300, deadline=None, derandomize=True) +def test_provenance_for_parser_produced_names(text: str) -> None: + pn = parse(text) + for t in pn.tokens: + assert t.span is not None + assert t.text == pn.original[t.span.start:t.span.end] + + +@given(st.text(alphabet=_ALPHABET, max_size=100)) +@settings(max_examples=200, deadline=None, derandomize=True) +def test_capitalized_idempotent(text: str) -> None: + once = parse(text).capitalized() + assert once.capitalized() == once + + +@given(st.text(alphabet=_ALPHABET, max_size=100)) +@settings(max_examples=200, deadline=None, derandomize=True) +def test_render_reparse_reaches_fixpoint(text: str) -> None: + # render/reparse legitimately takes several rounds to stabilize on + # comma-heavy input (each round can re-segment); the invariant is + # BOUNDED CONVERGENCE, not one-step idempotence + s = str(parse(text)) + for _ in range(10): + nxt = str(parse(s)) + if nxt == s: + break + s = nxt + assert str(parse(s)) == s, f"no fixpoint within 10 rounds: {s!r}" diff --git a/uv.lock b/uv.lock index fe847877..7ac65e14 100644 --- a/uv.lock +++ b/uv.lock @@ -321,6 +321,67 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/b2/50e9b292b5cac13e9e81272c7171301abc753a60460d21505b606e15cf21/furo-2025.12.19-py3-none-any.whl", hash = "sha256:bb0ead5309f9500130665a26bee87693c41ce4dbdff864dbfb6b0dae4673d24f", size = 339262, upload-time = "2025-12-19T17:34:38.905Z" }, ] +[[package]] +name = "hypothesis" +version = "6.156.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/20/83/8dbe89bdb8c6f25a7a52e7898af6d82fe35dfef08e5c702f6e33231ce6c6/hypothesis-6.156.6.tar.gz", hash = "sha256:96de02faefa3ce079873541da96f42595583bb001e8e4219294ed7d4501cc4cc", size = 476304, upload-time = "2026-07-10T20:56:49.96Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/dc/0c2a851f06c91d5ac9ef0f3b9615efc1ed650411d2eee23b6334f491c85e/hypothesis-6.156.6-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:caf6a93d011c10972da111c38ceb34ced20feaa8581e2b350c0655b022e27875", size = 747998, upload-time = "2026-07-10T20:56:16.311Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f8/59203ca978ab51595d12d6bc7e7a63300d7373431ab42ca3f1742e45db68/hypothesis-6.156.6-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:07f2bc9df1aeba80e12029c1618e2ee54abc440068c305d7075ffd6b85251843", size = 743073, upload-time = "2026-07-10T20:55:36.825Z" }, + { url = "https://files.pythonhosted.org/packages/68/d8/86a0023740434098d1b187a62bd5f99b198f098fb43e7fc58342283a8270/hypothesis-6.156.6-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7baca17f4803ad4aa151732326f3990baf54c3127df44aa872ac5bdf8a98a9a6", size = 1070169, upload-time = "2026-07-10T20:55:49.47Z" }, + { url = "https://files.pythonhosted.org/packages/9b/82/673453915fd0c67673f35a4876ba88f48c621335f293f3537d77b27d4286/hypothesis-6.156.6-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8083806645f84243aade727f4978185caaa0b7190af4318673999ee15fdbf424", size = 1121760, upload-time = "2026-07-10T20:55:53.502Z" }, + { url = "https://files.pythonhosted.org/packages/8a/c3/3a5557f52912f2fecc6ed59642dcf80dd8e89d0d9664502b68e23d66bf3d/hypothesis-6.156.6-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a922eedcd8618f9c2e17b79fa7b3f3f0b2df34e201958611cc3f0f46cca33c10", size = 1111440, upload-time = "2026-07-10T20:55:43.054Z" }, + { url = "https://files.pythonhosted.org/packages/38/a6/ae636d4ca7f996a1ccb4b3d5997d949f1718fba52b01559b3ab53b237b3f/hypothesis-6.156.6-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:5291bd33c4704d274d7c214d5c200e77f372a06644f5cbbe96dcbe53cb2fbf10", size = 1244944, upload-time = "2026-07-10T20:55:56.109Z" }, + { url = "https://files.pythonhosted.org/packages/1e/79/c425d22d734be0268ca60d120c6296299e4220a1783cb1a4cc76232807bb/hypothesis-6.156.6-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:55f3ec50161b4a95bae63bff2b5166e45935b493013d3be30ede279bf6192318", size = 1288808, upload-time = "2026-07-10T20:56:06.249Z" }, + { url = "https://files.pythonhosted.org/packages/c5/3a/cc9f479d22cbdd36ddfc55a978378eddadd183b09339ebdb81be33bb18e7/hypothesis-6.156.6-cp310-abi3-win32.whl", hash = "sha256:e96570ca5cdd9a5f2ff9e80a6fb2fd5420ebf33b833d7de5b09b6ebb26a3eb6c", size = 634868, upload-time = "2026-07-10T20:55:37.959Z" }, + { url = "https://files.pythonhosted.org/packages/d6/89/2008d287289841a936456cb13443ca89d88da6e4527d611d482e9544164d/hypothesis-6.156.6-cp310-abi3-win_amd64.whl", hash = "sha256:32710718c22fe8c5571464e898bb87d282837b02617d6ad68130abf7cb4843cb", size = 640382, upload-time = "2026-07-10T20:55:30.634Z" }, + { url = "https://files.pythonhosted.org/packages/13/64/e4a0796190d8089e85f06731e21fdddd7e8edd3a4e562101527a048e21c4/hypothesis-6.156.6-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:5baec7943a14d106e982121dd4f74cfc5ef45e37c17f94fe49338d3d1377f38c", size = 748988, upload-time = "2026-07-10T20:56:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a2/4a789b286cd2cced31992e1f683036b51dd6909b934ea007ffb43aa3a32f/hypothesis-6.156.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b5f519905ddeb10e23b8ba2c254541a5b1a8f146fe0551be94d972f4a77226f4", size = 743754, upload-time = "2026-07-10T20:56:09.113Z" }, + { url = "https://files.pythonhosted.org/packages/d2/f7/3dd36c1c03d24ae3ffc3c5b0eca8cc4ae90c07abc320f76509eceb37019a/hypothesis-6.156.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c108655960b58ded3ca71b2dc5c69fb2ba7e9c723aeb6106facec3892d09087", size = 1070732, upload-time = "2026-07-10T20:56:28.738Z" }, + { url = "https://files.pythonhosted.org/packages/57/51/befc4b816b471078034a875eb1ef69e0411ab84bcce582b4be173258785a/hypothesis-6.156.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c8d37bebb6924729bc0bbf5852689df568842948abe4d93dd0ad3377adf76fa", size = 1121988, upload-time = "2026-07-10T20:56:07.676Z" }, + { url = "https://files.pythonhosted.org/packages/05/b7/a796f5e3e4b7cb911ff346008d49720296d1f4073490b8bc1cce6b3fbb07/hypothesis-6.156.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:74137bef6d502305c3648b2ed1a9bb4bc05fb1025e96b30a2c092204c40fe097", size = 1245596, upload-time = "2026-07-10T20:55:50.662Z" }, + { url = "https://files.pythonhosted.org/packages/37/65/849c4cba44a6f6cc888fd931124429b24180234ccc4883abab8cad5fcfcb/hypothesis-6.156.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5e0afdf79cceed20fcf0a9fb80d4064a9b2b53d4d4eecbac0e21208a13f5a31b", size = 1289172, upload-time = "2026-07-10T20:55:58.915Z" }, + { url = "https://files.pythonhosted.org/packages/13/03/7106a110df29eb631d66776e8aa8128f82f04a9dd2b6b22b612e6025e3a2/hypothesis-6.156.6-cp311-cp311-win_amd64.whl", hash = "sha256:84dc89caaf741a02f904ca7bd02b1af99650c75552868162290208aeecb70858", size = 640222, upload-time = "2026-07-10T20:56:10.396Z" }, + { url = "https://files.pythonhosted.org/packages/8c/45/9f009005b9c796f4a40424484ac7e70847bc088456fd940a937f96bb4b6d/hypothesis-6.156.6-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a2a728b514fceb81e3f0464508911d5220fd74dadc3270f859427a686b60c4cf", size = 748844, upload-time = "2026-07-10T20:56:38.036Z" }, + { url = "https://files.pythonhosted.org/packages/02/2f/4d852bb8a9c73a68b18eca9b5b085285282122166e158f4d2a477639bfee/hypothesis-6.156.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7489b9a8f9df8227edd6c7cd8b9ccfab2483bab24da6a474c175973ca2294f58", size = 741936, upload-time = "2026-07-10T20:55:27.539Z" }, + { url = "https://files.pythonhosted.org/packages/74/89/b9968070ae042f9bf3149bb6ba6399d5f28f452e0fb7f638cafc69ff0b9a/hypothesis-6.156.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42760873d6db1069d6edbaa355a61b9078a9950259efcfc72fc695741d7db7cd", size = 1069749, upload-time = "2026-07-10T20:56:43.017Z" }, + { url = "https://files.pythonhosted.org/packages/00/a9/753806f5292b40aeab1d269e408e3a7e85be3c0d88828fb78ab4a34d6626/hypothesis-6.156.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4e66aaa7385538a5d617174d47c198ee807f06de99e282a67c6cb724c69340d", size = 1120983, upload-time = "2026-07-10T20:56:25.424Z" }, + { url = "https://files.pythonhosted.org/packages/85/88/8386d064d680be27e936eba94f1448bc93ef6fa05473ee5034139f1c4284/hypothesis-6.156.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:08796b674c0b31a5dd4119b2173823390055921588d13eb77324e861b00fd7f8", size = 1243911, upload-time = "2026-07-10T20:55:54.799Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8c/7524c1e5279e7728eb47c99f2357cbc5f08ae92e9bce49bf50118b53f9c9/hypothesis-6.156.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4ca8cc26ea2d31d22cf7710e92951cfaa921f0f8aa1b6db33a5176335f583a4f", size = 1287806, upload-time = "2026-07-10T20:56:02.176Z" }, + { url = "https://files.pythonhosted.org/packages/5a/b3/c347ad913e1c5f2988956fe17826c0400b4ce470b973e6c248e97b6a0acf/hypothesis-6.156.6-cp312-cp312-win_amd64.whl", hash = "sha256:c3363d3fb8015594636689572510bb6090602d8e8e838a5693c2d52d3b5b09d8", size = 637679, upload-time = "2026-07-10T20:55:39.056Z" }, + { url = "https://files.pythonhosted.org/packages/70/5d/9583fe153573523dac27226c89e041a86ad4aeeae08c868160cbb93d39d2/hypothesis-6.156.6-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:59a8def90d9a5a9b67e1ac529e903a2363ceb6cf873c209da6b4284c5daab671", size = 749264, upload-time = "2026-07-10T20:56:46.118Z" }, + { url = "https://files.pythonhosted.org/packages/86/35/e4113d06769b544f0fb77ffea9195b598b4c56a298905c21fd47c4eed388/hypothesis-6.156.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c574c3224563d730848bc5d1ef1683c4f83993400c0167899fe328f4bfcd4725", size = 742095, upload-time = "2026-07-10T20:56:41.412Z" }, + { url = "https://files.pythonhosted.org/packages/d8/5c/a47666ede10384e8978722cade7ab96a42df71d2ab577317092d0fed7c8a/hypothesis-6.156.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01bb8270c46b3ef53b0c2d23ff613ea506d609d06f936d823ea57c58b66b05f7", size = 1069917, upload-time = "2026-07-10T20:56:04.845Z" }, + { url = "https://files.pythonhosted.org/packages/79/93/75f6057dadd9dc0134f37c08d5d14d04d3cd7374debbcb0cc4569c6712f1/hypothesis-6.156.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4ea6559c13606e13b645927f2e0906e52b5ac5d99b40d3abaaeb2e8c7ceeb75", size = 1121204, upload-time = "2026-07-10T20:55:52.008Z" }, + { url = "https://files.pythonhosted.org/packages/62/87/308efef08bc60d1e673d035e8ca8e9663f4b6b3ba519c3cdebf6583c2b76/hypothesis-6.156.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2d47054d0230f0dd9b6868fc030126c7a6c25527144272ff376cc4e9c39f7540", size = 1244168, upload-time = "2026-07-10T20:55:40.288Z" }, + { url = "https://files.pythonhosted.org/packages/3b/66/de8fff5bd9a40a4056dafbe7f904887ef12632282bbbac90f1977c30dd3b/hypothesis-6.156.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:050c8c0815f88d47dd0875a92698d20d61639b7b721ee043a6d687c7f14ff7d8", size = 1288127, upload-time = "2026-07-10T20:56:00.541Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8d/794fb26e1fd3ff004978f8f18b7aa7e1c2270ba72e1f977b987a812064f8/hypothesis-6.156.6-cp313-cp313-win_amd64.whl", hash = "sha256:f0d73edab7b8a0051b3634f2d04d62b7e7282f8f274963b11188ee4957d672ef", size = 637954, upload-time = "2026-07-10T20:56:33.35Z" }, + { url = "https://files.pythonhosted.org/packages/a5/be/5b4b27984cb43c60e95f570b069660335dad34cb67f7d226017c5d35d31e/hypothesis-6.156.6-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:34a70a7b8226e34d658072d8fb81d03f97f0a75ceb536329a321b94ce2232fd6", size = 749312, upload-time = "2026-07-10T20:55:46.902Z" }, + { url = "https://files.pythonhosted.org/packages/31/11/709cceffc28666c9d4cb75ffc6df5ce30db8c7dd5cc2c8b38a2fd837427f/hypothesis-6.156.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f1969646beead7d8cf6a2537d2765af89d73056e2cb218e7fae92b83802250a3", size = 742332, upload-time = "2026-07-10T20:56:30.254Z" }, + { url = "https://files.pythonhosted.org/packages/84/52/cfc79b13d8dd3cd6de6b9df921c557efe8528a9c90a3a7cd93b37188d57e/hypothesis-6.156.6-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cbc2ec7b7d905e6b6ec1635f6340bfa52aaab718101c59f052bc012a6b486cd8", size = 1070109, upload-time = "2026-07-10T20:55:48.244Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ac/1da4def1f006b5ad01187ff96379e24c37439d659ec10c3e944c03436c0f/hypothesis-6.156.6-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9367ae25dfa6dc1af37904785e43c4f8fe1c4118cafdc2f06514154fbdd90992", size = 1121528, upload-time = "2026-07-10T20:56:39.665Z" }, + { url = "https://files.pythonhosted.org/packages/68/47/744e4f5e3d635dea20dbedf3fa486e2a6fa5210e0a52a0d5c4da56babd84/hypothesis-6.156.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:455f09107ec07c78f2a83cb8fc19e23879c9d51cdc831de6f9cb6ec4059cb9af", size = 1244690, upload-time = "2026-07-10T20:56:31.854Z" }, + { url = "https://files.pythonhosted.org/packages/25/8a/42252fcd5e521d140dac532f29c2a13ca8f22908cb545ffdd64b5e225680/hypothesis-6.156.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c76634c45a3ceee4c4fdfed39aebd08b8b822ec8b0c556877ef82846fd777730", size = 1288519, upload-time = "2026-07-10T20:56:03.429Z" }, + { url = "https://files.pythonhosted.org/packages/44/e7/176df9e47cf583d2b8d234b78c0aac3a47075ad5d147e60b2c21a1338bb1/hypothesis-6.156.6-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:eb7e9f8343bc6b948937e6ec12e6879ed25a17b53ceccbd2b84adadd3d511698", size = 586452, upload-time = "2026-07-10T20:56:22.285Z" }, + { url = "https://files.pythonhosted.org/packages/8c/75/2c8a0411bbe76429f3ae738ef9a00107201bf6146d9534350014ce369d98/hypothesis-6.156.6-cp314-cp314-win_amd64.whl", hash = "sha256:f9631cd604ae6032c3edf99160dc1b9e33873f2e52762246b24f07fb758652ae", size = 637774, upload-time = "2026-07-10T20:56:34.97Z" }, + { url = "https://files.pythonhosted.org/packages/2a/22/8115005e9aa72c8d63d90e9db5e0b8425fd8950fbc5d6e332805d4d32c9e/hypothesis-6.156.6-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:1f81163d36d3763b09ffaef7c3a71e88174ca3e6816201fca9d1d159f448fdb5", size = 747428, upload-time = "2026-07-10T20:56:44.611Z" }, + { url = "https://files.pythonhosted.org/packages/e8/c2/66bfe9337f4a4b1f7754ee6d01d950280152a81d0d797e6c1d9eb0909750/hypothesis-6.156.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:556b905767e36147918634a64356aa05d8c956576f00aee01eb351678f193908", size = 740889, upload-time = "2026-07-10T20:55:57.656Z" }, + { url = "https://files.pythonhosted.org/packages/95/3b/69f45af2d4f0950b7d1af3cdbdd800b88a6c2370331481eda79d6171fbe3/hypothesis-6.156.6-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3f2604b28d16d696aaaf4954d20f907b27e54034df98e64746a20c74c319f03", size = 1069270, upload-time = "2026-07-10T20:56:12.024Z" }, + { url = "https://files.pythonhosted.org/packages/f8/43/6b2549885da08f5e50ba34fb8e0d0a60b2f190ffd516fac220f8db5b5869/hypothesis-6.156.6-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffe012ad66dbe7b8e8ddef6f6992ab1b36719ea64430c2bf1ff7135521052a15", size = 1120409, upload-time = "2026-07-10T20:55:34.551Z" }, + { url = "https://files.pythonhosted.org/packages/70/97/745c778c3eb29befa2367b1ded8437eecfbbe6932359d0f831275bc32170/hypothesis-6.156.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5bfa3c7b758f7278081c6bfec5f89b43c4eb075c0c9eb095323f7a9eb019b513", size = 1243111, upload-time = "2026-07-10T20:56:17.83Z" }, + { url = "https://files.pythonhosted.org/packages/ab/d7/c5ec6a442dc9b822f47064bda4b6d3e739dccdd1c5bf44c9a57fb6136830/hypothesis-6.156.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d0db1f4573800c618773622f03cb6533bb3377430ef938c9476ba10c39d22591", size = 1287262, upload-time = "2026-07-10T20:56:23.749Z" }, + { url = "https://files.pythonhosted.org/packages/11/0c/c134d61710e14b68b010215dcf6bd57d2ec05cd169dff8bfab8fefc2d410/hypothesis-6.156.6-cp314-cp314t-win_amd64.whl", hash = "sha256:38cd0c4a7b9f809f1e23a4d15adfa9c5d99869b9afc327350a5e563350b78e48", size = 637862, upload-time = "2026-07-10T20:56:13.347Z" }, + { url = "https://files.pythonhosted.org/packages/59/9c/b94f3a31665527b6181616b72990fcf8d6d5fa82b4187aab104ab5f548f0/hypothesis-6.156.6-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:8893b4da90e06828846c1b100c3414a7729d047a020d854c0899ae9339df0e70", size = 749575, upload-time = "2026-07-10T20:55:29.371Z" }, + { url = "https://files.pythonhosted.org/packages/21/74/dcf695f79f526543ae5d0f8c1325508e9fe990a996c0e0853129a9a5d81d/hypothesis-6.156.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b7fc0b7df9b28d028e4cc295b2ac8fbbbc22e090a23382c92fff5e37696be74f", size = 744351, upload-time = "2026-07-10T20:56:36.46Z" }, + { url = "https://files.pythonhosted.org/packages/11/d3/5bff4c55c6995a6c43f66ec8e5866b56e34f03837fd0be0e4922f3bab168/hypothesis-6.156.6-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38ed3178526382d392d04ad699ad7a2e53845e521a09d40f1cbbc1e1ff63ba48", size = 1070916, upload-time = "2026-07-10T20:56:19.256Z" }, + { url = "https://files.pythonhosted.org/packages/ba/af/5ed42117a69221ea118caaff933d8212039a0ac0bc15afa915635f13984c/hypothesis-6.156.6-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ad94e28aabf4db0d479297d43b8a2a01e7caaa9bdfccfdac7a4a3717e05b993", size = 1122625, upload-time = "2026-07-10T20:56:14.758Z" }, + { url = "https://files.pythonhosted.org/packages/3b/d3/02499badc6e3f3e980941021edf5fd780c895d8d08c9015e78516340ed83/hypothesis-6.156.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:983a5cfd955994bffc7eb02976241f7a1f3c2d94dbc3389430c45858fa5c1ae0", size = 640823, upload-time = "2026-07-10T20:55:45.461Z" }, +] + [[package]] name = "idna" version = "3.18" @@ -575,6 +636,7 @@ source = { editable = "." } dev = [ { name = "dill" }, { name = "furo" }, + { name = "hypothesis" }, { name = "mypy" }, { name = "pytest" }, { name = "pytest-cov" }, @@ -590,6 +652,7 @@ dev = [ dev = [ { name = "dill", specifier = ">=0.2.5" }, { name = "furo" }, + { name = "hypothesis" }, { name = "mypy", specifier = ">=2.1" }, { name = "pytest", specifier = ">=8" }, { name = "pytest-cov", specifier = ">=6" }, @@ -734,6 +797,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4c/07/2ebca9b11fb9be7340a818d8d6f63feaebb146be2c4afbd6061701d6df6e/snowballstemmer-3.1.1-py3-none-any.whl", hash = "sha256:7e207fa178741da09cdee59d3ecec3827ad5f92b1fc5c9ff3755b639f71f5752", size = 104164, upload-time = "2026-06-03T00:56:38.614Z" }, ] +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + [[package]] name = "soupsieve" version = "2.8.4" From bacb42ba2376602c009a349138573bbd239eb526 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 13 Jul 2026 02:28:22 -0700 Subject: [PATCH 082/275] Apply type-design review fixes to the pipeline state contract The ParseState ownership map said roles belong to assign/post_rules exclusively, but group writes maiden roles -- an ownership contract wrong in one place teaches readers to distrust it everywhere. Fixed the map, noted post-group segments staleness and the sole-producer text invariant on WorkToken, documented PendingAmbiguity's dangling- index tolerance and Parser's None-default resolution, made assemble's role fallback say what it means, and pinned the whole ownership map mechanically: a new test folds the case corpus stage by stage and asserts each stage changes only the fields it owns. Co-Authored-By: Claude Fable 5 --- nameparser/_parser.py | 5 ++++- nameparser/_pipeline/_assemble.py | 3 ++- nameparser/_pipeline/_state.py | 11 +++++++--- tests/v2/pipeline/test_state.py | 36 +++++++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/nameparser/_parser.py b/nameparser/_parser.py index b94039ca..21e6073c 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -26,7 +26,10 @@ class Parser: """Immutable, thread-safe, picklable by construction (spec §4): all validity checking happens at construction; a Parser that constructs - successfully cannot fail at parse time on any str content.""" + successfully cannot fail at parse time on any str content. The None + field defaults resolve in __post_init__; after construction both + fields are always non-None (the annotations state the steady-state + truth, hence the assignment ignores on the defaults).""" lexicon: Lexicon = None # type: ignore[assignment] # None -> default() policy: Policy = None # type: ignore[assignment] # None -> Policy() diff --git a/nameparser/_pipeline/_assemble.py b/nameparser/_pipeline/_assemble.py index 0eb48975..485f0ad4 100644 --- a/nameparser/_pipeline/_assemble.py +++ b/nameparser/_pipeline/_assemble.py @@ -23,7 +23,8 @@ def assemble(state: ParseState) -> ParsedName: for i, t in enumerate(state.tokens): if i in dropped: continue - final[i] = Token(t.text, t.span, t.role or Role.GIVEN, t.tags) + role = t.role if t.role is not None else Role.GIVEN + final[i] = Token(t.text, t.span, role, t.tags) ambiguities = tuple( Ambiguity(p.kind, p.detail, tuple(final[i] for i in p.indices if i in final)) diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py index d2a8e2e0..afa50925 100644 --- a/nameparser/_pipeline/_state.py +++ b/nameparser/_pipeline/_state.py @@ -21,7 +21,9 @@ @dataclass(frozen=True, slots=True) class WorkToken: """One tokenized word. role stays None until assign; extracted - nickname/maiden tokens arrive with their role pre-set.""" + nickname/maiden tokens arrive with their role pre-set. text is + always the exact original slice (tokenize is the sole producer; + the anti-#100 invariant depends on it).""" text: str span: Span @@ -53,8 +55,11 @@ class ParseState: dataclasses.replace. Fields are filled progressively: extract_delimited -> extracted/masked; tokenize -> tokens (span- sorted)/comma_offsets; segment -> segments/structure; classify -> - token tags; group -> pieces/dropped; assign/post_rules -> token - roles; ambiguities accumulate anywhere.""" + token tags; group -> pieces/piece_tags/dropped AND maiden token + roles; assign/post_rules -> the remaining token roles; ambiguities + accumulate anywhere. Post-group, segments may retain indices of + dropped tokens -- assign iterates pieces, never segments. This + ownership map is pinned by tests/v2/pipeline/test_state.py.""" original: str lexicon: Lexicon diff --git a/tests/v2/pipeline/test_state.py b/tests/v2/pipeline/test_state.py index bbda440c..b01433c3 100644 --- a/tests/v2/pipeline/test_state.py +++ b/tests/v2/pipeline/test_state.py @@ -29,3 +29,39 @@ def test_state_is_frozen_and_replace_works() -> None: def test_worktoken_carries_optional_role() -> None: t = WorkToken("Jack", Span(6, 10), role=Role.NICKNAME) assert t.role is Role.NICKNAME + + +def test_stage_field_ownership() -> None: + # The ParseState docstring's ownership map, pinned mechanically: run + # the whole case corpus through the fold stage by stage and assert + # each stage only changes the fields it owns. Converts the prose + # contract into a test (a future stage clobbering another stage's + # field fails here, not in a distant assertion). + import dataclasses as _dc + + from nameparser import Lexicon as _Lexicon + from nameparser._pipeline import STAGES + + from ..cases import CASES + + ownership = { + "extract_delimited": {"extracted", "masked", "ambiguities"}, + "tokenize": {"tokens", "comma_offsets"}, + "segment": {"segments", "structure", "ambiguities"}, + "classify": {"tokens"}, + "group": {"tokens", "pieces", "piece_tags", "dropped"}, + "assign": {"tokens", "ambiguities"}, + "post_rules": {"tokens"}, + } + assert {s.__name__ for s in STAGES} == set(ownership) + for case in CASES: + state = ParseState(original=case.text, lexicon=_Lexicon.default(), + policy=case.policy or Policy()) + for stage in STAGES: + before = {f.name: getattr(state, f.name) + for f in _dc.fields(state)} + state = stage(state) + changed = {name for name, value in before.items() + if getattr(state, name) != value} + assert changed <= ownership[stage.__name__], ( + f"{case.id}: {stage.__name__} changed {changed - ownership[stage.__name__]}") From f65bd7ced6e504801227224ff1b65db9e21564de Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 13 Jul 2026 02:54:12 -0700 Subject: [PATCH 083/275] Exclude the ambiguous subset from the plain suffix-acronym test In the real data suffix_acronyms_ambiguous is a SUBSET of suffix_acronyms (v1 shape), so the plain membership test in classify and is_suffix_strict fired before the period gate could withhold vocab:suffix -- the gate was dead code for the overlap members ({'ed', 'jd'}), and 'Smith, Ed' silently dropped the given name into suffix. Both sites now exclude the ambiguous subset, Lexicon enforces the subset invariant like particles_ambiguous, the classify/vocab fixtures mirror the real subset shape instead of hiding the bug with disjoint sets, and three v1-verified case rows pin the parity. Found by cross-session PR review (C1/I1). Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 7 +++++++ nameparser/_pipeline/_classify.py | 8 +++++++- nameparser/_pipeline/_vocab.py | 5 ++++- tests/v2/cases.py | 7 +++++++ tests/v2/pipeline/test_classify.py | 12 +++++++++++- tests/v2/pipeline/test_vocab.py | 8 +++++++- tests/v2/test_lexicon.py | 8 ++++++++ 7 files changed, 51 insertions(+), 4 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index a1a08fda..1647546f 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -149,6 +149,13 @@ def __post_init__(self) -> None: f"particles_ambiguous must be a subset of particles; " f"not in particles: {extra}" ) + if not self.suffix_acronyms_ambiguous <= self.suffix_acronyms: + extra = ", ".join(sorted( + self.suffix_acronyms_ambiguous - self.suffix_acronyms)) + raise ValueError( + f"suffix_acronyms_ambiguous must be a subset of " + f"suffix_acronyms; not in suffix_acronyms: {extra}" + ) # -- constructors ---------------------------------------------------- diff --git a/nameparser/_pipeline/_classify.py b/nameparser/_pipeline/_classify.py index 60c35068..33226786 100644 --- a/nameparser/_pipeline/_classify.py +++ b/nameparser/_pipeline/_classify.py @@ -31,7 +31,13 @@ def _tags_for(token: WorkToken, state: ParseState) -> frozenset[str]: tags.add("vocab:title") if n in lex.given_name_titles: tags.add("vocab:given-title") - if n in lex.suffix_acronyms or n in lex.suffix_words: + # the ambiguous subset is EXCLUDED from the plain membership test: + # in the real data suffix_acronyms_ambiguous is a subset of + # suffix_acronyms, and without the exclusion the period gate below + # is dead code (bare 'Ed'/'Jd' would silently become suffixes) + if (n in lex.suffix_acronyms + and n not in lex.suffix_acronyms_ambiguous) \ + or n in lex.suffix_words: tags.add("vocab:suffix") if n in lex.suffix_words: tags.add("vocab:suffix-word") diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 48475fb2..57c77cd0 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -33,7 +33,10 @@ def is_suffix_strict(text: str, lexicon: Lexicon) -> bool: return True if is_initial(text): return False - return n in lexicon.suffix_acronyms or n in lexicon.suffix_words + # ambiguous subset excluded from the plain test (see _classify) + return (n in lexicon.suffix_acronyms + and n not in lexicon.suffix_acronyms_ambiguous) \ + or n in lexicon.suffix_words def is_suffix_lenient(text: str, lexicon: Lexicon) -> bool: diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 01579313..2becd4a2 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -102,6 +102,13 @@ class Case: policy=_TK), Case("empty", "", {}), Case("whitespace", " ", {}), + Case("bare_ambiguous_acronym", "John Ed", + {"given": "John", "family": "Ed"}, + notes="'ed' is an ambiguous acronym; bare form is a name (C1)"), + Case("comma_ambiguous_acronym", "Smith, Ed", + {"given": "Ed", "family": "Smith"}), + Case("ambiguous_acronym_with_suffix", "John Ed III", + {"given": "John", "family": "Ed", "suffix": "III"}), Case("unbalanced_quote", 'Jon "Nick Smith', {"given": "Jon", "middle": '"Nick', "family": "Smith"}, ambiguities=("unbalanced-delimiter",), diff --git a/tests/v2/pipeline/test_classify.py b/tests/v2/pipeline/test_classify.py index 1c4f0a4f..ef0ace19 100644 --- a/tests/v2/pipeline/test_classify.py +++ b/tests/v2/pipeline/test_classify.py @@ -9,7 +9,7 @@ _LEX = Lexicon( titles=frozenset({"dr", "sir"}), given_name_titles=frozenset({"sir"}), - suffix_acronyms=frozenset({"phd"}), + suffix_acronyms=frozenset({"phd", "ma"}), suffix_words=frozenset({"jr", "v"}), suffix_acronyms_ambiguous=frozenset({"ma"}), particles=frozenset({"de", "la", "van"}), @@ -66,3 +66,13 @@ def test_v_is_suffix_word_and_initial() -> None: # both tags present; assign applies the veto, not classify out = _classified("John V Smith") assert {"vocab:suffix", "vocab:suffix-word", "initial"} <= _tags(out, "V") + + +def test_bare_ambiguous_acronym_in_acronyms_is_not_suffix() -> None: + # the default lexicon has suffix_acronyms_ambiguous SUBSET OF + # suffix_acronyms (v1 data shape); the plain membership test must + # exclude the ambiguous members or the period gate is dead code + # and bare 'Ed'/'Jd' silently become suffixes (PR review C1) + out = _classified("Ma M.A.") + assert "vocab:suffix" not in _tags(out, "Ma") + assert "vocab:suffix" in _tags(out, "M.A.") diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py index 54d4cfc6..a8e21326 100644 --- a/tests/v2/pipeline/test_vocab.py +++ b/tests/v2/pipeline/test_vocab.py @@ -2,7 +2,7 @@ from nameparser._pipeline._vocab import is_initial, is_suffix_lenient, is_suffix_strict _LEX = Lexicon( - suffix_acronyms=frozenset({"phd"}), + suffix_acronyms=frozenset({"phd", "ma"}), suffix_words=frozenset({"jr", "v"}), suffix_acronyms_ambiguous=frozenset({"ma"}), ) @@ -32,3 +32,9 @@ def test_lenient_accepts_suffix_words_unconditionally() -> None: assert is_suffix_lenient("V", _LEX) assert is_suffix_lenient("V.", _LEX) assert not is_suffix_lenient("Ma", _LEX) + + +def test_strict_excludes_bare_ambiguous_even_when_in_acronyms() -> None: + # mirrors the real data shape: ambiguous is a SUBSET of acronyms + assert not is_suffix_strict("Ma", _LEX) + assert is_suffix_strict("M.A.", _LEX) diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 99bf0950..690fc927 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -186,3 +186,11 @@ def test_normalization_casefolds_and_strips_interior_periods() -> None: # A "simplify to .lower()/.strip('.')" regression must fail here. lex = Lexicon(titles=frozenset({"STRAßE", "Ph.D"})) assert lex.titles == frozenset({"strasse", "phd"}) + + +def test_suffix_ambiguous_must_be_subset_of_acronyms() -> None: + # same invariant as particles_ambiguous <= particles: the ambiguous + # set marks a subset of suffix_acronyms, and classify's period gate + # depends on the membership tests agreeing about it + with pytest.raises(ValueError, match="subset"): + Lexicon(suffix_acronyms_ambiguous=frozenset({"ma"})) From c66be143b72db8f924ae00d24fdde47f94525cf9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 13 Jul 2026 02:57:03 -0700 Subject: [PATCH 084/275] Heal the split Ph. D. credential across the pipeline (I2/I3) The group-stage merge reunited the two tokens as a piece, but the suffix view joins SUFFIX tokens with ', ' -- 'John Ph. D.' rendered as suffix 'Ph., D.' -- and a mid-name credential never reached the trailing suffix peel, landing in MIDDLE. Continuation tokens of a suffix-merged piece now carry a new stable 'joined' tag (spec's provenance-tag sketch anticipated it) which the suffix view uses to attach with a space; assign peels group-flagged suffix pieces at ANY position, matching v1's fix_phd which extracted the credential before parsing. Contract-test trigger added for the new stable tag; two v1-verified case rows pin the behavior. Found by cross-session PR review (I2/I3). Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_assign.py | 9 +++++++++ nameparser/_pipeline/_group.py | 8 ++++++++ nameparser/_types.py | 20 ++++++++++++++------ tests/v2/cases.py | 6 ++++++ tests/v2/pipeline/test_group.py | 5 +++++ tests/v2/test_contracts.py | 1 + tests/v2/test_parser.py | 15 +++++++++++++++ 7 files changed, 58 insertions(+), 6 deletions(-) diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 706294d6..e4a9aa26 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -92,6 +92,15 @@ def _assign_main(seg_idx: int, state: ParseState, continue break rest = list(range(n, len(pieces))) + if not rest: + return + # group-flagged suffix pieces (the ph-d merge) are suffixes at ANY + # position -- v1's fix_phd extracted the credential from the string + # before parsing, so position never mattered (PR review I3) + flagged = [k for k in rest if "suffix" in ptags[k]] + for k in flagged: + _set_roles(tokens, pieces[k], Role.SUFFIX) + rest = [k for k in rest if "suffix" not in ptags[k]] if not rest: return # v1 nickname rule (plan deviation #2) diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 4023c7e6..c64a779d 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -182,6 +182,14 @@ def group(state: ParseState) -> ParseState: additional = 1 if state.structure is Structure.FAMILY_COMMA else 0 for seg in state.segments: pieces, ptags = _group_segment(seg, additional, tuple(tokens)) + # continuation tokens of a suffix-merged piece (the ph-d merge) + # carry the stable "joined" tag: the suffix string view joins + # SUFFIX tokens with ", ", and the tag lets it heal the split + for piece, piece_tags_ in zip(pieces, ptags): + if "suffix" in piece_tags_ and len(piece) > 1: + for i in piece[1:]: + tokens[i] = dataclasses.replace( + tokens[i], tags=tokens[i].tags | {"joined"}) # maiden markers: a non-leading marker piece consumes following # pieces until a suffix; consumed tokens become MAIDEN, the # marker is dropped (#274) diff --git a/nameparser/_types.py b/nameparser/_types.py index af6789a3..b8065b67 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -54,8 +54,10 @@ def __add__(self, other: object) -> NoReturn: # type: ignore[override] #: Stable, documented tag vocabulary (API). All other tags are -#: namespaced ("vocab:...", "patronymic:...") and unstable. -STABLE_TAGS = frozenset({"particle", "conjunction", "initial"}) +#: namespaced ("vocab:...", "patronymic:...") and unstable. "joined" +#: marks a continuation token of a merged piece (the ph-d merge) and +#: drives the suffix view's space-vs-comma join. +STABLE_TAGS = frozenset({"particle", "conjunction", "initial", "joined"}) _E = TypeVar("_E", bound=Enum) @@ -302,8 +304,8 @@ def __repr__(self) -> str: def _text_for(self, *roles: Role, tag: str | None = None, without_tag: str | None = None) -> str: - joiner = ", " if roles == (Role.SUFFIX,) else " " - parts = [] + suffix_join = roles == (Role.SUFFIX,) + parts: list[str] = [] for tok in self.tokens: if tok.role not in roles: continue @@ -311,8 +313,14 @@ def _text_for(self, *roles: Role, tag: str | None = None, continue if without_tag is not None and without_tag in tok.tags: continue - parts.append(tok.text) - return joiner.join(parts) + # "joined" (stable tag) marks a continuation of the previous + # token ("Ph." + "D."): attach with a space so the suffix + # view's ", " join does not split one credential in two + if suffix_join and "joined" in tok.tags and parts: + parts[-1] += " " + tok.text + else: + parts.append(tok.text) + return (", " if suffix_join else " ").join(parts) @property def title(self) -> str: diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 2becd4a2..b0c174de 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -109,6 +109,12 @@ class Case: {"given": "Ed", "family": "Smith"}), Case("ambiguous_acronym_with_suffix", "John Ed III", {"given": "John", "family": "Ed", "suffix": "III"}), + Case("phd_split", "John Ph. D.", + {"given": "John", "suffix": "Ph. D."}, + notes="v1 fix_phd; healed via the stable 'joined' tag"), + Case("phd_split_mid_name", "Dr. John Ph. D. Smith", + {"title": "Dr.", "given": "John", "family": "Smith", + "suffix": "Ph. D."}), Case("unbalanced_quote", 'Jon "Nick Smith', {"given": "Jon", "middle": '"Nick', "family": "Smith"}, ambiguities=("unbalanced-delimiter",), diff --git a/tests/v2/pipeline/test_group.py b/tests/v2/pipeline/test_group.py index f5fecc4e..0d3150d2 100644 --- a/tests/v2/pipeline/test_group.py +++ b/tests/v2/pipeline/test_group.py @@ -90,6 +90,11 @@ def test_phd_split_across_tokens_merges_as_suffix() -> None: out = _grouped("John Smith Ph. D.") assert _piece_texts(out) == [["John", "Smith", "Ph. D."]] assert "suffix" in out.piece_tags[0][2] + # continuation tokens of the merged piece carry the stable "joined" + # tag so the suffix string view can heal the split (", " join would + # otherwise render 'Ph., D.') + d_tok = next(t for t in out.tokens if t.text == "D.") + assert "joined" in d_tok.tags def test_maiden_marker_consumes_tail() -> None: diff --git a/tests/v2/test_contracts.py b/tests/v2/test_contracts.py index 78f284d5..be4381b7 100644 --- a/tests/v2/test_contracts.py +++ b/tests/v2/test_contracts.py @@ -55,6 +55,7 @@ def test_every_patronymic_rule_has_a_trigger(rule: PatronymicRule) -> None: "particle": ("Juan de la Vega", "de"), "conjunction": ("Mr. and Mrs. John Smith", "and"), "initial": ("John A. Smith", "A."), + "joined": ("John Ph. D.", "D."), } diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index d82aa674..b215bd38 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -131,3 +131,18 @@ def test_matches_accepts_explicit_parser() -> None: pn = family_first.parse("Yamada Taro") assert pn.matches("Yamada Taro", parser=family_first) assert not pn.matches("Yamada Taro") # default parser reads given-first + + +def test_phd_split_heals_in_the_suffix_view() -> None: + # v1 parity via fix_phd: the split credential renders as one suffix + assert parse("John Ph. D.").suffix == "Ph. D." + assert parse("John Smith PhD MD").suffix == "PhD, MD" # unchanged + + +def test_phd_split_mid_name_is_a_suffix() -> None: + # v1 parity: fix_phd extracted the credential BEFORE parsing, so + # position never mattered; the merged piece is a suffix anywhere + pn = parse("Dr. John Ph. D. Smith") + assert pn.suffix == "Ph. D." + assert pn.family == "Smith" + assert pn.middle == "" From ec9e3a27f24a0f7d4284afbb403692f1bde50eeb Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 13 Jul 2026 02:58:40 -0700 Subject: [PATCH 085/275] Fold a leading never-given particle into the family name (I4) v1's handle_non_first_name_prefix was unported: 'de la Vega' produced given='de' with no ambiguity ('de' is not particles_ambiguous, so PARTICLE_OR_GIVEN never fires -- a silent misassignment). Port it as post_rules rule 1b keyed on the parsed given exactly like v1: a lone given token tagged particle-but-not-ambiguous with middles or family present folds into FAMILY; 'Jean de Mesnil' (non-leading) and bare 'de' (degenerate guard) stay untouched, and ambiguous 'van Gogh' keeps the given reading. v1-verified case row added. Found by cross-session PR review (I4). Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_post_rules.py | 16 ++++++++++++++++ tests/v2/cases.py | 4 ++++ tests/v2/pipeline/test_post_rules.py | 25 +++++++++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index 4edfb393..a9fb8fe8 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -71,6 +71,22 @@ def post_rules(state: ParseState) -> ParseState: for i in givens: _retag(tokens, i, Role.FAMILY) + # rule 1b: a leading particle that is NEVER a given name means the + # whole name is a surname -- fold given (and middles) into family + # (v1 handle_non_first_name_prefix; 'de la Vega' -> family, while + # ambiguous 'van Gogh' keeps the given reading). The middle/family + # guard leaves a degenerate bare 'de' as given rather than + # inventing a surname. + if len(givens) == 1 and (middles or families): + gtags = tokens[givens[0]].tags + if "particle" in gtags and "vocab:particle-ambiguous" not in gtags: + for i in givens + middles: + _retag(tokens, i, Role.FAMILY) + # downstream rules key on the role counts: recompute + givens = _idx(tokens, Role.GIVEN) + middles = _idx(tokens, Role.MIDDLE) + families = _idx(tokens, Role.FAMILY) + # v1 gates both rotations on `not self._had_comma` if state.structure is not Structure.NO_COMMA: return dataclasses.replace(state, tokens=tuple(tokens)) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index b0c174de..92fdb9c9 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -115,6 +115,10 @@ class Case: Case("phd_split_mid_name", "Dr. John Ph. D. Smith", {"title": "Dr.", "given": "John", "family": "Smith", "suffix": "Ph. D."}), + Case("leading_never_given_particle", "de la Vega", + {"family": "de la Vega"}, + notes="v1 handle_non_first_name_prefix: never-given leading " + "particle folds the whole name into family"), Case("unbalanced_quote", 'Jon "Nick Smith', {"given": "Jon", "middle": '"Nick', "family": "Smith"}, ambiguities=("unbalanced-delimiter",), diff --git a/tests/v2/pipeline/test_post_rules.py b/tests/v2/pipeline/test_post_rules.py index 12634ca5..a08567c1 100644 --- a/tests/v2/pipeline/test_post_rules.py +++ b/tests/v2/pipeline/test_post_rules.py @@ -7,6 +7,8 @@ _LEX = Lexicon( titles=frozenset({"mr", "sir"}), given_name_titles=frozenset({"sir"}), + particles=frozenset({"de", "la", "van"}), + particles_ambiguous=frozenset({"van"}), ) @@ -81,3 +83,26 @@ def test_turkic_rotation() -> None: assert _by_role(out, Role.GIVEN) == "Aygun" assert _by_role(out, Role.MIDDLE) == "Ali kizi" assert _by_role(out, Role.FAMILY) == "Mammadova" + + +def test_leading_never_given_particle_folds_into_family() -> None: + # v1 handle_non_first_name_prefix: a leading particle that is never + # a given name ('de') means the whole name is a surname + out = _parsed("de la Vega") + assert _by_role(out, Role.FAMILY) == "de la Vega" + assert not _by_role(out, Role.GIVEN) + + +def test_leading_ambiguous_particle_stays_given() -> None: + # 'van' is particles_ambiguous: the given reading stands (v1 parity) + out = _parsed("van Gogh") + assert _by_role(out, Role.GIVEN) == "van" + assert _by_role(out, Role.FAMILY) == "Gogh" + + +def test_degenerate_bare_particle_stays_given() -> None: + # v1's guard: with no middle or family, a bare 'de' keeps given='de' + # rather than inventing a surname + out = _parsed("de") + assert _by_role(out, Role.GIVEN) == "de" + assert not _by_role(out, Role.FAMILY) From 481f27afcf168139543782c438e54bb41b238ff4 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 13 Jul 2026 03:00:35 -0700 Subject: [PATCH 086/275] Wire the three dead Policy fields (I5) middle_as_family was validated but consumed by no stage -- ported v1's handle_middle_name_as_last as post_rules rule 4 (runs after the patronymic rotations, comma or not, matching v1's post_process order; span order reproduces v1's prepend). lenient_comma_suffixes now gates segment's post-comma predicate (strict mode vetoes initial-shaped suffix words). extra_suffix_delimiters cannot be honored until token re-splitting lands with the migration work, so setting it warns instead of silently doing nothing. Found by cross-session PR review (I5). Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_post_rules.py | 13 ++++++++++--- nameparser/_pipeline/_segment.py | 12 +++++++++--- nameparser/_policy.py | 9 +++++++++ tests/v2/pipeline/test_post_rules.py | 14 ++++++++++++++ tests/v2/pipeline/test_segment.py | 13 +++++++++++++ tests/v2/test_policy.py | 10 ++++++++++ 6 files changed, 65 insertions(+), 6 deletions(-) diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index a9fb8fe8..4b6dd1be 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -87,10 +87,12 @@ def post_rules(state: ParseState) -> ParseState: middles = _idx(tokens, Role.MIDDLE) families = _idx(tokens, Role.FAMILY) - # v1 gates both rotations on `not self._had_comma` - if state.structure is not Structure.NO_COMMA: - return dataclasses.replace(state, tokens=tuple(tokens)) + # v1 gates both rotations on `not self._had_comma`; the + # middle_as_family fold below runs comma or not (v1 order: + # patronymics first, then handle_middle_name_as_last) rules = state.policy.patronymic_rules + if state.structure is not Structure.NO_COMMA: + rules = frozenset() if PatronymicRule.EAST_SLAVIC in rules and \ len(givens) == 1 and len(middles) == 1 and len(families) == 1: tail = tokens[families[0]].text @@ -111,4 +113,9 @@ def post_rules(state: ParseState) -> ParseState: _retag(tokens, m2, Role.MIDDLE) _retag(tokens, f, Role.MIDDLE) _retag(tokens, g, Role.FAMILY) + # rule 4: opt-in fold of middles into family (v1 + # handle_middle_name_as_last; span order reproduces v1's prepend) + if state.policy.middle_as_family: + for i in _idx(tokens, Role.MIDDLE): + _retag(tokens, i, Role.FAMILY) return dataclasses.replace(state, tokens=tuple(tokens)) diff --git a/nameparser/_pipeline/_segment.py b/nameparser/_pipeline/_segment.py index 961f720e..01dcbe20 100644 --- a/nameparser/_pipeline/_segment.py +++ b/nameparser/_pipeline/_segment.py @@ -5,7 +5,8 @@ COMMA_STRUCTURE ambiguities for unrecognized extra segments. Reads: Lexicon suffix vocabulary (via _vocab.is_suffix_lenient) -- the suffix-comma decision is definitionally vocabulary-dependent -(recorded plan deviation #3); Policy is not consulted here. +(recorded plan deviation #3); reads Policy.lenient_comma_suffixes +to pick the lenient or strict predicate. Decision (v1 parity): >=1 comma and every post-first segment entirely lenient-suffix AND >1 word before the first comma -> SUFFIX_COMMA; @@ -19,7 +20,7 @@ import dataclasses from nameparser._pipeline._state import ParseState, PendingAmbiguity, Structure -from nameparser._pipeline._vocab import is_suffix_lenient +from nameparser._pipeline._vocab import is_suffix_lenient, is_suffix_strict from nameparser._types import AmbiguityKind @@ -44,8 +45,13 @@ def segment(state: ParseState) -> ParseState: return dataclasses.replace(state, segments=segs, structure=Structure.NO_COMMA) + # lenient_comma_suffixes=False drops the post-comma test back to + # the strict predicate (initial-shaped suffix words stop qualifying) + predicate = (is_suffix_lenient if state.policy.lenient_comma_suffixes + else is_suffix_strict) + def suffixy(seg: tuple[int, ...]) -> bool: - return all(is_suffix_lenient(state.tokens[i].text, state.lexicon) + return all(predicate(state.tokens[i].text, state.lexicon) for i in seg) rest = groups[1:] diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 3ef42336..312c956f 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -6,6 +6,7 @@ from __future__ import annotations import dataclasses +import warnings from dataclasses import dataclass, field from enum import Enum, StrEnum, auto @@ -125,6 +126,14 @@ def __post_init__(self) -> None: object.__setattr__( self, "extra_suffix_delimiters", frozenset(delimiters) ) + if delimiters: + # accepted-and-ignored would violate the fail-loud culture: + # the pipeline cannot honor these until token re-splitting + # lands with the migration work (v1 suffix_delimiter parity) + warnings.warn( + "extra_suffix_delimiters is not yet consumed by the " + "parse pipeline; it takes effect with the 2.0 migration " + "work", UserWarning, stacklevel=2) # Truthy strings ("no", "false") would silently invert the # caller's intent downstream; bools are the one field kind the # coercing checks above can't cover. diff --git a/tests/v2/pipeline/test_post_rules.py b/tests/v2/pipeline/test_post_rules.py index a08567c1..ca7da24d 100644 --- a/tests/v2/pipeline/test_post_rules.py +++ b/tests/v2/pipeline/test_post_rules.py @@ -106,3 +106,17 @@ def test_degenerate_bare_particle_stays_given() -> None: out = _parsed("de") assert _by_role(out, Role.GIVEN) == "de" assert not _by_role(out, Role.FAMILY) + + +def test_middle_as_family_folds_middles() -> None: + # v1 handle_middle_name_as_last, opt-in: middles prepend to family + out = _parsed("John Quincy Adams Smith", + Policy(middle_as_family=True)) + assert _by_role(out, Role.GIVEN) == "John" + assert not _by_role(out, Role.MIDDLE) + assert _by_role(out, Role.FAMILY) == "Quincy Adams Smith" + + +def test_middle_as_family_off_by_default() -> None: + out = _parsed("John Quincy Adams Smith") + assert _by_role(out, Role.MIDDLE) == "Quincy Adams" diff --git a/tests/v2/pipeline/test_segment.py b/tests/v2/pipeline/test_segment.py index aad05671..5aa5a9fe 100644 --- a/tests/v2/pipeline/test_segment.py +++ b/tests/v2/pipeline/test_segment.py @@ -3,6 +3,8 @@ from nameparser._pipeline._segment import segment from nameparser._pipeline._state import ParseState, Structure from nameparser._pipeline._tokenize import tokenize +import dataclasses + from nameparser._policy import Policy from nameparser._types import AmbiguityKind @@ -74,3 +76,14 @@ def test_comma_only_input_is_no_comma_structure() -> None: out = _segmented(",,,") assert out.structure is Structure.NO_COMMA assert out.segments == () + + +def test_strict_comma_suffixes_veto_lenient_only_members() -> None: + # lenient_comma_suffixes=False: the post-comma test drops back to + # the strict predicate, so initial-shaped suffix words no longer + # qualify and the structure reads FAMILY_COMMA + state = ParseState( + original="John Ingram, V", lexicon=_LEX, + policy=dataclasses.replace(Policy(), lenient_comma_suffixes=False)) + out = segment(tokenize(extract_delimited(state))) + assert out.structure is Structure.FAMILY_COMMA diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index b84ecac5..c8ff88ff 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -75,6 +75,7 @@ def test_policy_delimiters_do_not_alias_caller_containers() -> None: assert ("'", "'") not in p.nickname_delimiters +@pytest.mark.filterwarnings("ignore:extra_suffix_delimiters is not yet") def test_extra_suffix_delimiters_validated_and_coerced() -> None: with pytest.raises(TypeError, match="bare string"): Policy(extra_suffix_delimiters="ab") # type: ignore[arg-type] @@ -99,6 +100,7 @@ def test_policy_patch_mirrors_policy_field_types() -> None: assert patch_annotation == f"{f.type} | _Unset" +@pytest.mark.filterwarnings("ignore:extra_suffix_delimiters is not yet") def test_policy_patch_canonicalizes_union_fields() -> None: p = PolicyPatch(extra_suffix_delimiters=frozenset({"-"})) assert isinstance(p.extra_suffix_delimiters, frozenset) @@ -223,3 +225,11 @@ def test_name_order_rejects_bare_string() -> None: Policy(name_order="gmf") # type: ignore[arg-type] with pytest.raises(TypeError, match="bare string"): PolicyPatch(name_order="gmf") # type: ignore[arg-type] + + +def test_extra_suffix_delimiters_warns_not_yet_consumed() -> None: + # accepted-and-ignored would violate the fail-loud culture: until + # the migration work wires v1's suffix_delimiter expansion, setting + # the field warns instead of silently doing nothing + with pytest.warns(UserWarning, match="not yet consumed"): + Policy(extra_suffix_delimiters=frozenset({"-"})) From e7ccd670ed5bee7b56c32a5fcf3dc408b978868c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 13 Jul 2026 03:06:59 -0700 Subject: [PATCH 087/275] Restrict name_order to the exported orders; drop dangled ambiguities PR review I6/I7/I8/I9: - assemble omits an ambiguity whose referent tokens were ALL dropped (born-empty ambiguities like UNBALANCED_DELIMITER are kept -- they are token-independent by design) - Policy.name_order now accepts only GIVEN_FIRST / FAMILY_FIRST / FAMILY_FIRST_GIVEN_LAST: the unnamed permutations had no implemented assignment semantics and would silently misassign. Non-Role elements get the taxonomy's TypeError. - Span.__add__ comment no longer cites a nonexistent join stage; the real rationale is that no covering-span operation exists (anti-#100) - the stage ownership test now also pins token-level ownership: texts and spans fixed at tokenize, classify changes only tags, the role-assigning stages only roles (group also tags, for 'joined') - docstring polish: ParseState names the three ambiguity-recording stages; _types names the matches() call-time import; _assign's PARTICLE_OR_GIVEN wording is order-neutral Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_assemble.py | 17 ++++++++++++----- nameparser/_pipeline/_assign.py | 5 +++-- nameparser/_pipeline/_state.py | 5 +++-- nameparser/_policy.py | 25 ++++++++++++++++++------- nameparser/_types.py | 12 ++++++------ tests/v2/pipeline/test_assemble.py | 24 ++++++++++++++++++++++++ tests/v2/pipeline/test_state.py | 23 +++++++++++++++++++++++ tests/v2/test_parser.py | 4 +++- tests/v2/test_policy.py | 17 ++++++++++++++++- 9 files changed, 108 insertions(+), 24 deletions(-) diff --git a/nameparser/_pipeline/_assemble.py b/nameparser/_pipeline/_assemble.py index 485f0ad4..d3a6a169 100644 --- a/nameparser/_pipeline/_assemble.py +++ b/nameparser/_pipeline/_assemble.py @@ -25,10 +25,17 @@ def assemble(state: ParseState) -> ParsedName: continue role = t.role if t.role is not None else Role.GIVEN final[i] = Token(t.text, t.span, role, t.tags) - ambiguities = tuple( - Ambiguity(p.kind, p.detail, - tuple(final[i] for i in p.indices if i in final)) - for p in state.ambiguities) + ambiguities = [] + for pending in state.ambiguities: + materialized = tuple(final[i] for i in pending.indices + if i in final) + if pending.indices and not materialized: + # every referent was dropped: the ambiguity describes + # nothing that survives assembly. Born-empty ambiguities + # (unbalanced delimiters) are token-independent and kept. + continue + ambiguities.append( + Ambiguity(pending.kind, pending.detail, materialized)) return ParsedName(original=state.original, tokens=tuple(final.values()), - ambiguities=ambiguities) + ambiguities=tuple(ambiguities)) diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index e4a9aa26..d4b0e107 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -18,8 +18,9 @@ suffix; segments 2+ are suffixes (lenient -- segment already flagged non-suffixy ones COMMA_STRUCTURE). SUFFIX_COMMA: segment 0 as NO_COMMA; segments 1+ wholly SUFFIX. -Emits PARTICLE_OR_GIVEN when the given position consumed a leading -particles_ambiguous token with more pieces following ("Van Johnson"). +Emits PARTICLE_OR_GIVEN when the leading name piece is a lone +particles_ambiguous token with more pieces following ("Van Johnson") -- +whatever role name_order assigns that position. """ from __future__ import annotations diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py index afa50925..56eb7a25 100644 --- a/nameparser/_pipeline/_state.py +++ b/nameparser/_pipeline/_state.py @@ -56,8 +56,9 @@ class ParseState: extract_delimited -> extracted/masked; tokenize -> tokens (span- sorted)/comma_offsets; segment -> segments/structure; classify -> token tags; group -> pieces/piece_tags/dropped AND maiden token - roles; assign/post_rules -> the remaining token roles; ambiguities - accumulate anywhere. Post-group, segments may retain indices of + roles; assign/post_rules -> the remaining token roles; ambiguities are + recorded by extract/segment/assign only (pinned by the ownership + test). Post-group, segments may retain indices of dropped tokens -- assign iterates pieces, never segments. This ownership map is pinned by tests/v2/pipeline/test_state.py.""" diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 312c956f..0e76c7f1 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -64,11 +64,21 @@ class Policy: def __post_init__(self) -> None: _reject_bare_string_order(self.name_order) order = tuple(self.name_order) - if len(order) != 3 or set(order) != _NAME_ROLES: + for element in order: + if not isinstance(element, Role): + raise TypeError( + f"name_order elements must be Role members, " + f"got {element!r}" + ) + # Only the three exported orders have implemented assignment + # semantics; the unnamed permutations would silently misassign. + # Pre-2.0 strictness is free -- relaxing later is compatible. + if order not in (GIVEN_FIRST, FAMILY_FIRST, + FAMILY_FIRST_GIVEN_LAST): raise ValueError( - f"name_order must be a permutation of (Role.GIVEN, " - f"Role.MIDDLE, Role.FAMILY), got {order!r}; use " - f"GIVEN_FIRST, FAMILY_FIRST, or FAMILY_FIRST_GIVEN_LAST" + f"name_order must be one of the exported orders, got " + f"{order!r}; use GIVEN_FIRST, FAMILY_FIRST, or " + f"FAMILY_FIRST_GIVEN_LAST" ) object.__setattr__(self, "name_order", order) if isinstance(self.patronymic_rules, str): @@ -159,9 +169,10 @@ def __repr__(self) -> str: if value == f.default: continue if f.name == "name_order": - # Only 3 of the 6 possible role permutations have named - # constants; fall back to a compact role-name tuple for - # the rest so this can't KeyError on an unnamed order. + # __post_init__ restricts to the three named orders, so + # the fallback is unreachable via the constructor; kept + # because repr must never raise (e.g. a smuggled + # __setstate__ value -- layout is validated, values not). order_repr = constant_names.get( value, "(" + ", ".join(r.name for r in value) + ")") parts.append(f"name_order={order_repr}") diff --git a/nameparser/_types.py b/nameparser/_types.py index b8065b67..a04aca4d 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -3,8 +3,8 @@ Layering (enforced by tests/v2/test_layering.py): this module imports nothing from nameparser at module level -- it is the bottom of the module-import dependency graph. The rendering delegates import _render -at call time, and a TYPE_CHECKING-only _lexicon import supplies the -Lexicon annotation. +and matches() imports _parser at call time; TYPE_CHECKING-only imports +supply the Lexicon/Parser annotations. Repr policy (applies to every v2 type's __repr__, across this module and _lexicon.py/_policy.py/_locale.py): bounded output only. No repr may scale @@ -43,10 +43,10 @@ class Span(NamedTuple): end: int def __add__(self, other: object) -> NoReturn: # type: ignore[override] - # Inherited tuple + would concatenate two spans into a 4-tuple -- - # the natural but wrong spelling of "covering span". The real - # covering operation ships with its consumer, the pipeline's - # join stage. + # Inherited tuple + would concatenate two spans into a 4-tuple. + # There is deliberately NO covering-span operation: grouping is + # index-run based (spec §6, the anti-#100 invariant) and never + # merges spans. raise TypeError( "Span does not support +; tuple concatenation is not a " "covering span" diff --git a/tests/v2/pipeline/test_assemble.py b/tests/v2/pipeline/test_assemble.py index d1aaae9a..ba667825 100644 --- a/tests/v2/pipeline/test_assemble.py +++ b/tests/v2/pipeline/test_assemble.py @@ -50,3 +50,27 @@ def test_assemble_drops_structural_marker_tokens() -> None: def test_empty_parse_is_falsy() -> None: assert not _parse("") assert not _parse(" ") + + +def test_ambiguity_with_all_indices_dropped_is_omitted() -> None: + # an ambiguity whose referent tokens were ALL dropped describes + # nothing; emitting it hollow would mislead consumers. Born-empty + # ambiguities (unbalanced delimiters) are kept -- they are + # token-independent by design. + from nameparser._pipeline._state import PendingAmbiguity + from nameparser._types import AmbiguityKind as AK + import dataclasses + state = run(ParseState(original="Jane Smith née Jones", lexicon=_LEX, + policy=Policy())) + née_idx = next(i for i, t in enumerate(state.tokens) + if t.text == "née") + poisoned = dataclasses.replace( + state, ambiguities=state.ambiguities + ( + PendingAmbiguity(AK.ORDER, "refers only to the marker", + (née_idx,)), + PendingAmbiguity(AK.UNBALANCED_DELIMITER, "born empty", ()), + )) + pn = assemble(poisoned) + kinds = [a.kind for a in pn.ambiguities] + assert AK.ORDER not in kinds # fully dangled: omitted + assert AK.UNBALANCED_DELIMITER in kinds # born empty: kept diff --git a/tests/v2/pipeline/test_state.py b/tests/v2/pipeline/test_state.py index b01433c3..4ceff2bc 100644 --- a/tests/v2/pipeline/test_state.py +++ b/tests/v2/pipeline/test_state.py @@ -54,6 +54,17 @@ def test_stage_field_ownership() -> None: "post_rules": {"tokens"}, } assert {s.__name__ for s in STAGES} == set(ownership) + # Within the tokens themselves the contract is finer: texts and + # spans are fixed at tokenize (the anti-#100 invariant -- tokens + # are never re-created), classify touches only tags, and the + # role-assigning stages touch only roles (group also tags, for the + # ph-d "joined" marker). + token_ownership = { + "classify": {"tags"}, + "group": {"tags", "role"}, + "assign": {"role"}, + "post_rules": {"role"}, + } for case in CASES: state = ParseState(original=case.text, lexicon=_Lexicon.default(), policy=case.policy or Policy()) @@ -65,3 +76,15 @@ def test_stage_field_ownership() -> None: if getattr(state, name) != value} assert changed <= ownership[stage.__name__], ( f"{case.id}: {stage.__name__} changed {changed - ownership[stage.__name__]}") + if stage.__name__ not in token_ownership: + continue + allowed = token_ownership[stage.__name__] + assert len(state.tokens) == len(before["tokens"]), ( + f"{case.id}: {stage.__name__} changed the token count") + for old, new in zip(before["tokens"], state.tokens): + token_changed = { + f.name for f in _dc.fields(old) + if getattr(old, f.name) != getattr(new, f.name)} + assert token_changed <= allowed, ( + f"{case.id}: {stage.__name__} changed token fields " + f"{token_changed - allowed} on {old.text!r}") diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index b215bd38..5467cf70 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -103,7 +103,9 @@ def test_parser_for_wraps_pack_errors_with_identity() -> None: # latent in a perfectly constructible Locale until apply time bad = Locale(code="xx", lexicon=Lexicon.empty(), policy=PolicyPatch(name_order=(1, 2, 3))) # type: ignore[arg-type] - with pytest.raises(ValueError, match="while applying locale 'xx'"): + # the rewrap preserves the taxonomy's exception type (here the + # non-Role element TypeError) while adding the pack identity + with pytest.raises(TypeError, match="while applying locale 'xx'"): parser_for(bad) diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index c8ff88ff..902f9172 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -37,6 +37,21 @@ def test_name_order_must_be_permutation_and_error_names_constants() -> None: Policy(name_order=(Role.GIVEN, Role.GIVEN, Role.FAMILY)) +def test_name_order_restricted_to_the_three_exported_orders() -> None: + # _name_positions only implements the three exported semantics; the + # unnamed permutations would silently misassign (PR review I7). + # Pre-2.0 strictness is free: relaxing later is compatible. + with pytest.raises(ValueError, match="GIVEN_FIRST"): + Policy(name_order=(Role.MIDDLE, Role.GIVEN, Role.FAMILY)) + + +def test_name_order_rejects_non_role_elements_with_type_error() -> None: + # taxonomy: wrong element type -> TypeError, not the permutation + # ValueError (PR review polish) + with pytest.raises(TypeError, match="Role"): + Policy(name_order=(1, 2, 3)) # type: ignore[arg-type] + + def test_patronymic_rules_coerce_and_reject() -> None: p = Policy(patronymic_rules=frozenset({"east-slavic"})) # type: ignore[arg-type] assert p.patronymic_rules == frozenset({PatronymicRule.EAST_SLAVIC}) @@ -175,7 +190,7 @@ def test_apply_patch_revalidates_deferred_values() -> None: # PolicyPatch documents lazy validation: invalid values sit latent in # the patch and must fail when applied, not silently flow into Policy. bad_order = PolicyPatch(name_order=(Role.TITLE, Role.GIVEN, Role.FAMILY)) - with pytest.raises(ValueError, match="permutation"): + with pytest.raises(ValueError, match="exported orders"): apply_patch(Policy(), bad_order) bad_rules = PolicyPatch(patronymic_rules=frozenset({"klingon"})) # type: ignore[arg-type] with pytest.raises(ValueError, match="valid rules"): From ddd04161bac52f3b26f7c631dcee0ed55a6b3fcc Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 13 Jul 2026 03:10:07 -0700 Subject: [PATCH 088/275] Add the review's test-coverage batch (T1-T4, T6) - T1: three-piece FAMILY_FIRST_GIVEN_LAST pins given-from-the-END semantics (not a rotation of FAMILY_FIRST) - T2: reverse-coverage property -- every input char lies in a token span, a masked delimited span, or is individually ignorable; no character silently vanishes - T3: case row for post-comma non-suffix extras ('Smith, John, Extra, Jr.' -> suffix 'Extra, Jr.' + COMMA_STRUCTURE; v1 parity pinned live) - T4: multiple unbalanced delimiters are each reported; the scan does not stop at the first unmatched opener - T6: digits join the Hypothesis stress alphabet Co-Authored-By: Claude Fable 5 --- tests/v2/cases.py | 6 ++++++ tests/v2/test_parser.py | 24 +++++++++++++++++++++++- tests/v2/test_properties.py | 30 ++++++++++++++++++++++++++++-- 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 92fdb9c9..72816df7 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -40,6 +40,12 @@ class Case: {"given": "John", "family": "Smith"}), Case("suffix_comma", "John Smith, PhD", {"given": "John", "family": "Smith", "suffix": "PhD"}), + Case("comma_extras_become_suffixes", "Smith, John, Extra, Jr.", + {"given": "John", "family": "Smith", "suffix": "Extra, Jr."}, + ambiguities=("comma-structure",), + notes="post-comma segments land in suffix even when not " + "suffix-shaped; the ambiguity flags the guess (v1 " + "parity, pinned live 2026-07-13)"), Case("delavega", "Dr. Juan de la Vega III", {"title": "Dr.", "given": "Juan", "family": "de la Vega", "suffix": "III"}), diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 5467cf70..2ed3477b 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -3,7 +3,9 @@ import pytest from nameparser import Lexicon, Locale, Parser, Policy, PolicyPatch, parse, parser_for -from nameparser._policy import FAMILY_FIRST, PatronymicRule +from nameparser._policy import ( + FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, PatronymicRule, +) from nameparser._types import AmbiguityKind @@ -128,6 +130,26 @@ def test_matches_component_wise_case_insensitive() -> None: pn.matches(42) # type: ignore[arg-type] +def test_family_first_given_last_places_middle_between() -> None: + # T1: the three-piece FAMILY_FIRST_GIVEN_LAST assignment -- family + # from the front, given from the END, middle between (not a rotation + # of FAMILY_FIRST) + p = Parser(policy=Policy(name_order=FAMILY_FIRST_GIVEN_LAST)) + pn = p.parse("Zeng Xiao Long") + assert (pn.family, pn.middle, pn.given) == ("Zeng", "Xiao", "Long") + + +def test_multiple_unbalanced_delimiters_each_reported() -> None: + # T4: the extract scan continues past the first unmatched opener; + # each one is reported and treated as literal text + pn = parse('John "Jack (Smith') + unbalanced = [a for a in pn.ambiguities + if a.kind is AmbiguityKind.UNBALANCED_DELIMITER] + assert len(unbalanced) == 2 + assert pn.given == "John" and pn.family == "(Smith" + assert not pn.nickname + + def test_matches_accepts_explicit_parser() -> None: family_first = Parser(policy=Policy(name_order=FAMILY_FIRST)) pn = family_first.parse("Yamada Taro") diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index 157116d1..0e4440b0 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -9,10 +9,12 @@ from hypothesis import given, settings from hypothesis import strategies as st -from nameparser import parse +from nameparser import Lexicon, Policy, parse +from nameparser._pipeline import run +from nameparser._pipeline._state import ParseState _ALPHABET = st.sampled_from( - 'abcdefgh ABC .,،，\'"()«»‏‏\U0001f600éñßЖ-') + 'abcdefgh ABC 12 .,،，\'"()«»‏‏\U0001f600éñßЖ-') @given(st.text(alphabet=_ALPHABET, max_size=200)) @@ -50,3 +52,27 @@ def test_render_reparse_reaches_fixpoint(text: str) -> None: break s = nxt assert str(parse(s)) == s, f"no fixpoint within 10 rounds: {s!r}" + + +@given(st.text(alphabet=_ALPHABET, max_size=100)) +@settings(max_examples=300, deadline=None, derandomize=True) +def test_every_original_char_is_accounted_for(text: str) -> None: + # Reverse coverage (the dual of provenance): no character of the + # input silently vanishes. Every char lies in a token span, a + # masked delimited span, or is individually ignorable -- whitespace, + # a structural comma, or a char the strip options remove. Checked on + # the pre-assembly state because dropped/extracted tokens keep their + # spans there. + state = run(ParseState(original=text, lexicon=Lexicon.default(), + policy=Policy())) + covered: set[int] = set() + for tok in state.tokens: + covered.update(range(tok.span.start, tok.span.end)) + for span in state.masked: + covered.update(range(span.start, span.end)) + ignorable = {",", "،", "，", "\U0001f600", "‏"} + for i, ch in enumerate(text): + if i in covered or ch.isspace() or ch in ignorable: + continue + raise AssertionError( + f"char {ch!r} at {i} in {text!r} is unaccounted for") From d64cb6ab4931bcd67e9a065a61093860ebb29a4c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 00:49:59 -0700 Subject: [PATCH 089/275] Single-source the suffix-as-written rule and the comma set /simplify (all four reviewers converged on the first): the trickiest membership expression in the vocabulary layer -- unambiguous acronym OR suffix word, ambiguous acronym only with periods -- was spelled in full in both classify and is_suffix_strict, aligned only by comments. Drift would make the comma-structure decision disagree with piece classification for the same token. Now _vocab.suffix_as_written is the single source; classify and the strict/lenient predicates compose it (and normalize each token once instead of twice on the lenient path). The three-comma set was likewise defined in tokenize and extract under different names with no sync note; it moves to _state.COMMA_CHARS. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_classify.py | 12 ++-------- nameparser/_pipeline/_extract.py | 8 +++---- nameparser/_pipeline/_state.py | 5 +++++ nameparser/_pipeline/_tokenize.py | 10 ++++----- nameparser/_pipeline/_vocab.py | 37 ++++++++++++++++++++++--------- 5 files changed, 43 insertions(+), 29 deletions(-) diff --git a/nameparser/_pipeline/_classify.py b/nameparser/_pipeline/_classify.py index 33226786..94d26bc7 100644 --- a/nameparser/_pipeline/_classify.py +++ b/nameparser/_pipeline/_classify.py @@ -20,7 +20,7 @@ from nameparser._lexicon import _normalize from nameparser._pipeline._state import ParseState, WorkToken -from nameparser._pipeline._vocab import is_initial +from nameparser._pipeline._vocab import is_initial, suffix_as_written def _tags_for(token: WorkToken, state: ParseState) -> frozenset[str]: @@ -31,20 +31,12 @@ def _tags_for(token: WorkToken, state: ParseState) -> frozenset[str]: tags.add("vocab:title") if n in lex.given_name_titles: tags.add("vocab:given-title") - # the ambiguous subset is EXCLUDED from the plain membership test: - # in the real data suffix_acronyms_ambiguous is a subset of - # suffix_acronyms, and without the exclusion the period gate below - # is dead code (bare 'Ed'/'Jd' would silently become suffixes) - if (n in lex.suffix_acronyms - and n not in lex.suffix_acronyms_ambiguous) \ - or n in lex.suffix_words: + if suffix_as_written(n, token.text, lex): tags.add("vocab:suffix") if n in lex.suffix_words: tags.add("vocab:suffix-word") if n in lex.suffix_acronyms_ambiguous: tags.add("vocab:suffix-ambiguous") - if "." in token.text: - tags.add("vocab:suffix") if n in lex.particles: tags.add("particle") if n in lex.particles_ambiguous: diff --git a/nameparser/_pipeline/_extract.py b/nameparser/_pipeline/_extract.py index 1d0560f3..50cbb4db 100644 --- a/nameparser/_pipeline/_extract.py +++ b/nameparser/_pipeline/_extract.py @@ -22,11 +22,11 @@ import dataclasses -from nameparser._pipeline._state import ParseState, PendingAmbiguity +from nameparser._pipeline._state import ( + COMMA_CHARS, ParseState, PendingAmbiguity, +) from nameparser._types import AmbiguityKind, Role, Span -_CLOSE_BOUNDARY_EXTRAS = {",", "،", "，"} # comma chars end a close - def _open_ok(text: str, i: int) -> bool: return i == 0 or text[i - 1].isspace() @@ -34,7 +34,7 @@ def _open_ok(text: str, i: int) -> bool: def _close_ok(text: str, j: int, width: int) -> bool: k = j + width - return k >= len(text) or text[k].isspace() or text[k] in _CLOSE_BOUNDARY_EXTRAS + return k >= len(text) or text[k].isspace() or text[k] in COMMA_CHARS def _overlaps(span: Span, taken: list[Span]) -> bool: diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py index 56eb7a25..30d5e5f0 100644 --- a/nameparser/_pipeline/_state.py +++ b/nameparser/_pipeline/_state.py @@ -18,6 +18,11 @@ from nameparser._types import AmbiguityKind, Role, Span +# The comma characters (ASCII/Arabic/fullwidth, #265). Shared here so +# tokenize (separators/segmentation) and extract (close-quote +# boundaries) cannot drift apart. +COMMA_CHARS = frozenset({",", "\u060c", "\uff0c"}) + @dataclass(frozen=True, slots=True) class WorkToken: """One tokenized word. role stays None until assign; extracted diff --git a/nameparser/_pipeline/_tokenize.py b/nameparser/_pipeline/_tokenize.py index 712e0943..67c1378d 100644 --- a/nameparser/_pipeline/_tokenize.py +++ b/nameparser/_pipeline/_tokenize.py @@ -21,11 +21,11 @@ import dataclasses import re -from nameparser._pipeline._state import ParseState, WorkToken +from nameparser._pipeline._state import ( + COMMA_CHARS, ParseState, WorkToken, +) from nameparser._types import Role, Span -_COMMA_CHARS = {",", "،", "，"} # ASCII, Arabic, fullwidth - # Ported verbatim from v1 (nameparser/config/regexes.py, "emoji" and # "bidi") -- layering forbids importing the config package here, so the # patterns are duplicated by design with this provenance note. When @@ -52,12 +52,12 @@ def _tokenize_region(state: ParseState, start: int, end: int, tok_start: int | None = None for i in range(start, end): ch = text[i] - if ch in _COMMA_CHARS or _ignorable(ch, state): + if ch in COMMA_CHARS or _ignorable(ch, state): if tok_start is not None: tokens.append(WorkToken(text[tok_start:i], Span(tok_start, i), role=role)) tok_start = None - if ch in _COMMA_CHARS and record_commas: + if ch in COMMA_CHARS and record_commas: commas.append(i) continue if tok_start is None: diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 57c77cd0..788e6696 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -23,25 +23,42 @@ def is_initial(text: str) -> bool: return bool(_INITIAL.fullmatch(text)) -def is_suffix_strict(text: str, lexicon: Lexicon) -> bool: - """v1's is_suffix: suffix vocabulary with the initial veto ('V.' in - 'John V. Smith' is a middle initial, not roman five); ambiguous - acronyms count only when written with periods ('M.A.' yes, 'Ma' no). +def suffix_as_written(n: str, text: str, lexicon: Lexicon) -> bool: + """Counts as a suffix as written, with NO initial veto (the veto + differs by caller): unambiguous suffix vocabulary, or an ambiguous + acronym written with periods ('M.A.' yes, 'Ma' no). `n` is + _normalize(text), passed in so callers normalize once. + + Single source for classify's "vocab:suffix" tag and the segment/ + assign predicates. The ambiguous subset is EXCLUDED from the plain + membership test: in the real data suffix_acronyms_ambiguous is a + subset of suffix_acronyms, and without the exclusion the period + gate is dead code (bare 'Ed'/'Jd' would silently become suffixes). """ - n = _normalize(text) if "." in text and n in lexicon.suffix_acronyms_ambiguous: return True - if is_initial(text): - return False - # ambiguous subset excluded from the plain test (see _classify) return (n in lexicon.suffix_acronyms and n not in lexicon.suffix_acronyms_ambiguous) \ or n in lexicon.suffix_words +def _is_suffix_strict_n(n: str, text: str, lexicon: Lexicon) -> bool: + if is_initial(text): + # period-written ambiguous acronyms are exempt from the veto + return "." in text and n in lexicon.suffix_acronyms_ambiguous + return suffix_as_written(n, text, lexicon) + + +def is_suffix_strict(text: str, lexicon: Lexicon) -> bool: + """v1's is_suffix: suffix_as_written with the initial veto ('V.' in + 'John V. Smith' is a middle initial, not roman five).""" + return _is_suffix_strict_n(_normalize(text), text, lexicon) + + def is_suffix_lenient(text: str, lexicon: Lexicon) -> bool: """v1's is_suffix_lenient: suffix_words accepted unconditionally, bypassing the initial veto -- only safe in unambiguous positions (after a comma).""" - return _normalize(text) in lexicon.suffix_words or \ - is_suffix_strict(text, lexicon) + n = _normalize(text) + return n in lexicon.suffix_words \ + or _is_suffix_strict_n(n, text, lexicon) From 222b079df8cd5f543ad3badaedd42ac228dff6f0 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 00:51:40 -0700 Subject: [PATCH 090/275] Factor the shared piece machinery in group and assign /simplify: the piece predicates were typed for group's internal mutable lists, so every cross-stage call paid a triple list/set/tuple conversion (including O(n) token-tuple copies inside assign's peel loops and group's maiden scan). They now take Sequence/Set and the call-site churn is gone. Also: the leading-title peel was spelled twice (assign's main path and the FAMILY_COMMA branch, the latter with a dead 'not given_done' guard) -- extracted as _peel_leading_titles; the trailing all-suffix segment loops collapse into one loop over the structure's tail; the five hand-rolled pieces/ptags parallel-slice merges in _group_segment become one merge() closure that keeps the arrays in lockstep; post_rules' 'others' list was only ever tested for truthiness (any()); the patronymic comma gate is a named rotations_apply boolean instead of falsifying the policy value; and the module docstring records the GIVEN_FIRST positional assumption the rules inherit from v1 (#270). Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_assign.py | 58 ++++++++++++++-------------- nameparser/_pipeline/_group.py | 60 ++++++++++++++--------------- nameparser/_pipeline/_post_rules.py | 16 +++++--- 3 files changed, 68 insertions(+), 66 deletions(-) diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index d4b0e107..4ff3d7ab 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -50,12 +50,29 @@ def _set_roles(tokens: list[WorkToken], piece: tuple[int, ...], def _is_leading_title(piece: tuple[int, ...], ptags: frozenset[str], tokens: list[WorkToken]) -> bool: - if _is_title_piece(list(piece), set(ptags), tuple(tokens)): + if _is_title_piece(piece, ptags, tokens): return True return (len(piece) == 1 and bool(_PERIOD_ABBREV.match(tokens[piece[0]].text))) +def _peel_leading_titles(pieces: tuple[tuple[int, ...], ...], + ptags: tuple[frozenset[str], ...], + tokens: list[WorkToken]) -> int: + """Assign TITLE to the leading title pieces and return the first + non-title index. A title needs a following piece, unless the whole + segment is one title (v1 parity).""" + n = 0 + while n < len(pieces): + if ((n + 1 < len(pieces) or len(pieces) == 1) + and _is_leading_title(pieces[n], ptags[n], tokens)): + _set_roles(tokens, pieces[n], Role.TITLE) + n += 1 + continue + break + return n + + def _name_positions(order: tuple[Role, Role, Role], count: int) -> list[Role]: """Roles for `count` name pieces (titles/suffixes already peeled), @@ -82,16 +99,7 @@ def _assign_main(seg_idx: int, state: ParseState, pieces = state.pieces[seg_idx] ptags = state.piece_tags[seg_idx] has_nickname = any(t.role is Role.NICKNAME for t in tokens) - # peel leading titles - n = 0 - while n < len(pieces): - has_next = n + 1 < len(pieces) - if ((has_next or len(pieces) == 1) - and _is_leading_title(pieces[n], ptags[n], tokens)): - _set_roles(tokens, pieces[n], Role.TITLE) - n += 1 - continue - break + n = _peel_leading_titles(pieces, ptags, tokens) rest = list(range(n, len(pieces))) if not rest: return @@ -116,7 +124,7 @@ def _assign_main(seg_idx: int, state: ParseState, while k > 0: piece = pieces[rest[k - 1]] tags = ptags[rest[k - 1]] - if _is_suffix_piece(list(piece), set(tags), tuple(tokens)): + if _is_suffix_piece(piece, tags, tokens): k -= 1 continue if (k == len(rest) and k >= 2 and len(piece) == 1 @@ -153,11 +161,10 @@ def assign(state: ParseState) -> ParseState: return state if state.structure is Structure.NO_COMMA: _assign_main(0, state, tokens, ambiguities) + tail = len(state.segments) elif state.structure is Structure.SUFFIX_COMMA: _assign_main(0, state, tokens, ambiguities) - for seg_idx in range(1, len(state.segments)): - for piece in state.pieces[seg_idx]: - _set_roles(tokens, piece, Role.SUFFIX) + tail = 1 else: # FAMILY_COMMA # PARTICLE_OR_GIVEN is deliberately not emitted here: after a # comma the family is already fixed, so a leading given-position @@ -167,27 +174,20 @@ def assign(state: ParseState) -> ParseState: if len(state.segments) > 1: pieces = state.pieces[1] ptags = state.piece_tags[1] + n = _peel_leading_titles(pieces, ptags, tokens) given_done = False - n = 0 - while n < len(pieces): - if (not given_done - and _is_leading_title(pieces[n], ptags[n], tokens) - and (n + 1 < len(pieces) or len(pieces) == 1)): - _set_roles(tokens, pieces[n], Role.TITLE) - n += 1 - continue - break for m in range(n, len(pieces)): - if _is_suffix_piece(list(pieces[m]), set(ptags[m]), - tuple(tokens)): + if _is_suffix_piece(pieces[m], ptags[m], tokens): _set_roles(tokens, pieces[m], Role.SUFFIX) elif not given_done: _set_roles(tokens, pieces[m], Role.GIVEN) given_done = True else: _set_roles(tokens, pieces[m], Role.MIDDLE) - for seg_idx in range(2, len(state.segments)): - for piece in state.pieces[seg_idx]: - _set_roles(tokens, piece, Role.SUFFIX) + tail = 2 + # segments past the structure's name segments are wholly suffixes + for seg_idx in range(tail, len(state.segments)): + for piece in state.pieces[seg_idx]: + _set_roles(tokens, piece, Role.SUFFIX) return dataclasses.replace(state, tokens=tuple(tokens), ambiguities=tuple(ambiguities)) diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index c64a779d..acafb37d 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -19,6 +19,7 @@ import dataclasses import re +from collections.abc import Sequence, Set from nameparser._pipeline._state import ParseState, Structure, WorkToken from nameparser._types import Role @@ -29,22 +30,22 @@ Piece = list[int] -def _is_title_piece(piece: Piece, ptags: set[str], - tokens: tuple[WorkToken, ...]) -> bool: +def _is_title_piece(piece: Sequence[int], ptags: Set[str], + tokens: Sequence[WorkToken]) -> bool: if "title" in ptags: return True return len(piece) == 1 and "vocab:title" in tokens[piece[0]].tags -def _is_prefix_piece(piece: Piece, ptags: set[str], - tokens: tuple[WorkToken, ...]) -> bool: +def _is_prefix_piece(piece: Sequence[int], ptags: Set[str], + tokens: Sequence[WorkToken]) -> bool: if "prefix" in ptags: return True return len(piece) == 1 and "particle" in tokens[piece[0]].tags -def _is_suffix_piece(piece: Piece, ptags: set[str], - tokens: tuple[WorkToken, ...]) -> bool: +def _is_suffix_piece(piece: Sequence[int], ptags: Set[str], + tokens: Sequence[WorkToken]) -> bool: if "suffix" in ptags: return True if len(piece) != 1: @@ -53,15 +54,15 @@ def _is_suffix_piece(piece: Piece, ptags: set[str], return "vocab:suffix" in tags and "initial" not in tags -def _is_conj_piece(piece: Piece, ptags: set[str], - tokens: tuple[WorkToken, ...]) -> bool: +def _is_conj_piece(piece: Sequence[int], ptags: Set[str], + tokens: Sequence[WorkToken]) -> bool: if "conjunction" in ptags: return True return len(piece) == 1 and "conjunction" in tokens[piece[0]].tags -def _is_rootname(piece: Piece, ptags: set[str], - tokens: tuple[WorkToken, ...]) -> bool: +def _is_rootname(piece: Sequence[int], ptags: Set[str], + tokens: Sequence[WorkToken]) -> bool: if len(piece) == 1 and "initial" in tokens[piece[0]].tags: return False return not (_is_title_piece(piece, ptags, tokens) @@ -70,7 +71,7 @@ def _is_rootname(piece: Piece, ptags: set[str], def _group_segment(seg: tuple[int, ...], additional: int, - tokens: tuple[WorkToken, ...], + tokens: Sequence[WorkToken], ) -> tuple[list[Piece], list[set[str]]]: pieces: list[Piece] = [[i] for i in seg] ptags: list[set[str]] = [set() for _ in seg] @@ -87,6 +88,13 @@ def suffix(k: int) -> bool: def conj(k: int) -> bool: return _is_conj_piece(pieces[k], ptags[k], tokens) + def merge(lo: int, hi: int, add: Set[str] = frozenset(), + drop: Set[str] = frozenset()) -> None: + # pieces/ptags are parallel arrays; every merge must update + # both in lockstep + pieces[lo:hi] = [[i for piece in pieces[lo:hi] for i in piece]] + ptags[lo:hi] = [(set().union(*ptags[lo:hi]) | add) - drop] + # ph-d merge first: "Ph." "D." adjacent -> one suffix piece (plan # deviation #1; v1 fix_phd did this by regex on the raw string) k = 0 @@ -95,9 +103,7 @@ def conj(k: int) -> bool: if (len(a) == 1 and len(b) == 1 and _PH.fullmatch(tokens[a[0]].text) and _D.fullmatch(tokens[b[0]].text)): - pieces[k:k + 2] = [a + b] - merged = ptags[k] | ptags[k + 1] | {"suffix"} - ptags[k:k + 2] = [merged] + merge(k, k + 2, add={"suffix"}) else: k += 1 @@ -108,8 +114,7 @@ def conj(k: int) -> bool: k = 0 while k < len(pieces) - 1: if conj(k) and conj(k + 1): - pieces[k:k + 2] = [pieces[k] + pieces[k + 1]] - ptags[k:k + 2] = [ptags[k] | ptags[k + 1] | {"conjunction"}] + merge(k, k + 2, add={"conjunction"}) else: k += 1 # each conjunction joins its neighbors (v1 issue #11 carve-out: @@ -127,14 +132,12 @@ def conj(k: int) -> bool: start = max(0, k - 1) end = min(len(pieces), k + 2) neighbor = start if start < k else end - 1 - new_tags = set().union(*ptags[start:end]) + derived = set() if title(neighbor): - new_tags.add("title") + derived.add("title") if prefix(neighbor): - new_tags.add("prefix") - merged_piece = [i for p in pieces[start:end] for i in p] - pieces[start:end] = [merged_piece] - ptags[start:end] = [new_tags] + derived.add("prefix") + merge(start, end, add=derived) k = start + 1 # prefix chains: a non-leading prefix run absorbs everything to # the next prefix or suffix (v1's leading_first_name rule keeps @@ -149,10 +152,7 @@ def conj(k: int) -> bool: j += 1 while j < len(pieces) and not prefix(j) and not suffix(j): j += 1 - merged_piece = [i for p in pieces[k:j] for i in p] - merged_tags = set().union(*ptags[k:j]) - {"prefix"} - pieces[k:j] = [merged_piece] - ptags[k:j] = [merged_tags] + merge(k, j, drop={"prefix"}) k += 1 # bound given names: first non-title piece joins the next when # enough rootname pieces remain (v1 reserve_last) @@ -166,9 +166,7 @@ def conj(k: int) -> bool: non_suffix = sum(1 for k in range(len(pieces)) if not title(k) and not suffix(k)) if non_suffix >= 3: - bg = first_name_k - pieces[bg:bg + 2] = [pieces[bg] + pieces[bg + 1]] - ptags[bg:bg + 2] = [ptags[bg] | ptags[bg + 1]] + merge(first_name_k, first_name_k + 2) return pieces, ptags @@ -181,7 +179,7 @@ def group(state: ParseState) -> ParseState: # parts (parser.py:1333); the SUFFIX_COMMA pre-comma segment gets 0. additional = 1 if state.structure is Structure.FAMILY_COMMA else 0 for seg in state.segments: - pieces, ptags = _group_segment(seg, additional, tuple(tokens)) + pieces, ptags = _group_segment(seg, additional, tokens) # continuation tokens of a suffix-merged piece (the ph-d merge) # carry the stable "joined" tag: the suffix string view joins # SUFFIX tokens with ", ", and the tag lets it heal the split @@ -202,7 +200,7 @@ def group(state: ParseState) -> ParseState: j = m + 1 consumed: list[int] = [] while j < len(pieces) and not _is_suffix_piece( - pieces[j], ptags[j], tuple(tokens)): + pieces[j], ptags[j], tokens): consumed.extend(pieces[j]) j += 1 if consumed: diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index 4b6dd1be..70fba69e 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -21,6 +21,11 @@ Both rotations fire only on Structure.NO_COMMA (v1 gates them on `not self._had_comma`): a comma already established the family. + +These rules reconstruct token POSITION from roles, which is faithful +to v1 only under the default GIVEN_FIRST order; their interaction with +other name_order values is an open design question for the locale-pack +work (#270). """ from __future__ import annotations @@ -61,8 +66,8 @@ def post_rules(state: ParseState) -> ParseState: givens = _idx(tokens, Role.GIVEN) middles = _idx(tokens, Role.MIDDLE) families = _idx(tokens, Role.FAMILY) - others = [t for t in tokens - if t.role in (Role.SUFFIX, Role.NICKNAME, Role.MAIDEN)] + others = any(t.role in (Role.SUFFIX, Role.NICKNAME, Role.MAIDEN) + for t in tokens) # rule 1: title + lone given -> family (v1 handle_firstnames) if titles and givens and not middles and not families and not others: @@ -91,9 +96,8 @@ def post_rules(state: ParseState) -> ParseState: # middle_as_family fold below runs comma or not (v1 order: # patronymics first, then handle_middle_name_as_last) rules = state.policy.patronymic_rules - if state.structure is not Structure.NO_COMMA: - rules = frozenset() - if PatronymicRule.EAST_SLAVIC in rules and \ + rotations_apply = state.structure is Structure.NO_COMMA + if rotations_apply and PatronymicRule.EAST_SLAVIC in rules and \ len(givens) == 1 and len(middles) == 1 and len(families) == 1: tail = tokens[families[0]].text mid = tokens[middles[0]].text @@ -104,7 +108,7 @@ def post_rules(state: ParseState) -> ParseState: _retag(tokens, m, Role.GIVEN) _retag(tokens, f, Role.MIDDLE) _retag(tokens, g, Role.FAMILY) - if PatronymicRule.TURKIC in rules and \ + if rotations_apply and PatronymicRule.TURKIC in rules and \ len(givens) == 1 and len(middles) == 2 and len(families) == 1: tail = tokens[families[0]].text if _TURKIC.match(tail) or _TURKIC_CYR.match(tail): From 2d5d8c25d731a4deaa9d153b08f56f1892e3a2a8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 00:52:20 -0700 Subject: [PATCH 091/275] Trim constant-factor waste on the tokenize hot path /simplify efficiency findings: with the default strip flags every non-space character paid two failing regex calls in _ignorable; both strip classes are entirely non-ASCII, so an isascii() fast path skips them for virtually all real input. Span is a NamedTuple and already sorts as a tuple, so the tuple(...) sort keys in tokenize/extract were pure allocation. The case runner's seven-field list now derives from Role, whose declaration order the type declares canonical -- a new Role member fails the runner instead of being silently skipped. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_extract.py | 4 ++-- nameparser/_pipeline/_tokenize.py | 6 +++++- tests/v2/test_cases.py | 5 ++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/nameparser/_pipeline/_extract.py b/nameparser/_pipeline/_extract.py index 50cbb4db..4ac1ca66 100644 --- a/nameparser/_pipeline/_extract.py +++ b/nameparser/_pipeline/_extract.py @@ -92,8 +92,8 @@ def extract_delimited(state: ParseState) -> ParseState: extracted.append((role, inner)) masked.append(full) pos = j + len(close) - extracted.sort(key=lambda pair: tuple(pair[1])) - masked.sort(key=tuple) + extracted.sort(key=lambda pair: pair[1]) + masked.sort() return dataclasses.replace( state, extracted=tuple(extracted), masked=tuple(masked), ambiguities=state.ambiguities + tuple(ambiguities)) diff --git a/nameparser/_pipeline/_tokenize.py b/nameparser/_pipeline/_tokenize.py index 67c1378d..ab0390dd 100644 --- a/nameparser/_pipeline/_tokenize.py +++ b/nameparser/_pipeline/_tokenize.py @@ -40,6 +40,10 @@ def _ignorable(ch: str, state: ParseState) -> bool: if ch.isspace(): return True + if ch.isascii(): + # both strip classes are entirely non-ASCII (bidi >= U+061C, + # emoji >= U+2600): skip two failing regex calls per letter + return False if state.policy.strip_bidi and _BIDI.match(ch): return True return bool(state.policy.strip_emoji and _EMOJI.match(ch)) @@ -81,6 +85,6 @@ def tokenize(state: ParseState) -> ParseState: for role, inner in state.extracted: _tokenize_region(state, inner.start, inner.end, role, False, tokens, commas) - tokens.sort(key=lambda t: tuple(t.span)) + tokens.sort(key=lambda t: t.span) return dataclasses.replace(state, tokens=tuple(tokens), comma_offsets=tuple(sorted(commas))) diff --git a/tests/v2/test_cases.py b/tests/v2/test_cases.py index aea0e19a..bb14e671 100644 --- a/tests/v2/test_cases.py +++ b/tests/v2/test_cases.py @@ -2,12 +2,11 @@ runner (migration plan) consumes the same CASES.""" import pytest -from nameparser import Parser +from nameparser import Parser, Role from .cases import CASES, Case -_FIELDS = ("title", "given", "middle", "family", "suffix", "nickname", - "maiden") +_FIELDS = tuple(r.value for r in Role) # declaration order is canonical @pytest.mark.parametrize("case", CASES, ids=lambda c: c.id) From 12298df4d4568c8b31be7961302d53febe69faa6 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 01:47:06 -0700 Subject: [PATCH 092/275] =?UTF-8?q?Add=20the=20shim=20SetManager=20(migrat?= =?UTF-8?q?ion=20spec=20=C2=A73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- nameparser/_config_shim.py | 128 +++++++++++++++++++++++++++++++++++ tests/v2/test_config_shim.py | 82 ++++++++++++++++++++++ 2 files changed, 210 insertions(+) create mode 100644 nameparser/_config_shim.py create mode 100644 tests/v2/test_config_shim.py diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py new file mode 100644 index 00000000..cbfbdeb2 --- /dev/null +++ b/nameparser/_config_shim.py @@ -0,0 +1,128 @@ +"""v1 ``Constants`` compatibility shim over Lexicon/Policy (migration +spec §3). ``nameparser.config`` re-exports these names from the swap +commit onward; the whole module is deleted in 3.0 with the facade. + +Layering: facade layer -- may import anything public; here that's just +``nameparser.util`` for ``lc()``. +""" +from __future__ import annotations + +from collections.abc import Callable, Iterable, Iterator + +from nameparser.util import lc + + +class SetManager: + """v1 ``SetManager`` surface over a plain set of ``lc()``-normalized + strings. Mutations call ``_on_change`` (the owning Constants' + generation bump, wired by a later task). ``__call__`` and the + missing-member-tolerant ``remove()`` are gone per the #243 schedule + (warned 1.3.0, removed 2.0): ``remove()`` of a missing member raises + ``KeyError``, matching ``set.remove``. + """ + + _elements: set[str] + _on_change: Callable[[], None] | None + + def __init__(self, elements: Iterable[str] = (), + _on_change: Callable[[], None] | None = None) -> None: + self._elements = {lc(e) for e in elements} + self._on_change = _on_change + + def _changed(self) -> None: + if self._on_change is not None: + self._on_change() + + def add(self, *strings: str) -> SetManager: + """Add the normalized string arguments to the set. Returns + ``self`` for chaining.""" + # notify only on real change (v1 parity): a no-op add must not + # bump the owner's generation + changed = False + for s in strings: + normalized = lc(s) + if normalized not in self._elements: + self._elements.add(normalized) + changed = True + if changed: + self._changed() + return self + + def remove(self, *strings: str) -> SetManager: + """Remove the normalized string arguments from the set. + Raises ``KeyError`` if any argument is not a member. Returns + ``self`` for chaining.""" + changed = False + try: + for s in strings: + self._elements.remove(lc(s)) # KeyError on missing (#243) + changed = True + finally: + # a KeyError mid-list still leaves earlier removals applied, + # so the owner must hear about them or its cache goes stale + if changed: + self._changed() + return self + + def __contains__(self, item: object) -> bool: + return isinstance(item, str) and lc(item) in self._elements + + def __iter__(self) -> Iterator[str]: + return iter(self._elements) + + def __len__(self) -> int: + return len(self._elements) + + def __eq__(self, other: object) -> bool: + if isinstance(other, SetManager): + return self._elements == other._elements + if isinstance(other, (set, frozenset)): + return self._elements == other + return NotImplemented + + __hash__ = None # type: ignore[assignment] # mutable; v1 parity + + def _as_operand(self, other: object) -> set[str]: + if isinstance(other, SetManager): + return other._elements + if isinstance(other, (set, frozenset)): + return {lc(e) if isinstance(e, str) else e for e in other} + raise TypeError(f"unsupported operand type for SetManager: {other!r}") + + def __or__(self, other: object) -> set[str]: + return self._elements | self._as_operand(other) + + __ror__ = __or__ + + def __and__(self, other: object) -> set[str]: + return self._elements & self._as_operand(other) + + __rand__ = __and__ + + def __sub__(self, other: object) -> set[str]: + return self._elements - self._as_operand(other) + + def __rsub__(self, other: object) -> set[str]: + return self._as_operand(other) - self._elements + + def __repr__(self) -> str: + # Sorted so repr is stable across runs -- set() iteration order + # depends on per-process string hash randomization. + elements = ", ".join(repr(e) for e in sorted(self._elements)) + return f"SetManager({{{elements}}})" if self._elements else "SetManager(set())" + + # -- pickle interop with v1 blobs --------------------------------------- + + def __getstate__(self) -> dict[str, object]: + return {"_elements": set(self._elements)} + + def __setstate__(self, state: dict[str, object]) -> None: + # v1 SetManager stored its set under `elements` (plain __dict__ + # pickling); the shim stores `_elements`. Accept both, so a + # v1.3/1.4 Constants blob's embedded managers unpickle straight + # into shim instances. Re-normalize: nothing guarantees an + # incoming blob's elements passed through lc(). + elements: Iterable[str] = state.get( # type: ignore[assignment] + "_elements", state.get("elements", ())) + self._elements = {lc(e) for e in elements} + self._on_change = None # rewired by the owning Constants diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py new file mode 100644 index 00000000..ec980417 --- /dev/null +++ b/tests/v2/test_config_shim.py @@ -0,0 +1,82 @@ +"""Shim Constants/SetManager/TupleManager (migration spec §3).""" +import pickle + +import pytest + +from nameparser._config_shim import SetManager + + +def test_set_manager_normalizes_and_holds_membership() -> None: + s = SetManager(["Dr", "MRS."]) + assert "dr" in s and "Dr" in s # lc() normalization, both ways + assert "mrs" in s + assert len(s) == 2 + assert sorted(s) == ["dr", "mrs"] + + +def test_set_manager_add_remove_chain_and_keyerror() -> None: + s = SetManager() + assert s.add("Dame", "Fra") is s # chainable, v1 parity + assert "dame" in s + assert s.remove("Dame") is s + assert "dame" not in s + with pytest.raises(KeyError): # 1.3.0 grace period ended (#243) + s.remove("never-there") + + +def test_set_manager_call_is_removed() -> None: + s = SetManager(["dr"]) + with pytest.raises(TypeError): # #243: __call__ removed in 2.0 + s() # type: ignore[operator] + + +def test_set_manager_operators_and_equality() -> None: + a, b = SetManager(["a", "b"]), SetManager(["b", "c"]) + assert a | b == {"a", "b", "c"} + assert a & b == {"b"} + assert a - b == {"a"} + assert a | {"z"} == {"a", "b", "z"} + assert {"z"} | a == {"a", "b", "z"} # reflected: set op manager + assert {"a", "b", "c"} - a == {"c"} # operand order matters here + assert a == {"a", "b"} + assert a == SetManager(["a", "b"]) and a != b + with pytest.raises(TypeError): # mutable, unhashable (v1 parity) + hash(a) + + +def test_set_manager_reports_mutations_to_owner() -> None: + bumps = [] + s = SetManager(["a"], _on_change=lambda: bumps.append(1)) + s.add("b") + s.remove("a") + assert len(bumps) == 2 + s.add("b") # no-op: already present + assert len(bumps) == 2 # must not bump (v1 parity) + + +def test_set_manager_partial_remove_still_notifies_owner() -> None: + bumps = [] + s = SetManager(["a", "b"], _on_change=lambda: bumps.append(1)) + with pytest.raises(KeyError): + s.remove("a", "missing") # "a" IS removed before the raise + assert "a" not in s + assert len(bumps) == 1 # the real removal was reported + + +def test_set_manager_accepts_v1_pickle_state() -> None: + s = SetManager.__new__(SetManager) + s.__setstate__({"elements": {"dr", "mr"}, "_on_change": None}) + assert "dr" in s and len(s) == 2 + # the shim's own key spelling, with un-normalized elements: loading + # must re-normalize rather than trust the blob passed through lc() + s2 = SetManager.__new__(SetManager) + s2.__setstate__({"_elements": {"Dr", "MRS."}}) + assert "dr" in s2 and "mrs" in s2 + assert sorted(s2) == ["dr", "mrs"] + + +def test_set_manager_pickle_round_trip() -> None: + # in-process round trip of a blob we just built; pickle is not a + # security boundary here (same stance as the v2 pickle guards) + t = pickle.loads(pickle.dumps(SetManager(["Dr", "Mrs."]))) + assert t == SetManager(["dr", "mrs"]) From 95de0074e49c3168e7bacfbfc355a13e2b639fbe Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 02:10:36 -0700 Subject: [PATCH 093/275] Add shim TupleManager, delimiter managers, and the read-only regexes proxy Co-Authored-By: Claude Fable 5 --- nameparser/_config_shim.py | 199 ++++++++++++++++++++++++++++++++++- tests/v2/test_config_shim.py | 109 ++++++++++++++++++- 2 files changed, 304 insertions(+), 4 deletions(-) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index cbfbdeb2..a8361c47 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -2,13 +2,16 @@ spec §3). ``nameparser.config`` re-exports these names from the swap commit onward; the whole module is deleted in 3.0 with the facade. -Layering: facade layer -- may import anything public; here that's just -``nameparser.util`` for ``lc()``. +Layering: facade layer -- may import anything public; here that's +``nameparser.util`` for ``lc()`` and ``nameparser.config.regexes`` for +the read-only regexes proxy's underlying compiled patterns. """ from __future__ import annotations -from collections.abc import Callable, Iterable, Iterator +from collections.abc import Callable, Iterable, Iterator, KeysView +from typing import Self +from nameparser.config.regexes import REGEXES from nameparser.util import lc @@ -126,3 +129,193 @@ def __setstate__(self, state: dict[str, object]) -> None: "_elements", state.get("elements", ())) self._elements = {lc(e) for e in elements} self._on_change = None # rewired by the owning Constants + + +class TupleManager(dict[str, object]): + """v1 ``TupleManager``: a dict with dot-notation access. Backs + ``capitalization_exceptions``. Unknown-key attribute access raises + ``AttributeError`` naming the key (#256, warned 1.4, enforced 2.0 -- + the v1 ``DeprecationWarning`` is gone, this shim only speaks 2.0). + Mutations call ``_on_change`` (the owning Constants' generation + bump, wired by a later task). + """ + + _on_change: Callable[[], None] | None + + def __init__(self, *args: object, + _on_change: Callable[[], None] | None = None, + **kwargs: object) -> None: + super().__init__(*args, **kwargs) + self._on_change = _on_change + + def _changed(self) -> None: + if self._on_change is not None: + self._on_change() + + def __getattr__(self, name: str) -> object: + # Only reached for a missing attribute -- real instance attrs + # (_on_change) and dict methods (keys, get, ...) resolve first + # without ever hitting this. Dunder/underscore probes (pickling, + # copy.deepcopy, IPython's _repr_html_) are never config keys. + if name.startswith("_"): + raise AttributeError(name) + try: + return self[name] + except KeyError: + raise AttributeError(f"no key {name!r} in this manager") from None + + def __setitem__(self, key: str, value: object) -> None: + super().__setitem__(key, value) + self._changed() + + def __delitem__(self, key: str) -> None: + super().__delitem__(key) + self._changed() + + def pop(self, *args: object) -> object: + # bump only on a real removal -- pop(key, default) on a missing + # key is a no-op read, not a mutation, same rule as SetManager's + # no-op add() + present = bool(args) and args[0] in self + result = super().pop(*args) # type: ignore[call-overload] + if present: + self._changed() + return result + + def popitem(self) -> tuple[str, object]: + result = super().popitem() # KeyError when empty: no bump + self._changed() + return result + + def clear(self) -> None: + had_items = bool(self) + super().clear() + if had_items: # clearing an empty dict is a no-op, not a change + self._changed() + + def update(self, *args: object, **kwargs: object) -> None: + # dict.update's C path skips a subclass __setitem__; route every + # item through it so subclass validation (_DelimiterManager's + # sentinel rule) and the owner notification hold here too + for key, value in dict(*args, **kwargs).items(): + self[key] = value + + def setdefault(self, key: str, default: object = None) -> object: + if key in self: + return self[key] # existing key: a read, not a mutation + self[key] = default # validated + notifying path + return default + + # in-place |= must validate/notify like update; dict's C path would + # skip both. mypy flags any non-overloaded __ior__ as inconsistent + # with dict.__or__'s overloads -- the runtime behavior is the plain + # dict |= contract, so the ignore is about typeshed shape only. + def __ior__(self, other: object) -> Self: # type: ignore[override, misc] + self.update(other) + return self + + # -- pickle interop ------------------------------------------------- + + def __reduce__(self) -> tuple[type[TupleManager], tuple[()], dict[str, object]]: + return (type(self), (), dict(self)) + + def __setstate__(self, state: dict[str, object]) -> None: + # routes through __setitem__ (validated for _DelimiterManager); + # _on_change is still None here, so no spurious bumps + self.update(state) + self._on_change = None # rewired by the owning Constants + + +_DELIMITER_SENTINELS = ("quoted_word", "double_quotes", "parenthesis") + + +class _DelimiterManager(TupleManager): + """v1 ``nickname_delimiters``/``maiden_delimiters`` bucket. In 2.0 + only the three named sentinels exist (spec §3) -- each maps to the + name of a ``_RegexesProxy`` entry it stays linked to; assigning any + other key raises so a caller reaches for a custom-delimiter Policy + kwarg instead of a dict entry that silently does nothing. ``pop()``/ + ``__setitem__``/``__delitem__`` stay open (inherited) for the + documented bucket-move idiom, e.g. + ``maiden_delimiters['parenthesis'] = nickname_delimiters.pop('parenthesis')``. + """ + + def __init__(self, *args: object, + _on_change: Callable[[], None] | None = None, + **kwargs: object) -> None: + # dict's C-level __init__ never calls a subclass __setitem__, so + # collect and validate the initial items here -- BEFORE any item + # lands -- or the sentinel rule silently misses the constructor + items: dict[str, object] = dict(*args, **kwargs) + for key in items: + self._reject_non_sentinel(key) + super().__init__(items, _on_change=_on_change) + + @staticmethod + def _reject_non_sentinel(key: str) -> None: + if key not in _DELIMITER_SENTINELS: + raise TypeError( + f"2.0 delimiter managers accept only the named sentinels " + f"{_DELIMITER_SENTINELS}; for custom delimiter pairs use " + f"Policy(nickname_delimiters=...) / maiden_delimiters" + ) + + def __setitem__(self, key: str, value: object) -> None: + self._reject_non_sentinel(key) + super().__setitem__(key, value) + # update/setdefault/|= inherit TupleManager's routing through + # __setitem__, so they validate (and notify) for free + + +class _RegexesProxy: + """Read-only view over the v1 compiled patterns + (``nameparser.config.regexes.REGEXES``). Reads keep working -- + ``CONSTANTS.regexes.word`` stays informational -- but 2.0 configures + parsing behavior through named ``Policy`` flags, not by mutating a + regex, so any attribute *or* item assignment raises ``TypeError`` + (spec §3's uniform read-only rule). + """ + + def __getattr__(self, name: str) -> object: + if name.startswith("_"): + raise AttributeError(name) + try: + return REGEXES[name] + except KeyError: + raise AttributeError(f"no regex named {name!r}") from None + + def __getitem__(self, name: str) -> object: + return REGEXES[name] + + def __contains__(self, name: object) -> bool: + return name in REGEXES + + def __iter__(self) -> Iterator[str]: + return iter(REGEXES) + + def keys(self) -> KeysView[str]: + return REGEXES.keys() + + def __setattr__(self, name: str, value: object) -> None: + self._raise_readonly(name) + + def __setitem__(self, name: str, value: object) -> None: + self._raise_readonly(name) + + @staticmethod + def _raise_readonly(name: str) -> None: + # bidi/emoji are the two regexes v1 code toggled directly + # (`CONSTANTS.regexes.bidi = False`) to opt out of stripping; + # point those two at their named Policy replacement, everything + # else gets the generic pointer. + hints = { + "bidi": "use Policy(strip_bidi=False) to keep bidi marks", + "emoji": "use Policy(strip_emoji=False) to keep emoji", + } + hint = hints.get( + name, "parsing behavior is configured through named Policy " + "flags in 2.0; if none fits, open an issue") + raise TypeError( + f"assigning CONSTANTS.regexes.{name} is not supported in " + f"2.0: {hint}" + ) diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index ec980417..79c4b940 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -1,9 +1,12 @@ """Shim Constants/SetManager/TupleManager (migration spec §3).""" +import copy import pickle import pytest -from nameparser._config_shim import SetManager +from nameparser._config_shim import ( + SetManager, TupleManager, _DelimiterManager, _RegexesProxy, +) def test_set_manager_normalizes_and_holds_membership() -> None: @@ -80,3 +83,107 @@ def test_set_manager_pickle_round_trip() -> None: # security boundary here (same stance as the v2 pickle guards) t = pickle.loads(pickle.dumps(SetManager(["Dr", "Mrs."]))) assert t == SetManager(["dr", "mrs"]) + + +def test_tuple_manager_attribute_access_and_unknown_key() -> None: + t = TupleManager({"mcdonald": "McDonald"}) + assert t.mcdonald == "McDonald" + assert t["mcdonald"] == "McDonald" + with pytest.raises(AttributeError, match="mcdonalds"): # #256 + t.mcdonalds + + +def test_tuple_manager_mutations_bump_owner() -> None: + bumps = [] + t = TupleManager({"a": "A"}, _on_change=lambda: bumps.append(1)) + t["b"] = "B" + del t["a"] + t.pop("b") + assert len(bumps) == 3 + + +def test_tuple_manager_pop_default_on_missing_key_is_noop() -> None: + bumps = [] + t = TupleManager({"a": "A"}, _on_change=lambda: bumps.append(1)) + assert t.pop("missing", None) is None + assert len(bumps) == 0 # no-op: key was never present + + +def test_tuple_manager_bulk_mutations_bump_owner() -> None: + # dict's C fast paths (update, setdefault, |=, clear, popitem) must + # notify too, or a cached parser built from the owner goes stale + bumps = [] + t = TupleManager(_on_change=lambda: bumps.append(1)) + t.update({"a": "A", "b": "B"}) + assert len(bumps) == 2 # per-key, via __setitem__ + assert t.setdefault("c", "C") == "C" + assert len(bumps) == 3 + assert t.setdefault("a", "ignored") == "A" + assert len(bumps) == 3 # existing key: read, not write + t |= {"d": "D"} + assert len(bumps) == 4 + assert t.popitem()[0] in "abcd" + assert len(bumps) == 5 + t.clear() + assert len(bumps) == 6 + t.clear() + assert len(bumps) == 6 # already empty: no-op + + +def test_tuple_manager_pickle_round_trip() -> None: + t = pickle.loads(pickle.dumps(TupleManager({"a": "A"}))) + assert dict(t) == {"a": "A"} + assert t._on_change is None + + +def test_delimiter_manager_sentinels_only() -> None: + d = _DelimiterManager({"parenthesis": "parenthesis"}) + moved = d.pop("parenthesis") + d2 = _DelimiterManager() + d2["parenthesis"] = moved # the documented bucket-move idiom + assert "parenthesis" in d2 + with pytest.raises(TypeError, match="quoted_word"): + d2["angle_brackets"] = "custom" # spec §3: custom keys raise + + +def test_delimiter_manager_no_bypass_via_constructor_or_update() -> None: + # dict's C fast paths (dict.__init__, dict.update) skip a subclass's + # __setitem__ -- the sentinel rule must hold on every mutation path + with pytest.raises(TypeError, match="quoted_word"): + _DelimiterManager({"angle_brackets": "x"}) + with pytest.raises(TypeError, match="quoted_word"): + _DelimiterManager(angle_brackets="x") + with pytest.raises(TypeError, match="quoted_word"): + _DelimiterManager().update({"angle_brackets": "x"}) + with pytest.raises(TypeError, match="quoted_word"): + _DelimiterManager().setdefault("angle_brackets", "x") + with pytest.raises(TypeError, match="quoted_word"): + d0 = _DelimiterManager() + d0 |= {"angle_brackets": "x"} + # update through the validated path still notifies the owner per key + bumps = [] + d = _DelimiterManager(_on_change=lambda: bumps.append(1)) + d.update({"quoted_word": "quoted_word", "parenthesis": "parenthesis"}) + assert dict(d) == {"quoted_word": "quoted_word", + "parenthesis": "parenthesis"} + assert len(bumps) == 2 + + +def test_regexes_reads_ok_assignment_raises() -> None: + r = _RegexesProxy() + assert r.word.match("Smith") # type: ignore[attr-defined] # reads keep working + with pytest.raises(TypeError, match="strip_bidi"): + r.bidi = None # slot-aware message + with pytest.raises(TypeError, match="Policy"): + r.roman_numeral = None # generic message + + +def test_regexes_membership_iteration_and_deepcopy() -> None: + r = _RegexesProxy() + assert "word" in r # membership is a read + assert "nope" not in r + assert "word" in sorted(r) # iteration is a read + assert "word" in r.keys() + # dunder probes must raise AttributeError, not resolve as regex + # names -- the classic copy.deepcopy regression (see AGENTS.md) + assert isinstance(copy.deepcopy(r), _RegexesProxy) From e2979f67cedaea1ee5c87755a9051435f5058bd7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 02:39:00 -0700 Subject: [PATCH 094/275] Add the shim Constants with generation tracking and pickle acceptance Co-Authored-By: Claude Fable 5 --- nameparser/_config_shim.py | 246 +++++++++++++++++++++++++++++ tests/v2/data/constants_v14.pickle | Bin 0 -> 13562 bytes tests/v2/test_config_shim.py | 126 ++++++++++++++- 3 files changed, 371 insertions(+), 1 deletion(-) create mode 100644 tests/v2/data/constants_v14.pickle diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index a8361c47..d65ba722 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -8,6 +8,7 @@ """ from __future__ import annotations +import warnings from collections.abc import Callable, Iterable, Iterator, KeysView from typing import Self @@ -319,3 +320,248 @@ def _raise_readonly(name: str) -> None: f"assigning CONSTANTS.regexes.{name} is not supported in " f"2.0: {hint}" ) + + +_SET_FIELDS = ( + "prefixes", "suffix_acronyms", "suffix_not_acronyms", + "suffix_acronyms_ambiguous", "titles", "first_name_titles", + "conjunctions", "bound_first_names", "non_first_name_prefixes", +) +_MANAGER_FIELDS = _SET_FIELDS + ( + "capitalization_exceptions", "nickname_delimiters", "maiden_delimiters", +) +_SCALAR_DEFAULTS: dict[str, object] = { + "patronymic_name_order": False, + "middle_name_as_last": False, + "capitalize_name": False, + "force_mixed_case_capitalization": False, + "string_format": "{title} {first} {middle} {last} {suffix} ({nickname})", + "initials_format": "{first} {middle} {last}", + "initials_delimiter": ".", + "initials_separator": " ", + "suffix_delimiter": None, +} + +# distinguishes "attribute not set yet" from any real scalar value +# (None is a legitimate value for string_format/suffix_delimiter) +_UNSET = object() + +_SHARED_MUTATION_MESSAGE = ( + "mutating the shared CONSTANTS singleton is deprecated and will be " + "removed in 3.0; build a Lexicon/Policy (or a private Constants " + "passed as HumanName(constants=...)) instead. See the migration " + "guide." +) + + +def _default_vocab() -> dict[str, set[str]]: + # v1 data modules stay the single vocabulary source through 2.x + # (same rule as Lexicon.default()). + from nameparser.config.bound_first_names import BOUND_FIRST_NAMES + from nameparser.config.conjunctions import CONJUNCTIONS + from nameparser.config.prefixes import ( + NON_FIRST_NAME_PREFIXES, PREFIXES, + ) + from nameparser.config.suffixes import ( + SUFFIX_ACRONYMS, SUFFIX_ACRONYMS_AMBIGUOUS, SUFFIX_NOT_ACRONYMS, + ) + from nameparser.config.titles import FIRST_NAME_TITLES, TITLES + return { + "prefixes": PREFIXES, + "suffix_acronyms": SUFFIX_ACRONYMS, + "suffix_not_acronyms": SUFFIX_NOT_ACRONYMS, + "suffix_acronyms_ambiguous": SUFFIX_ACRONYMS_AMBIGUOUS, + "titles": TITLES, + "first_name_titles": FIRST_NAME_TITLES, + "conjunctions": CONJUNCTIONS, + "bound_first_names": BOUND_FIRST_NAMES, + "non_first_name_prefixes": NON_FIRST_NAME_PREFIXES, + } + + +class Constants: + """v1 ``Constants`` shim: a mutable container whose state resolves to + a frozen ``(Lexicon, Policy, _RenderDefaults)`` snapshot on demand + (added in a later task). ``_generation`` increments on every + mutation; facades compare it against a cached value to decide + whether their snapshot is stale (dirty-tracking, spec §3). + + The module-level ``CONSTANTS`` singleton (below) has ``_shared`` + flipped to ``True``: any mutation reached through it emits + ``DeprecationWarning`` pointing at ``Lexicon``/``Policy`` and + ``HumanName(constants=...)``. A private ``Constants()`` never + warns -- only the shared instance is on the 3.0 removal path. + """ + + _shared = False # the CONSTANTS singleton flips this to True + _generation: int + + prefixes: SetManager + suffix_acronyms: SetManager + suffix_not_acronyms: SetManager + suffix_acronyms_ambiguous: SetManager + titles: SetManager + first_name_titles: SetManager + conjunctions: SetManager + bound_first_names: SetManager + non_first_name_prefixes: SetManager + capitalization_exceptions: TupleManager + nickname_delimiters: _DelimiterManager + maiden_delimiters: _DelimiterManager + regexes: _RegexesProxy + + patronymic_name_order: bool + middle_name_as_last: bool + capitalize_name: bool + force_mixed_case_capitalization: bool + string_format: str | None + initials_format: str + initials_delimiter: str + initials_separator: str + suffix_delimiter: str | None + + def __init__(self) -> None: + vocab = _default_vocab() + object.__setattr__(self, "_generation", 0) + for name in _SET_FIELDS: + object.__setattr__( + self, name, SetManager(vocab[name], _on_change=self._bump)) + from nameparser.config.capitalization import ( + CAPITALIZATION_EXCEPTIONS, + ) + object.__setattr__(self, "capitalization_exceptions", TupleManager( + CAPITALIZATION_EXCEPTIONS, _on_change=self._bump)) + object.__setattr__(self, "nickname_delimiters", _DelimiterManager( + {name: name for name in _DELIMITER_SENTINELS}, + _on_change=self._bump)) + object.__setattr__(self, "maiden_delimiters", _DelimiterManager( + _on_change=self._bump)) + object.__setattr__(self, "regexes", _RegexesProxy()) + for name, value in _SCALAR_DEFAULTS.items(): + object.__setattr__(self, name, value) + + def _bump(self) -> None: + # stacklevel=3 is exact for the direct scalar-assignment path + # (user code -> Constants.__setattr__ -> here) and lands one + # frame short -- inside the manager's own add()/remove()/ + # __setitem__ -- for the indirect manager-mutation path (user + # code -> manager method -> _changed() -> here), since a + # single warn() call can't be exact for both call depths at + # once. Either way the warning still fires from this module, + # not the manager's true caller, which is enough: only the + # DeprecationWarning's presence/category/message are load- + # bearing (see the specified test), not the reported line. + if self._shared: + warnings.warn(_SHARED_MUTATION_MESSAGE, DeprecationWarning, + stacklevel=3) + object.__setattr__(self, "_generation", self._generation + 1) + + def __setattr__(self, name: str, value: object) -> None: + if name == "empty_attribute_default": + raise AttributeError( + "empty_attribute_default was removed in 2.0 (#255): " + "empty attributes are always ''" + ) + if name == "regexes": + raise TypeError( + "replacing CONSTANTS.regexes is not supported in 2.0; " + "parsing behavior is configured through named Policy " + "flags" + ) + if name in _SET_FIELDS: + # v1 allowed wholesale reassignment (c.titles = {...}) + value = SetManager(value, _on_change=self._bump) # type: ignore[arg-type] + elif name == "capitalization_exceptions": + value = TupleManager(value, _on_change=self._bump) # type: ignore[arg-type] + elif name in ("nickname_delimiters", "maiden_delimiters"): + value = _DelimiterManager(value, _on_change=self._bump) # type: ignore[arg-type] + elif name in _SCALAR_DEFAULTS and \ + getattr(self, name, _UNSET) == value: + # no-op scalar assignment: managers already suppress no-op + # mutations, so re-assigning the current scalar value must + # not bump the generation (or warn on the shared singleton) + # either. __init__ writes via object.__setattr__, so this + # only runs on real user assignments -- _UNSET never + # actually matches, it just keeps a not-yet-set attribute + # from raising here. Manager-field reassignment above stays + # an unconditional bump: comparing manager contents isn't + # worth it. + object.__setattr__(self, name, value) + return + object.__setattr__(self, name, value) + if name in _MANAGER_FIELDS or name in _SCALAR_DEFAULTS: + self._bump() + + def copy(self) -> Constants: # #260 + # An independent instance with its own generation counter and + # its own manager callbacks -- not a shared-state alias like a + # naive attribute-for-attribute copy would produce. + new = Constants() + for name in _SET_FIELDS: + object.__setattr__( + new, name, + SetManager(getattr(self, name), _on_change=new._bump)) + object.__setattr__(new, "capitalization_exceptions", TupleManager( + dict(self.capitalization_exceptions), _on_change=new._bump)) + for bucket in ("nickname_delimiters", "maiden_delimiters"): + object.__setattr__(new, bucket, _DelimiterManager( + dict(getattr(self, bucket)), _on_change=new._bump)) + for name in _SCALAR_DEFAULTS: + object.__setattr__(new, name, getattr(self, name)) + return new + + # -- pickle ----------------------------------------------------------- + + def __getstate__(self) -> dict[str, object]: + state: dict[str, object] = {} + for name in _SET_FIELDS: + state[name] = set(getattr(self, name)) + state["capitalization_exceptions"] = dict( + self.capitalization_exceptions) + state["nickname_delimiters"] = dict(self.nickname_delimiters) + state["maiden_delimiters"] = dict(self.maiden_delimiters) + for name in _SCALAR_DEFAULTS: + state[name] = getattr(self, name) + return state + + def __setstate__(self, state: dict[str, object]) -> None: + if "suffixes_prefixes_titles" in state: + # pre-1.3.0 blob: its dir()-sweep __getstate__ captured this + # computed property. The 1.4 DeprecationWarning promised + # ValueError in 2.0 (#279). + raise ValueError( + "this pickle was written by nameparser <= 1.2.x (#279); " + "re-pickle under 1.3/1.4 to migrate, or re-create the " + "configuration. See " + "https://github.com/derek73/python-nameparser/issues/279" + ) + # Accepts BOTH shapes with a single overlay: the shim's own + # state and v1.3/1.4 state (public field names -> manager/ + # scalar values) share every key that matters, so no shape + # marker is needed. empty_attribute_default is accepted and + # DROPPED (#255: empty is always '' in 2.0). + state = {k: v for k, v in state.items() + if k != "empty_attribute_default"} + self.__init__() # type: ignore[misc] # defaults, then overlay + # (managers re-wrapped below so _on_change points at THIS + # instance, not whatever produced the incoming state) + for name in _SET_FIELDS: + if name in state: + object.__setattr__(self, name, SetManager( + state[name], _on_change=self._bump)) # type: ignore[arg-type] + if "capitalization_exceptions" in state: + object.__setattr__( + self, "capitalization_exceptions", TupleManager( + state["capitalization_exceptions"], # type: ignore[arg-type] + _on_change=self._bump)) + for bucket in ("nickname_delimiters", "maiden_delimiters"): + if bucket in state: + object.__setattr__(self, bucket, _DelimiterManager( + state[bucket], _on_change=self._bump)) # type: ignore[arg-type] + for name in _SCALAR_DEFAULTS: + if name in state: + object.__setattr__(self, name, state[name]) + + +CONSTANTS = Constants() +CONSTANTS._shared = True # type: ignore[attr-defined] diff --git a/tests/v2/data/constants_v14.pickle b/tests/v2/data/constants_v14.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0e94446528a7ad1a3c32ba57ff685a5978488b36 GIT binary patch literal 13562 zcmaKz%a0}JS;p zT~1F=CX%BdP?Q#lltv;D#T$eaBoHP-v64U*&I+*@XG6BkKaj9s!H(baesy}r#=)|x z>r{Q0_xik-ujZBLFMjm=WA@+q6IEPhP26@_d(G9=DqmlmFMgx0x<0P@?&2#K&wTFU zvlmy-=S`cf@;g~~v3cx#`TJS_$+(KwS^LO{tjNktlfK0GaGMnu=fTtc#d(;%@%LQg zVZ}91s*CeUniWNUaUN|lR?yG2eg@l=5!<>PN5nj;8%<3Hu1wN>%mPu8SGKaI^Mc8? zX2!2=&43`)da+0-cxW$cew5C}YedHaLA=$H8Nb7RqX98QM(o7iVQTYJJx;Z960_TU z)n@r7Yqjwv(;6vbCmrI_oHEu9{Ic=(8rp-qo&h+E*oVcIyJd14DVitSxY9mT`|BBd znq|_uV})K6^u{mD^T0@CsJ7$fm(Q0+b#>%6akXaXJQ?z$&nu?R)>$9-y;QXc&)bGE77(tq+G@?X6aSHGoOX51iuchv*VwSN^}2CcSISac!+_~(gS8`)&~13g zwARRs8*SB=-HG1Cw#aQUIB{Jks-hED;uukp_fQwP&UT2q90nFu;!?LcXqX$dDUxH_ zDognrB+wX0Ac13KP-&^QZ}}588#Ej>YrT!)repZDYHnVC;g-ltJb6ZC>P+-m*&igl ztF^Zjao{D%byB*G*qWw!Q^)`#*Y>tT-byacZM3Cp#Dm+2D*T8E;p$Y^xDhR#C&`^w zdcQU*%g&2eT8k>zOAF3v4jbvd7x~DM_dy1a(W>0?B3i}vzTHAbVA3ke?v(pj3sx!S zcs`eI51hmu<&A6vC$#-eV}8uKzRa^sb{`k4KSR(5n_Qbazqw=_b$PBKZgg#<2YIqp z!}CCR6F3A19AJpzM1~O*h%LNrCYv-^VB|_O zx5ArNqJ2Vlhjk}uoUFXWR+2&&OMWxsv(p2&lD&qjOuL*oPr2kBrCSp~o64aX#x+YJ zY`%`O?B@07Wfmvit8y(GwsMBB-Rj;I;0p}4`mt_BW7;+wew5zdO#hYq4bwLk)}nJR zqsl+>!pAiGOyb16pe8sG%P!|OY|KLQ)He0Xa-IBUk`trdGKGaHPY!Dd*PyQkU` zW01*2wYg2`VkTpgW@b*BmS5@5?Ugo}#w)~T(sCjO7}209tc>Ou%gG|9GCEp&*DTYG zyrcETeOr1n0Tkx`Ss~t>bRP|!a2&oQW2b;;uqM$JGFQTl-ZM;_G^hOG5rZ|%)@(L# zHEYWJfCn1XdBrsH892+uMYD-p6L^`4%(9fj1s;sdx4Pb}66ewAu6!z&LQ69o=OfTc zG=#A&(>U2o6U*&`R$_Ijbcpw2GBdZ0Z{D~eD_M(}up|(}%uoE@YQ*V*`V*kwu{ZgG9HIT%42oy3Bm)#F?7isWIL-#X5`PvB9QW*D$@k^r*g^zG1(}5f}gUKz#LW9zc^vQM}SwJ&WbEcpOPuy>jSaS@aiHnM_)5jC@KUt2 zmYnRulaqUesZ1v9Oqx!CBTUT`Gc%>3bHdb8>1b#{IyB#z6V=A-A-t{ND2Yl~!CFML z9=HtY$gWue5jT469c*h7CEVK!GyGL-n0nh;ygil;w8{=-Z003(VnCBxDpOkcJuw>& zESXw+pH59<25=;0>=+f6r2vcyQKctLSu|OIAxCPWcL_`g^0hIt)K&2ztPCAH9|TSW zA0gQ!%I}C<2V-o=6tv=Atgmve(|ong9h*$57UAoHJaPIB_)GQRo3_MO$!UL%of50#JO-qDp!^ z$mJ9Bmb8^{gVqc*X{{P?l_c$Ag=smS7OgDnB?X2IMB=uJMMODSoey5cR(!?l&Y<6d zl2V(%A^E7RPMv`o#Prwh+=9p^e3b zU7V3dT`$9l66m$H^#_&OeyZuPRIBMiH<*mPIjhtLPZhNAJh20Am@eB?(6tphL58i_ zc;{2BnD$9~;Pt$=Nh`Cb)tKm&B2}e`F}UG6-3dQX5JRd~9+VbcO0^sU3s0$;P3ThT z#L0Smk0X(p&Bn0G{K7{AVY-SCHXtRCpAcdt+qFT&Oqoh$v2}weSgf|qX;x^Y3^=f4 zbX^KZEe{VHVGXp4J6oU=Vg;-8;>(-g+Wht=+~l{4!O}`sZfyeFtFUuGQh`g@UzpiV=j*?RoL1x$4Z<6>bbX5GwrG7)zErN zqhmtnYXu+#17Uwwo}uve?BaYcsZ(mtxbJ0svodZexmkF@-AP8H1?PQN6Bk)s^>tCx ziGT$hFwat>x5HjaFN(s)WqeWo?1tzL06OWUC=bvJi)nWeda zaN4n%4TI8NfqlX9tV0KiEkRcK;)Dr?-)fvpu6Mm{>gt14Jz1@HZne-pxdbwZ zdrAW~VB&y$4I5U4Xr;a~7tFY-cO)6~nYNTr>&8-ZRUt+y@~h#3ZI+VlPB_z&{(=NL zr-{4HjF%<0ie_>I%F1P-taB;*I&Jf{x=K_Ht!P?ovx;33>ymC!0~;^8?2Xr$Y!z7% zr*J6KD5PmXSQfqCZ0f#Vx0Hz@75!A}!I(t3H3KGPe5UaWnx^937;5V?37u9UJ1=AM zhpvlL^X@i-kO3;MTK>3FK|Q=h*}#zzYxmsRVPp~YUafw8Xke1uI7&EOjL3nMb)c{y~Wlk6pJ9jBna zaTArb(PvHrqkT(-Ihfv`@Gg4L)zun-d*%sr>hjLxbXz;9#8}pC{l=@SH@<6{c#pry z+Pf&$Z6w6SNmZW~S*nKa#LOJd@MRN7l$a0Fn6J~p>~=9V zB(bXLT(2QvdV;p(Ff-JS=_r@!Gi-}jJGh;1AwuRVa~P7ifC;8f;v`W>SRmEn4D?`? z@U*6XG|lc{uDqJlUq-~HRlpI@mT3YNj zM5Bq>%S~O?!)T>KyBgGz1bHtbogss*X(N!mhc~oJBrc(=!)@ivbC22bzB%^A{56Ih zZSV)CIE9lrZNI$Btr@?GkB^0DGG~Nj>~t8rLC>VgdO1FoDfCah`-={GK5@-@ff^$7 zbm(yx?Kmsz#GB2OHSVhBpPkk z!WH=XnN6QHK)6{ngN3-2DF`fXi%eRfUS42R*l>f0KeLJjZ2pP>LFg6%DHmUuPl9{~YGF%7N22u5G2+bZW!Wfydp z9D4NaKuyZaAVJcoRe+*E^e$Xs)7B6(hG=>jP`7W|VN7q)khYn8q5&D06DVKCYO4(# z>TUpKMFBXeXNHqg{+OS0@PhZ70wTnE-7FBvVHArIl%;d9+6`U`m{`^&XEu8&zIV9= zk$$up%+=1*t~|5`BXmw7nr0U07s=4#&7cGnt5Iip&>X11090JfJw8gn)I3V%XK_JX z)`NLhuliGh7Pr*BB8+T+0~gk@p^Fs1FEoi8@sQ7vL2BXKwb;vqPy;w{pkwLb`W$?J@Fn-9qmZVwI8`ep3jS+yjv z+^NK|h@F&r@0UI}^FUEyY8A}Kd-k1{HP{B*p{v<@h6S%ZTKWV;&X%oK!~`sDj)#wa zg(XYAraxTJBo~1SP!^_6ipWYND-q#aIao?!&I)8AV`9BfQGM;e0z_2As*0w&+#>S% zMs#X#$?t{f$0jG7Ge^^~L6=BJQh*`Ziq}T8u?QrfK(g#@N!ktu1^YxdL`pn5Q)vM_ z5BO`M^mIq0D&_x0{&l1MByVdB6%3g(A`qM$8Mu?6Z(S zhClO*b7wGYcZzYKM)XI=0UJA(oyv_F2f&>|DZAZ#Ye(W)V zxKHI^?$+X}P(dcb13g)yVsjOto^f-D^K-%nXrm=(YYK;z@~yB`n|)H&cHBQN&H!$2 z%;1c{C}##f5uX&j`QxO`66-U_tv#no{>+pX*qfrK0Y;js9I=q11d(i?{W3cv#}|PN ze#>QvC>aKPC6;F&UsHXr(MmliukVLs$zM-X7mjv?}Gj^$u{v8bC<*xR$EH$sBBkoM6qRT zVWYH!=!y+#Rk8(@h`z=WQWl03iZfK0YTqkAlfV$bv;jm7UIZFYyD0L6+RFmYM0tm@ z0A#B0tLELTY*Z0pRw%?gS-FA&o`4S2p<{(Hn6#H96g@!ZHA-o3**ch}9+xaTyrDvs z#Iy!$A$``>MQTVsApohrL1jyUBoi4{z3kS?U_Fm3MdgrW5wlZK1lSpRTCZTzjfDiQ z%v|}O@d%6wRrf9CUPAg6SL3Z5rHbRiZT&qHuVFb*}#3H_~ zpbHj=3KSNW3Ss-Iu=)4}!dy4o)N@3xwQa3cTlW(#)D{+vm z>;fvZW2G-+fH%}RoM@~$@~kBL-$=WA9SQS8%BUb1PVi7&!^~xY#Zmy6h3+@%Gbmj$ zg(Lq45fMBEs>p<>0IhX;n(Jt0o&kSelze8QN|N>7i?O%sH@@*y`ZTQ+Xi+GMsP@@5 zO!eKvm0gl_iq?_kB&)VHQkW(~nyS?}X9>zx8Ml3?6QL*EYl-Y7YSZep9DMwWu7A|R zi9(oUJacEx=Zec?$_Auw=C*IPcNw>qh?3_%XW@`=fT|yzj&p5aCZaY`)G3M$kc~$tFVwkcM zVX7#}J09;5%qS&cLzW`v+9;!@6f<=!yE7b@+@}&vL?|yo%!yiA%Y@)0#(9?nY0P@} zu))liI)odN&YV`U7x4mzL-*z#L98xvf~m^i()!E-*tWKaFj-?D_+ONf#G8~#Oj-Mp zPmrM-ZA3X-L`Fl%#UaP&NoN~Q77)=6)v07T$tdobyG-i(#F9*Y?Bg<)tfO(pkTa_< zA!`Z^Sl)|+z&30W%IV2ivyGTMK7=;D7(${!Nku}sNXqTeKU~{)UU)U)<(nn7yB3eM z2IkpGwobeQVDvsyqW%Jo1si6BI;|}r zX;A`xV`38euN{Od2twMw3(cp>(1{`5N2$%!{HBox!YkNHShLI2dqF*p`}-~AB#3Ll zhX-};Grg^kjee2C!j0wT$*yFP-1hZ zqX^%*yIfgGn~L9g1xkA~Su_J?ppF0xQOwb^HdK;yZlD@Ep|78K6esknP*J>5o$I>= z$jV0$Eui>sjLprsEKpBrc=>K+&4+oG9YjTcj3@7Tq(}LZ77<0pmXi=`hd@V+vj!9K zzNlu?2!_OU{F0c$i0}o8BoG)AP%UfN_qp`0E1=YQd+i{tF~&Vo36eZ|Bg!6UV@d+! zs)=GBpK0EXL3OD%e{1NYv}*w=5RF)}z>l`qJm_)=Ax{qMTRp+71P(?E(R2_-mDtiNzBFf3_>3^_4aQDd z-t#al>?AR#wTjHYhqiEb&TJ4(!3>uqaik@)+ zc9}UEYSr{YdF-QB=DRy~%oB17!1K$#tenISd&cx$5mR~gSoXyOJWI(Rv$(jcibEgQ zu&c4?zwF3Q5Uf55c;w^O>8>xyyEI3t;qzuOI{uj>4T7Myn8_2ZC#{;oi$h7M4xE>0 zz2b8hd&;S{;fmz-%eDo8JZ+x3- zS7w20V-LmZF7|tvT1Ou7F3Zqedtm47inK~&X(9T`V+2iBZ5`DaQe`S`IOb}RlYne`*<9pQiD$KY={JF~B+ z?VE4^!Sp+P^8F6qdi!s@?OW>q=ezDlKB49lp{HTnzKh;{+3i1#*7w|_5Z?E;@0pJ{ zZpha|O@sKY2Y!LRkkPlRyNLh?`9_i7wKVTYEsbVW%46sIe=syf_Q~;3fzRVVWuM2} zL1ggKOOWNiM|(;4I(0k`Uivf_xlQ!RYrlDo5x%?lyBqyN@iTHyaueu!?EHyO>HeMf zn09=aq~s^CKHU*{4jszu;wvn9*nRlo6Hn=w^_;FFS>*A5P9A&ge17%t^xomMXD(mb z_v!xKyI;Oq#xJ^O81k7L&wloqi%)#O=FjN=(FQHXB^Y*idU);L&0jnC<6Fq=S^HnSs}+>Ki-4nJ`9 zng)MhH*Al8kwD=EfO2@?$z2>?^@AR+)v9{nwdj876k6-=jmwx=}U+LzrzxUuTUpaWdpRe)f>)*R|sRhiO zmX@FQzMDV%+JjgA{J|?<`a>^xgAYyVpO%0akwmVAG{N zQg)|*2ta$^uk`CzYo2fJ)!Qn*NAbXIrdw`wr2^u~$#9n$n|sA@qCYDgLhFxyJ=DW5 zyz@p!j%E+F@ovZCtM~u){y*J+{mnnSXE*Zr}V9mhw;S)}B73?C0p6LF*&j9}%vVhIt|U-pg0(burwlzyGSP z>*7q8{9b+Y=?AZT_20g8YrVVS;cZI5d$+&;+THKJcJD;@Cueuh?yYqhm^ci@@Nnn% z|D>JYE|rs)C6;f(>*pmF-F%UoAAH50y!mzA+<*NZ9P^5s``>=^i^zg8-yCm#@Yk0& z_g}yNEk9VR{NOcP?9Ia9I{d@K^yrpQ#z?fpCE>{0Xhzhv2V`ET9X<$rXi3ky7<@-3^5-f?My=9B-j2IL}7vC literal 0 HcmV?d00001 diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index 79c4b940..85e7537f 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -1,13 +1,18 @@ """Shim Constants/SetManager/TupleManager (migration spec §3).""" import copy import pickle +import warnings +from pathlib import Path import pytest from nameparser._config_shim import ( - SetManager, TupleManager, _DelimiterManager, _RegexesProxy, + CONSTANTS, Constants, SetManager, TupleManager, _DelimiterManager, + _RegexesProxy, ) +_DATA_DIR = Path(__file__).parent / "data" + def test_set_manager_normalizes_and_holds_membership() -> None: s = SetManager(["Dr", "MRS."]) @@ -187,3 +192,122 @@ def test_regexes_membership_iteration_and_deepcopy() -> None: # dunder probes must raise AttributeError, not resolve as regex # names -- the classic copy.deepcopy regression (see AGENTS.md) assert isinstance(copy.deepcopy(r), _RegexesProxy) + + +def test_constants_default_fields_present() -> None: + c = Constants() + assert "dr" in c.titles + assert "phd" in c.suffix_acronyms + assert "van" in c.prefixes + assert c.patronymic_name_order is False + assert c.middle_name_as_last is False + assert c.capitalize_name is False + assert c.force_mixed_case_capitalization is False + assert c.string_format == "{title} {first} {middle} {last} {suffix} ({nickname})" + assert c.suffix_delimiter is None + assert set(c.nickname_delimiters) == {"quoted_word", "double_quotes", + "parenthesis"} + assert len(c.maiden_delimiters) == 0 + + +def test_constants_mutation_bumps_generation() -> None: + c = Constants() + g0 = c._generation + c.titles.add("zqxtitle") # not a default title (real bump) + assert c._generation > g0 + g1 = c._generation + c.patronymic_name_order = True + assert c._generation > g1 + + +def test_private_constants_do_not_warn_shared_singleton_does() -> None: + c = Constants() + with warnings.catch_warnings(): + warnings.simplefilter("error") + c.titles.add("zqxtitle") # private: silent + with pytest.deprecated_call(match="Lexicon"): + CONSTANTS.titles.add("zqxtest") + with pytest.deprecated_call(): + CONSTANTS.titles.remove("zqxtest") # leave the singleton clean + + +def test_scalar_noop_assignment_neither_bumps_nor_warns() -> None: + # managers already suppress no-op mutations (no-op add, pop with + # default); re-assigning a scalar's current value must match -- + # neither a generation bump nor, on the shared singleton, the + # deprecation warning + c = Constants() + g0 = c._generation + c.string_format = c.string_format + assert c._generation == g0 + g_shared = CONSTANTS._generation + with warnings.catch_warnings(): + warnings.simplefilter("error") + CONSTANTS.string_format = CONSTANTS.string_format + assert CONSTANTS._generation == g_shared + + +def test_empty_attribute_default_assignment_raises() -> None: + c = Constants() + with pytest.raises(AttributeError, match="#255"): + c.empty_attribute_default = None + + +def test_regexes_attribute_assignment_raises() -> None: + with pytest.raises(TypeError, match="Policy"): + Constants().regexes = {} # type: ignore[assignment] + + +def test_constants_copy_is_independent() -> None: # #260 + c = Constants() + d = c.copy() + d.titles.add("zqxtitle") # not a default title (real addition) + assert "zqxtitle" in d.titles and "zqxtitle" not in c.titles + assert d._generation != -1 # a real instance with its own counter + + +def test_constants_pickle_round_trip() -> None: + c = Constants() + c.titles.add("zqxtitle") + loaded = pickle.loads(pickle.dumps(c)) + assert "zqxtitle" in loaded.titles + loaded.titles.add("fra") # mutations still tracked + assert "fra" in loaded.titles + + +def test_pre_1_3_constants_blob_raises() -> None: # #279 + # pre-1.3.0 blobs are recognizable by the computed property their + # dir()-sweep __getstate__ captured; the 1.4 warning promised + # ValueError in 2.0 + with pytest.raises(ValueError, match="#279|1.3"): + Constants.__new__(Constants).__setstate__( + {"prefixes": set(), "suffixes_prefixes_titles": set()}) + + +def test_v14_constants_state_shape_accepted() -> None: + # v1.4 __getstate__ emitted public field names -> manager values; + # scalars ride along. empty_attribute_default is accepted and + # DROPPED (#255: empty is always '' in 2.0). + c = Constants.__new__(Constants) + c.__setstate__({"prefixes": {"van", "de"}, + "string_format": "{first} {last}", + "empty_attribute_default": None}) + assert "van" in c.prefixes + assert c.string_format == "{first} {last}" + assert not hasattr(c, "empty_attribute_default") or True # dropped + + +@pytest.mark.xfail( + reason="resolves to the v1 class until the M11 swap", strict=True) +def test_v14_constants_blob_unpickles_into_shim() -> None: + # tests/v2/data/constants_v14.pickle was produced by a real 1.4.0 + # install (`uv run --no-project --with "nameparser==1.4.*" ...`), + # not synthesized here -- it is the actual bytes a caller upgrading + # from 1.4 to 2.0 would hand to pickle.load(). Until the M11 swap + # replaces nameparser.config.Constants with this shim, the blob's + # pickled class reference still resolves to the live v1 class, so + # this assertion fails today by construction. + with open(_DATA_DIR / "constants_v14.pickle", "rb") as f: + loaded = pickle.load(f) + assert isinstance(loaded, Constants) + assert "van" in loaded.prefixes From 8b22f1dd30c3adf3b7c840dc078e4e8166d40543 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 02:54:08 -0700 Subject: [PATCH 095/275] Resolve shim Constants to (Lexicon, Policy, defaults) snapshots with a shared parser cache Co-Authored-By: Claude Fable 5 --- nameparser/_config_shim.py | 113 ++++++++++++++++++++++++++++++++--- nameparser/_lexicon.py | 2 + tests/v2/test_config_shim.py | 62 ++++++++++++++++++- 3 files changed, 168 insertions(+), 9 deletions(-) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index d65ba722..fdb0de66 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -3,15 +3,21 @@ commit onward; the whole module is deleted in 3.0 with the facade. Layering: facade layer -- may import anything public; here that's -``nameparser.util`` for ``lc()`` and ``nameparser.config.regexes`` for -the read-only regexes proxy's underlying compiled patterns. +``nameparser.util`` for ``lc()``, ``nameparser.config.regexes`` for +the read-only regexes proxy's underlying compiled patterns, and +``nameparser._lexicon``/``_policy``/``_parser`` for the ``_snapshot()`` +translation and the shared parser cache. """ from __future__ import annotations +import functools import warnings from collections.abc import Callable, Iterable, Iterator, KeysView -from typing import Self +from typing import NamedTuple, Self +from nameparser._lexicon import Lexicon +from nameparser._parser import Parser +from nameparser._policy import PatronymicRule, Policy from nameparser.config.regexes import REGEXES from nameparser.util import lc @@ -227,7 +233,17 @@ def __setstate__(self, state: dict[str, object]) -> None: self._on_change = None # rewired by the owning Constants -_DELIMITER_SENTINELS = ("quoted_word", "double_quotes", "parenthesis") +#: v1's three named delimiter buckets, translated to the ``Policy`` +#: (open, close) pairs they stand for (spec §3). +_SENTINEL_PAIRS = { + "quoted_word": ("'", "'"), + "double_quotes": ('"', '"'), + "parenthesis": ("(", ")"), +} + +#: derived, so the manager's accepted keys and _snapshot()'s +#: translation table can never drift apart +_DELIMITER_SENTINELS = tuple(_SENTINEL_PAIRS) class _DelimiterManager(TupleManager): @@ -379,12 +395,35 @@ def _default_vocab() -> dict[str, set[str]]: } +class _RenderDefaults(NamedTuple): + """v1 scalar rendering knobs that have no home on ``Policy`` + (spec §3): ``__str__``/initials formatting and capitalization stay + per-Constants defaults, layered onto a shared ``Parser`` by the + facade (a later task) rather than folded into the cache key.""" + + string_format: str | None + initials_format: str + initials_delimiter: str + initials_separator: str + suffix_delimiter: str | None + capitalize_name: bool + force_mixed_case_capitalization: bool + + +@functools.lru_cache(maxsize=64) +def _cached_parser(lexicon: Lexicon, policy: Policy) -> Parser: + # keyed on hashable value objects: shared across every facade whose + # Constants resolve to the same snapshot (spec §3) + return Parser(lexicon=lexicon, policy=policy) + + class Constants: """v1 ``Constants`` shim: a mutable container whose state resolves to - a frozen ``(Lexicon, Policy, _RenderDefaults)`` snapshot on demand - (added in a later task). ``_generation`` increments on every - mutation; facades compare it against a cached value to decide - whether their snapshot is stale (dirty-tracking, spec §3). + a frozen ``(Lexicon, Policy, _RenderDefaults)`` snapshot via + ``_snapshot()``. ``_generation`` increments on every mutation; + facades compare it against a cached value to decide whether their + snapshot is stale (dirty-tracking, spec §3 -- the facade itself is + a later task). The module-level ``CONSTANTS`` singleton (below) has ``_shared`` flipped to ``True``: any mutation reached through it emits @@ -510,6 +549,64 @@ def copy(self) -> Constants: # #260 object.__setattr__(new, name, getattr(self, name)) return new + # -- snapshot ----------------------------------------------------------- + + def _snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: + """Resolve this v1-shaped, mutable Constants into the frozen + 2.0 value objects it corresponds to (spec §3). A pure read: no + generation bump, no deprecation warning even on the shared + singleton -- only direct attribute mutation is on the 3.0 + removal path. + """ + from nameparser.config.maiden_markers import MAIDEN_MARKERS + acronyms = frozenset(self.suffix_acronyms) + # keep in sync with _lexicon._default_lexicon() (pinned by the + # default-Constants equality test in tests/v2/test_config_shim.py) + lexicon = Lexicon( + titles=frozenset(self.titles), + given_name_titles=frozenset(self.first_name_titles), + suffix_acronyms=acronyms, + suffix_words=frozenset(self.suffix_not_acronyms), + # intersect: Lexicon enforces ambiguous <= acronyms; v1 + # behaves the same when an acronym is deleted but its + # ambiguous entry lingers (the entry simply stops mattering) + suffix_acronyms_ambiguous=frozenset( + self.suffix_acronyms_ambiguous) & acronyms, + particles=frozenset(self.prefixes), + # complement translation: v1 marks the never-given subset; + # v2 marks the may-be-given subset + particles_ambiguous=frozenset(self.prefixes) + - frozenset(self.non_first_name_prefixes), + conjunctions=frozenset(self.conjunctions), + bound_given_names=frozenset(self.bound_first_names), + # v1 Constants has no manager for these (#274 is 2.0 + # behavior); the data module is the only source + maiden_markers=frozenset(MAIDEN_MARKERS), + # TupleManager is dict[str, object] (v1 parity: values were + # never statically str-typed); every real entry is a str, + # same assumption _DelimiterManager's sentinel lookup makes + capitalization_exceptions=tuple( + sorted(self.capitalization_exceptions.items())), # type: ignore[arg-type] + ) + rules = frozenset({PatronymicRule.EAST_SLAVIC, PatronymicRule.TURKIC}) \ + if self.patronymic_name_order else frozenset() + policy = Policy( + patronymic_rules=rules, + middle_as_family=self.middle_name_as_last, + nickname_delimiters=frozenset( + _SENTINEL_PAIRS[k] for k in self.nickname_delimiters), + maiden_delimiters=frozenset( + _SENTINEL_PAIRS[k] for k in self.maiden_delimiters), + # suffix_delimiter is a _RenderDefaults-only field here; the + # facade layers it onto extra_suffix_delimiters per instance + # (a later task) -- _snapshot() itself stays pure translation + ) + defaults = _RenderDefaults( + self.string_format, self.initials_format, self.initials_delimiter, + self.initials_separator, self.suffix_delimiter, + self.capitalize_name, self.force_mixed_case_capitalization) + return lexicon, policy, defaults + # -- pickle ----------------------------------------------------------- def __getstate__(self) -> dict[str, object]: diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 1647546f..dea3ed5f 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -296,6 +296,8 @@ def _default_lexicon() -> Lexicon: # v1 data modules export plain `set[str]`; wrap each at this call site # so the strictly-typed frozenset[str] fields never see a bare set. + # keep in sync with _config_shim.Constants._snapshot() (pinned by the + # default-Constants equality test in tests/v2/test_config_shim.py) return Lexicon( titles=frozenset(TITLES), given_name_titles=frozenset(FIRST_NAME_TITLES), diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index 85e7537f..d548d1dc 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -6,9 +6,10 @@ import pytest +from nameparser import GIVEN_FIRST, Lexicon, PatronymicRule, Policy from nameparser._config_shim import ( CONSTANTS, Constants, SetManager, TupleManager, _DelimiterManager, - _RegexesProxy, + _RegexesProxy, _cached_parser, ) _DATA_DIR = Path(__file__).parent / "data" @@ -311,3 +312,62 @@ def test_v14_constants_blob_unpickles_into_shim() -> None: loaded = pickle.load(f) assert isinstance(loaded, Constants) assert "van" in loaded.prefixes + + +def test_snapshot_field_translation() -> None: + c = Constants() + lexicon, policy, defaults = c._snapshot() + # drift guard: a default Constants must resolve to EXACTLY the v2 + # defaults -- a field populated in _default_lexicon() but forgotten + # in _snapshot() (or vice versa) fails loudly here + assert lexicon == Lexicon.default() + assert policy == Policy() + assert isinstance(lexicon, Lexicon) and isinstance(policy, Policy) + assert lexicon.suffix_words == frozenset(c.suffix_not_acronyms) + # complement translation: v1 marks never-given, v2 marks may-be-given + assert lexicon.particles_ambiguous == \ + frozenset(c.prefixes) - frozenset(c.non_first_name_prefixes) + assert lexicon.suffix_acronyms_ambiguous <= lexicon.suffix_acronyms + assert policy.name_order == GIVEN_FIRST + assert policy.patronymic_rules == frozenset() + assert defaults.string_format == c.string_format + # a snapshot is a pure read: no generation bump, no deprecation + # warning even on the shared singleton + g0 = CONSTANTS._generation + with warnings.catch_warnings(): + warnings.simplefilter("error") + CONSTANTS._snapshot() + assert CONSTANTS._generation == g0 + + +def test_snapshot_patronymic_and_middle_flags() -> None: + c = Constants() + c.patronymic_name_order = True + c.middle_name_as_last = True + _, policy, _ = c._snapshot() + assert policy.patronymic_rules == frozenset( + {PatronymicRule.EAST_SLAVIC, PatronymicRule.TURKIC}) + assert policy.middle_as_family is True + + +def test_snapshot_delimiter_bucket_move() -> None: + c = Constants() + c.maiden_delimiters["parenthesis"] = c.nickname_delimiters.pop("parenthesis") + _, policy, _ = c._snapshot() + assert ("(", ")") in policy.maiden_delimiters + assert ("(", ")") not in policy.nickname_delimiters + + +def test_snapshot_ambiguous_removed_acronym_intersects() -> None: + c = Constants() + c.suffix_acronyms.remove("jd") # 'jd' stays in ambiguous + lexicon, _, _ = c._snapshot() + assert "jd" not in lexicon.suffix_acronyms + assert "jd" not in lexicon.suffix_acronyms_ambiguous # subset holds + + +def test_parser_cache_shared_across_equal_snapshots() -> None: + a, b = Constants(), Constants() + la, pa, _ = a._snapshot() + lb, pb, _ = b._snapshot() + assert _cached_parser(la, pa) is _cached_parser(lb, pb) From 273e3735725d76cb60262228d132f7c98e78c3ad Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 21:33:21 -0700 Subject: [PATCH 096/275] Wire Policy.extra_suffix_delimiters through segment and group (v1 suffix_delimiter parity) Migration plan M5. v1's suffix_delimiter fired at three parse sites (expand_suffix_delimiter, parser.py:1318/1332/1409): post-comma suffix-segment DETECTION and tail-segment CONSUMPTION. The v2 wiring: - segment: a delimiter-core token (configured delimiter, whitespace stripped) is transparent in the all-suffix tests, and a token that a no-whitespace core splits into all-suffix parts counts as a suffix (splits_into_suffixes) -- covers the SUFFIX_COMMA determination and suppresses the spurious COMMA_STRUCTURE flag v1 never emitted - group: tail segments (FAMILY_COMMA >=2, SUFFIX_COMMA >=1) drop delimiter-core tokens via the same structural mechanism as maiden markers; assemble already omits dropped tokens - the 'not yet consumed' Policy warning is gone: the field is consumed Five case rows pinned live against v1.4 (2026-07-16); the one intentional divergence -- a no-whitespace core keeps the token whole (anti-#100), rendering 'RN/CRNA' where v1 rendered 'RN, CRNA' -- is classified fix(suffix-delimiter-rendering) for the release log. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_group.py | 21 ++++++++++++++++++-- nameparser/_pipeline/_segment.py | 23 +++++++++++++++++---- nameparser/_pipeline/_vocab.py | 24 ++++++++++++++++++++++ nameparser/_policy.py | 9 --------- tests/v2/cases.py | 34 ++++++++++++++++++++++++++++++++ tests/v2/test_policy.py | 10 ---------- 6 files changed, 96 insertions(+), 25 deletions(-) diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index acafb37d..c747c396 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -7,7 +7,8 @@ Reads: token tags (from classify); Policy is not consulted. The v1 "derived titles/prefixes" registration becomes piece_tags entries -- per-parse state that dissolves with the state (v1 kept per-parse sets -for the same reason). +for the same reason). Reads Policy.extra_suffix_delimiters: tail +segments drop delimiter-core tokens (v1 suffix_delimiter parity). Ports v1's join_on_conjunctions + prefix chains + _join_bound_first_name plus two additions: the "Ph. D."-split merge (v1 fix_phd, recorded plan @@ -22,6 +23,7 @@ from collections.abc import Sequence, Set from nameparser._pipeline._state import ParseState, Structure, WorkToken +from nameparser._pipeline._vocab import delimiter_cores from nameparser._types import Role _PH = re.compile(r"^ph\.?$", re.IGNORECASE) @@ -178,8 +180,23 @@ def group(state: ParseState) -> ParseState: # v1 parity: additional_parts_count=1 applies only to FAMILY_COMMA # parts (parser.py:1333); the SUFFIX_COMMA pre-comma segment gets 0. additional = 1 if state.structure is Structure.FAMILY_COMMA else 0 - for seg in state.segments: + # v1 expand_suffix_delimiter parity (#191): tail segments (wholly + # consumed as suffixes by assign) drop delimiter-core tokens, the + # same structural mechanism as the maiden marker below + cores = delimiter_cores(state.policy.extra_suffix_delimiters) + tail_start = {Structure.SUFFIX_COMMA: 1, + Structure.FAMILY_COMMA: 2}.get(state.structure) + for seg_idx, seg in enumerate(state.segments): pieces, ptags = _group_segment(seg, additional, tokens) + if cores and tail_start is not None and seg_idx >= tail_start: + kept = [k for k in range(len(pieces)) + if not (len(pieces[k]) == 1 + and tokens[pieces[k][0]].text in cores)] + if len(kept) != len(pieces): + dropped.extend(i for k in range(len(pieces)) + if k not in kept for i in pieces[k]) + pieces = [pieces[k] for k in kept] + ptags = [ptags[k] for k in kept] # continuation tokens of a suffix-merged piece (the ph-d merge) # carry the stable "joined" tag: the suffix string view joins # SUFFIX tokens with ", ", and the tag lets it heal the split diff --git a/nameparser/_pipeline/_segment.py b/nameparser/_pipeline/_segment.py index 01dcbe20..f81e8f10 100644 --- a/nameparser/_pipeline/_segment.py +++ b/nameparser/_pipeline/_segment.py @@ -6,7 +6,9 @@ Reads: Lexicon suffix vocabulary (via _vocab.is_suffix_lenient) -- the suffix-comma decision is definitionally vocabulary-dependent (recorded plan deviation #3); reads Policy.lenient_comma_suffixes -to pick the lenient or strict predicate. +to pick the lenient or strict predicate, and +Policy.extra_suffix_delimiters for v1 suffix_delimiter parity (a +delimiter-core token is transparent in the all-suffix tests). Decision (v1 parity): >=1 comma and every post-first segment entirely lenient-suffix AND >1 word before the first comma -> SUFFIX_COMMA; @@ -20,7 +22,10 @@ import dataclasses from nameparser._pipeline._state import ParseState, PendingAmbiguity, Structure -from nameparser._pipeline._vocab import is_suffix_lenient, is_suffix_strict +from nameparser._pipeline._vocab import ( + delimiter_cores, is_suffix_lenient, is_suffix_strict, + splits_into_suffixes, +) from nameparser._types import AmbiguityKind @@ -49,10 +54,20 @@ def segment(state: ParseState) -> ParseState: # the strict predicate (initial-shaped suffix words stop qualifying) predicate = (is_suffix_lenient if state.policy.lenient_comma_suffixes else is_suffix_strict) + # v1 expand_suffix_delimiter parity (#191): a configured delimiter + # is TRANSPARENT in the all-suffix tests -- v1 split the part string + # on the delimiter before checking, so the delimiter never counted + cores = delimiter_cores(state.policy.extra_suffix_delimiters) + + def counts_as_suffix(text: str) -> bool: + if text in cores: + return True + return predicate(text, state.lexicon) or ( + bool(cores) + and splits_into_suffixes(text, cores, state.lexicon)) def suffixy(seg: tuple[int, ...]) -> bool: - return all(predicate(state.tokens[i].text, state.lexicon) - for i in seg) + return all(counts_as_suffix(state.tokens[i].text) for i in seg) rest = groups[1:] if all(suffixy(s) for s in rest) and len(groups[0]) > 1: diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 788e6696..dfc8cd8b 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -62,3 +62,27 @@ def is_suffix_lenient(text: str, lexicon: Lexicon) -> bool: n = _normalize(text) return n in lexicon.suffix_words \ or _is_suffix_strict_n(n, text, lexicon) + + +def delimiter_cores(policy_delimiters: frozenset[str]) -> frozenset[str]: + """Configured suffix delimiters with surrounding whitespace + stripped: ' - ' -> '-'. Whitespace-padded delimiters surface as + standalone tokens; the stripped core is what tokenize produced.""" + return frozenset(d.strip() for d in policy_delimiters if d.strip()) + + +def splits_into_suffixes(text: str, cores: frozenset[str], + lexicon: Lexicon) -> bool: + """v1 expand_suffix_delimiter parity for delimiters WITHOUT + whitespace ('RN/CRNA' with '/'): the token counts as a suffix when + some core splits it into >=2 non-empty parts that are all suffixes. + The token text is never rewritten (anti-#100): it takes Role.SUFFIX + whole, which renders 'RN/CRNA' where v1 rendered 'RN, CRNA' -- the + documented divergence, release-log classified.""" + for core in cores: + if core in text: + parts = [part for part in text.split(core) if part] + if len(parts) >= 2 and all( + is_suffix_lenient(part, lexicon) for part in parts): + return True + return False diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 0e76c7f1..b6ad6a48 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -6,7 +6,6 @@ from __future__ import annotations import dataclasses -import warnings from dataclasses import dataclass, field from enum import Enum, StrEnum, auto @@ -136,14 +135,6 @@ def __post_init__(self) -> None: object.__setattr__( self, "extra_suffix_delimiters", frozenset(delimiters) ) - if delimiters: - # accepted-and-ignored would violate the fail-loud culture: - # the pipeline cannot honor these until token re-splitting - # lands with the migration work (v1 suffix_delimiter parity) - warnings.warn( - "extra_suffix_delimiters is not yet consumed by the " - "parse pipeline; it takes effect with the 2.0 migration " - "work", UserWarning, stacklevel=2) # Truthy strings ("no", "false") would silently invert the # caller's intent downstream; bools are the one field kind the # coercing checks above can't cover. diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 72816df7..03886b5d 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -33,6 +33,7 @@ class Case: _ES = Policy(patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})) _TK = Policy(patronymic_rules=frozenset({PatronymicRule.TURKIC})) +_SD = Policy(extra_suffix_delimiters=frozenset({" - "})) CASES: tuple[Case, ...] = ( Case("plain", "John Smith", {"given": "John", "family": "Smith"}), @@ -40,6 +41,39 @@ class Case: {"given": "John", "family": "Smith"}), Case("suffix_comma", "John Smith, PhD", {"given": "John", "family": "Smith", "suffix": "PhD"}), + Case("suffix_delimiter_tail_segment", "Doe, John, RN - CRNA", + {"given": "John", "family": "Doe", "suffix": "RN, CRNA"}, + policy=_SD, + notes="v1 suffix_delimiter parity (#191): the delimiter token " + "is dropped from consumed tail segments (pinned live " + "2026-07-16)"), + Case("suffix_delimiter_detection", "Doe, John RN - CRNA", + {"given": "John", "middle": "-", "family": "Doe", + "suffix": "RN, CRNA"}, + policy=_SD, + notes="the delimiter fires only at suffix sites; the stray " + "token keeps its per-piece walk role (v1 parity, pinned " + "live 2026-07-16)"), + Case("suffix_delimiter_suffix_comma", "John Smith, RN - CRNA", + {"given": "John", "family": "Smith", "suffix": "RN, CRNA"}, + policy=_SD, + notes="delimiter transparency in the SUFFIX_COMMA " + "determination: the post-comma segment counts as " + "all-suffix (v1 parity, pinned live 2026-07-16)"), + Case("suffix_delimiter_no_space_core", "John Smith, RN/CRNA", + {"given": "John", "family": "Smith", "suffix": "RN/CRNA"}, + policy=Policy(extra_suffix_delimiters=frozenset({"/"})), + classification="fix(suffix-delimiter-rendering)", + notes="v1 split 'RN/CRNA' and rendered 'RN, CRNA'; v2 keeps " + "the token whole (anti-#100) with Role.SUFFIX -- role " + "assignment matches, rendering differs (migration plan " + "deviation 5, release-log classified)"), + Case("suffix_delimiter_name_segment_untouched", "Doe, Mary - Kate, RN", + {"given": "Mary", "middle": "- Kate", "family": "Doe", + "suffix": "RN"}, + policy=_SD, + notes="a delimiter token in a NAME segment is kept (v1 parity, " + "pinned live 2026-07-16)"), Case("comma_extras_become_suffixes", "Smith, John, Extra, Jr.", {"given": "John", "family": "Smith", "suffix": "Extra, Jr."}, ambiguities=("comma-structure",), diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 902f9172..eedce628 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -90,7 +90,6 @@ def test_policy_delimiters_do_not_alias_caller_containers() -> None: assert ("'", "'") not in p.nickname_delimiters -@pytest.mark.filterwarnings("ignore:extra_suffix_delimiters is not yet") def test_extra_suffix_delimiters_validated_and_coerced() -> None: with pytest.raises(TypeError, match="bare string"): Policy(extra_suffix_delimiters="ab") # type: ignore[arg-type] @@ -115,7 +114,6 @@ def test_policy_patch_mirrors_policy_field_types() -> None: assert patch_annotation == f"{f.type} | _Unset" -@pytest.mark.filterwarnings("ignore:extra_suffix_delimiters is not yet") def test_policy_patch_canonicalizes_union_fields() -> None: p = PolicyPatch(extra_suffix_delimiters=frozenset({"-"})) assert isinstance(p.extra_suffix_delimiters, frozenset) @@ -240,11 +238,3 @@ def test_name_order_rejects_bare_string() -> None: Policy(name_order="gmf") # type: ignore[arg-type] with pytest.raises(TypeError, match="bare string"): PolicyPatch(name_order="gmf") # type: ignore[arg-type] - - -def test_extra_suffix_delimiters_warns_not_yet_consumed() -> None: - # accepted-and-ignored would violate the fail-loud culture: until - # the migration work wires v1's suffix_delimiter expansion, setting - # the field warns instead of silently doing nothing - with pytest.warns(UserWarning, match="not yet consumed"): - Policy(extra_suffix_delimiters=frozenset({"-"})) From 3db907ec957da02840a0df3b290f6f3a6ec132cf Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 21:38:10 -0700 Subject: [PATCH 097/275] Add the HumanName facade skeleton: construction, config binding, dirty-tracked parsing Co-Authored-By: Claude Fable 5 --- AGENTS.md | 2 +- nameparser/_facade.py | 196 ++++++++++++++++++++++++++++++++++++++++ tests/v2/test_facade.py | 100 ++++++++++++++++++++ 3 files changed, 297 insertions(+), 1 deletion(-) create mode 100644 nameparser/_facade.py create mode 100644 tests/v2/test_facade.py diff --git a/AGENTS.md b/AGENTS.md index 5d62952e..84cd03dc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,7 +119,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). - **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. - **Pickling**: v2 types must round-trip (`Parser` will be picklable by construction, and it holds a `Lexicon`). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, and values are deliberately NOT re-validated (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` keeps its own copy of the guard (layering) plus the `mappingproxy` slot rebuild; a new unpicklable slot type needs the same treatment plus a round-trip test. -- **One sanctioned global**: the (future) cached default `Parser`. Lazily cached FROZEN singletons (`Lexicon.default()`'s `functools.cache`, the future default parser) are constants, not state; any second piece of module-level MUTABLE state requires amending the conventions doc, on purpose, in review. +- **One sanctioned global**: the (future) cached default `Parser`. Lazily cached FROZEN singletons (`Lexicon.default()`'s `functools.cache`, the future default parser) are constants, not state; any second piece of module-level MUTABLE state requires amending the conventions doc, on purpose, in review. Sanctioned exceptions, facade layer only: `_config_shim.CONSTANTS` (the v1 shared singleton, mutable by design) and `_facade._WARNED_SUBCLASSES` (the once-per-subclass hook-warning dedup set) — both deleted with the layer in 3.0. - **Tests**: all v2 tests live in the `tests/v2/` package (its `conftest.py` overrides the v1 dual-run fixture — v2 code never reads shared `CONSTANTS`), one test module per source module plus cross-cutting `test_reprs.py` and `test_layering.py`, names stating behavior. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. ## Extension Patterns diff --git a/nameparser/_facade.py b/nameparser/_facade.py new file mode 100644 index 00000000..1ea777bc --- /dev/null +++ b/nameparser/_facade.py @@ -0,0 +1,196 @@ +"""The 2.0 ``HumanName`` facade (migration spec §2): a mutable wrapper +over a frozen ParsedName, delegating parsing to the core Parser resolved +from the bound Constants shim. Keeps every v1 spelling. Deleted in 3.0. + +Layering: facade layer -- may import anything public plus _render. +""" +from __future__ import annotations + +import dataclasses +import warnings + +from nameparser._config_shim import CONSTANTS, Constants, _cached_parser +from nameparser._parser import Parser +from nameparser._types import ParsedName + +_V2_FIELD = {"first": "given", "last": "family"} # v1 name -> v2 name +_MEMBERS = ("title", "first", "middle", "last", "suffix", "nickname", + "maiden") + +#: v1 parsing hooks the facade never calls (spec §2 exception 2 / #280). +_V1_HOOKS = ( + "pre_process", "post_process", "parse_full_name", "parse_pieces", + "parse_nicknames", "join_on_conjunctions", "squash_emoji", + "handle_firstnames", "handle_middle_name_as_last", + "is_title", "is_conjunction", "is_prefix", "is_roman_numeral", + "is_suffix", "is_suffix_lenient", "is_an_initial", +) +# Module-level mutable state (sanctioned exception, AGENTS.md "One +# sanctioned global"): strong-references every distinct HumanName +# subclass for process lifetime so the hook warning fires once per +# class. Fine in practice -- subclasses are statically defined, and the +# whole module is deleted with the facade layer in 3.0. +_WARNED_SUBCLASSES: set[type] = set() + + +def _empty_parsed() -> ParsedName: + return ParsedName(original="", tokens=(), ambiguities=()) + + +class HumanName: + """v1 ``HumanName`` facade: a mutable wrapper over a frozen + ``ParsedName``, delegating all parsing to the core ``Parser`` + resolved from the bound ``Constants`` shim (dirty-tracked via its + ``_generation``). Keeps every v1 spelling (``first``/``last`` over + the core ``given``/``family``); the v1 parsing hooks are never + called (#280). Deleted with the facade layer in 3.0. + """ + + def __init__( + self, + full_name: str = "", + constants: Constants | None = CONSTANTS, + string_format: str | None = None, + initials_format: str | None = None, + initials_delimiter: str | None = None, + initials_separator: str | None = None, + suffix_delimiter: str | None = None, + first: str | list[str] | None = None, + middle: str | list[str] | None = None, + last: str | list[str] | None = None, + title: str | list[str] | None = None, + suffix: str | list[str] | None = None, + nickname: str | list[str] | None = None, + maiden: str | list[str] | None = None, + ) -> None: + if constants is None: + raise TypeError( + "constants=None was removed in 2.0 (#261): pass a " + "Constants instance, or use the new Parser/Lexicon/" + "Policy API for per-call configuration" + ) + if not isinstance(constants, Constants): + raise TypeError( + f"constants must be a Constants instance, got {constants!r}" + ) + self._warn_overridden_hooks() + self._C = constants + self._snapshot_gen = -1 # forces first resolve + _, _, defaults = constants._snapshot() + self.string_format = (string_format if string_format is not None + else defaults.string_format) + self.initials_format = (initials_format if initials_format is not None + else defaults.initials_format) + self.initials_delimiter = ( + initials_delimiter if initials_delimiter is not None + else defaults.initials_delimiter) + self.initials_separator = ( + initials_separator if initials_separator is not None + else defaults.initials_separator) + # STALENESS TRAP until M9's validating setters land: these are + # plain attributes, so reassigning one after construction (e.g. + # n.suffix_delimiter = " - ") does NOT invalidate _snapshot_gen + # and a cached snapshot keeps the old value. The M9 setters + # reset _snapshot_gen to -1 on assignment. + self.suffix_delimiter = (suffix_delimiter if suffix_delimiter is not None + else defaults.suffix_delimiter) + self._full_name = "" + self._parsed = _empty_parsed() + if first or middle or last or title or suffix or nickname or maiden: + # NOTE (M6): these are plain instance attributes until the + # field-setter properties land in M7 -- no parsing happens + # and full_name stays "". Component-kwarg construction is + # not exercised by this task's tests; see tests/v2/test_facade.py. + self.first = first + self.middle = middle + self.last = last + self.title = title + self.suffix = suffix + self.nickname = nickname + self.maiden = maiden + else: + self._apply_full_name(full_name) + + @classmethod + def _warn_overridden_hooks(cls) -> None: + if cls is HumanName or cls in _WARNED_SUBCLASSES: + return + overridden = [h for h in _V1_HOOKS + if getattr(cls, h, None) is not getattr( + HumanName, h, None)] + _WARNED_SUBCLASSES.add(cls) + if overridden: + warnings.warn( + f"{cls.__name__} overrides v1 parsing hooks " + f"({', '.join(overridden)}) that the 2.0 facade never " + f"calls; parsing is delegated to the core Parser. " + f"Migrate to the Lexicon/Policy API. See " + f"https://github.com/derek73/python-nameparser/issues/280", + DeprecationWarning, stacklevel=3) + + # -- config / parsing --------------------------------------------------- + + def _resolve(self) -> Parser: + """Dirty-tracked parser resolution (spec §3): rebuild the + snapshot only when the bound Constants' generation moved.""" + gen = self._C._generation + if self._snapshot_gen != gen: + lexicon, policy, _ = self._C._snapshot() + if self.suffix_delimiter: + policy = dataclasses.replace( + policy, + extra_suffix_delimiters=frozenset( + {self.suffix_delimiter})) + self._lexicon, self._policy = lexicon, policy + self._snapshot_gen = gen + return _cached_parser(self._lexicon, self._policy) + + def _apply_full_name(self, value: str) -> None: + if isinstance(value, bytes): + raise TypeError( + "bytes input was removed in 2.0 (#245): decode first, " + "e.g. HumanName(raw.decode('utf-8'))" + ) + if not isinstance(value, str): + raise TypeError(f"full_name must be a str, got {value!r}") + self._full_name = value + self._parsed = self._resolve().parse(value) + if self._C.capitalize_name: + self.capitalize() # v1 parser.py:1653 parity + + def capitalize(self) -> None: + """Minimal M6 body (M10 ports the full force-semantics rules): + re-capitalize the current parse against the bound lexicon.""" + self._resolve() + self._parsed = self._parsed.capitalized(self._lexicon, force=True) + + @property + def full_name(self) -> str: + return self._full_name + + @full_name.setter + def full_name(self, value: str) -> None: + self._apply_full_name(value) + + @property + def original(self) -> str: + return self._parsed.original or self._full_name + + def __getattr__(self, name: str) -> str: + # Only reached when normal lookup fails -- an instance attribute + # set by the component-kwargs branch (self.first = ... in + # __init__) shadows this, so that path is untouched. Field + # setters land in M7 as real properties; until then, reading a + # v1 field name after full_name-driven parsing delegates here. + if name in _MEMBERS: + return getattr(self._parsed, _V2_FIELD.get(name, name)) + raise AttributeError( + f"{type(self).__name__!r} object has no attribute {name!r}") + + @property + def C(self) -> Constants: + return self._C + + @property + def has_own_config(self) -> bool: + return self._C is not CONSTANTS diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py new file mode 100644 index 00000000..fc6630c9 --- /dev/null +++ b/tests/v2/test_facade.py @@ -0,0 +1,100 @@ +"""The 2.0 HumanName facade (migration spec §2).""" +import warnings + +import pytest + +from nameparser._config_shim import CONSTANTS, Constants +from nameparser._facade import HumanName + + +def test_basic_parse_and_v1_spellings() -> None: + n = HumanName("Dr. Juan de la Vega III") + assert n.title == "Dr." + assert n.first == "Juan" # v1 spelling of core 'given' + assert n.last == "de la Vega" # v1 spelling of core 'family' + assert n.suffix == "III" + assert n.original == "Dr. Juan de la Vega III" + assert n.full_name == "Dr. Juan de la Vega III" + + +def test_bytes_raises_with_decode_hint() -> None: # #245 + with pytest.raises(TypeError, match="decode"): + HumanName(b"John Smith") # type: ignore[arg-type] + + +def test_constants_none_raises_migration_hint() -> None: # #261 + with pytest.raises(TypeError, match="constants"): + HumanName("John Smith", constants=None) + + +def test_full_name_assignment_reparses() -> None: + n = HumanName("John Smith") + n.full_name = "Jane Doe" + assert (n.first, n.last) == ("Jane", "Doe") + + +def test_private_constants_mutation_honored_lazily() -> None: + # NB: the task spec's example used "dame" as a marker not in the + # default titles, but nameparser/config/titles.py already lists + # "dame" -- swapped in a nonsense marker to keep the test's premise + # (mutate a private Constants, next parse sees the change) true. + c = Constants() + n = HumanName("Zzqtitle Judy Dench", constants=c) + assert n.title == "" # 'zzqtitle' not a default title + c.titles.add("zzqtitle") + n.full_name = "Zzqtitle Judy Dench" # next parse sees the change + assert n.title == "Zzqtitle" + + +def test_capitalize_name_flag_capitalizes_on_parse() -> None: + c = Constants() + c.capitalize_name = True + n = HumanName("john smith", constants=c) + assert (n.first, n.last) == ("John", "Smith") + + +def test_constructor_suffix_delimiter_layers_onto_policy() -> None: + n = HumanName("Doe, John, RN - CRNA", constants=Constants(), + suffix_delimiter=" - ") + assert n.suffix == "RN, CRNA" + + +def test_subclass_overriding_no_hooks_never_warns() -> None: + class Plain(HumanName): + pass + + with warnings.catch_warnings(): + warnings.simplefilter("error") + Plain("John Smith") + + +def test_subclass_hook_override_warns_once_per_class() -> None: # #280 + class Custom(HumanName): + def parse_pieces(self, parts: object, additional_parts_count: int = 0) -> object: + return parts + + with pytest.deprecated_call(match="parse_pieces"): + Custom("John Smith") + with warnings.catch_warnings(): + warnings.simplefilter("error") + Custom("Jane Doe") # second instance: silent + + +def test_c_property_and_has_own_config() -> None: + n = HumanName("John Smith") + assert n.C is CONSTANTS + assert n.has_own_config is False + m = HumanName("John Smith", constants=Constants()) + assert m.has_own_config is True + + +def test_dirty_tracking_reuses_cached_parser_across_unchanged_generation() -> None: + n = HumanName("John Smith") + assert n._resolve() is n._resolve() + + c = Constants() + m = HumanName("John Smith", constants=c) + parser_before = m._resolve() + c.titles.add("zzq") # bumps the generation + parser_after = m._resolve() + assert parser_before is not parser_after From 0719ae066bf044ca37434860c70f27b2a80bb8c1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 21:57:14 -0700 Subject: [PATCH 098/275] Add facade field properties, list snapshots, and derived views Co-Authored-By: Claude Fable 5 --- nameparser/_facade.py | 186 +++++++++++++++++++++++++++++++++++++--- tests/v2/test_facade.py | 71 +++++++++++++++ 2 files changed, 245 insertions(+), 12 deletions(-) diff --git a/nameparser/_facade.py b/nameparser/_facade.py index 1ea777bc..6a7d39b4 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -10,8 +10,9 @@ import warnings from nameparser._config_shim import CONSTANTS, Constants, _cached_parser +from nameparser._lexicon import _normalize from nameparser._parser import Parser -from nameparser._types import ParsedName +from nameparser._types import ParsedName, Role _V2_FIELD = {"first": "given", "last": "family"} # v1 name -> v2 name _MEMBERS = ("title", "first", "middle", "last", "suffix", "nickname", @@ -176,17 +177,6 @@ def full_name(self, value: str) -> None: def original(self) -> str: return self._parsed.original or self._full_name - def __getattr__(self, name: str) -> str: - # Only reached when normal lookup fails -- an instance attribute - # set by the component-kwargs branch (self.first = ... in - # __init__) shadows this, so that path is untouched. Field - # setters land in M7 as real properties; until then, reading a - # v1 field name after full_name-driven parsing delegates here. - if name in _MEMBERS: - return getattr(self._parsed, _V2_FIELD.get(name, name)) - raise AttributeError( - f"{type(self).__name__!r} object has no attribute {name!r}") - @property def C(self) -> Constants: return self._C @@ -194,3 +184,175 @@ def C(self) -> Constants: @property def has_own_config(self) -> bool: return self._C is not CONSTANTS + + # -- fields --------------------------------------------------------- + + def _get_field(self, member: str) -> str: + return getattr(self._parsed, _V2_FIELD.get(member, member)) + + def _set_field(self, member: str, value: str | list[str] | None) -> None: + if value is None: + joined = "" + elif isinstance(value, list): + for element in value: + if not isinstance(element, str): + raise TypeError( + f"name parts must be strings, got {element!r}") + joined = " ".join(value) + elif isinstance(value, str): + joined = value + else: + raise TypeError( + f"{member} must be a str, list, or None, got {value!r}") + self._parsed = self._parsed.replace( + **{_V2_FIELD.get(member, member): joined}) + + def _list_for(self, member: str) -> list[str]: + # A "joined" continuation token ("Ph." + "D.") belongs to its + # predecessor's part, matching v1's fix_phd (suffix_list had ONE + # "Ph. D." element). ParsedName._text_for heals only the suffix + # string view (the ", " join); the facade list view heals for + # every role -- a continuation is never its own list element. + role = Role(_V2_FIELD.get(member, member)) + parts: list[str] = [] + for tok in self._parsed.tokens_for(role): + if "joined" in tok.tags and parts: + parts[-1] += " " + tok.text + else: + parts.append(tok.text) + return parts + + @property + def title(self) -> str: + return self._get_field("title") + + @title.setter + def title(self, value: str | list[str] | None) -> None: + self._set_field("title", value) + + @property + def title_list(self) -> list[str]: + return self._list_for("title") + + @property + def first(self) -> str: + return self._get_field("first") + + @first.setter + def first(self, value: str | list[str] | None) -> None: + self._set_field("first", value) + + @property + def first_list(self) -> list[str]: + return self._list_for("first") + + @property + def middle(self) -> str: + return self._get_field("middle") + + @middle.setter + def middle(self, value: str | list[str] | None) -> None: + self._set_field("middle", value) + + @property + def middle_list(self) -> list[str]: + return self._list_for("middle") + + @property + def last(self) -> str: + return self._get_field("last") + + @last.setter + def last(self, value: str | list[str] | None) -> None: + self._set_field("last", value) + + @property + def last_list(self) -> list[str]: + return self._list_for("last") + + @property + def suffix(self) -> str: + return self._get_field("suffix") + + @suffix.setter + def suffix(self, value: str | list[str] | None) -> None: + self._set_field("suffix", value) + + @property + def suffix_list(self) -> list[str]: + return self._list_for("suffix") + + @property + def nickname(self) -> str: + return self._get_field("nickname") + + @nickname.setter + def nickname(self, value: str | list[str] | None) -> None: + self._set_field("nickname", value) + + @property + def nickname_list(self) -> list[str]: + return self._list_for("nickname") + + @property + def maiden(self) -> str: + return self._get_field("maiden") + + @maiden.setter + def maiden(self, value: str | list[str] | None) -> None: + self._set_field("maiden", value) + + @property + def maiden_list(self) -> list[str]: + return self._list_for("maiden") + + # -- derived views ---------------------------------------------------- + + @property + def surnames_list(self) -> list[str]: + return self.middle_list + self.last_list + + @property + def surnames(self) -> str: + return " ".join(self.surnames_list) + + @property + def given_names_list(self) -> list[str]: + return self.first_list + self.middle_list + + @property + def given_names(self) -> str: + return " ".join(self.given_names_list) + + def _is_particle(self, text: str) -> bool: + self._resolve() + return _normalize(text) in self._lexicon.particles + + def _split_last(self) -> tuple[list[str], list[str]]: + # v1 parser.py _split_last, verbatim: vocabulary lookup at ACCESS + # time (so assigned last names split too), with the all-particle + # guard (a family name is assumed not to consist entirely of + # particles, e.g. surname "Do" which also appears in PREFIXES) + words = " ".join(self.last_list).split() + i = 0 + while i < len(words) and self._is_particle(words[i]): + i += 1 + if i == len(words): + return [], words + return words[:i], words[i:] + + @property + def last_prefixes_list(self) -> list[str]: + return self._split_last()[0] + + @property + def last_prefixes(self) -> str: + return " ".join(self._split_last()[0]) + + @property + def last_base_list(self) -> list[str]: + return self._split_last()[1] + + @property + def last_base(self) -> str: + return " ".join(self._split_last()[1]) diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index fc6630c9..3db49120 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -98,3 +98,74 @@ def test_dirty_tracking_reuses_cached_parser_across_unchanged_generation() -> No c.titles.add("zzq") # bumps the generation parser_after = m._resolve() assert parser_before is not parser_after + + +def test_component_kwargs_bypass_parsing() -> None: + n = HumanName(first="John", last="de la Vega") + assert n.first == "John" and n.last == "de la Vega" + assert n.full_name == "" + + +def test_field_assignment_str_list_none() -> None: + n = HumanName("John Smith") + n.first = "Jane" + assert n.first == "Jane" + n.middle = ["Q", "Xavier"] # lists join with space + assert n.middle == "Q Xavier" + n.middle = None # None clears + assert n.middle == "" + assert n.last == "Smith" # untouched fields survive + assert n.full_name == "John Smith" # no re-parse (v1 parity) + + +def test_list_attributes_are_snapshots() -> None: # spec §2 exc. 1 + n = HumanName("John Quincy Adams Smith") + lst = n.middle_list + lst.append("HACKED") + assert "HACKED" not in n.middle_list + + +def test_derived_views() -> None: + n = HumanName("Juan Q. de la Vega") + assert n.surnames == "Q. de la Vega" # middle + last (v1 shape) + assert n.given_names == "Juan Q." # first + middle + assert n.last_prefixes == "de la" + assert n.last_base == "Vega" + + +def test_split_last_all_prefix_guard() -> None: + n = HumanName(last="de la") # every word is a particle + assert n.last_prefixes == "" # v1 guard: no stripping + assert n.last_base == "de la" + + +@pytest.mark.parametrize("member", [ + "title", "first", "middle", "last", "suffix", "nickname", "maiden", +]) +def test_every_member_set_get_and_list(member: str) -> None: + n = HumanName() + setattr(n, member, "Alpha Beta") + # the suffix string view joins parts with ", " (v1 parity) + joined = "Alpha, Beta" if member == "suffix" else "Alpha Beta" + assert getattr(n, member) == joined + assert getattr(n, f"{member}_list") == ["Alpha", "Beta"] + + +def test_list_assignment_rejects_non_str_elements() -> None: + n = HumanName("John Smith") + with pytest.raises(TypeError, match="strings"): + n.first = [1, 2] # type: ignore[list-item] + with pytest.raises(TypeError, match="strings"): + n.first = [None] # type: ignore[list-item] + + +def test_list_assignment_multiword_elements_join() -> None: + n = HumanName("John Smith") + n.middle = ["ab", "cd ef"] + assert n.middle == "ab cd ef" + + +def test_suffix_list_heals_joined_continuations() -> None: # v1 fix_phd + n = HumanName("John Ph. D.") + assert n.suffix == "Ph. D." + assert n.suffix_list == ["Ph. D."] # ONE element, v1 parity From ea497784a5ecc969013cbd4ff504bab1f535ba4e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 22:09:08 -0700 Subject: [PATCH 099/275] Add facade dunders: v1 str/repr/iteration, #258 item semantics, identity equality Co-Authored-By: Claude Fable 5 --- nameparser/_facade.py | 58 ++++++++++++++++++++++++++++++++++++++++ tests/v2/test_facade.py | 59 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) diff --git a/nameparser/_facade.py b/nameparser/_facade.py index 6a7d39b4..09edb13d 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -7,7 +7,9 @@ from __future__ import annotations import dataclasses +import re import warnings +from collections.abc import Iterator from nameparser._config_shim import CONSTANTS, Constants, _cached_parser from nameparser._lexicon import _normalize @@ -18,6 +20,11 @@ _MEMBERS = ("title", "first", "middle", "last", "suffix", "nickname", "maiden") +_SPACES = re.compile(r"\s+") +_SPACE_BEFORE_COMMA = re.compile(r"\s+,") +# the three comma characters, same set as the pipeline's COMMA_CHARS +_TRAILING_COMMA = re.compile(r"[,،，]$") + #: v1 parsing hooks the facade never calls (spec §2 exception 2 / #280). _V1_HOOKS = ( "pre_process", "post_process", "parse_full_name", "parse_pieces", @@ -356,3 +363,54 @@ def last_base_list(self) -> list[str]: @property def last_base(self) -> str: return " ".join(self._split_last()[1]) + + # -- dunders ------------------------------------------------------------ + + def collapse_whitespace(self, string: str) -> str: + # v1 parser.py:976 verbatim (regexes.spaces / regexes.commas) + string = _SPACES.sub(" ", string.strip()) + if string and _TRAILING_COMMA.search(string): + string = string[:-1] + return string + + def __str__(self) -> str: + if self.string_format is not None: + _s = self.string_format.format( + **{k: v or "" for k, v in self.as_dict().items()}) + _s = _s.replace(" ()", "").replace(" ''", "").replace(' ""', "") + _s = _SPACE_BEFORE_COMMA.sub(",", _s) + return self.collapse_whitespace(_s).strip(", ") + return " ".join(self) + + def __repr__(self) -> str: + attrs = ( + f" title: {self.title or ''!r}\n" + f" first: {self.first or ''!r}\n" + f" middle: {self.middle or ''!r}\n" + f" last: {self.last or ''!r}\n" + f" suffix: {self.suffix or ''!r}\n" + f" nickname: {self.nickname or ''!r}\n" + f" maiden: {self.maiden or ''!r}" + ) + return f"<{self.__class__.__name__} : [\n{attrs}\n]>" + + def __iter__(self) -> Iterator[str]: + return (value for member in _MEMBERS + if (value := getattr(self, member))) + + def __len__(self) -> int: + return sum(1 for member in _MEMBERS if getattr(self, member)) + + def __getitem__(self, key: str) -> str: + if isinstance(key, slice): + raise TypeError( + "slicing a HumanName was removed in 2.0 (#258); access " + "the named attributes instead" + ) + return getattr(self, key) + + def as_dict(self, include_empty: bool = True) -> dict[str, str]: + d = {member: getattr(self, member) for member in _MEMBERS} + if include_empty: + return d + return {k: v for k, v in d.items() if v} diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index 3db49120..75b58e1a 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -169,3 +169,62 @@ def test_suffix_list_heals_joined_continuations() -> None: # v1 fix_phd n = HumanName("John Ph. D.") assert n.suffix == "Ph. D." assert n.suffix_list == ["Ph. D."] # ONE element, v1 parity + + +def test_str_uses_string_format_with_v1_cleanup() -> None: + n = HumanName("Dr. Juan de la Vega III") + assert str(n) == "Dr. Juan de la Vega III" + n2 = HumanName("John Smith") # empty nickname: no ' ()' + assert str(n2) == "John Smith" + n2.string_format = "{last}, {first}" + assert str(n2) == "Smith, John" + + +def test_str_none_format_falls_back_to_space_join() -> None: + # v1-identical: the format wraps the nickname in parens, the plain + # member join does not + n = HumanName('John "Jack" Kennedy') + assert str(n) == "John Kennedy (Jack)" + n.string_format = None + assert str(n) == "John Kennedy Jack" + + +def test_getitem_unknown_key_raises_attribute_error() -> None: + n = HumanName("John Smith") + with pytest.raises(AttributeError): # v1 behavior, not KeyError + n["nope"] + + +def test_repr_v1_shape() -> None: + r = repr(HumanName("John Smith")) + assert r.startswith(" None: + n = HumanName("John Smith") + assert list(n) == ["John", "Smith"] + assert len(n) == 2 + assert len(HumanName("")) == 0 # documented emptiness check + + +def test_getitem_str_ok_slice_raises() -> None: # #258 + n = HumanName("John Smith") + assert n["first"] == "John" + with pytest.raises(TypeError, match="#258"): + n[1:-1] # type: ignore[index] + with pytest.raises(TypeError): + n["first"] = "Jane" # type: ignore[index] # no __setitem__ in 2.0 + + +def test_eq_and_hash_are_object_identity() -> None: # #223 + a, b = HumanName("John Smith"), HumanName("John Smith") + assert a != b and a == a + assert hash(a) != hash(b) or a is b # default object hash + + +def test_as_dict_v1_keys() -> None: + n = HumanName("Dr. John Smith") + d = n.as_dict() + assert d["title"] == "Dr." and d["first"] == "John" and d["last"] == "Smith" + assert set(n.as_dict(include_empty=False)) == {"title", "first", "last"} From bd2abc7533b0dfb5735bf99fdf1aeda61ba4766e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 22:18:08 -0700 Subject: [PATCH 100/275] Port v1 initials to the facade and validate render-default assignment Co-Authored-By: Claude Fable 5 --- nameparser/_facade.py | 139 +++++++++++++++++++++++++++++++++++++--- tests/v2/test_facade.py | 30 +++++++++ 2 files changed, 160 insertions(+), 9 deletions(-) diff --git a/nameparser/_facade.py b/nameparser/_facade.py index 09edb13d..e7c01d20 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -95,20 +95,19 @@ def __init__( self.initials_separator = ( initials_separator if initials_separator is not None else defaults.initials_separator) - # STALENESS TRAP until M9's validating setters land: these are - # plain attributes, so reassigning one after construction (e.g. - # n.suffix_delimiter = " - ") does NOT invalidate _snapshot_gen - # and a cached snapshot keeps the old value. The M9 setters - # reset _snapshot_gen to -1 on assignment. + # These five assignments route through the validating properties + # below. The suffix_delimiter setter resets _snapshot_gen to -1 + # on every assignment (including this one, harmlessly -- it's + # already -1 above), so reassigning it post-construction (e.g. + # n.suffix_delimiter = " - ") correctly forces the next + # _resolve() to rebuild the Policy with the new delimiter. self.suffix_delimiter = (suffix_delimiter if suffix_delimiter is not None else defaults.suffix_delimiter) self._full_name = "" self._parsed = _empty_parsed() if first or middle or last or title or suffix or nickname or maiden: - # NOTE (M6): these are plain instance attributes until the - # field-setter properties land in M7 -- no parsing happens - # and full_name stays "". Component-kwarg construction is - # not exercised by this task's tests; see tests/v2/test_facade.py. + # These route through the field-setter properties (None + # clears the field); no full-string parse, full_name stays "". self.first = first self.middle = middle self.last = last @@ -136,6 +135,70 @@ def _warn_overridden_hooks(cls) -> None: f"https://github.com/derek73/python-nameparser/issues/280", DeprecationWarning, stacklevel=3) + # -- render defaults ----------------------------------------------------- + # One-line validating setters (spec §2): assigning a non-str (or, for + # the two fields that allow it, non-str-non-None) raises TypeError at + # assignment time instead of failing later inside .format(). + + @property + def string_format(self) -> str | None: + return self._string_format + + @string_format.setter + def string_format(self, value: str | None) -> None: + if value is not None and not isinstance(value, str): + raise TypeError( + f"string_format must be a str or None, got {value!r}") + self._string_format = value + + @property + def initials_format(self) -> str: + return self._initials_format + + @initials_format.setter + def initials_format(self, value: str) -> None: + if not isinstance(value, str): + raise TypeError( + f"initials_format must be a str, got {value!r}") + self._initials_format = value + + @property + def initials_delimiter(self) -> str: + return self._initials_delimiter + + @initials_delimiter.setter + def initials_delimiter(self, value: str) -> None: + if not isinstance(value, str): + raise TypeError( + f"initials_delimiter must be a str, got {value!r}") + self._initials_delimiter = value + + @property + def initials_separator(self) -> str: + return self._initials_separator + + @initials_separator.setter + def initials_separator(self, value: str) -> None: + if not isinstance(value, str): + raise TypeError( + f"initials_separator must be a str, got {value!r}") + self._initials_separator = value + + @property + def suffix_delimiter(self) -> str | None: + return self._suffix_delimiter + + @suffix_delimiter.setter + def suffix_delimiter(self, value: str | None) -> None: + if value is not None and not isinstance(value, str): + raise TypeError( + f"suffix_delimiter must be a str or None, got {value!r}") + self._suffix_delimiter = value + # Invalidate the cached Policy: _resolve() layers suffix_delimiter + # onto extra_suffix_delimiters, so a stale snapshot would keep + # parsing against the old delimiter. + self._snapshot_gen = -1 + # -- config / parsing --------------------------------------------------- def _resolve(self) -> Parser: @@ -335,6 +398,10 @@ def _is_particle(self, text: str) -> bool: self._resolve() return _normalize(text) in self._lexicon.particles + def _is_conjunction(self, text: str) -> bool: + self._resolve() + return _normalize(text) in self._lexicon.conjunctions + def _split_last(self) -> tuple[list[str], list[str]]: # v1 parser.py _split_last, verbatim: vocabulary lookup at ACCESS # time (so assigned last names split too), with the all-particle @@ -364,6 +431,60 @@ def last_base_list(self) -> list[str]: def last_base(self) -> str: return " ".join(self._split_last()[1]) + # -- initials ------------------------------------------------------------- + + def _process_initial(self, name_part: str, firstname: bool = False) -> str: + # v1 parser.py:427 verbatim: particles/conjunctions are filtered + # from initials unless the part is a first name. split() rather + # than split(" "): *_list attributes assigned directly bypass + # whitespace normalization, and split(" ") yields empty strings + # for repeated spaces (#232). + parts = name_part.split() + initials = [] + for part in parts: + if not (self._is_particle(part) + or self._is_conjunction(part)) or firstname: + initials.append(part[0]) + if len(initials) > 0: + return self.initials_separator.join(initials) + # Return '' (never empty_attribute_default, which may be None) + # when a part has no initialable words, e.g. a middle name + # consisting only of prefixes ("de la"). Callers drop these + # parts entirely. + return "" + + def _initials_lists(self) -> tuple[list[str], list[str], list[str]]: + """Initials for the first, middle and last name groups. Parts + that yield no initials (e.g. a prefix-only middle name like + "de la") are dropped rather than kept as empty strings. + """ + def group_initials(names: list[str], + firstname: bool = False) -> list[str]: + return [i for i in (self._process_initial(n, firstname) + for n in names if n) if i] + return (group_initials(self.first_list, True), + group_initials(self.middle_list), + group_initials(self.last_list)) + + def initials_list(self) -> list[str]: + first, middle, last = self._initials_lists() + return first + middle + last + + def initials(self) -> str: + first, middle, last = self._initials_lists() + joiner = self.initials_delimiter + self.initials_separator + + def group(items: list[str]) -> str: + return joiner.join(items) + self.initials_delimiter \ + if items else "" + + # A fully-empty result renders as "" -- the v1 fallback to + # C.empty_attribute_default (which may be None) is dropped per + # #255. + _s = self.initials_format.format( + first=group(first), middle=group(middle), last=group(last)) + return self.collapse_whitespace(_s) + # -- dunders ------------------------------------------------------------ def collapse_whitespace(self, string: str) -> str: diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index 75b58e1a..070f6d5d 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -228,3 +228,33 @@ def test_as_dict_v1_keys() -> None: d = n.as_dict() assert d["title"] == "Dr." and d["first"] == "John" and d["last"] == "Smith" assert set(n.as_dict(include_empty=False)) == {"title", "first", "last"} + + +def test_initials_v1_semantics() -> None: + assert HumanName("Sir Bob Andrew Dole").initials() == "B. A. D." + assert HumanName("Sir Bob Andrew Dole").initials_list() == ["B", "A", "D"] + n = HumanName("Doe, John A.", initials_delimiter="", initials_separator="") + assert n.initials() == "J A D" + # prefixes/conjunctions are filtered except in first names (v1 rule) + assert HumanName("Juan de la Vega").initials() == "J. V." + + +def test_initials_format_kwarg() -> None: + n = HumanName("Sir Bob Andrew Dole", initials_format="{first} {middle}") + assert n.initials() == "B. A." + + +def test_render_default_setters_validate() -> None: + n = HumanName("John Smith") + with pytest.raises(TypeError, match="initials_delimiter"): + n.initials_delimiter = 5 # type: ignore[assignment] + with pytest.raises(TypeError, match="initials_format"): + n.initials_format = 5 # type: ignore[assignment] + with pytest.raises(TypeError, match="initials_separator"): + n.initials_separator = None # type: ignore[assignment] + with pytest.raises(TypeError, match="string_format"): + n.string_format = 5 # type: ignore[assignment] + with pytest.raises(TypeError, match="suffix_delimiter"): + n.suffix_delimiter = 5 # type: ignore[assignment] + n.string_format = None # None allowed for these two + n.suffix_delimiter = None From 0cdd7872df8cb5db9c7620f49e3d8ee35b156d1a Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 22:32:23 -0700 Subject: [PATCH 101/275] Add facade capitalize/matches/comparison_key and v1-compatible pickling Co-Authored-By: Claude Fable 5 --- nameparser/_facade.py | 77 +++++++++++++++++++++++++++-- tests/v2/data/humanname_v14.pickle | Bin 0 -> 590 bytes tests/v2/test_facade.py | 67 +++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 4 deletions(-) create mode 100644 tests/v2/data/humanname_v14.pickle diff --git a/nameparser/_facade.py b/nameparser/_facade.py index e7c01d20..f91a540f 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -10,6 +10,7 @@ import re import warnings from collections.abc import Iterator +from typing import Any from nameparser._config_shim import CONSTANTS, Constants, _cached_parser from nameparser._lexicon import _normalize @@ -229,11 +230,16 @@ def _apply_full_name(self, value: str) -> None: if self._C.capitalize_name: self.capitalize() # v1 parser.py:1653 parity - def capitalize(self) -> None: - """Minimal M6 body (M10 ports the full force-semantics rules): - re-capitalize the current parse against the bound lexicon.""" + def capitalize(self, force: bool | None = None) -> None: + """Re-capitalize the current parse against the bound lexicon. + force=None reads the bound Constants' render default + (force_mixed_case_capitalization); the core's capitalized() + implements the single-case gate (v1 parity) -- not + re-implemented here.""" self._resolve() - self._parsed = self._parsed.capitalized(self._lexicon, force=True) + if force is None: + force = self._C.force_mixed_case_capitalization + self._parsed = self._parsed.capitalized(self._lexicon, force=force) @property def full_name(self) -> str: @@ -485,6 +491,17 @@ def group(items: list[str]) -> str: first=group(first), middle=group(middle), last=group(last)) return self.collapse_whitespace(_s) + # -- comparison ----------------------------------------------------------- + + def matches(self, other: str | HumanName) -> bool: + """Component-wise case-insensitive comparison (v1 parity); a + str argument is parsed with this instance's resolved parser.""" + target = other._parsed if isinstance(other, HumanName) else other + return self._parsed.matches(target, parser=self._resolve()) + + def comparison_key(self) -> tuple[str, ...]: + return self._parsed.comparison_key() + # -- dunders ------------------------------------------------------------ def collapse_whitespace(self, string: str) -> str: @@ -535,3 +552,55 @@ def as_dict(self, include_empty: bool = True) -> dict[str, str]: if include_empty: return d return {k: v for k, v in d.items() if v} + + # -- pickle (v1-shaped state; one path for 1.4 and 2.x blobs) ----------- + + def __getstate__(self) -> dict[str, Any]: + # The emitted key set matches v1.4's pickle shape (minus + # encoding/_had_comma/_derived_*, which are v1-internal and + # ignored on read), so one __setstate__ path serves both eras. + state: dict[str, Any] = { + "_full_name": self._full_name, + "original": self.original, + "C": None if self._C is CONSTANTS else self._C, + "string_format": self.string_format, + "initials_format": self.initials_format, + "initials_delimiter": self.initials_delimiter, + "initials_separator": self.initials_separator, + "suffix_delimiter": self.suffix_delimiter, + } + for member in _MEMBERS: + state[f"{member}_list"] = getattr(self, f"{member}_list") + return state + + def __setstate__(self, state: dict[str, Any]) -> None: + c = state.get("C") + self._C = CONSTANTS if c is None else c + self._snapshot_gen = -1 + defaults = self._C._snapshot()[2] + self._string_format = state.get("string_format", + defaults.string_format) + self._initials_format = state.get("initials_format", + defaults.initials_format) + self._initials_delimiter = state.get("initials_delimiter", + defaults.initials_delimiter) + self._initials_separator = state.get("initials_separator", + defaults.initials_separator) + self._suffix_delimiter = state.get("suffix_delimiter", + defaults.suffix_delimiter) + self._full_name = state.get("_full_name", "") + # components come back exactly as pickled (spec §2): synthetic + # tokens via replace(), never a re-parse. Known edge: replace() + # re-splits on whitespace without the "joined" tag, so joined-tag + # healing is lost for multi-word list elements -- v1's fix_phd + # suffix pickles as ["Ph. D."] but round-trips to ["Ph.", "D."], + # rendering the suffix as "Ph., D.". Classify in the differential + # harness / M12 if it surfaces. + parsed = ParsedName(original=str(state.get("original", "")), + tokens=(), ambiguities=()) + fields = {} + for member in _MEMBERS: + values = state.get(f"{member}_list") or [] + fields[_V2_FIELD.get(member, member)] = " ".join(values) + self._parsed = parsed.replace( + **{k: v for k, v in fields.items() if v}) diff --git a/tests/v2/data/humanname_v14.pickle b/tests/v2/data/humanname_v14.pickle new file mode 100644 index 0000000000000000000000000000000000000000..718585a38e4ad25b3c53d36988c4707ff3198015 GIT binary patch literal 590 zcmaJ-!Ab)$5Vcytwv<{ydd;ytxHrLjMXZOO)SE1EldYpkwj@~*Nx@1<=N2*Vc&UM0cl=X%{}4q zsvf##s~oKJRn#DalGR7?I&T(Ov+>G6_7HAbPY4HKzynt=h#i}xJa`?Yfi7YS6Enp;fio{Y7=%*a>ImVinD9E;A}RV zQPq-{N~K}ve`&P^sRHNehH$6SBaYqLkchU!gbMC8{zSYJY>HX;dxgOLIwLBFtvL2K TM#i?D<{+erd234Z@;3Pd(yiy( literal 0 HcmV?d00001 diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index 070f6d5d..9bf6d3ef 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -1,11 +1,15 @@ """The 2.0 HumanName facade (migration spec §2).""" +import pickle import warnings +from pathlib import Path import pytest from nameparser._config_shim import CONSTANTS, Constants from nameparser._facade import HumanName +_DATA_DIR = Path(__file__).parent / "data" + def test_basic_parse_and_v1_spellings() -> None: n = HumanName("Dr. Juan de la Vega III") @@ -258,3 +262,66 @@ def test_render_default_setters_validate() -> None: n.suffix_delimiter = 5 # type: ignore[assignment] n.string_format = None # None allowed for these two n.suffix_delimiter = None + + +def test_capitalize_gate_and_force() -> None: + n = HumanName("bob v. de la macdole-eisenhower phd") + n.capitalize() + assert str(n) == "Bob V. de la MacDole-Eisenhower Ph.D." + m = HumanName("Shirley Maclaine") # mixed case: untouched + m.capitalize() + assert str(m) == "Shirley Maclaine" + m.capitalize(force=True) + assert str(m) == "Shirley MacLaine" + + +def test_force_mixed_case_flag_feeds_default() -> None: + c = Constants() + c.force_mixed_case_capitalization = True + n = HumanName("Shirley Maclaine", constants=c) + n.capitalize() # force=None -> flag -> True + assert str(n) == "Shirley MacLaine" + + +def test_matches_and_comparison_key() -> None: + # NB: the task spec's example compared "Dr. John A. Smith" against + # "John Smith" -- but matches()/comparison_key() are component-wise + # over all seven fields (v1 parity, verified against live 1.4), so a + # title/middle-initial mismatch always fails the match; swapped in + # case/order variations that actually exercise "case-insensitive, + # component-wise" without dropping fields. + n = HumanName("Dr. John A. Smith") + assert n.matches("dr. john a. smith") + assert n.matches(HumanName("smith, dr. john a.")) + assert not n.matches("Jane Smith") + assert n.comparison_key() == HumanName("DR. JOHN A. SMITH").comparison_key() + + +def test_pickle_round_trip_preserves_components() -> None: + n = HumanName("Dr. Juan de la Vega III") + n.first = "Johan" # mutated state must survive + loaded = pickle.loads(pickle.dumps(n)) + assert loaded.first == "Johan" + assert loaded.last == "de la Vega" + assert loaded.original == "Dr. Juan de la Vega III" + assert loaded.C is CONSTANTS # shared sentinel restored + + +@pytest.mark.xfail( + reason="resolves to the v1 class until the M11 swap", strict=True) +def test_v14_humanname_blob_unpickles() -> None: + # tests/v2/data/humanname_v14.pickle was produced by a real 1.4.0 + # install (`uv run --no-project --with "nameparser==1.4.*" ...`) on + # HumanName("Dr. Juan de la Vega III"); components come back exactly + # as pickled, NOT re-parsed. Until the M11 swap replaces + # nameparser.parser.HumanName with this facade, the blob's pickled + # class reference still resolves to the live v1 class, so this + # assertion fails today by construction (mirrors + # test_v14_constants_blob_unpickles_into_shim in test_config_shim.py). + # pickle.load is safe here: humanname_v14.pickle is a repo-controlled + # fixture generated by this task from a real 1.4.0 install (Step 2 + # above), not data from an untrusted source. + with open(_DATA_DIR / "humanname_v14.pickle", "rb") as f: + loaded = pickle.load(f) + assert isinstance(loaded, HumanName) + assert loaded.first == "Juan" and loaded.last == "de la Vega" From 75e321902b7c22c013e38af5f15438e34ef149ee Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 22:58:12 -0700 Subject: [PATCH 102/275] Swap parser.py and config to the 2.0 facade and shim (v1 suite reconciliation follows) Co-Authored-By: Claude Fable 5 --- nameparser/_config_shim.py | 63 +- nameparser/_facade.py | 16 + nameparser/config/__init__.py | 1007 +------------------- nameparser/parser.py | 1658 +-------------------------------- pyproject.toml | 4 + tests/conftest.py | 6 + tests/v2/test_config_shim.py | 15 +- tests/v2/test_facade.py | 10 +- tests/v2/test_layering.py | 20 +- 9 files changed, 138 insertions(+), 2661 deletions(-) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index fdb0de66..ff16584e 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -7,18 +7,26 @@ the read-only regexes proxy's underlying compiled patterns, and ``nameparser._lexicon``/``_policy``/``_parser`` for the ``_snapshot()`` translation and the shared parser cache. + +All ``nameparser.config.`` imports below are deferred +(function-local), never module-level: ``nameparser/config/__init__.py`` +itself imports ``CONSTANTS``/``Constants``/etc. from this module, so a +module-level ``from nameparser.config.regexes import REGEXES`` here +would need the ``nameparser.config`` package's ``__init__.py`` to run +to completion first -- which needs this module to already be fully +initialized. Importing the data submodules lazily, on first use, breaks +that cycle. """ from __future__ import annotations import functools import warnings -from collections.abc import Callable, Iterable, Iterator, KeysView +from collections.abc import Callable, Iterable, Iterator, KeysView, Mapping from typing import NamedTuple, Self from nameparser._lexicon import Lexicon from nameparser._parser import Parser from nameparser._policy import PatronymicRule, Policy -from nameparser.config.regexes import REGEXES from nameparser.util import lc @@ -246,6 +254,21 @@ def __setstate__(self, state: dict[str, object]) -> None: _DELIMITER_SENTINELS = tuple(_SENTINEL_PAIRS) +class RegexTupleManager(TupleManager): + """Pickle-compat alias only: v1.4's ``Constants.regexes`` field was a + ``nameparser.config.RegexTupleManager`` instance (a ``TupleManager`` + subclass whose ``__getattr__`` fell back to ``EMPTY_REGEX`` for an + unknown key). Unpickling a v1.4 blob resolves and constructs this + class -- via ``TupleManager.__reduce__``'s ``(type(self), (), state)`` + -- before ``Constants.__setstate__`` runs, so the name must exist + here even though the shim's ``Constants._snapshot()`` never reads it: + ``regexes`` is a read-only ``_RegexesProxy`` in 2.0 (see above), and + ``Constants.__setstate__`` below deliberately ignores an incoming + ``regexes`` key rather than restoring it from this reconstructed + (and otherwise unused) instance. + """ + + class _DelimiterManager(TupleManager): """v1 ``nickname_delimiters``/``maiden_delimiters`` bucket. In 2.0 only the three named sentinels exist (spec §3) -- each maps to the @@ -293,25 +316,32 @@ class _RegexesProxy: (spec §3's uniform read-only rule). """ + @staticmethod + def _regexes() -> Mapping[str, object]: + # deferred import: see the module docstring's note on why every + # nameparser.config. import in this file is lazy + from nameparser.config.regexes import REGEXES + return REGEXES + def __getattr__(self, name: str) -> object: if name.startswith("_"): raise AttributeError(name) try: - return REGEXES[name] + return self._regexes()[name] except KeyError: raise AttributeError(f"no regex named {name!r}") from None def __getitem__(self, name: str) -> object: - return REGEXES[name] + return self._regexes()[name] def __contains__(self, name: object) -> bool: - return name in REGEXES + return name in self._regexes() def __iter__(self) -> Iterator[str]: - return iter(REGEXES) + return iter(self._regexes()) def keys(self) -> KeysView[str]: - return REGEXES.keys() + return self._regexes().keys() def __setattr__(self, name: str, value: object) -> None: self._raise_readonly(name) @@ -479,6 +509,25 @@ def __init__(self) -> None: for name, value in _SCALAR_DEFAULTS.items(): object.__setattr__(self, name, value) + def _invalidate_pst(self) -> None: + """Pickle-compat alias only, never called at runtime: v1's four + cached-union ``SetManager`` fields (``prefixes``, + ``suffix_acronyms``, ``suffix_not_acronyms``, ``titles``) stored + their ``_on_change`` as the bound method + ``Constants._invalidate_pst``. A pickled bound method serializes + as a back-reference to its ``__self__`` (this same ``Constants`` + instance, mid-unpickle) plus the method NAME -- reconstructed via + ``getattr(constants_obj, '_invalidate_pst')`` before + ``Constants.__setstate__`` ever runs (same two-phase-unpickling + story as ``RegexTupleManager`` above). Without this name, loading + a v1.4 blob raises ``AttributeError`` looking up the method. + ``SetManager.__setstate__`` (shim) never restores ``_on_change`` + from pickled state regardless -- it always resets to ``None`` and + is rewired by ``Constants.__setstate__`` below -- so this bound + method value is reconstructed only to satisfy the pickle format; + it is discarded immediately and never invoked. + """ + def _bump(self) -> None: # stacklevel=3 is exact for the direct scalar-assignment path # (user code -> Constants.__setattr__ -> here) and lands one diff --git a/nameparser/_facade.py b/nameparser/_facade.py index f91a540f..c1626f13 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -12,6 +12,22 @@ from collections.abc import Iterator from typing import Any +# Import order matters here -- breaks a real import cycle. nameparser. +# config's package __init__ re-exports CONSTANTS/Constants/etc. from +# _config_shim (the v1 nameparser.config.Constants compat path), while +# _config_shim's own default CONSTANTS singleton needs nameparser.config's +# DATA submodules (titles, prefixes, ...), imported lazily -- see +# _config_shim.py's module docstring. If _config_shim were the first of +# the two ever touched, building its CONSTANTS would need to import +# nameparser.config, whose __init__ would in turn need _config_shim's +# (not-yet-built) CONSTANTS: ImportError. Importing the config package +# here first lets its __init__ run to completion; when IT then imports +# _config_shim to build the default CONSTANTS, nameparser.config is +# already registered in sys.modules, so its data-submodule imports +# resolve directly instead of re-entering (and failing on) its own +# still-executing __init__. +import nameparser.config # noqa: F401 + from nameparser._config_shim import CONSTANTS, Constants, _cached_parser from nameparser._lexicon import _normalize from nameparser._parser import Parser diff --git a/nameparser/config/__init__.py b/nameparser/config/__init__.py index a0bbdcea..05dbae09 100644 --- a/nameparser/config/__init__.py +++ b/nameparser/config/__init__.py @@ -1,991 +1,20 @@ +"""v1 import-path preservation (migration spec §3): the Constants shim +lives in nameparser._config_shim. The vocabulary data modules in this +package (titles, suffixes, ...) remain the single source through 2.x. +This package is deleted in 3.0. + +``RegexTupleManager`` is re-exported unchanged from the shim purely for +pickle compatibility: a v1.4 ``Constants`` blob's ``regexes`` field was +pickled as ``nameparser.config.RegexTupleManager(...)``, and unpickling +resolves and reconstructs that nested object before +``Constants.__setstate__`` ever runs. Without this name, loading such a +blob raises ``AttributeError`` looking up the class, not a clean +compatibility failure. """ -The :py:mod:`nameparser.config` module manages the configuration of the -nameparser. +from nameparser._config_shim import CONSTANTS as CONSTANTS +from nameparser._config_shim import Constants as Constants +from nameparser._config_shim import RegexTupleManager as RegexTupleManager +from nameparser._config_shim import SetManager as SetManager +from nameparser._config_shim import TupleManager as TupleManager -:py:class:`~nameparser.config.Constants` is for application-level -configuration, set once at startup. ``CONSTANTS``, the module-level instance -used by every ``HumanName`` created without its own config, is the only -channel that reaches parses happening in code you don't own (helpers, -pipelines, a third-party library using nameparser internally) -- the same -role ``logging`` and ``locale`` play elsewhere. Import it and change it -directly: - -:: - - >>> from nameparser.config import CONSTANTS - >>> CONSTANTS.titles.remove('hon').add('chemistry','dean') # doctest: +SKIP - -For anything scoped -- one dataset, one library, one test -- pass your own -:py:class:`Constants` instance as the second argument upon instantiation -instead: ``Constants()`` for fresh library defaults, or ``CONSTANTS.copy()`` -for a private snapshot of the current module config. - -:: - - >>> from nameparser import HumanName - >>> from nameparser.config import Constants - >>> hn = HumanName("Dean Robert Johns", Constants()) - >>> hn.C.titles.add('dean') # doctest: +SKIP - >>> hn.parse_full_name() # need to run this again after config changes - -Mixing the two up is where the surprises come from, not the API itself: if -you do not pass your own :py:class:`Constants` instance as the second -argument, ``hn.C`` will be a reference to the module config, and a change -there reaches every other instance sharing it. See `Customizing the Parser -`_. - -.. deprecated:: 1.4.0 - Passing ``None`` as the second argument also builds a fresh - ``Constants()``, but is deprecated in favor of the explicit spellings - above; it will raise ``TypeError`` in 2.0 (issue #260). -""" -import copy -import inspect -import re -import sys -import warnings -from collections.abc import Callable, Iterable, Iterator, Mapping, Set -from typing import Any, Self, TypeVar, overload - -from nameparser.util import lc -from nameparser.config.prefixes import PREFIXES, NON_FIRST_NAME_PREFIXES -from nameparser.config.bound_first_names import BOUND_FIRST_NAMES -from nameparser.config.capitalization import CAPITALIZATION_EXCEPTIONS -from nameparser.config.conjunctions import CONJUNCTIONS -from nameparser.config.suffixes import SUFFIX_ACRONYMS -from nameparser.config.suffixes import SUFFIX_NOT_ACRONYMS -from nameparser.config.suffixes import SUFFIX_ACRONYMS_AMBIGUOUS -from nameparser.config.titles import TITLES -from nameparser.config.titles import FIRST_NAME_TITLES -from nameparser.config.regexes import EMPTY_REGEX, REGEXES - -DEFAULT_ENCODING = 'UTF-8' - - -def _reject_bare_str_or_bytes(value: object, expected: str) -> None: - # A bare string is an iterable of its characters, so e.g. set('dr') or - # dict('ab') would silently shred it, and bytes iterates to ints, which - # can never match parsed str tokens -- shared by SetManager's constructor/ - # operands (#238) and TupleManager's constructor (#242). - if isinstance(value, bytes): - raise TypeError( - f"expected {expected}, got a single bytes; " - f"decode it first: [{value!r}.decode()]" - ) - if isinstance(value, str): - raise TypeError( - f"expected {expected}, got a single str; wrap it in a list: [{value!r}]" - ) - - -class SetManager(Set): - ''' - Easily add and remove config variables per module or instance. Subclass of - ``collections.abc.Set``. - - Special functionality beyond that provided by set() is to normalize - constants for comparison (lowercase, leading/trailing periods stripped) - when they are add()ed and remove()d, and to allow passing multiple - string arguments to the :py:func:`add()` and :py:func:`remove()` - methods. The constructor and the set operators apply the same - normalization to their elements and operands, so every entry is stored - in the form the parser's lookups expect, and they reject a bare string - with ``TypeError``, since e.g. ``set('dr')`` would silently build a set - of single characters. - - ''' - - _on_change: Callable[[], None] | None - - @classmethod - def _normalized_elements(cls, elements: Iterable[str]) -> set[str]: - # a SetManager's elements were validated and normalized when it was - # built, so copy them instead of re-validating — this is what keeps - # chained unions (suffixes_prefixes_titles) and default Constants() - # construction from re-checking ~1,400 entries per step - if isinstance(elements, SetManager): - return set(elements.elements) - _reject_bare_str_or_bytes(elements, "an iterable of strings") - # apply the same lc() normalization (lowercase, strip leading/ - # trailing periods) that add() applies, and reject junk elements: - # lc() on bytes or int crashes without naming the culprit, and - # lc(None) silently transmutes to ''. Divergence from add() is - # deliberate: add_with_encoding() decodes bytes for back-compat, - # bulk boundaries stay strict. - normalized = set() - for s in elements: - if isinstance(s, bytes): - raise TypeError( - f"expected str elements, got bytes; decode it first: {s!r}.decode()" - ) - if not isinstance(s, str): - raise TypeError( - f"expected str elements, got {type(s).__name__}: {s!r}" - ) - normalized.add(lc(s)) - return normalized - - @classmethod - def _from_normalized(cls, elements: set[str]) -> 'SetManager': - # Private fast constructor: bypasses __init__ so results aren't - # re-validated element by element. This performs NO validation or - # normalization of `elements` -- the caller is fully responsible - # for guaranteeing every element is already a str that has passed - # through lc(). Only call this with a set built from other - # SetManagers' already-normalized .elements (operator results, - # prebuilt default copies); passing anything else silently defeats - # the constructor's #238 guarantees with no error raised here. - obj = cls.__new__(cls) - obj.elements = elements - obj._on_change = None - return obj - - def __init__(self, elements: Iterable[str]) -> None: - self.elements = self._normalized_elements(elements) - # Optional invalidation hook, wired by an owning Constants so that - # in-place add()/remove() can clear its cached suffixes_prefixes_titles - # union. None when the manager is used standalone. - self._on_change = None - - def __call__(self) -> Set[str]: - """ - .. deprecated:: 1.3.0 - Removed in 2.0 (see issue #243). Returns the raw underlying set, - so mutating it bypasses normalization and cache invalidation; - iterate the manager or copy with ``set(manager)`` instead. - """ - warnings.warn( - "Calling a SetManager to get the raw underlying set is " - "deprecated and will be removed in 2.0; iterate the manager or " - "copy it with set(manager) instead. See " - "https://github.com/derek73/python-nameparser/issues/243", - DeprecationWarning, - stacklevel=2, - ) - return self.elements - - def __repr__(self) -> str: - # Sorted so repr is stable across runs -- set() iteration order - # depends on string hash randomization, which varies per process. - elements = "{" + ", ".join(repr(e) for e in sorted(self.elements)) + "}" if self.elements else "set()" - return f"SetManager({elements})" # used for docs - - def __iter__(self) -> Iterator[str]: - return iter(self.elements) - - def __contains__(self, value: object) -> bool: - # add()/remove()/the constructor/the operators all normalize (lowercase, - # strip leading/trailing periods) before comparing; without the same - # normalization here, `'Dr.' in c.titles` returns False even though - # every other operation on the same value succeeds (#244). The parser's - # own lookups (e.g. `piece.lower() in self.C.conjunctions`) already pass - # an lc()-normalized value, which is the hot path during parsing, so - # try the raw value first and only pay for lc() on a miss. - if value in self.elements: - return True - return isinstance(value, str) and lc(value) in self.elements - - def __len__(self) -> int: - return len(self.elements) - - # The ABC mixins compare raw operand elements against stored (normalized) - # ones, and their __or__/__and__ accept a bare str as Iterable, so every - # operand is validated and normalized here. Results are built with plain - # set ops on already-normalized elements instead of delegating to the - # mixins, whose _from_iterable would re-validate the whole result - # through __init__. - # - # the runtime ABC accepts any Iterable operand, so annotate honestly and - # ignore typeshed's narrower AbstractSet declarations - def __or__(self, other: Iterable[str]) -> 'SetManager': # type: ignore[override] - return self._from_normalized(self.elements | self._normalized_elements(other)) - - __ror__ = __or__ - - def __and__(self, other: Iterable[str]) -> 'SetManager': # type: ignore[override] - return self._from_normalized(self.elements & self._normalized_elements(other)) - - __rand__ = __and__ - - def __sub__(self, other: Iterable[str]) -> 'SetManager': # type: ignore[override] - return self._from_normalized(self.elements - self._normalized_elements(other)) - - def __rsub__(self, other: Iterable[str]) -> 'SetManager': - return self._from_normalized(self._normalized_elements(other) - self.elements) - - def __xor__(self, other: Iterable[str]) -> 'SetManager': # type: ignore[override] - return self._from_normalized(self.elements ^ self._normalized_elements(other)) - - __rxor__ = __xor__ - - def _add_normalized(self, s: str | bytes, encoding: str | None, *, stacklevel: int) -> None: - # Shared by add() and add_with_encoding() so each can call it - # directly with a stacklevel that attributes the warning to *its own* - # caller -- add() delegating to add_with_encoding() would otherwise - # add a frame and misattribute the warning to this module. - stdin_encoding = None - if sys.stdin: - stdin_encoding = sys.stdin.encoding - encoding = encoding or stdin_encoding or DEFAULT_ENCODING - if isinstance(s, bytes): - warnings.warn( - "Passing bytes to SetManager.add()/add_with_encoding() is " - "deprecated and will raise TypeError in 2.0; decode it " - "first, e.g. value.decode('utf-8'). See " - "https://github.com/derek73/python-nameparser/issues/245", - DeprecationWarning, - stacklevel=stacklevel, - ) - s = s.decode(encoding) - normalized = lc(s) - if normalized not in self.elements: - self.elements.add(normalized) - if self._on_change: - self._on_change() - - def add_with_encoding(self, s: str | bytes, encoding: str | None = None) -> None: - """ - Add the lowercased, leading/trailing-periods-stripped version of the string to the set. Pass an - explicit `encoding` parameter to specify the encoding of binary strings that - are not DEFAULT_ENCODING (UTF-8). - - .. deprecated:: 1.3.0 - ``bytes`` arguments will raise ``TypeError`` in 2.0 (see issue - #245); decode before adding. - - .. deprecated:: 1.4.0 - The method itself is removed in 2.0 (see issue #245); use - :py:func:`add` instead, decoding bytes first. - """ - warnings.warn( - "SetManager.add_with_encoding() is deprecated and will be " - "removed in 2.0; use add() instead (decode bytes first). See " - "https://github.com/derek73/python-nameparser/issues/245", - DeprecationWarning, - stacklevel=2, - ) - self._add_normalized(s, encoding, stacklevel=3) - - def add(self, *strings: str) -> Self: - """ - Add the lowercased, leading/trailing-periods-stripped version of the string arguments to the set. - Returns ``self`` for chaining. - - .. deprecated:: 1.3.0 - ``bytes`` arguments will raise ``TypeError`` in 2.0 (see issue - #245); decode before adding. - """ - for s in strings: - self._add_normalized(s, None, stacklevel=3) - - return self - - def remove(self, *strings: str) -> Self: - """ - Remove the lower case and no-period version of the string arguments from the set. - Returns ``self`` for chaining. - - .. deprecated:: 1.3.0 - Removing a *missing* member currently does nothing but will - raise ``KeyError`` in 2.0, matching ``set.remove`` (see issue - #243); use :py:func:`discard` to ignore missing members. - """ - changed = False - for s in strings: - if (lower := lc(s)) in self.elements: - self.elements.remove(lower) - changed = True - else: - warnings.warn( - "SetManager.remove() of a missing member currently does " - "nothing, but will raise KeyError in 2.0; use discard() " - "to ignore missing members. See " - "https://github.com/derek73/python-nameparser/issues/243", - DeprecationWarning, - stacklevel=2, - ) - if changed and self._on_change: - self._on_change() - return self - - def discard(self, *strings: str) -> Self: - """ - Remove the lower case and no-period version of the string arguments - from the set if present; missing members are ignored, like - ``set.discard``. Returns ``self`` for chaining. - """ - changed = False - for s in strings: - if (lower := lc(s)) in self.elements: - self.elements.remove(lower) - changed = True - if changed and self._on_change: - self._on_change() - return self - - def clear(self) -> Self: - """Remove all entries from the set. Returns ``self`` for chaining.""" - if self.elements: - self.elements.clear() - if self._on_change: - self._on_change() - return self - - -T = TypeVar('T') - - -def _is_dunder(attr: str) -> bool: - # Dunder names are Python's protocol probes (copy looks up __deepcopy__, - # inspect.unwrap looks up __wrapped__, typing's GenericAlias.__call__ sets - # __orig_class__, ...), never config keys. The TupleManager attribute hooks - # all route dunders to normal object-attribute behavior so those probes - # work instead of being mistaken for dict entries. - return attr.startswith("__") and attr.endswith("__") - - -# The default config sets are module constants that never change, so -# validate and normalize each one exactly once at import. Constants() -# copies these via _normalized_elements' SetManager fast path instead of -# re-checking ~1,400 elements per construction — a cost that otherwise -# repeats on the per-instance-config path, HumanName(constants=Constants()). -# -# This snapshot is taken once, at import time: mutating a raw constant -# (e.g. `TITLES.add('x')`) after import is *not* picked up by Constants() -# built afterward, since the identity check in Constants.__init__ reuses -# this frozen SetManager rather than re-wrapping the (now-changed) raw -# set. That's a behavior change from re-wrapping every time, but the -# documented customization path mutates the SetManager wrapper on a -# Constants instance (``CONSTANTS.titles.add(...)``), not the raw -# constant, so this only affects an unsupported/undocumented pattern. -_DEFAULT_PREFIXES = SetManager(PREFIXES) -_DEFAULT_SUFFIX_ACRONYMS = SetManager(SUFFIX_ACRONYMS) -_DEFAULT_SUFFIX_NOT_ACRONYMS = SetManager(SUFFIX_NOT_ACRONYMS) -_DEFAULT_SUFFIX_ACRONYMS_AMBIGUOUS = SetManager(SUFFIX_ACRONYMS_AMBIGUOUS) -_DEFAULT_TITLES = SetManager(TITLES) -_DEFAULT_FIRST_NAME_TITLES = SetManager(FIRST_NAME_TITLES) -_DEFAULT_CONJUNCTIONS = SetManager(CONJUNCTIONS) -_DEFAULT_BOUND_FIRST_NAMES = SetManager(BOUND_FIRST_NAMES) -_DEFAULT_NON_FIRST_NAME_PREFIXES = SetManager(NON_FIRST_NAME_PREFIXES) - - -class TupleManager(dict[str, T]): - ''' - A dictionary with dot.notation access. Subclass of ``dict``. Wraps the - mapping config constants (``capitalization_exceptions``, ``regexes``, and - the nickname/maiden delimiter buckets). The name is historical: before - 1.3.0 these constants were tuples of pairs. - ''' - - def __init__( - self, - arg: Mapping[str, T] | Iterable[tuple[str, T]] = (), - **kwargs: T, - ) -> None: - # dict.__init__ accepts a bare str/bytes as an iterable-of-pairs - # argument (each character iterates further, and dict() only - # complains once it hits a "pair" of the wrong length) and accepts an - # iterable of 2-character strings as if each one were a (key, value) - # pair, silently shredding it -- mirrors SetManager's guard against - # the same class of mistake (#238), applied to the mapping - # constructor's own failure modes (#242). - _reject_bare_str_or_bytes(arg, "a mapping or iterable of (key, value) pairs") - if not isinstance(arg, Mapping): - checked = [] - for item in arg: - if isinstance(item, (str, bytes)): - raise TypeError( - "expected (key, value) pairs, got a " - f"{'bytes' if isinstance(item, bytes) else 'str'} " - f"element {item!r}; a 2-character string silently " - "splits into a key and a value" - ) - checked.append(item) - arg = checked - super().__init__(arg, **kwargs) - - def _warn_unknown_key(self, attr: str) -> None: - # Deprecated 1.4.0, raises AttributeError in 2.0 (#256): a misspelled - # key otherwise degrades silently with no traceback pointing at the - # typo. - warnings.warn( - f"{attr!r} is not a known key ({', '.join(sorted(self))}); " - "unknown-key attribute access is deprecated and will raise " - "AttributeError in 2.0. Use .get() for intentional soft access. " - "See https://github.com/derek73/python-nameparser/issues/256", - DeprecationWarning, - stacklevel=3, - ) - - def __getattr__(self, attr: str) -> T | None: - # Otherwise the dict default (None) is mistaken for a real protocol hook. - if _is_dunder(attr): - raise AttributeError(attr) - # Single-underscore introspection probes (IPython/Jupyter's - # _repr_html_, _ipython_canary_method_should_not_exist_, etc.) are - # never config keys either -- no real config key starts with '_'. - if attr not in self and not attr.startswith('_'): - self._warn_unknown_key(attr) - return self.get(attr) - - def __setattr__(self, attr: str, value: T) -> None: - # Fall back to normal object attribute storage for dunders; everything - # else keeps the dict-backed dot-notation behavior this class exists - # for. Concretely: constructing a subscripted generic, e.g. - # TupleManager[re.Pattern[str] | str](...), makes typing's - # GenericAlias.__call__ set `__orig_class__` on the new instance right - # after __init__ returns. Without this guard that assignment falls - # through to dict.__setitem__ and silently inserts a bogus - # '__orig_class__' entry into the dict itself, corrupting - # .values()/iteration. - if _is_dunder(attr): - object.__setattr__(self, attr, value) - else: - self[attr] = value - - def __delattr__(self, attr: str) -> None: - if _is_dunder(attr): - object.__delattr__(self, attr) - else: - del self[attr] - - def __getstate__(self) -> Mapping[str, T]: - return dict(self) - - def __setstate__(self, state: Mapping[str, T]) -> None: - self.update(state) - - def __reduce__(self) -> tuple[type, tuple[()], Mapping[str, T]]: - # Use type(self), not TupleManager, so subclasses such as - # RegexTupleManager survive a pickle round-trip instead of being - # downgraded to a plain TupleManager (which loses the EMPTY_REGEX - # default for unknown keys). - return (type(self), (), self.__getstate__()) - - -class RegexTupleManager(TupleManager[re.Pattern[str]]): - def __getattr__(self, attr: str) -> re.Pattern[str]: - # Otherwise EMPTY_REGEX is returned for a dunder probe; copy.deepcopy - # then tries to call the returned re.Pattern and raises TypeError. - if _is_dunder(attr): - raise AttributeError(attr) - if attr not in self and not attr.startswith('_'): - self._warn_unknown_key(attr) - return self.get(attr, EMPTY_REGEX) - - -class _SetManagerAttribute: - """Descriptor enforcing ``isinstance(value, SetManager)`` on assignment. - - Backs the five plain SetManager attributes (``first_name_titles``, - ``conjunctions``, ``bound_first_names``, ``non_first_name_prefixes``, - ``suffix_acronyms_ambiguous``). Without this guard, e.g. - ``c.conjunctions = 'and'`` is accepted silently, and every later - ``piece.lower() in self.C.conjunctions`` becomes a substring test against - the plain str instead of a set membership test (#241). - - ``_CachedUnionMember`` subclasses this to add ``_pst`` cache invalidation - for the four attributes whose union ``Constants`` caches. - """ - - _attr: str - - def __set_name__(self, owner: type, name: str) -> None: - self._attr = '_' + name - - @overload - def __get__(self, obj: None, objtype: type | None = None) -> '_SetManagerAttribute': ... - @overload - def __get__(self, obj: 'Constants', objtype: type | None = None) -> SetManager: ... - - def __get__(self, obj: 'Constants | None', objtype: type | None = None) -> 'SetManager | _SetManagerAttribute': - if obj is None: - return self - return getattr(obj, self._attr) - - def _validate(self, value: SetManager) -> None: - if not isinstance(value, SetManager): - raise TypeError( - f"Expected a SetManager instance, got {type(value).__name__!r}. " - "Wrap your iterable: SetManager(['mr', 'ms'])" - ) - - def __set__(self, obj: 'Constants', value: SetManager) -> None: - self._validate(value) - setattr(obj, self._attr, value) - - -class _CachedUnionMember(_SetManagerAttribute): - """Descriptor for the four ``SetManager`` attributes whose union ``Constants`` - caches in ``_pst`` (``prefixes``, ``suffix_acronyms``, ``suffix_not_acronyms``, - ``titles``). - - Assigning a new manager — or mutating one in place via ``add()`` / ``remove()`` - — invalidates that cache. Keeping the behavior on a descriptor scopes it to - exactly these attributes, beside their declarations, rather than spreading it - across a catch-all ``__setattr__`` and a separate attribute-name list. - """ - - def __set__(self, obj: 'Constants', value: SetManager) -> None: - self._validate(value) - previous = getattr(obj, self._attr, None) - if isinstance(previous, SetManager): - previous._on_change = None # detach the replaced manager so it no longer invalidates - value._on_change = obj._invalidate_pst - obj._invalidate_pst() - setattr(obj, self._attr, value) - - -class _EmptyAttributeDefaultAttribute: - """Descriptor backing ``Constants.empty_attribute_default``. - - .. deprecated:: 1.4.0 - Assignment is deprecated (see issue #255): the only legal value - left once ``None`` support goes in 2.0 is the default ``''``, so a - dial with one position isn't configuration. - """ - - _attr = '_empty_attribute_default' - - def __get__(self, obj: 'Constants | None', objtype: type | None = None) -> str: - # Annotated `str`, not `str | None`, to match the pre-descriptor - # plain-attribute inference: None is documented/supported (see the - # class docstring), but typing it honestly cascades `| None` - # through every public str-typed name accessor (title, first, ...). - # Returning '' rather than `self` on class access (unlike - # _SetManagerAttribute, which returns `self`) is also load-bearing - # for Constants.__repr__'s `getattr(type(self), name)` default - # comparison in _repr_scalar_attrs -- returning `self` there would - # make every Constants() show this attribute as "customized". - if obj is None: - return '' - return getattr(obj, self._attr, '') - - def __set__(self, obj: 'Constants', value: str | None) -> None: - if value is not None and not isinstance(value, str): - raise TypeError( - f"empty_attribute_default must be a str or None, got " - f"{type(value).__name__!r}" - ) - warnings.warn( - "Assigning Constants.empty_attribute_default is deprecated and " - "will raise TypeError in 2.0; empty attributes will always " - "return ''. See " - "https://github.com/derek73/python-nameparser/issues/255", - DeprecationWarning, - stacklevel=2, - ) - setattr(obj, self._attr, value) - - -class Constants: - """ - An instance of this class hold all of the configuration constants for the parser. - - :param set prefixes: - :py:attr:`prefixes` wrapped with :py:class:`SetManager`. - :param set titles: - :py:attr:`titles` wrapped with :py:class:`SetManager`. - :param set first_name_titles: - :py:attr:`~titles.FIRST_NAME_TITLES` wrapped with :py:class:`SetManager`. - :param set suffix_acronyms: - :py:attr:`~suffixes.SUFFIX_ACRONYMS` wrapped with :py:class:`SetManager`. - :param set suffix_not_acronyms: - :py:attr:`~suffixes.SUFFIX_NOT_ACRONYMS` wrapped with :py:class:`SetManager`. - :param set suffix_acronyms_ambiguous: - :py:attr:`~suffixes.SUFFIX_ACRONYMS_AMBIGUOUS` wrapped with :py:class:`SetManager`. - :param set conjunctions: - :py:attr:`conjunctions` wrapped with :py:class:`SetManager`. - :param set bound_first_names: - :py:attr:`~bound_first_names.BOUND_FIRST_NAMES` wrapped with :py:class:`SetManager`. - :param set non_first_name_prefixes: - :py:attr:`~prefixes.NON_FIRST_NAME_PREFIXES` wrapped with :py:class:`SetManager`. - The subset of prefixes that are never a first name, so a *leading* one - marks the whole name as a surname. Must stay disjoint from - ``bound_first_names``. - :type capitalization_exceptions: dict or iterable of (key, value) tuples - :param capitalization_exceptions: - :py:attr:`~capitalization.CAPITALIZATION_EXCEPTIONS` wrapped with :py:class:`TupleManager`. - :type regexes: dict or iterable of (name, compiled pattern) tuples - :param regexes: - :py:attr:`~regexes.REGEXES` wrapped with :py:class:`RegexTupleManager`. - - :py:attr:`nickname_delimiters` and :py:attr:`maiden_delimiters` are not - constructor arguments -- they're always set in ``__init__`` (see the - comment there for the string-sentinel-vs-compiled-pattern mechanism) -- - but are documented here since they're the two `Constants` attributes a - caller is most likely to want to look up: per-bucket - :py:class:`TupleManager` collections that :py:meth:`~nameparser.parser.HumanName.parse_nicknames` - consults to route delimited content into ``nickname``/``maiden``. See - the "Adding Custom Nickname Delimiters" and "Routing to Maiden Name" - sections of the customization docs. - """ - - prefixes = _CachedUnionMember() - suffix_acronyms = _CachedUnionMember() - suffix_not_acronyms = _CachedUnionMember() - titles = _CachedUnionMember() - first_name_titles = _SetManagerAttribute() - conjunctions = _SetManagerAttribute() - bound_first_names = _SetManagerAttribute() - non_first_name_prefixes = _SetManagerAttribute() - suffix_acronyms_ambiguous = _SetManagerAttribute() - capitalization_exceptions: TupleManager[str] - regexes: RegexTupleManager - nickname_delimiters: TupleManager[re.Pattern[str] | str] - maiden_delimiters: TupleManager[re.Pattern[str] | str] - _pst: Set[str] | None - - string_format: str | None = "{title} {first} {middle} {last} {suffix} ({nickname})" - """ - The default string format use for all new `HumanName` instances. - """ - - initials_format = "{first} {middle} {last}" - """ - The default initials format used for all new `HumanName` instances. - """ - - initials_delimiter = "." - """ - The default initials delimiter used for all new `HumanName` instances. - Will be used to add a delimiter between each initial. - """ - - initials_separator = " " - """ - The default separator placed between consecutive initials within a name - group (first, middle, or last). Distinct from ``initials_delimiter``, - which is the trailing character after each individual initial. - - With defaults ``initials_delimiter="."`` and ``initials_separator=" "``, - ``initials()`` produces ``"J. A. D."``. Setting ``initials_separator=""`` - with ``initials_delimiter="."`` and ``initials_format="{first}{middle}{last}"`` - produces ``"J.A.D."``. With the default ``initials_format``, group-level - spacing from the template is still applied. - """ - - suffix_delimiter: str | None = None - """ - If set, an additional delimiter used to split suffix groups after - comma-splitting. For example, setting ``suffix_delimiter=" - "`` allows - ``"RN - CRNA"`` to be parsed as two separate suffixes. Default is - ``None`` (no additional splitting beyond the standard comma split). - - Note: setting this to ``","`` or ``", "`` has no additional effect — - the full name is already split on comma characters first (including the - Arabic ``،`` and fullwidth ``，`` variants), and each resulting part is - stripped of surrounding whitespace before this step runs. - - The delimiter is only applied to parts once they've been identified as - a suffix group, so it never leaks into a first- or middle-name part. For - example, in inverted format (``"Last, First, suffix"``) a hyphenated - given name like ``"Doe, Mary - Kate, RN"`` with ``suffix_delimiter=" - "`` - does not get mistaken for a suffix split. - """ - - empty_attribute_default = _EmptyAttributeDefaultAttribute() - """ - Default return value for empty attributes. - - .. deprecated:: 1.4.0 - Assignment emits ``DeprecationWarning``; the option is removed in - 2.0 (see issue #255) and empty attributes will always return ``''``. - - .. doctest:: - - >>> import warnings - >>> from nameparser.config import CONSTANTS - >>> with warnings.catch_warnings(): - ... warnings.simplefilter('ignore', DeprecationWarning) - ... CONSTANTS.empty_attribute_default = None - >>> name = HumanName("John Doe") - >>> print(name.title) - None - >>> name.first - 'John' - >>> with warnings.catch_warnings(): - ... warnings.simplefilter('ignore', DeprecationWarning) - ... CONSTANTS.empty_attribute_default = '' - - """ - - capitalize_name = False - """ - If set, applies :py:meth:`~nameparser.parser.HumanName.capitalize` to - :py:class:`~nameparser.parser.HumanName` instance. - - .. doctest:: - - >>> from nameparser.config import CONSTANTS - >>> CONSTANTS.capitalize_name = True - >>> name = HumanName("bob v. de la macdole-eisenhower phd") - >>> str(name) - 'Bob V. de la MacDole-Eisenhower Ph.D.' - >>> CONSTANTS.capitalize_name = False - - """ - - force_mixed_case_capitalization = False - """ - If set, forces the capitalization of mixed case strings when - :py:meth:`~nameparser.parser.HumanName.capitalize` is called. - - .. doctest:: - - >>> from nameparser.config import CONSTANTS - >>> CONSTANTS.force_mixed_case_capitalization = True - >>> name = HumanName('Shirley Maclaine') - >>> name.capitalize() - >>> str(name) - 'Shirley MacLaine' - >>> CONSTANTS.force_mixed_case_capitalization = False - - """ - - patronymic_name_order = False - """ - If set, detects names in Russian formal order (``Surname GivenName Patronymic``) - by recognizing a trailing East-Slavic patronymic suffix on the last token, and - rotates the three name parts so that ``first``/``middle``/``last`` map to - given name / patronymic / surname respectively. Detection requires exactly one - token in each of first, middle, and last; names with multi-part given names or - multiple middle names are left unchanged. - - Also detects reversed-order Azerbaijani/Central-Asian Turkic patronymics - (``Surname GivenName PatronymicRoot Marker``, e.g. ``oglu``/``qizi``), a - structurally different, standalone-marker-word patronymic family. Detection - requires exactly one token in each of first and last, exactly two tokens in - middle, and the last token a recognised Turkic marker. - - Opt-in because a Western person whose surname happens to end in a patronymic - suffix (e.g. ``"David Michael Abramovich"``) will be reordered incorrectly - when the flag is on. Enable only when your data is predominantly Russian - formal-order names. - - For per-instance control without a shared ``Constants``, pass a dedicated - instance: ``HumanName("...", constants=Constants(patronymic_name_order=True))``. - - .. doctest:: - - >>> from nameparser import HumanName - >>> from nameparser.config import Constants - >>> C = Constants(patronymic_name_order=True) - >>> hn = HumanName("Ivanov Ivan Ivanovich", constants=C) - >>> hn.first, hn.middle, hn.last - ('Ivan', 'Ivanovich', 'Ivanov') - >>> hn2 = HumanName("Aliyev Vusal Said oglu", constants=C) - >>> hn2.first, hn2.middle, hn2.last - ('Vusal', 'Said oglu', 'Aliyev') - - """ - - middle_name_as_last = False - """ - If set, folds middle names into the last name: ``middle_list`` is prepended - to ``last_list`` and ``middle_list`` is cleared, so ``.last`` becomes what - ``.surnames`` already was and ``.middle`` becomes empty. Useful for naming - systems with no middle-name concept, where everything after the given name - is lineage/family (e.g. Arabic patronymic chaining: given + father + - grandfather + family). - - The fold is uniform across both no-comma and comma ("Last, First Middle") - input, so the two written forms of a name converge on the same result. - - For per-instance control without a shared ``Constants``, pass a dedicated - instance: ``HumanName("...", constants=Constants(middle_name_as_last=True))``. - - .. doctest:: - - >>> from nameparser import HumanName - >>> from nameparser.config import Constants - >>> C = Constants(middle_name_as_last=True) - >>> hn = HumanName("Mohamad Ahmad Ali Hassan", constants=C) - >>> hn.first, hn.middle, hn.last - ('Mohamad', '', 'Ahmad Ali Hassan') - - """ - - def __init__(self, - prefixes: Iterable[str] = PREFIXES, - suffix_acronyms: Iterable[str] = SUFFIX_ACRONYMS, - suffix_not_acronyms: Iterable[str] = SUFFIX_NOT_ACRONYMS, - suffix_acronyms_ambiguous: Iterable[str] = SUFFIX_ACRONYMS_AMBIGUOUS, - titles: Iterable[str] = TITLES, - first_name_titles: Iterable[str] = FIRST_NAME_TITLES, - conjunctions: Iterable[str] = CONJUNCTIONS, - bound_first_names: Iterable[str] = BOUND_FIRST_NAMES, - non_first_name_prefixes: Iterable[str] = NON_FIRST_NAME_PREFIXES, - capitalization_exceptions: Mapping[str, str] | Iterable[tuple[str, str]] = CAPITALIZATION_EXCEPTIONS, - regexes: Mapping[str, re.Pattern[str]] | Iterable[tuple[str, re.Pattern[str]]] = REGEXES, - patronymic_name_order: bool = False, - middle_name_as_last: bool = False, - ) -> None: - # These four descriptor assignments call _CachedUnionMember.__set__, which - # calls _invalidate_pst() and establishes self._pst. They must come before - # any read of suffixes_prefixes_titles. - # untouched defaults (identity check) copy the prebuilt module-level - # managers instead of re-validating the raw constants element by - # element; user-supplied iterables still get the full check - self.prefixes = SetManager(_DEFAULT_PREFIXES if prefixes is PREFIXES else prefixes) - self.suffix_acronyms = SetManager(_DEFAULT_SUFFIX_ACRONYMS if suffix_acronyms is SUFFIX_ACRONYMS else suffix_acronyms) - self.suffix_not_acronyms = SetManager(_DEFAULT_SUFFIX_NOT_ACRONYMS if suffix_not_acronyms is SUFFIX_NOT_ACRONYMS else suffix_not_acronyms) - self.titles = SetManager(_DEFAULT_TITLES if titles is TITLES else titles) - self.first_name_titles = SetManager(_DEFAULT_FIRST_NAME_TITLES if first_name_titles is FIRST_NAME_TITLES else first_name_titles) - self.conjunctions = SetManager(_DEFAULT_CONJUNCTIONS if conjunctions is CONJUNCTIONS else conjunctions) - self.bound_first_names = SetManager(_DEFAULT_BOUND_FIRST_NAMES if bound_first_names is BOUND_FIRST_NAMES else bound_first_names) - self.non_first_name_prefixes = SetManager(_DEFAULT_NON_FIRST_NAME_PREFIXES if non_first_name_prefixes is NON_FIRST_NAME_PREFIXES else non_first_name_prefixes) - self.suffix_acronyms_ambiguous = SetManager(_DEFAULT_SUFFIX_ACRONYMS_AMBIGUOUS if suffix_acronyms_ambiguous is SUFFIX_ACRONYMS_AMBIGUOUS else suffix_acronyms_ambiguous) - self.capitalization_exceptions = TupleManager(capitalization_exceptions) - self.regexes = RegexTupleManager(regexes) - # Per-bucket delimiter collections that parse_nicknames() consults to - # route delimited content into nickname_list / maiden_list. Each value - # is either a compiled re.Pattern (a custom delimiter a caller adds -- - # the old extra_nickname_delimiters use case, see issue #112) or the - # string name of a self.regexes entry to resolve live at parse time. - # The latter is how the three built-ins (quoted_word, double_quotes, - # parenthesis) stay linked to self.regexes, so overriding e.g. - # self.regexes.parenthesis keeps affecting nickname/maiden parsing - # exactly as before. Move a key between the two dicts - # (`maiden_delimiters['parenthesis'] = - # nickname_delimiters.pop('parenthesis')`) to change which bucket it - # routes to without losing that live link. maiden_delimiters starts - # empty -- maiden is off until a caller routes a delimiter to it. - # See issue #22. - # Only seed a built-in name if it's actually present in self.regexes -- - # a caller who overrides regexes with a minimal custom set (dropping - # e.g. "parenthesis" entirely) shouldn't end up with a dangling - # string sentinel that parse_nicknames() would treat as a mistake. - # See parse_nicknames()'s fail-loud check on an unresolvable sentinel. - self.nickname_delimiters = TupleManager[re.Pattern[str] | str]({ - name: name for name in ('quoted_word', 'double_quotes', 'parenthesis') - if name in self.regexes - }) - self.maiden_delimiters = TupleManager[re.Pattern[str] | str]() - self.patronymic_name_order = patronymic_name_order - self.middle_name_as_last = middle_name_as_last - - def _invalidate_pst(self) -> None: - self._pst = None - - @property - def suffixes_prefixes_titles(self) -> Set[str]: - if self._pst is None: - self._pst = self.prefixes | self.suffix_acronyms | self.suffix_not_acronyms | self.titles - return self._pst - - _repr_collection_attrs = ( - 'prefixes', 'suffix_acronyms', 'suffix_not_acronyms', 'titles', - 'first_name_titles', 'conjunctions', 'bound_first_names', - 'non_first_name_prefixes', 'suffix_acronyms_ambiguous', - ) - _repr_scalar_attrs = ( - 'string_format', 'initials_format', 'initials_delimiter', - 'initials_separator', 'suffix_delimiter', 'empty_attribute_default', - 'capitalize_name', 'force_mixed_case_capitalization', - 'patronymic_name_order', 'middle_name_as_last', - ) - - def __repr__(self) -> str: - # Collections (some with hundreds of entries, e.g. titles/prefixes) - # are summarized as counts rather than dumped in full. Scalars are - # only shown when they differ from the class default, so a plain - # Constants() reads as just the collection sizes. - lines = [f" {name}: {len(getattr(self, name))}" for name in self._repr_collection_attrs] - lines += [ - f" {name}: {value!r}" for name in self._repr_scalar_attrs - if (value := getattr(self, name)) != getattr(type(self), name) - ] - return "" - - def copy(self) -> 'Constants': - """ - Return a detached deep copy of this ``Constants`` instance, preserving - its current customizations -- unlike :py:class:`Constants`'s own - constructor, which always starts from library defaults. Useful for - snapshotting the shared module-level ``CONSTANTS`` (including - whatever it's been customized with) into a private instance, e.g. - ``CONSTANTS.copy()``. Relies on the same ``__getstate__``/``__setstate__`` - pair pickling uses, so it's as cheap and correct as pickle round-tripping. - """ - return copy.deepcopy(self) - - def __setstate__(self, state: Mapping[str, Any]) -> None: - # Restore each saved attribute directly. The previous implementation - # passed the whole state dict to __init__ as the ``prefixes`` argument, - # which silently reverted every collection to its module default on - # unpickling. - self._pst = None - legacy_format = False - for name, value in state.items(): - # inspect.getattr_static, not getattr, so descriptors are - # inspected directly rather than triggering their __get__. - descriptor = inspect.getattr_static(type(self), name, None) - # Migration shim: pickles written before this fix (1.3.0 and earlier, - # including 1.2.1) used a dir() sweep for __getstate__, so their state - # carries the read-only ``suffixes_prefixes_titles`` property. Skip any - # such computed property rather than raising AttributeError on its - # missing setter; the real config is restored from the other keys. We - # don't promise to read pre-fix blobs forever — this only smooths - # migration for anyone persisting them, and can be dropped a release - # or two after 1.3.0 once they've re-pickled. - if isinstance(descriptor, property): - legacy_format = True - continue - if isinstance(descriptor, _EmptyAttributeDefaultAttribute): - # Bypass the descriptor's setter: restoring saved state isn't - # a user assignment, so it shouldn't emit #255's deprecation - # warning on every unpickle/copy() of a customized instance. - setattr(self, descriptor._attr, value) - continue - setattr(self, name, value) - if legacy_format: - # Once per __setstate__ call, not once per skipped key (see - # issue #279): the 2.0 removal turns this into a ValueError - # naming the same fix. - warnings.warn( - "Loading a legacy-format Constants pickle (written by " - "nameparser <= 1.2.x, before the 1.3.0 pickle fix) is " - "deprecated and will raise ValueError in 2.0; re-pickle " - "under 1.3/1.4 to migrate. See " - "https://github.com/derek73/python-nameparser/issues/279", - DeprecationWarning, - stacklevel=2, - ) - # Verify each descriptor-backed attr was restored. Without this, a missing - # key surfaces later as AttributeError: 'Constants' object has no attribute - # '_prefixes' — the private mangled name, not the public one, making it - # very hard to diagnose. - for attr in (n for n, v in vars(type(self)).items() if isinstance(v, _SetManagerAttribute)): - if not hasattr(self, '_' + attr): - raise ValueError( - f"Pickle state is missing required field {attr!r}. " - "The state blob may be truncated or from an incompatible version." - ) - - def __getstate__(self) -> Mapping[str, Any]: - # Pickle the instance's own configuration: the collections built in - # __init__ plus any instance-level scalar overrides. - # _CachedUnionMember descriptors store their values with a leading - # underscore (e.g. `_prefixes` for `prefixes`) so that the descriptor's - # __set__ owns assignment. We map those back to the public names so - # __setstate__ can restore them through the descriptor, re-wiring the - # invalidation callbacks. All other underscore-prefixed names (_pst, etc.) - # are private/cache and are intentionally excluded. - state: dict[str, Any] = {} - for name, val in self.__dict__.items(): - if name.startswith('_'): - public = name[1:] - descriptor = inspect.getattr_static(type(self), public, None) - if isinstance(descriptor, (_SetManagerAttribute, _EmptyAttributeDefaultAttribute)): - state[public] = val - else: - state[name] = val - return state - - -#: A module-level instance of the :py:class:`Constants()` class. -#: Provides a common instance for the module to share -#: to easily adjust configuration for the entire module. -#: See `Customizing the Parser with Your Own Configuration `_. -CONSTANTS = Constants() +__all__ = ["CONSTANTS", "Constants", "RegexTupleManager", "SetManager", "TupleManager"] diff --git a/nameparser/parser.py b/nameparser/parser.py index 1e1e7865..aa85bfe4 100644 --- a/nameparser/parser.py +++ b/nameparser/parser.py @@ -1,1654 +1,6 @@ -import re -import warnings -from collections.abc import Callable, Iterable, Iterator -from operator import itemgetter -from itertools import groupby +"""v1 import-path preservation (migration spec §3): the 2.0 HumanName +facade lives in nameparser._facade. This module is deleted in 3.0. +""" +from nameparser._facade import HumanName as HumanName -from typing import overload - -from nameparser.util import HumanNameAttributeT, lc -from nameparser.util import log -from nameparser.config import CONSTANTS -from nameparser.config import Constants -from nameparser.config import DEFAULT_ENCODING - -def group_contiguous_integers(data: Iterable[int]) -> list[tuple[int, int]]: - """ - return list of tuples containing first and last index - position of contiguous numbers in a series - """ - ranges: list[tuple[int, int]] = [] - for key, group_with_indices in groupby(enumerate(data), lambda i: i[0] - i[1]): - group = list(map(itemgetter(1), group_with_indices)) - if len(group) > 1: - ranges.append((group[0], group[-1])) - return ranges - - -class HumanName: - """ - Parse a person's name into individual components. - - Instantiation assigns to ``full_name``, and assignment to - :py:attr:`full_name` triggers :py:func:`parse_full_name`. After parsing the - name, these instance attributes are available. Alternatively, you can pass - any of the instance attributes to the constructor method and skip the parsing - process. If any of the the instance attributes are passed to the constructor - as keywords, :py:func:`parse_full_name` will not be performed. - - **HumanName Instance Attributes** - - * :py:attr:`title` - * :py:attr:`first` - * :py:attr:`middle` - * :py:attr:`last` - * :py:attr:`suffix` - * :py:attr:`nickname` - * :py:attr:`maiden` - * :py:attr:`surnames` - * :py:attr:`given_names` - - :param str full_name: The name string to be parsed. - :param constants: - a :py:class:`~nameparser.config.Constants` instance (subclasses are - honored). Defaults to the shared module-level ``CONSTANTS``. For - `per-instance config `_, pass ``Constants()`` for - fresh library defaults, or ``CONSTANTS.copy()`` for a private - snapshot of the current shared config. Passing ``None`` also builds - a fresh ``Constants()``, but is deprecated (warns; raises - ``TypeError`` in 2.0, see issue #260) since it silently discards any - customizations the caller may have expected to carry over. Anything - else raises ``TypeError``. - :param str encoding: string representing the encoding of your input - (deprecated with ``bytes`` input, removal in 2.0 — decode before - passing; see issue #245) - :param str string_format: python string formatting - :param str initials_format: python initials string formatting - :param str initials_delimter: string delimiter for initials - :param str initials_separator: string separator between consecutive initials - :param str suffix_delimiter: additional delimiter to split post-comma parts - before suffix detection, e.g. ``" - "`` for ``"RN - CRNA"`` - :param str first: first name - :param str middle: middle name - :param str last: last name - :param str title: The title or prenominal - :param str suffix: The suffix or postnominal - :param str nickname: Nicknames - :param str maiden: Maiden name - """ - - original: str | bytes = '' - """ - The original string, untouched by the parser. - """ - - _members = ['title', 'first', 'middle', 'last', 'suffix', 'nickname', 'maiden'] - _full_name = '' - - title_list: list[str] - first_list: list[str] - middle_list: list[str] - last_list: list[str] - suffix_list: list[str] - nickname_list: list[str] - maiden_list: list[str] - _had_comma: bool - - def __init__( - self, - full_name: str | bytes = "", - constants: Constants | None = CONSTANTS, - encoding: str = DEFAULT_ENCODING, - string_format: str | None = None, - initials_format: str | None = None, - initials_delimiter: str | None = None, - initials_separator: str | None = None, - suffix_delimiter: str | None = None, - first: str | list[str] | None = None, - middle: str | list[str] | None = None, - last: str | list[str] | None = None, - title: str | list[str] | None = None, - suffix: str | list[str] | None = None, - nickname: str | list[str] | None = None, - maiden: str | list[str] | None = None, - ) -> None: - # calls _validate_constants directly (not through the C setter) so - # the deprecation warning below attributes to this constructor's - # caller rather than to the setter, mirroring _apply_full_name below - self._C = self._validate_constants(constants, stacklevel=3) - - # Lookup entries derived while parsing this instance (period-joined - # titles/suffixes like "Lt.Gov.", conjunction-joined pieces like - # "Mr. and Mrs." or "von und zu"). Kept separate from self.C so that - # parsing never writes into the config — which is usually the shared - # module-level CONSTANTS — keeping results independent of what was - # parsed before and config reads safe across threads. Values are - # lc()-normalized, mirroring how SetManager stores them. Reset at the - # start of each parse_full_name() run. - self._derived_titles: set[str] = set() - self._derived_suffixes: set[str] = set() - self._derived_conjunctions: set[str] = set() - self._derived_prefixes: set[str] = set() - - self.encoding = encoding - self.string_format = string_format if string_format is not None else self.C.string_format - self.initials_format = initials_format if initials_format is not None else self.C.initials_format - self.initials_delimiter = initials_delimiter if initials_delimiter is not None else self.C.initials_delimiter - self.initials_separator = initials_separator if initials_separator is not None else self.C.initials_separator - self.suffix_delimiter = suffix_delimiter if suffix_delimiter is not None else self.C.suffix_delimiter - self._had_comma = False - if (first or middle or last or title or suffix or nickname or maiden): - self.first = first - self.middle = middle - self.last = last - self.title = title - self.suffix = suffix - self.nickname = nickname - self.maiden = maiden - else: - # calls _apply_full_name directly (not the setter) so the - # deprecation warning below attributes to this constructor's - # caller rather than to the setter - self._apply_full_name(full_name, stacklevel=3) - - @staticmethod - def _validate_constants(constants: 'Constants | None', *, stacklevel: int) -> 'Constants': - # Shared by the constructor and the C setter so both assignment paths - # give the same immediate TypeError instead of one bypassing the - # other and failing far from the cause (#239). - if constants is None: - # deprecated 1.4.0, raises TypeError in 2.0 (#260, removal #261): - # None means "build a fresh private Constants()", the opposite of - # what None conventionally means (the default is the *shared* - # CONSTANTS) -- an easy trap since customizing CONSTANTS then - # passing None elsewhere silently drops those customizations with - # no error. CONSTANTS.copy() is the explicit spelling for the - # other reading: a private snapshot of the current shared config. - warnings.warn( - "Passing constants=None is deprecated and will raise " - "TypeError in 2.0; use constants=Constants() for fresh " - "library defaults, or constants=CONSTANTS.copy() to snapshot " - "the current shared config. See " - "https://github.com/derek73/python-nameparser/issues/260", - DeprecationWarning, - stacklevel=stacklevel, - ) - return Constants() - if not isinstance(constants, Constants): - # passing the class itself is the likeliest mistake, and - # reporting it as "got type" would only add confusion - hint = (" (a class was passed; did you mean Constants()?)" - if isinstance(constants, type) else "") - raise TypeError( - "constants must be a Constants instance or None, " - f"got {type(constants).__name__}{hint}" - ) - return constants - - @property - def C(self) -> 'Constants': - """ - A reference to the configuration for this instance, which may or may not be - a reference to the shared, module-wide instance at - :py:mod:`~nameparser.config.CONSTANTS`. See `Customizing the Parser - `_. - - Assigning a non-``Constants`` value (besides ``None``, which builds a - fresh private ``Constants()`` and emits a ``DeprecationWarning`` -- - see :py:meth:`~nameparser.parser.HumanName.__init__`) raises the same - ``TypeError`` as passing an invalid ``constants`` argument to the - constructor (#239). - """ - return self._C - - @C.setter - def C(self, constants: 'Constants | None') -> None: - self._C = self._validate_constants(constants, stacklevel=3) - - def __getstate__(self) -> dict: - state = self.__dict__.copy() - c = state.pop('_C') - state['C'] = None if c is CONSTANTS else c # sentinel: restore shared singleton on load - return state - - def __setstate__(self, state: dict) -> None: - state = dict(state) - c = state.pop('C', None) - self._C = CONSTANTS if c is None else c - self.__dict__.update(state) - # pickles from before the per-parse derived sets existed lack them; - # backfill so the is_* predicates work without a re-parse - for attr in ('_derived_titles', '_derived_suffixes', - '_derived_conjunctions', '_derived_prefixes'): - self.__dict__.setdefault(attr, set()) - - def __iter__(self) -> Iterator[str]: - return (value for member in self._members - if (value := getattr(self, member))) - - def __len__(self) -> int: - return sum(1 for member in self._members if getattr(self, member)) - - def __eq__(self, other: object) -> bool: - """ - .. deprecated:: 1.3.0 - Removed in 2.0 (see issue #223); use :py:meth:`matches`. - - HumanName instances are equal to other objects whose - lower case unicode representation is the same. Note the - differences from :py:meth:`matches`: this compares formatted - output, so it depends on ``string_format`` and cannot see - ``maiden``, and it stringifies operands of any type. - """ - warnings.warn( - "HumanName == comparison is deprecated and will be removed in " - "2.0; use matches() instead. See " - "https://github.com/derek73/python-nameparser/issues/223", - DeprecationWarning, - stacklevel=2, - ) - return str(self).lower() == str(other).lower() - - @overload - def __getitem__(self, key: slice) -> list[str]: ... - @overload - def __getitem__(self, key: str) -> str: ... - def __getitem__(self, key: slice | str) -> str | list[str]: - """ - .. deprecated:: 1.4.0 - Slice access (``name[1:-3]``) is removed in 2.0 (see issue - #258); field access by position has no real use case. - String-key access (``name['first']``) is unaffected. - """ - if isinstance(key, slice): - warnings.warn( - "Slicing a HumanName by position is deprecated and will be " - "removed in 2.0; access the named attributes instead. See " - "https://github.com/derek73/python-nameparser/issues/258", - DeprecationWarning, - stacklevel=2, - ) - return [getattr(self, x) for x in self._members[key]] - else: - return getattr(self, key) - - def __setitem__(self, key: str, value: str | list[str] | None) -> None: - """ - .. deprecated:: 1.4.0 - Removed in 2.0 (see issue #258); it duplicates plain attribute - assignment. Use ``name.first = value`` instead. - """ - warnings.warn( - "HumanName item assignment is deprecated and will be removed " - "in 2.0; it duplicates plain attribute assignment, use " - "name.first = value instead. See " - "https://github.com/derek73/python-nameparser/issues/258", - DeprecationWarning, - stacklevel=2, - ) - if key in self._members: - self._set_list(key, value) - else: - raise KeyError("Not a valid HumanName attribute", key) - - def __str__(self) -> str: - if self.string_format is not None: - # string_format = "{title} {first} {middle} {last} {suffix} ({nickname})" - # Empty attributes must render as '' (not empty_attribute_default, - # which may be None) so str.format does not interpolate the - # literal "None" into the output, which cannot be scrubbed - # afterward without corrupting name text containing the same - # substring (#254). - _s = self.string_format.format(**{k: v or '' for k, v in self.as_dict().items()}) - # remove trailing punctuation from missing nicknames - _s = _s.replace(" ()", "").replace(" ''", "").replace(' ""', "") - _s = self.C.regexes.space_before_comma.sub(',', _s) - return self.collapse_whitespace(_s).strip(', ') - return " ".join(self) - - def __hash__(self) -> int: - """ - .. deprecated:: 1.3.0 - Removed in 2.0 (see issue #223); use :py:meth:`comparison_key` - for sets, dicts, and dedup. - """ - warnings.warn( - "hash(HumanName) is deprecated and will be removed in 2.0; use " - "comparison_key() for sets and dicts. See " - "https://github.com/derek73/python-nameparser/issues/223", - DeprecationWarning, - stacklevel=2, - ) - # __eq__ compares lowercased strings, so hash the lowercased string - # to keep equal instances in the same hash bucket. - return hash(str(self).lower()) - - def __repr__(self) -> str: - attrs = ( - f" title: {self.title or ''!r}\n" - f" first: {self.first or ''!r}\n" - f" middle: {self.middle or ''!r}\n" - f" last: {self.last or ''!r}\n" - f" suffix: {self.suffix or ''!r}\n" - f" nickname: {self.nickname or ''!r}\n" - f" maiden: {self.maiden or ''!r}" - ) - return f"<{self.__class__.__name__} : [\n{attrs}\n]>" - - def as_dict(self, include_empty: bool = True) -> dict[str, str]: - """ - Return the parsed name as a dictionary of its attributes. - - :param bool include_empty: Include keys in the dictionary for empty name attributes. - :rtype: dict - - .. doctest:: - - >>> name = HumanName("Bob Dole") - >>> name.as_dict() - {'title': '', 'first': 'Bob', 'middle': '', 'last': 'Dole', 'suffix': '', 'nickname': '', 'maiden': ''} - >>> name.as_dict(False) - {'first': 'Bob', 'last': 'Dole'} - - """ - d = {} - for m in self._members: - if include_empty: - d[m] = getattr(self, m) - else: - val = getattr(self, m) - if val: - d[m] = val - return d - - def comparison_key(self) -> tuple[str, ...]: - """ - The seven name components (title, first, middle, last, suffix, - nickname, maiden) as a lowercased tuple: a canonical, hashable - identity for the parsed name. Use it for dedup, dict keys, and - sorting or grouping, e.g. - ``unique = {n.comparison_key(): n for n in names}.values()``. - - Built from the ``*_list`` attributes, so it is unaffected by - display settings like ``string_format`` and - ``empty_attribute_default``. - - Empty or unparsable input yields the all-empty key, so such names - all compare equal and collide in dedup; screen them out with - ``len(name) == 0`` first. - - .. doctest:: - - >>> HumanName("Dr. Juan Q. Xavier de la Vega III").comparison_key() - ('dr.', 'juan', 'q. xavier', 'de la vega', 'iii', '', '') - - """ - return tuple( - " ".join(getattr(self, member + "_list")).lower() - for member in self._members - ) - - def matches(self, other: 'str | HumanName') -> bool: - """ - Compare parsed components case-insensitively; the semantic - replacement for the deprecated ``==``. A ``str`` argument is parsed - first, using this instance's configuration, so any written form of - the same name matches; a ``HumanName`` argument is compared as - already parsed — its own configuration determined its components. - Two empty or unparsable names match each other; check - ``len(name) == 0`` to screen them. - - .. doctest:: - - >>> name = HumanName("Dr. Juan Q. Xavier de la Vega III") - >>> name.matches("de la vega, dr. juan Q. xavier III") - True - >>> name.matches("Juan de la Vega") - False - - Unlike the deprecated ``==``, all seven components participate - (including ``maiden``, which the default ``string_format`` omits) - and display settings have no effect. Raises ``TypeError`` for - anything that is not a ``str`` or ``HumanName``; guard optional - values explicitly, e.g. ``x is not None and name.matches(x)``. - - Parses string arguments on every call. When matching one name - against many candidates, parse the candidates once or compare - :py:meth:`comparison_key` values instead. - """ - if isinstance(other, HumanName): - return self.comparison_key() == other.comparison_key() - if isinstance(other, str): - return self.comparison_key() == type(self)(other, self.C).comparison_key() - raise TypeError( - f"matches() requires a str or HumanName, got {type(other).__name__}" - ) - - def _process_initial(self, name_part: str, firstname: bool = False) -> str: - """ - Name parts may include prefixes or conjunctions. This function filters these from the name unless it is - a first name, since first names cannot be conjunctions or prefixes. - """ - # split() rather than split(" "): *_list attributes assigned directly - # bypass parse_pieces whitespace normalization, and split(" ") yields - # empty strings for repeated spaces (#232) - parts = name_part.split() - initials = [] - for part in parts: - if not (self.is_prefix(part) or self.is_conjunction(part)) or firstname: - initials.append(part[0]) - if len(initials) > 0: - return self.initials_separator.join(initials) - # Return '' (never empty_attribute_default, which may be None) when a - # part has no initialable words, e.g. a middle name consisting only of - # prefixes ("de la"). Callers drop these parts entirely. - return '' - - def _initials_lists(self) -> tuple[list[str], list[str], list[str]]: - """Initials for the first, middle and last name groups. Parts that - yield no initials (e.g. a prefix-only middle name like "de la") are - dropped rather than kept as empty strings. - """ - def group_initials(names: list[str], firstname: bool = False) -> list[str]: - return [i for i in (self._process_initial(n, firstname) for n in names if n) if i] - return (group_initials(self.first_list, True), - group_initials(self.middle_list), - group_initials(self.last_list)) - - def initials_list(self) -> list[str]: - """ - Returns the initials as a list - - .. doctest:: - - >>> name = HumanName("Sir Bob Andrew Dole") - >>> name.initials_list() - ['B', 'A', 'D'] - >>> name = HumanName("J. Doe") - >>> name.initials_list() - ['J', 'D'] - """ - first_initials_list, middle_initials_list, last_initials_list = self._initials_lists() - return first_initials_list + middle_initials_list + last_initials_list - - def initials(self) -> str: - """ - Return formatted initials for the name, controlled by - ``initials_format``, ``initials_delimiter``, and ``initials_separator``. - - ``initials_delimiter`` is appended after each individual initial. - ``initials_separator`` is placed between consecutive initials within - a name group (first, middle, or last). Both can be set as - ``Constants`` attributes or as ``HumanName`` constructor kwargs. - - .. doctest:: - - >>> name = HumanName("Sir Bob Andrew Dole") - >>> name.initials() - 'B. A. D.' - >>> name = HumanName("Sir Bob Andrew Dole", initials_format="{first} {middle}") - >>> name.initials() - 'B. A.' - >>> name = HumanName("Doe, John A.", initials_delimiter="", initials_separator="") - >>> name.initials() - 'J A D' - """ - - first_initials_list, middle_initials_list, last_initials_list = self._initials_lists() - - # Empty name groups must render as '' (not empty_attribute_default, - # which may be None) so str.format does not interpolate the literal - # "None" into the output. A fully-empty result falls back to - # empty_attribute_default, matching the other attribute accessors - # (e.g. ``first``). - initials_dict = { - "first": (self.initials_delimiter + self.initials_separator).join(first_initials_list) + self.initials_delimiter - if len(first_initials_list) else "", - "middle": (self.initials_delimiter + self.initials_separator).join(middle_initials_list) + self.initials_delimiter - if len(middle_initials_list) else "", - "last": (self.initials_delimiter + self.initials_separator).join(last_initials_list) + self.initials_delimiter - if len(last_initials_list) else "" - } - - _s = self.initials_format.format(**initials_dict) # noqa: UP032 - return self.collapse_whitespace(_s) or self.C.empty_attribute_default - - @property - def has_own_config(self) -> bool: - """ - True if this instance is not using the shared module-level - configuration. - """ - return self.C is not CONSTANTS - - # attributes - - @property - def title(self) -> str: - """ - The person's titles. Any string of consecutive pieces in - :py:mod:`~nameparser.config.titles` or - :py:mod:`~nameparser.config.conjunctions` - at the beginning of :py:attr:`full_name`. - """ - return " ".join(self.title_list) or self.C.empty_attribute_default - - @title.setter - def title(self, value: str | list[str] | None) -> None: - self._set_list('title', value) - - @property - def first(self) -> str: - """ - The person's first name. The first name piece after any known - :py:attr:`title` pieces parsed from :py:attr:`full_name`. - """ - return " ".join(self.first_list) or self.C.empty_attribute_default - - @first.setter - def first(self, value: str | list[str] | None) -> None: - self._set_list('first', value) - - @property - def middle(self) -> str: - """ - The person's middle names. All name pieces after the first name and - before the last name parsed from :py:attr:`full_name`. - """ - return " ".join(self.middle_list) or self.C.empty_attribute_default - - @middle.setter - def middle(self, value: str | list[str] | None) -> None: - self._set_list('middle', value) - - @property - def last(self) -> str: - """ - The person's last name. The last name piece parsed from - :py:attr:`full_name`. - """ - return " ".join(self.last_list) or self.C.empty_attribute_default - - @last.setter - def last(self, value: str | list[str] | None) -> None: - self._set_list('last', value) - - @property - def suffix(self) -> str: - """ - The persons's suffixes. Pieces at the end of the name that are found in - :py:mod:`~nameparser.config.suffixes`, or pieces that are at the end - of comma separated formats, e.g. - "Lastname, Title Firstname Middle[,] Suffix [, Suffix]" parsed - from :py:attr:`full_name`. - """ - return ", ".join(self.suffix_list) or self.C.empty_attribute_default - - @suffix.setter - def suffix(self, value: str | list[str] | None) -> None: - self._set_list('suffix', value) - - @property - def nickname(self) -> str: - """ - The person's nicknames. Any text found inside of quotes (``""``) or - parenthesis (``()``) - """ - return " ".join(self.nickname_list) or self.C.empty_attribute_default - - @nickname.setter - def nickname(self, value: str | list[str] | None) -> None: - self._set_list('nickname', value) - - @property - def maiden(self) -> str: - """ - The person's maiden (alternate/prior) last name. Empty unless a - delimiter has been routed to it via - :py:attr:`~nameparser.config.Constants.maiden_delimiters` -- see the - "Routing to Maiden Name" section of the customization docs. - """ - return " ".join(self.maiden_list) or self.C.empty_attribute_default - - @maiden.setter - def maiden(self, value: str | list[str] | None) -> None: - self._set_list('maiden', value) - - @property - def surnames_list(self) -> list[str]: - """ - List of middle names followed by last name. - """ - return self.middle_list + self.last_list - - @property - def surnames(self) -> str: - """ - A string of all middle names followed by the last name. - """ - return " ".join(self.surnames_list) or self.C.empty_attribute_default - - @property - def given_names_list(self) -> list[str]: - """ - List of first name followed by middle names. - """ - return self.first_list + self.middle_list - - @property - def given_names(self) -> str: - """ - A string of the first name followed by all middle names. - """ - return " ".join(self.given_names_list) or self.C.empty_attribute_default - - def _split_last(self) -> tuple[list[str], list[str]]: - """Return (prefix_particles, base_words) split from the last name. - - The base_words list is never empty: if every word in the last name - matches a prefix particle, the guard fires and all words are returned - as the base with an empty prefix list (heuristic: a family name is - assumed not to consist entirely of particles). - - >>> HumanName("Vincent van Gogh")._split_last() - (['van'], ['Gogh']) - >>> HumanName("Anh Do")._split_last() - ([], ['Do']) - """ - words = " ".join(self.last_list).split() - i = 0 - while i < len(words) and self.is_prefix(words[i]): - i += 1 - if i == len(words): - # Heuristic: assume a family name isn't entirely composed of - # particles (e.g. surname "Do" which also appears in PREFIXES). - # Don't strip — treat the whole last name as the base. - return [], words - return words[:i], words[i:] - - @property - def last_prefixes_list(self) -> list[str]: - """ - List of leading prefix particles in the last name (the *tussenvoegsel*). - Returns ``[]`` when there are none, including the case where every word - in the last name matches a prefix — see :py:meth:`_split_last`. - - >>> HumanName("Juan de la Vega").last_prefixes_list - ['de', 'la'] - """ - return self._split_last()[0] - - @property - def last_base_list(self) -> list[str]: - """ - List of last-name words after stripping leading prefix particles. - Never empty: when every word matches a prefix, no stripping occurs and - the full last name is returned — see :py:meth:`_split_last`. - - >>> HumanName("Vincent van Gogh").last_base_list - ['Gogh'] - """ - return self._split_last()[1] - - @property - def last_base(self) -> str: - """ - The last name with leading prefix particles removed (the core surname). - For ``"van Gogh"`` this is ``"Gogh"``; for ``"Smith"`` it is ``"Smith"``. - ``last`` is always unchanged. When every word in the last name matches a - prefix particle, no stripping occurs and the full last name is returned. - - >>> HumanName("Vincent van Gogh").last_base - 'Gogh' - >>> HumanName("John Smith").last_base - 'Smith' - """ - return " ".join(self.last_base_list) or self.C.empty_attribute_default - - @property - def last_prefixes(self) -> str: - """ - The leading prefix particle(s) of the last name (the *tussenvoegsel*). - Returns ``""`` (or ``empty_attribute_default``) when there are none, - including when every word in the last name matches a prefix particle - (the all-particles guard; see :py:meth:`_split_last`). - - >>> HumanName("Vincent van Gogh").last_prefixes - 'van' - >>> HumanName("Juan de la Vega").last_prefixes - 'de la' - """ - return " ".join(self.last_prefixes_list) or self.C.empty_attribute_default - - # setter methods - - def _set_list(self, attr: str, value: str | list[str] | None) -> None: - if isinstance(value, list): - val = value - elif isinstance(value, (str, bytes)): - val = [value] - elif value is None: - val = [] - else: - raise TypeError( - "Can only assign strings, lists or None to name attributes." - f" Got {type(value)}") - setattr(self, attr+"_list", self.parse_pieces(val)) - - # Parse helpers - def is_title(self, value: str) -> bool: - """Is in the :py:data:`~nameparser.config.titles.TITLES` set or was - derived as a title earlier in this parse (e.g. ``"Lt.Gov."``, - ``"Mr. and Mrs."``).""" - word = lc(value) - return word in self.C.titles or word in self._derived_titles - - def is_leading_title(self, piece: str) -> bool: - """ - True if ``piece`` is a known title, or an unrecognized multi-letter - word ending in a single trailing period (e.g. ``"Major."``). The - ``{2,}`` in the ``period_abbreviation`` regex, not a separate - ``is_an_initial()`` check, is what excludes single-letter initials - like ``"J."``. Only meaningful for pieces in the title position - (before the first name is set) — a period-abbreviation appearing - later in the name is left as a middle name. The match is not - registered in ``C.titles`` or the per-parse derived titles, so - matching ``"Major."`` here never makes ``"Major"`` (or ``"Major."``) - a recognized title elsewhere, even within the same parse. - """ - return self.is_title(piece) or bool(self.C.regexes.period_abbreviation.match(piece)) - - def is_conjunction(self, piece: str | list[str]) -> bool: - """Is in the conjunctions set — config or derived earlier in this - parse (e.g. ``"of the"``) — and not :py:func:`is_an_initial()`.""" - if isinstance(piece, list): - for item in piece: - if self.is_conjunction(item): - return True - return False - return (piece.lower() in self.C.conjunctions - or piece.lower() in self._derived_conjunctions) \ - and not self.is_an_initial(piece) - - def is_prefix(self, piece: str | list[str]) -> bool: - """ - Lowercased, leading/trailing-periods-stripped version of piece is in the - :py:data:`~nameparser.config.prefixes.PREFIXES` set, or was derived as - a prefix earlier in this parse (e.g. ``"von und"``). - """ - if isinstance(piece, list): - for item in piece: - if self.is_prefix(item): - return True - return False - word = lc(piece) - return word in self.C.prefixes or word in self._derived_prefixes - - def is_bound_first_name(self, piece: str) -> bool: - """Lowercased, leading/trailing-periods-stripped version of piece is in :py:attr:`~nameparser.config.Constants.bound_first_names`.""" - return lc(piece) in self.C.bound_first_names - - def is_non_first_name_prefix(self, piece: str) -> bool: - """Lowercased, leading/trailing-periods-stripped version of piece is in - :py:attr:`~nameparser.config.Constants.non_first_name_prefixes`.""" - return lc(piece) in self.C.non_first_name_prefixes - - def _join_bound_first_name(self, pieces: list[str], reserve_last: bool) -> list[str]: - """Join a first-name prefix to its following piece. - - Finds the first non-title piece; if it is in ``bound_first_names``, - merges it with the next piece — unless ``reserve_last`` is True and no - further piece would remain for the last name. - """ - fi = next((i for i, p in enumerate(pieces) if not self.is_title(p)), None) - if fi is None: - return pieces - if not self.is_bound_first_name(pieces[fi]): - return pieces - next_i = fi + 1 - if next_i >= len(pieces): - return pieces - if reserve_last: - # Count non-suffix pieces from next_i onward; need ≥2 so the join - # target and at least one last-name piece both exist. - non_suffix_remaining = sum( - 1 for p in pieces[next_i:] if not self.is_suffix(p) - ) - if non_suffix_remaining <= 1: - return pieces - pieces[fi] = pieces[fi] + " " + pieces[next_i] - del pieces[next_i] - return pieces - - def is_roman_numeral(self, value: str) -> bool: - """ - Matches the ``roman_numeral`` regular expression in - :py:data:`~nameparser.config.regexes.REGEXES`. - """ - return bool(self.C.regexes.roman_numeral.match(value)) - - def is_suffix(self, piece: str | list[str]) -> bool: - """ - Is in the suffixes set — or was derived as a period-joined suffix - earlier in this parse (e.g. ``"JD.CPA"``) — and not - :py:func:`is_an_initial()`. - - Some suffixes may be acronyms (M.B.A) while some are not (Jr.), - so we remove the periods from `piece` when testing against - `C.suffix_acronyms`. - """ - # suffixes may have periods inside them like "M.D." - if isinstance(piece, list): - for item in piece: - if self.is_suffix(item): - return True - return False - else: - word = lc(piece) - return ((word.replace('.', '') in self.C.suffix_acronyms) - or (word in self.C.suffix_not_acronyms) - or (word in self._derived_suffixes)) \ - and not self.is_an_initial(piece) - - def are_suffixes(self, pieces: Iterable[str]) -> bool: - """Return True if all pieces are suffixes. - - Vacuously True for an empty iterable — the piece loops in - :py:func:`parse_full_name` rely on this to route the final piece - to the last-name branch. - """ - for piece in pieces: - if not self.is_suffix(piece): - return False - return True - - def is_suffix_lenient(self, piece: str) -> bool: - """Like is_suffix(), but suffix_not_acronyms members are accepted - unconditionally, bypassing is_suffix()'s is_an_initial() veto. - - This covers all suffix_not_acronyms members (i, ii, iii, iv, v, jr, - sr, etc.), case-insensitively, including single-letter entries that - is_suffix() would otherwise reject. Only safe for pieces in - unambiguous positions, e.g. after a comma ("John Ingram, V"). - """ - return lc(piece) in self.C.suffix_not_acronyms or self.is_suffix(piece) - - def expand_suffix_delimiter(self, part: str) -> list[str]: - """Split a single post-comma part on :py:attr:`suffix_delimiter`, - if configured. Used only at suffix-consumption sites, where a part - has already been identified as a suffix group, so splitting it - further can't misparse an unrelated name segment. Returns ``[part]`` - unchanged if no delimiter is configured. - """ - if not self.suffix_delimiter: - return [part] - return [p for p in (p.strip() for p in part.split(self.suffix_delimiter)) if p] - - def are_suffixes_after_comma(self, pieces: Iterable[str]) -> bool: - """Return True if all pieces are suffixes by the lenient - :py:func:`is_suffix_lenient` test. Used when detecting suffix-comma - format (e.g. "John Ingram, V") where the post-comma position is - unambiguous. - """ - return all(self.is_suffix_lenient(piece) for piece in pieces) - - def is_rootname(self, piece: str) -> bool: - """ - Is not a known title, suffix or prefix. Just first, middle, last names. - """ - word = lc(piece) - return word not in self.C.suffixes_prefixes_titles \ - and word not in self._derived_titles \ - and word not in self._derived_suffixes \ - and word not in self._derived_prefixes \ - and not self.is_an_initial(piece) - - def is_an_initial(self, value: str) -> bool: - """ - Words with a single period at the end, or a single uppercase letter. - - Matches the ``initial`` regular expression in - :py:data:`~nameparser.config.regexes.REGEXES`. - """ - return bool(self.C.regexes.initial.match(value)) - - def is_east_slavic_patronymic(self, piece: str) -> bool: - """ - Return True if ``piece`` ends with a recognised East-Slavic patronymic - suffix, checked against both Latin-script and Cyrillic patterns in - ``self.C.regexes``. Latin suffixes: ``-ovich``, ``-ovna``, ``-evich``, - ``-evna``, ``-ichna``, and the irregular forms ``-ilyich``, ``-kuzmich``, - ``-lukich``, ``-fomich``, ``-fokich``. Cyrillic equivalents are matched - by a separate pattern. - """ - return bool( - self.C.regexes.east_slavic_patronymic.search(piece) - or self.C.regexes.east_slavic_patronymic_cyrillic.search(piece) - ) - - def is_turkic_patronymic_marker(self, piece: str) -> bool: - """ - Return True if ``piece`` is exactly a recognised Turkic patronymic - marker word (e.g. ``oglu``, ``qizi``, ``uly``), checked against both - Latin-script and Cyrillic patterns in ``self.C.regexes``. Unlike - East-Slavic patronymics, these are standalone marker words, not - suffixes, so the match is whole-word rather than a suffix search. - """ - return bool( - self.C.regexes.turkic_patronymic_marker.match(piece) - or self.C.regexes.turkic_patronymic_marker_cyrillic.match(piece) - ) - - # full_name parser - - @property - def full_name(self) -> str: - """The string output of the HumanName instance.""" - return str(self) - - @full_name.setter - def full_name(self, value: str | bytes) -> None: - self._apply_full_name(value, stacklevel=3) - - def _apply_full_name(self, value: str | bytes, *, stacklevel: int) -> None: - # Shared by the setter and the constructor so each can call it - # directly with a stacklevel that attributes the warning to *its own* - # caller -- the constructor going through the setter would otherwise - # add a frame and misattribute the warning to this module. - self.original = value - - if isinstance(value, bytes): - # deprecated 1.3.0, raises TypeError in 2.0 (#245) - warnings.warn( - "Passing bytes to HumanName is deprecated and will raise " - "TypeError in 2.0; decode it first, e.g. " - "value.decode('utf-8'). See " - "https://github.com/derek73/python-nameparser/issues/245", - DeprecationWarning, - stacklevel=stacklevel, - ) - self._full_name = value.decode(self.encoding) - else: - self._full_name = value - - self.parse_full_name() - - def collapse_whitespace(self, string: str) -> str: - # collapse multiple spaces into single space - string = self.C.regexes.spaces.sub(" ", string.strip()) - if string and self.C.regexes.commas.fullmatch(string[-1]): - string = string[:-1] - return string - - def pre_process(self) -> None: - """ - - This method happens at the beginning of the :py:func:`parse_full_name` - before any other processing of the string aside from unicode - normalization, so it's a good place to do any custom handling in a - subclass. Runs :py:func:`squash_bidi`, :py:func:`parse_nicknames` and - :py:func:`squash_emoji`. - - """ - self.squash_bidi() - self.fix_phd() - self.parse_nicknames() - self.squash_emoji() - - def handle_east_slavic_patronymic_name_order(self) -> None: - """ - When patronymic_name_order is enabled, detect Russian formal order - (Surname GivenName Patronymic) and rotate to Western order. - Fires only for no-comma, single-token first/middle/last where the last - token is a patronymic and the middle token is not. Title, suffix, and - nickname parts do not affect this guard — reordering proceeds regardless - of whether they are present. - """ - if ( - not self._had_comma - and len(self.first_list) == 1 - and len(self.middle_list) == 1 - and len(self.last_list) == 1 - and self.is_east_slavic_patronymic(self.last_list[0]) - and not self.is_east_slavic_patronymic(self.middle_list[0]) - ): - self.first_list, self.middle_list, self.last_list = ( - self.middle_list, - self.last_list, - self.first_list, - ) - - def handle_turkic_patronymic_name_order(self) -> None: - """ - When patronymic_name_order is enabled, detect the reversed Turkic - formal order (Surname GivenName PatronymicRoot Marker) and rotate to - Western order. Fires only for the strict 4-token, no-comma shape: - single-token first/last and exactly two middle tokens, where the last - token is a recognised Turkic patronymic marker. - """ - if ( - not self._had_comma - and len(self.first_list) == 1 - and len(self.middle_list) == 2 - and len(self.last_list) == 1 - and self.is_turkic_patronymic_marker(self.last_list[0]) - ): - self.first_list, self.middle_list, self.last_list = ( - [self.middle_list[0]], - [self.middle_list[1], self.last_list[0]], - self.first_list, - ) - - def handle_non_first_name_prefix(self) -> None: - """ - A leading prefix that is never a first name means the whole name is a - surname -- fold first (and any middle) into last. Keys on the parsed - first name, so a non-leading particle ("Jean de Mesnil") is untouched - and title/suffix are preserved. The middle_list/last_list guard leaves a - degenerate bare "de" as first="de" rather than inventing a surname. - """ - if (len(self.first_list) == 1 - and self.is_non_first_name_prefix(self.first_list[0]) - and (self.middle_list or self.last_list)): - self.last_list = self.first_list + self.middle_list + self.last_list - self.first_list = [] - self.middle_list = [] - - def handle_middle_name_as_last(self) -> None: - """ - When middle_name_as_last is enabled, fold middle_list into last_list - (prepended, preserving order) and clear middle_list. No-op when - middle_list is already empty. - """ - self.last_list = self.middle_list + self.last_list - self.middle_list = [] - - def post_process(self) -> None: - """ - This happens at the end of the :py:func:`parse_full_name` after - all other processing has taken place. Runs :py:func:`handle_firstnames` - and :py:func:`handle_capitalization`. - """ - self.handle_firstnames() - self.handle_non_first_name_prefix() - if self.C.patronymic_name_order: - self.handle_east_slavic_patronymic_name_order() - self.handle_turkic_patronymic_name_order() - if self.C.middle_name_as_last: - self.handle_middle_name_as_last() - self.handle_capitalization() - - def fix_phd(self) -> None: - _re = self.C.regexes.phd - - if match := _re.search(self._full_name): - self.suffix_list.extend(match.groups()) - self._full_name = _re.sub("", self._full_name) - - def parse_nicknames(self) -> None: - """ - Delimited content in the name is routed to either the nickname or - maiden bucket, based on which of - :py:attr:`~nameparser.config.Constants.nickname_delimiters` / - :py:attr:`~nameparser.config.Constants.maiden_delimiters` the matching - delimiter belongs to -- unless that content is suffix-shaped -- an - unambiguous suffix_not_acronyms/suffix_acronyms member, or content - ending in a period -- in which case it's left in place (undelimited) - for normal downstream suffix/title/word parsing instead. This happens - before any other processing of the name. - - Single quotes cannot span white space characters and must border - white space to allow for quotes in names like O'Connor and Kawai'ae'a. - Double quotes and parenthesis can span white space. - - By default, ``nickname_delimiters`` holds the three built-in - delimiters (``quoted_word``, ``double_quotes`` and ``parenthesis``, - resolved live from :py:attr:`~nameparser.config.Constants.regexes` so - overriding e.g. ``CONSTANTS.regexes.parenthesis`` keeps affecting - nickname parsing) and ``maiden_delimiters`` is empty. Move a key - between the two dicts, e.g. - ``maiden_delimiters['parenthesis'] = nickname_delimiters.pop('parenthesis')``, - to route it to ``maiden`` instead, or add a new compiled pattern under - any key to recognize an additional delimiter -- see the "Adding - Custom Nickname Delimiters" and "Routing to Maiden Name" sections of - the customization docs. - """ - - def handle_match(target_list: list[str]) -> Callable[['re.Match[str]'], str]: - def _handle(m: 're.Match[str]') -> str: - # Fall back to the whole match when the regex has no capturing - # group (e.g. a custom override regex without one, like - # EMPTY_REGEX) -- mirrors the old code's use of findall(), which - # returns the whole match for group-less patterns. - content = m.group(1) if m.lastindex else m.group(0) - stripped = lc(content) - # Inlined rather than calling self.is_suffix(content): is_suffix() - # also rejects single-letter initials via is_an_initial(), which - # isn't relevant here, and the suffix_acronyms_ambiguous exclusion - # needs to be interleaved into the acronym branch specifically. - # Acronym suffixes may have periods between every letter (e.g. - # "M.D", "Ph.D") that aren't necessarily trailing, so -- exactly - # like is_suffix() -- strip all periods before checking - # suffix_acronyms/suffix_acronyms_ambiguous membership. Bare - # `stripped` (lc() only strips leading/trailing periods) is still - # used for suffix_not_acronyms, matching is_suffix()'s asymmetry. - acronym_stripped = stripped.replace('.', '') - is_unambiguous_suffix = ( - stripped in self.C.suffix_not_acronyms - or (acronym_stripped in self.C.suffix_acronyms - and acronym_stripped not in self.C.suffix_acronyms_ambiguous) - ) - if is_unambiguous_suffix or content.endswith('.'): - # Leave the bare content -- no delimiters -- so downstream - # word-splitting/suffix-matching sees it exactly as if it had - # never been wrapped in parens/quotes. is_suffix()/lc() only - # strip periods, never parens/quotes, so returning m.group(0) - # here (e.g. literal "(Ret)") would never match - # suffix_not_acronyms ("ret"). - return content - target_list.append(content) - return '' - return _handle - - # Same handle_match for every delimiter: suffix-shaped content is rare - # in quotes but not impossible, and the logic is delimiter-agnostic, - # so there's no reason to special-case parenthesis here. A string - # delimiter value names a Constants.regexes entry, resolved via - # getattr() so overriding e.g. self.C.regexes.parenthesis keeps - # working; anything else is already a compiled pattern, added - # directly by a caller. Unlike a caller directly querying - # self.C.regexes (where RegexTupleManager's EMPTY_REGEX default for - # an unknown attribute is harmless -- the caller sees the pattern and - # can react to it), a bad string here is an internal cross-reference - # the delimiter dict itself is responsible for keeping valid. - # EMPTY_REGEX matches the empty string at every position, so - # silently falling back to it would not just skip the delimiter -- - # handle_match() would fire on every zero-width match and append '' - # into the bucket's list repeatedly, producing a truthy - # whitespace-only nickname/maiden while leaving the real delimited - # content (e.g. literal parentheses) unstripped. Fail loudly instead. - for bucket, delimiters in ( - ('nickname', self.C.nickname_delimiters), - ('maiden', self.C.maiden_delimiters), - ): - target_list = getattr(self, bucket + '_list') - _handle_match = handle_match(target_list) - for raw_pattern in delimiters.values(): - if isinstance(raw_pattern, re.Pattern): - _re = raw_pattern - elif raw_pattern in self.C.regexes: - _re = getattr(self.C.regexes, raw_pattern) - else: - raise ValueError( - f"{bucket}_delimiters references unknown regexes key {raw_pattern!r}. " - f"Known regexes keys: {sorted(self.C.regexes)}" - ) - self._full_name = _re.sub(_handle_match, self._full_name) - - def squash_emoji(self) -> None: - """ - Remove emoji from the input string. - """ - re_emoji = self.C.regexes.emoji - if re_emoji and re_emoji.search(self._full_name): - self._full_name = re_emoji.sub('', self._full_name) - - def squash_bidi(self) -> None: - """ - Remove invisible bidirectional control characters from the input - string. They carry no name content but stick to the parts they - surround, so parsed attributes stop comparing equal to the clean name. - """ - re_bidi = self.C.regexes.bidi - if re_bidi and re_bidi.search(self._full_name): - self._full_name = re_bidi.sub('', self._full_name) - - def handle_firstnames(self) -> None: - """ - If there are only two parts and one is a title, assume it's a last name - instead of a first name. e.g. Mr. Johnson. Unless it's a special title - like "Sir", then when it's followed by a single name that name is always - a first name. - """ - if self.title \ - and len(self) == 2 \ - and lc(self.title) not in self.C.first_name_titles: - self.last, self.first = self.first, self.last - - def parse_full_name(self) -> None: - """ - - The main parse method for the parser. This method is run upon - assignment to the :py:attr:`full_name` attribute or instantiation. - - Basic flow is to hand off to :py:func:`pre_process` to handle - nicknames. It then splits on commas and chooses a code path depending - on the number of commas. - - :py:func:`parse_pieces` then splits those parts on spaces and - :py:func:`join_on_conjunctions` joins any pieces next to conjunctions. - """ - - self.title_list = [] - self.first_list = [] - self.middle_list = [] - self.last_list = [] - self.suffix_list = [] - self.nickname_list = [] - self.maiden_list = [] - - # each parse derives these from scratch; entries from a previous - # full_name must not influence this one - self._derived_titles = set() - self._derived_suffixes = set() - self._derived_conjunctions = set() - self._derived_prefixes = set() - - self.pre_process() - - self._full_name = self.collapse_whitespace(self._full_name) - - # break up full_name by commas. A missing "commas" key in a custom - # regexes dict falls back to RegexTupleManager's EMPTY_REGEX, whose - # .split() matches between every character rather than not - # splitting at all -- guard against that so a custom regexes dict - # that omits "commas" disables the comma split instead of shattering - # the name into single characters. - commas = self.C.regexes.commas - parts = [x.strip() for x in (commas.split(self._full_name) if commas.pattern else [self._full_name])] - self._had_comma = len(parts) > 1 - - log.debug("full_name: %s", self._full_name) - log.debug("parts: %s", parts) - - if len(parts) == 1: - - # no commas, title first middle middle middle last suffix - # part[0] - - pieces = self.parse_pieces(parts) - pieces = self._join_bound_first_name(pieces, reserve_last=True) - p_len = len(pieces) - for i, piece in enumerate(pieces): - try: - nxt = pieces[i + 1] - except IndexError: - nxt = None - - # title must have a next piece, unless it's just a title - if not self.first \ - and (nxt or p_len == 1) \ - and self.is_leading_title(piece): - self.title_list.append(piece) - continue - if not self.first: - if p_len == 1 and self.nickname: - self.last_list.append(piece) - continue - self.first_list.append(piece) - continue - if self.are_suffixes(pieces[i+1:]) or \ - ( - # if the next piece is the last piece and a roman - # numeral but this piece is not an initial - nxt is not None and \ - self.is_roman_numeral(nxt) and i == p_len - 2 - and not self.is_an_initial(piece) - ): - # any piece reaching this check as the final piece lands - # here: are_suffixes() is vacuously True for the empty - # tail, making this the last-name branch as well as the - # suffix branch - self.last_list.append(piece) - self.suffix_list += pieces[i+1:] - break - - self.middle_list.append(piece) - else: - # if all the end parts are suffixes and there is more than one piece - # in the first part. (Suffixes will never appear after last names - # only, and allows potential first names to be in suffixes, e.g. - # "Johnson, Bart" - - post_comma_pieces = self.parse_pieces(parts[1].split(' '), 1) - - # Detection must see the delimiter-expanded words too, or a - # delimiter-joined suffix group like "RN - CRNA" would never be - # recognized as suffix-comma format in the first place. - suffix_delimiter_pieces = [word for part in self.expand_suffix_delimiter(parts[1]) - for word in part.split(' ')] - - if self.are_suffixes_after_comma(suffix_delimiter_pieces) \ - and len(parts[0].split(' ')) > 1: - - # suffix comma: - # title first middle last [suffix], suffix [suffix] [, suffix] - # parts[0], parts[1:...] - - for part in parts[1:]: - # skip empty segments from doubled commas, mirroring the - # parts[2:] guard in the lastname-comma path below - if part: - self.suffix_list += self.expand_suffix_delimiter(part) - pieces = self.parse_pieces(parts[0].split(' ')) - pieces = self._join_bound_first_name(pieces, reserve_last=True) - log.debug("pieces: %s", str(pieces)) - for i, piece in enumerate(pieces): - try: - nxt = pieces[i + 1] - except IndexError: - nxt = None - - if not self.first \ - and (nxt or len(pieces) == 1) \ - and self.is_leading_title(piece): - self.title_list.append(piece) - continue - if not self.first: - self.first_list.append(piece) - continue - if self.are_suffixes(pieces[i+1:]): - # any piece reaching this check as the final piece - # lands here: are_suffixes() is vacuously True for the - # empty tail, making this the last-name branch as well - # as the suffix branch - self.last_list.append(piece) - self.suffix_list = pieces[i+1:] + self.suffix_list - break - self.middle_list.append(piece) - else: - - # lastname comma: - # last [suffix], title first middles[,] suffix [,suffix] - # parts[0], parts[1], parts[2:...] - - log.debug("post-comma pieces: %s", str(post_comma_pieces)) - post_comma_pieces = self._join_bound_first_name(post_comma_pieces, reserve_last=False) - - # lastname part may have suffixes in it - lastname_pieces = self.parse_pieces(parts[0].split(' '), 1) - for piece in lastname_pieces: - # the first one is always a last name, even if it looks like - # a suffix - if self.is_suffix(piece) and len(self.last_list) > 0: - self.suffix_list.append(piece) - else: - self.last_list.append(piece) - - for i, piece in enumerate(post_comma_pieces): - try: - nxt = post_comma_pieces[i + 1] - except IndexError: - nxt = None - - if not self.first \ - and (nxt or len(post_comma_pieces) == 1) \ - and self.is_leading_title(piece): - self.title_list.append(piece) - continue - if not self.first: - self.first_list.append(piece) - continue - # A trailing token in a two-part lastname-comma name is - # unambiguously positioned, so use the lenient test that - # accepts suffix_not_acronyms members is_suffix() would - # veto as initials. When parts[2] exists the caller - # already declared an explicit suffix via comma (e.g. - # 'Doe, Rev. John V, Jr.'), making the trailing token - # more likely a middle initial. - if self.is_suffix(piece) or \ - (nxt is None and len(parts) == 2 - and self.is_suffix_lenient(piece)): - self.suffix_list.append(piece) - continue - self.middle_list.append(piece) - for part in parts[2:]: - # skip empty segments from doubled commas ("Doe, John,, Jr.") - # without dropping the segments that follow them - if part: - self.suffix_list += self.expand_suffix_delimiter(part) - - self.post_process() - - def parse_pieces(self, parts: Iterable[str], additional_parts_count: int = 0) -> list[str]: - """ - Split parts on spaces and remove commas, join on conjunctions and - lastname prefixes. Tokens that are empty after stripping spaces and - commas are dropped, so the returned pieces never contain empty - strings. If parts have periods in the middle, try splitting - on periods and check if the parts are titles or suffixes. If they are, - register the periods-joined part as a derived title/suffix for this - parse so it will be recognized; the constants are not modified. - - :param list parts: name part strings from the comma split - :param int additional_parts_count: - - if the comma format contains other parts, we need to know - how many there are to decide if things should be considered a - conjunction. - :return: pieces split on spaces and joined on conjunctions - :rtype: list - """ - - output: list[str] = [] - for part in parts: - if not isinstance(part, (str, bytes)): - raise TypeError("Name parts must be strings. " - f" Got {type(part)}") - # drop tokens that strip to nothing (e.g. from a bare "," input or - # an empty comma segment) so no empty piece reaches the parse - # loops and the public *_list attributes - output += [s for s in (x.strip(' ,') for x in part.split(' ')) if s] - - # If part contains periods, check if it's multiple titles or suffixes - # together without spaces. If so, register the periods-joined part as - # a derived title/suffix for this parse so it gets recognized later - for part in output: - # if this part has a period not at the beginning or end - if self.C.regexes.period_not_at_end and self.C.regexes.period_not_at_end.match(part): - # split on periods, any of the split pieces titles or suffixes? - # ("Lt.Gov.") - period_chunks = part.split(".") - titles = list(filter(self.is_title, period_chunks)) - suffixes = list(filter(self.is_suffix, period_chunks)) - - # register the part so it will be found by the is_* checks - if titles: - self._derived_titles.add(lc(part)) - continue - if suffixes: - self._derived_suffixes.add(lc(part)) - continue - - return self.join_on_conjunctions(output, additional_parts_count) - - def join_on_conjunctions(self, pieces: list[str], additional_parts_count: int = 0) -> list[str]: - """ - Join conjunctions to surrounding pieces. Title- and prefix-aware. e.g.: - - ['Mr.', 'and', 'Mrs.', 'John', 'Doe'] ==> - ['Mr. and Mrs.', 'John', 'Doe'] - - ['The', 'Secretary', 'of', 'State', 'Hillary', 'Clinton'] ==> - ['The Secretary of State', 'Hillary', 'Clinton'] - - When joining titles, registers the newly formed piece as a derived - title for the current parse so it will be recognized correctly later - in the same parse. E.g. while parsing the example names above, - 'The Secretary of State' and 'Mr. and Mrs.' are treated as titles. - The configuration in ``self.C`` is never modified. - - :param list pieces: name pieces strings after split on spaces - :param int additional_parts_count: - :return: new list with piece next to conjunctions merged into one piece - with spaces in it. - :rtype: list - - """ - length = len(pieces) + additional_parts_count - # don't join on conjunctions if there's only 2 parts - if length < 3: - return pieces - - rootname_pieces = [p for p in pieces if self.is_rootname(p)] - total_length = len(rootname_pieces) + additional_parts_count - - # find all the conjunctions, join any conjunctions that are next to each - # other, then join those newly joined conjunctions and any single - # conjunctions to the piece before and after it - conj_index = [i for i, piece in enumerate(pieces) - if self.is_conjunction(piece)] - - contiguous_conj_i = group_contiguous_integers(conj_index) - - # process ranges in reverse so deleting one range doesn't shift the - # indices of ranges still to be processed - for cont_i in reversed(contiguous_conj_i): - new_piece = " ".join(pieces[cont_i[0]: cont_i[1]+1]) - pieces[cont_i[0]:cont_i[1]+1] = [new_piece] - # register newly joined conjunctions to be found later this parse - self._derived_conjunctions.add(lc(new_piece)) - - if len(pieces) == 1: - # if there's only one piece left, nothing left to do - return pieces - - # refresh conjunction index locations - conj_index = [i for i, piece in enumerate(pieces) if self.is_conjunction(piece)] - - def register_joined_piece(new_piece: str, neighbor: str) -> None: - if self.is_title(neighbor): - # when joining to a title, make new_piece a title too - self._derived_titles.add(lc(new_piece)) - if self.is_prefix(neighbor): - # when joining to a prefix, make new_piece a prefix too, so - # e.g. "von" + "und" bridges into "von und" and can still - # chain onto a following prefix/lastname (see "von und zu") - self._derived_prefixes.add(lc(new_piece)) - - def shift_conj_index(past: int, by: int) -> None: - # after removing pieces at/after `past`, indices of the - # remaining conjunctions need to shift down by `by` - for j, val in enumerate(conj_index): - if val > past: - conj_index[j] = val - by - - for i in conj_index: - if len(pieces[i]) == 1 and total_length < 4 and pieces[i].isalpha(): - # if there are only 3 total parts (minus known titles, suffixes - # and prefixes) and this conjunction is a single letter, prefer - # treating it as an initial rather than a conjunction. - # http://code.google.com/p/python-nameparser/issues/detail?id=11 - continue - - start = max(0, i - 1) - end = min(len(pieces), i + 2) - new_piece = " ".join(pieces[start:end]) - neighbor = pieces[start] if start < i else pieces[end - 1] - register_joined_piece(new_piece, neighbor) - pieces[start:end] = [new_piece] - shift_conj_index(past=i, by=end - start - 1) - - # join prefixes to following lastnames: ['de la Vega'], ['van Buren'] - i = 0 - while i < len(pieces): - # total_length >= 1 covers essentially all real input, so this - # treats any leading piece as a first name rather than a prefix. - leading_first_name = i == 0 and total_length >= 1 - if not self.is_prefix(pieces[i]) or leading_first_name: - i += 1 - continue - - # absorb any immediately-adjacent prefixes into one contiguous run - # e.g. "von und zu der" ==> chain them all before looking further - j = i + 1 - while j < len(pieces) and self.is_prefix(pieces[j]): - j += 1 - - # then join everything after the run until the next prefix or suffix - while j < len(pieces) and not self.is_prefix(pieces[j]) and not self.is_suffix(pieces[j]): - j += 1 - - pieces[i:j] = [' '.join(pieces[i:j])] - i += 1 - - log.debug("pieces: %s", pieces) - return pieces - - # Capitalization Support - - def cap_word(self, word: str, attribute: HumanNameAttributeT) -> str: - if (self.is_prefix(word) and attribute in ('last', 'middle')) \ - or self.is_conjunction(word): - return word.lower() - exceptions = self.C.capitalization_exceptions - key = lc(word) - for k in (key, key.replace('.', '')): - if k in exceptions: - return exceptions[k] - mac_match = self.C.regexes.mac.match(word) - if mac_match: - def cap_after_mac(m: re.Match) -> str: - return m.group(1).capitalize() + m.group(2).capitalize() - return self.C.regexes.mac.sub(cap_after_mac, word) - else: - return word.capitalize() - - def cap_piece(self, piece: str, attribute: HumanNameAttributeT) -> str: - if not piece: - return "" - - def replacement(m: re.Match) -> str: - return self.cap_word(m.group(0), attribute) - - return self.C.regexes.word.sub(replacement, piece) - - def capitalize(self, force: bool | None = None) -> None: - """ - The HumanName class can try to guess the correct capitalization of name - entered in all upper or lower case. By default, it will not adjust the - case of names entered in mixed case. To run capitalization on all names - pass the parameter `force=True`. - - :param bool force: Forces capitalization of mixed case strings. This - parameter overrides rules set within - :py:class:`~nameparser.config.CONSTANTS`. - - **Usage** - - .. doctest:: capitalize - - >>> name = HumanName('bob v. de la macdole-eisenhower phd') - >>> name.capitalize() - >>> str(name) - 'Bob V. de la MacDole-Eisenhower Ph.D.' - >>> # Don't touch good names - >>> name = HumanName('Shirley Maclaine') - >>> name.capitalize() - >>> str(name) - 'Shirley Maclaine' - >>> name.capitalize(force=True) - >>> str(name) - 'Shirley MacLaine' - - """ - name = str(self) - force = self.C.force_mixed_case_capitalization \ - if force is None else force - - if not force and not (name == name.upper() or name == name.lower()): - return - self.title_list = self.cap_piece(self.title, 'title').split() - self.first_list = self.cap_piece(self.first, 'first').split() - self.middle_list = self.cap_piece(self.middle, 'middle').split() - self.last_list = self.cap_piece(self.last, 'last').split() - # suffix is stored comma-separated ("Ph.D., J.D."), not space-separated - self.suffix_list = [s for s in self.cap_piece(self.suffix, 'suffix').split(', ') if s] - - def handle_capitalization(self) -> None: - """ - Handles capitalization configurations set within - :py:class:`~nameparser.config.CONSTANTS`. - """ - if self.C.capitalize_name: - self.capitalize() +__all__ = ["HumanName"] diff --git a/pyproject.toml b/pyproject.toml index 6a86e4b1..8ad0c1c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,10 @@ dev = [ [tool.mypy] packages = ["nameparser", "tests"] +# TEMPORARY (M11 -> M12): the v1 suite is being reconciled against the +# 2.0 facade; lifted with tests/conftest.py's collect_ignore_glob. Must +# be gone before the branch leaves draft. tests/v2/ stays checked. +exclude = '^tests/(test_[^/]*\.py|base\.py)$' [[tool.mypy.overrides]] module = [ diff --git a/tests/conftest.py b/tests/conftest.py index 79dbfd92..4d2124f2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,12 @@ from nameparser.config import CONSTANTS +# TEMPORARY (Task M11 -> M12): the v1 suite is being reconciled against +# the 2.0 facade file by file. Lifted in Task M12 together with its mypy +# twin (the `exclude` under [tool.mypy] in pyproject.toml); both must be +# GONE before this branch leaves draft. +collect_ignore_glob = ["test_*.py"] + # Scalar (non-collection) config attributes that individual tests mutate on the # global CONSTANTS singleton. Several tests change these without restoring them; # the original suite only survived because unittest happens to run methods in diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index d548d1dc..811f5a87 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -298,20 +298,23 @@ def test_v14_constants_state_shape_accepted() -> None: assert not hasattr(c, "empty_attribute_default") or True # dropped -@pytest.mark.xfail( - reason="resolves to the v1 class until the M11 swap", strict=True) def test_v14_constants_blob_unpickles_into_shim() -> None: # tests/v2/data/constants_v14.pickle was produced by a real 1.4.0 # install (`uv run --no-project --with "nameparser==1.4.*" ...`), # not synthesized here -- it is the actual bytes a caller upgrading - # from 1.4 to 2.0 would hand to pickle.load(). Until the M11 swap - # replaces nameparser.config.Constants with this shim, the blob's - # pickled class reference still resolves to the live v1 class, so - # this assertion fails today by construction. + # from 1.4 to 2.0 would hand to pickle.load(). Since the M11 swap + # replaced nameparser.config.Constants with this shim, the blob's + # pickled class reference now resolves here. Its nested `regexes` + # field pickles as nameparser.config.RegexTupleManager, reconstructed + # by the unpickler before Constants.__setstate__ runs -- see + # RegexTupleManager's docstring in _config_shim.py. with open(_DATA_DIR / "constants_v14.pickle", "rb") as f: loaded = pickle.load(f) assert isinstance(loaded, Constants) assert "van" in loaded.prefixes + assert "dr" in loaded.titles + assert "jr" in loaded.suffix_not_acronyms + assert loaded.string_format == "{title} {first} {middle} {last} {suffix} ({nickname})" def test_snapshot_field_translation() -> None: diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index 9bf6d3ef..8b63b844 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -307,16 +307,13 @@ def test_pickle_round_trip_preserves_components() -> None: assert loaded.C is CONSTANTS # shared sentinel restored -@pytest.mark.xfail( - reason="resolves to the v1 class until the M11 swap", strict=True) def test_v14_humanname_blob_unpickles() -> None: # tests/v2/data/humanname_v14.pickle was produced by a real 1.4.0 # install (`uv run --no-project --with "nameparser==1.4.*" ...`) on # HumanName("Dr. Juan de la Vega III"); components come back exactly - # as pickled, NOT re-parsed. Until the M11 swap replaces + # as pickled, NOT re-parsed. Since the M11 swap replaced # nameparser.parser.HumanName with this facade, the blob's pickled - # class reference still resolves to the live v1 class, so this - # assertion fails today by construction (mirrors + # class reference now resolves here (mirrors # test_v14_constants_blob_unpickles_into_shim in test_config_shim.py). # pickle.load is safe here: humanname_v14.pickle is a repo-controlled # fixture generated by this task from a real 1.4.0 install (Step 2 @@ -325,3 +322,6 @@ def test_v14_humanname_blob_unpickles() -> None: loaded = pickle.load(f) assert isinstance(loaded, HumanName) assert loaded.first == "Juan" and loaded.last == "de la Vega" + assert loaded.title == "Dr." and loaded.suffix == "III" + assert loaded.original == "Dr. Juan de la Vega III" + assert loaded.C is CONSTANTS # shared sentinel restored (v1.4's C: None) diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 5b852188..3be5d8a0 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -17,7 +17,8 @@ "_pipeline/_vocab.py", "_pipeline/_segment.py", "_pipeline/_classify.py", "_pipeline/_group.py", "_pipeline/_assign.py", "_pipeline/_post_rules.py", - "_pipeline/_assemble.py", "_parser.py"} + "_pipeline/_assemble.py", "_parser.py", "_facade.py", + "_config_shim.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", @@ -60,6 +61,23 @@ "_parser.py": ("nameparser._types", "nameparser._lexicon", "nameparser._policy", "nameparser._locale", "nameparser._pipeline"), + # facade layer (migration spec §2/§3): may import anything public + # plus _render (unused yet) and each other. _facade delegates + # parsing to the core Parser resolved from the bound Constants shim. + # The bare "nameparser.config" import (not just its submodules) is + # deliberate -- it resolves an import cycle, see _facade.py's comment. + "_facade.py": ("nameparser._config_shim", "nameparser._lexicon", + "nameparser._parser", "nameparser._types", + "nameparser._render", "nameparser.util", + "nameparser.config"), + # the Constants/SetManager/TupleManager shim: config DATA modules + # stay the single vocabulary source through 2.x (nameparser.config.*) + "_config_shim.py": ("nameparser._lexicon", "nameparser._parser", + "nameparser._policy", "nameparser.util", + "nameparser.config"), + # v1 import-path preservation: thin re-exports of the facade/shim + "parser.py": ("nameparser._facade",), + "config/__init__.py": ("nameparser._config_shim",), } From 2d3fb93d373e6302079ebe1a74f0bf229b88013d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 23:24:09 -0700 Subject: [PATCH 103/275] M12 batch 1: drop the dead empty_attribute_default dual-run fixture (bucket A) empty_attribute_default was removed in 2.0 (#255; nameparser/_config_shim.py's Constants.__setattr__ now raises AttributeError on assignment, and there is no stored default to read either). conftest.py's autouse fixture used to parametrize every test over ['', None] to reproduce tests.py's old dual-path regression check, plus snapshot/restore CONSTANTS around each test. The dual path no longer exists, so only the isolation half remains, un-parametrized. Also drops "regexes" from the restored collection attrs: it's a read-only _RegexesProxy in 2.0, so it can neither be mutated by a test nor reassigned by this fixture's restore loop. tests/base.py's `m()` helper had the same dependency (falling back to hn.C.empty_attribute_default, which no longer exists); empty attributes are always '' now, so it falls back to '' directly. pyproject.toml/conftest.py's TEMPORARY exclude lists are re-synced to the current not-yet-reconciled file set for this batch (test_python_api.py and test_constants.py go back on the list -- see the batch report for why). Co-Authored-By: Claude Fable 5 --- pyproject.toml | 2 +- tests/base.py | 10 ++++-- tests/conftest.py | 87 +++++++++++++++++++++++++++++++---------------- 3 files changed, 67 insertions(+), 32 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8ad0c1c0..d7d862cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ packages = ["nameparser", "tests"] # TEMPORARY (M11 -> M12): the v1 suite is being reconciled against the # 2.0 facade; lifted with tests/conftest.py's collect_ignore_glob. Must # be gone before the branch leaves draft. tests/v2/ stays checked. -exclude = '^tests/(test_[^/]*\.py|base\.py)$' +exclude = '^tests/(test_(bound_first_names|brute_force|capitalization|comma_variants|conjunctions|constants|east_slavic_patronymic_order|first_name|initials|middle_name_as_last|nicknames|output_format|parser_util|prefixes|python_api|suffixes|titles|turkic_patronymic_order|variations)\.py)$' [[tool.mypy.overrides]] module = [ diff --git a/tests/base.py b/tests/base.py index b563fb3f..283e09d1 100644 --- a/tests/base.py +++ b/tests/base.py @@ -16,8 +16,14 @@ class HumanNameTestBase(Generic[T]): """ def m(self, actual: T, expected: T, hn: HumanName) -> None: - """assertEqual with a better message and awareness of hn.C.empty_attribute_default""" - expected_ = expected or hn.C.empty_attribute_default + """assertEqual with a better message. + + ``expected or ''`` used to fall back to ``hn.C.empty_attribute_default``, + which could be '' or None depending on which of the (formerly dual-run) + settings a test ran under. That setting was removed in 2.0 (#255): + empty attributes are always ''. + """ + expected_ = expected or '' try: assert actual == expected_, f"{actual!r} != {expected!r} for {hn.original!r}\n{hn!r}" except UnicodeDecodeError: diff --git a/tests/conftest.py b/tests/conftest.py index 4d2124f2..9889a6bc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,7 +10,38 @@ # the 2.0 facade file by file. Lifted in Task M12 together with its mypy # twin (the `exclude` under [tool.mypy] in pyproject.toml); both must be # GONE before this branch leaves draft. -collect_ignore_glob = ["test_*.py"] +# +# Explicit list of not-yet-reconciled files. Narrow this as each file is +# reconciled; remove entirely once the whole suite runs against the facade. +# +# test_python_api.py and test_constants.py are BACK on this list (not +# reconciled this batch): both are blocked by confirmed facade/pipeline bugs +# (missing Constants(**kwargs) constructor support, missing +# HumanName.parse_full_name()/C setter, a "Doe,, Jr." title/suffix routing +# regression -- see the M12 batch report) that this task must not paper over +# by editing nameparser/ source or weakening the tests. Re-narrow once those +# are fixed upstream. +collect_ignore_glob = [ + "test_bound_first_names.py", + "test_brute_force.py", + "test_capitalization.py", + "test_comma_variants.py", + "test_conjunctions.py", + "test_constants.py", + "test_east_slavic_patronymic_order.py", + "test_first_name.py", + "test_initials.py", + "test_middle_name_as_last.py", + "test_nicknames.py", + "test_output_format.py", + "test_parser_util.py", + "test_prefixes.py", + "test_python_api.py", + "test_suffixes.py", + "test_titles.py", + "test_turkic_patronymic_order.py", + "test_variations.py", +] # Scalar (non-collection) config attributes that individual tests mutate on the # global CONSTANTS singleton. Several tests change these without restoring them; @@ -18,8 +49,12 @@ # alphabetical order, so a later test reset the value. pytest runs in definition # order, so we snapshot and restore these around every test to keep tests # isolated regardless of order. +# +# empty_attribute_default is gone from this list: it was removed in 2.0 (#255, +# see nameparser/_config_shim.py's Constants.__setattr__), so there is no +# longer a second parsing path to snapshot/restore or dual-run against. Empty +# attributes are always ''. _SCALAR_CONFIG_ATTRS = ( - "empty_attribute_default", "string_format", "initials_format", "initials_delimiter", @@ -34,6 +69,10 @@ # these in place, so a shallow snapshot of the reference would not protect later # tests. We snapshot independent copies and restore them, making collection # mutations order-independent too. +# +# regexes is gone from this list: it is a read-only _RegexesProxy in 2.0 (set +# once in Constants.__init__), and reassigning it raises TypeError, so it can +# neither be mutated by a test nor restored by this fixture. _COLLECTION_CONFIG_ATTRS = ( "prefixes", "suffix_acronyms", @@ -44,45 +83,35 @@ "bound_first_names", "non_first_name_prefixes", "capitalization_exceptions", - "regexes", "nickname_delimiters", "maiden_delimiters", ) -@pytest.fixture(autouse=True, params=['', None], ids=['default', 'none']) -def empty_attribute_default(request: pytest.FixtureRequest) -> Iterator[str | None]: - """Run every test under both empty_attribute_default settings, isolating global config. +@pytest.fixture(autouse=True) +def _isolate_constants() -> Iterator[None]: + """Snapshot and restore the shared CONSTANTS singleton around every test. - Reproduces the original tests.py __main__ block, which ran the whole suite - twice — once with the default ('') and once with None — as a regression - check that the three parsing code paths agree. The surrounding snapshot of - both the scalar and the collection CONSTANTS attributes restores any global - config a test mutates, so tests do not leak state into one another (the - original relied on unittest's alphabetical method ordering to mask such - leaks). + Formerly also drove a parametrized dual-run over empty_attribute_default + settings (reproducing the original tests.py __main__ block, which ran the + whole suite twice as a regression check that the three parsing code paths + agreed). That setting no longer exists in 2.0 (#255: empty attributes are + always ''), so only the isolation half of the original fixture remains. """ scalar_snapshot = {attr: getattr(CONSTANTS, attr) for attr in _SCALAR_CONFIG_ATTRS} collection_snapshot = { attr: copy.deepcopy(getattr(CONSTANTS, attr)) for attr in _COLLECTION_CONFIG_ATTRS } - # empty_attribute_default assignment is deprecated (#255); this fixture's - # own systematic exercise of both settings across the whole suite is - # infrastructure, not a test of the deprecation itself, so it's silenced - # here (narrowly, around just the assignment) rather than at every one of - # its ~1500 call sites. Must not wrap `yield` -- that would suppress - # DeprecationWarning for the test body itself, hiding real ones. + yield + # Restoring through the shared singleton is itself a mutation and would + # otherwise emit the shared-CONSTANTS DeprecationWarning (#262) on every + # single test's teardown; this fixture is infrastructure; it is not + # exercising that deprecation, so it's silenced narrowly here rather than + # at every test. with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) - CONSTANTS.empty_attribute_default = request.param - yield request.param - for attr, value in scalar_snapshot.items(): - if attr == "empty_attribute_default": - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - setattr(CONSTANTS, attr, value) - else: + for attr, value in scalar_snapshot.items(): + setattr(CONSTANTS, attr, value) + for attr, value in collection_snapshot.items(): setattr(CONSTANTS, attr, value) - for attr, value in collection_snapshot.items(): - setattr(CONSTANTS, attr, value) From a689e1d8a1df6baf25696a6dd8aebd13e0019add Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 23:24:23 -0700 Subject: [PATCH 104/275] M12 batch 1: delete test_config_attribute_docstrings.py (bucket A) This file used ast to recover Sphinx "attribute docstring" doctests (bare string literals following an attribute assignment) from the Constants class in nameparser.config, since that convention is invisible to Python's own doctest.DocTestFinder. After the M11 swap, nameparser/config/__init__.py is a pure re-export shim -- it no longer defines a Constants class at all (the real one lives in nameparser/_config_shim.py), so the module-level AST walk raised ValueError on import: `(class_node,) = (... 0 matches ...)`. Even pointed at the right module, there is nothing left to discover: the shim's Constants fields are bare type annotations (`patronymic_name_order: bool`), not `attr = default` followed by a prose+doctest string literal, so the mechanism this file existed to protect no longer has any doctest examples to run. Deleting rather than porting, per bucket A. Co-Authored-By: Claude Fable 5 --- tests/test_config_attribute_docstrings.py | 79 ----------------------- 1 file changed, 79 deletions(-) delete mode 100644 tests/test_config_attribute_docstrings.py diff --git a/tests/test_config_attribute_docstrings.py b/tests/test_config_attribute_docstrings.py deleted file mode 100644 index 4b1f4c79..00000000 --- a/tests/test_config_attribute_docstrings.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Exercise the doctest examples embedded in Constants attribute docstrings. - -nameparser.config.Constants documents several scalar attributes (e.g. -patronymic_name_order, middle_name_as_last) with a bare string literal placed -right after the class-attribute assignment -- Sphinx's "attribute docstring" -convention, picked up by autodoc's source-level analysis for the built docs. - -That convention is invisible to Python at runtime: only module/class/function/ -method docstrings become a real __doc__, so a bare string following -`attr = value` is evaluated and discarded. doctest.DocTestFinder walks __doc__ -attributes, so pytest's --doctest-modules (see pyproject.toml addopts) never -finds the `.. doctest::` examples inside them -- they can go stale silently. - -This module parses the source with `ast` to recover those literals (the same -information Sphinx's static analysis relies on) and runs any doctest examples -found inside them explicitly, so a stale example fails the suite. -""" -import ast -import doctest -import io -from pathlib import Path - -import pytest - -import nameparser.config as config_module -from nameparser import HumanName -from nameparser.config import CONSTANTS, Constants - -CONFIG_SOURCE_PATH = Path(config_module.__file__) - - -def _constants_attribute_docstrings() -> dict[str, str]: - """Map attribute name -> bare-string-literal docstring, Constants class body only.""" - tree = ast.parse(CONFIG_SOURCE_PATH.read_text(), filename=str(CONFIG_SOURCE_PATH)) - (class_node,) = ( - node for node in ast.walk(tree) - if isinstance(node, ast.ClassDef) and node.name == 'Constants' - ) - docstrings = {} - body = class_node.body - for stmt, following in zip(body, body[1:]): - if not (isinstance(stmt, ast.Assign) and len(stmt.targets) == 1 - and isinstance(stmt.targets[0], ast.Name)): - continue - if isinstance(following, ast.Expr) and isinstance(following.value, ast.Constant) \ - and isinstance(following.value.value, str): - docstrings[stmt.targets[0].id] = following.value.value - return docstrings - - -ATTRIBUTE_DOCSTRINGS = _constants_attribute_docstrings() - -DOCTEST_GLOBS = {'HumanName': HumanName, 'CONSTANTS': CONSTANTS, 'Constants': Constants} - - -def _attrs_with_doctest_examples() -> list[str]: - parser = doctest.DocTestParser() - return sorted( - attr for attr, docstring in ATTRIBUTE_DOCSTRINGS.items() - if parser.get_examples(docstring) - ) - - -@pytest.mark.parametrize("attr", _attrs_with_doctest_examples()) -def test_constants_attribute_docstring_examples(attr: str) -> None: - docstring = ATTRIBUTE_DOCSTRINGS[attr] - test = doctest.DocTestParser().get_doctest( - docstring, DOCTEST_GLOBS, attr, str(CONFIG_SOURCE_PATH), 0, - ) - output = io.StringIO() - failures, _ = doctest.DocTestRunner().run(test, out=output.write) - assert failures == 0, output.getvalue() - - -def test_found_expected_attributes_with_doctest_examples() -> None: - """Guard the discovery mechanism itself: if this drops to zero, the AST - walk above stopped matching Constants's attribute-docstring pattern and - every parametrized case above is silently skipped rather than run.""" - assert _attrs_with_doctest_examples() From b959112c9f2e3c10771255a24cde9ff532055f44 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 23:28:56 -0700 Subject: [PATCH 105/275] Preserve empty comma segments' structural position (v1 parity) M12 reconciliation surfaced this: dropping empty buckets renumbered the segments, so in 'Doe,, Jr.' the tail suffix landed in the given segment and the lone-post-comma-title heuristic claimed it (title='Jr.'). v1's mechanics, pinned live: collapse_whitespace strips exactly ONE trailing comma as cosmetic; every other empty part keeps its position -- an empty given segment ('Doe,,'), a failed suffix-comma detection ('John Smith,, MD'), silent consumption of empty tails ('Doe, John,, Jr.'). Five case rows pin the matrix. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_segment.py | 24 +++++++++++++++++++----- tests/v2/cases.py | 20 ++++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/nameparser/_pipeline/_segment.py b/nameparser/_pipeline/_segment.py index f81e8f10..6372eda4 100644 --- a/nameparser/_pipeline/_segment.py +++ b/nameparser/_pipeline/_segment.py @@ -1,7 +1,8 @@ """Stage: segment. Consumes: tokens (role-None main stream), comma_offsets. -Produces: segments (runs of main-token indices), structure, +Produces: segments (runs of main-token indices; interior segments may +be EMPTY -- doubled commas keep their structural position), structure, COMMA_STRUCTURE ambiguities for unrecognized extra segments. Reads: Lexicon suffix vocabulary (via _vocab.is_suffix_lenient) -- the suffix-comma decision is definitionally vocabulary-dependent @@ -44,9 +45,16 @@ def segment(state: ParseState) -> ParseState: start = state.tokens[i].span.start bucket = bisect.bisect_left(state.comma_offsets, start) buckets[bucket].append(i) - groups = [tuple(b) for b in buckets if b] + groups = [tuple(b) for b in buckets] + # v1 strips exactly ONE trailing comma as cosmetic (parser.py's + # collapse_whitespace); every other empty bucket is STRUCTURAL and + # keeps its position -- in 'Doe,, Jr.' the given segment is empty, + # so 'Jr.' stays a tail suffix instead of masquerading as a lone + # post-comma title (v1 parity, pinned live 2026-07-16) + if len(groups) > 1 and not groups[-1]: + groups.pop() if len(groups) <= 1: - segs = tuple(groups) if groups else (tuple(main),) + segs = tuple(groups) if groups and groups[0] else (tuple(main),) return dataclasses.replace(state, segments=segs, structure=Structure.NO_COMMA) @@ -67,7 +75,11 @@ def counts_as_suffix(text: str) -> bool: and splits_into_suffixes(text, cores, state.lexicon)) def suffixy(seg: tuple[int, ...]) -> bool: - return all(counts_as_suffix(state.tokens[i].text) for i in seg) + # an EMPTY segment is not suffix-shaped: v1's suffix-comma + # detection fails on an empty parts[1] ('John Smith,, MD' is a + # family-comma parse) + return bool(seg) and all( + counts_as_suffix(state.tokens[i].text) for i in seg) rest = groups[1:] if all(suffixy(s) for s in rest) and len(groups[0]) > 1: @@ -75,7 +87,9 @@ def suffixy(seg: tuple[int, ...]) -> bool: structure=Structure.SUFFIX_COMMA) ambiguities = list(state.ambiguities) for seg in groups[2:]: - if not suffixy(seg): + # empty segments are consumed silently (v1 skips them without + # comment); only non-empty non-suffix tails get flagged + if seg and not suffixy(seg): texts = " ".join(state.tokens[i].text for i in seg) ambiguities.append(PendingAmbiguity( AmbiguityKind.COMMA_STRUCTURE, diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 03886b5d..9f830a83 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -41,6 +41,26 @@ class Case: {"given": "John", "family": "Smith"}), Case("suffix_comma", "John Smith, PhD", {"given": "John", "family": "Smith", "suffix": "PhD"}), + Case("doubled_comma_suffix", "Doe,, Jr.", + {"family": "Doe", "suffix": "Jr."}, + notes="the EMPTY given segment keeps its position: 'Jr.' is a " + "tail suffix, not a lone post-comma title (v1 parity, " + "pinned live 2026-07-16)"), + Case("doubled_comma_given_kept", "Doe, John,, Jr.", + {"given": "John", "family": "Doe", "suffix": "Jr."}), + Case("single_trailing_comma_cosmetic", "John,", + {"given": "John"}, + notes="v1 collapse_whitespace strips exactly ONE trailing " + "comma before parsing"), + Case("double_trailing_comma_structural", "Doe,,", + {"family": "Doe"}, + notes="one trailing comma is cosmetic, the second is " + "structural: an empty given segment makes this a " + "family-comma parse (v1 parity, pinned live 2026-07-16)"), + Case("doubled_comma_blocks_suffix_comma", "John Smith,, MD", + {"family": "John Smith", "suffix": "MD"}, + notes="an empty segment 1 fails the suffix-comma detection " + "(v1 parity)"), Case("suffix_delimiter_tail_segment", "Doe, John, RN - CRNA", {"given": "John", "family": "Doe", "suffix": "RN, CRNA"}, policy=_SD, From 2e64c481c9c9f388e78f852cd2509bc05a72a2fe Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 23:33:41 -0700 Subject: [PATCH 106/275] Close four v1-compat gaps: Constants kwargs, parse_full_name, C setter, matches message Co-Authored-By: Claude Fable 5 --- nameparser/_config_shim.py | 81 +++++++++++++++++++++++++++++++++--- nameparser/_facade.py | 38 +++++++++++++++++ tests/v2/test_config_shim.py | 51 +++++++++++++++++++++++ tests/v2/test_facade.py | 55 ++++++++++++++++++++++++ 4 files changed, 219 insertions(+), 6 deletions(-) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index ff16584e..ebf6bfee 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -392,6 +392,21 @@ def _raise_readonly(name: str) -> None: # (None is a legitimate value for string_format/suffix_delimiter) _UNSET = object() +# distinguishes "kwarg not passed to Constants()" (use library defaults) +# from any real value a caller might pass, including a falsy one like "" +_UNSET_KWARG = object() + + +def _reject_bare_str_for_field(value: object, field: str) -> None: + # A bare string is an iterable of its characters, so e.g. + # SetManager("dr") would silently shred it into {'d', 'r'} instead of + # raising -- shared by every Constants() set-field kwarg (#238/#244). + if isinstance(value, (str, bytes)): + raise TypeError( + f"{field} must be an iterable of strings, not a single " + f"str/bytes: {value!r}; wrap it in a list, e.g. [{value!r}]" + ) + _SHARED_MUTATION_MESSAGE = ( "mutating the shared CONSTANTS singleton is deprecated and will be " "removed in 3.0; build a Lexicon/Policy (or a private Constants " @@ -489,17 +504,71 @@ class Constants: initials_separator: str suffix_delimiter: str | None - def __init__(self) -> None: + def __init__( + self, + *, + prefixes: Iterable[str] | object = _UNSET_KWARG, + suffix_acronyms: Iterable[str] | object = _UNSET_KWARG, + suffix_not_acronyms: Iterable[str] | object = _UNSET_KWARG, + suffix_acronyms_ambiguous: Iterable[str] | object = _UNSET_KWARG, + titles: Iterable[str] | object = _UNSET_KWARG, + first_name_titles: Iterable[str] | object = _UNSET_KWARG, + conjunctions: Iterable[str] | object = _UNSET_KWARG, + bound_first_names: Iterable[str] | object = _UNSET_KWARG, + non_first_name_prefixes: Iterable[str] | object = _UNSET_KWARG, + capitalization_exceptions: + Mapping[str, str] | Iterable[tuple[str, str]] | object + = _UNSET_KWARG, + regexes: object = _UNSET_KWARG, + ) -> None: + # v1.4 parity constructor kwargs (#238/#242/#244 hardening); the + # signature is spelled out rather than **kwargs so an unknown + # keyword raises Python's own TypeError with no help from here. + # `regexes` is the one deliberate 2.0 divergence: v1.4 accepted it + # (RegexTupleManager(regexes)), but constructor injection is + # assignment by another door, and __setattr__ above already + # forbids `c.regexes = ...` post-construction -- the uniform 2.0 + # rule is that parsing behavior is configured through Policy, not + # by handing Constants a compiled-pattern mapping either way. + if regexes is not _UNSET_KWARG: + raise TypeError( + "Constants(regexes=...) is not supported in 2.0; parsing " + "behavior is configured through named Policy flags, not " + "by constructing Constants with a regex mapping. See the " + "migration guide." + ) + overrides = { + "prefixes": prefixes, + "suffix_acronyms": suffix_acronyms, + "suffix_not_acronyms": suffix_not_acronyms, + "suffix_acronyms_ambiguous": suffix_acronyms_ambiguous, + "titles": titles, + "first_name_titles": first_name_titles, + "conjunctions": conjunctions, + "bound_first_names": bound_first_names, + "non_first_name_prefixes": non_first_name_prefixes, + } vocab = _default_vocab() object.__setattr__(self, "_generation", 0) for name in _SET_FIELDS: + value = overrides[name] + if value is _UNSET_KWARG: + value = vocab[name] + else: + # a caller-supplied value REPLACES that field's default + # vocabulary wholesale (v1 parity) -- validated/normalized + # by SetManager below, once past the bare-str guard + _reject_bare_str_for_field(value, name) object.__setattr__( - self, name, SetManager(vocab[name], _on_change=self._bump)) - from nameparser.config.capitalization import ( - CAPITALIZATION_EXCEPTIONS, - ) + self, name, SetManager(value, _on_change=self._bump)) # type: ignore[arg-type] + if capitalization_exceptions is _UNSET_KWARG: + from nameparser.config.capitalization import ( + CAPITALIZATION_EXCEPTIONS, + ) + capitalization_exceptions = CAPITALIZATION_EXCEPTIONS object.__setattr__(self, "capitalization_exceptions", TupleManager( - CAPITALIZATION_EXCEPTIONS, _on_change=self._bump)) + capitalization_exceptions, # type: ignore[arg-type] + _on_change=self._bump)) object.__setattr__(self, "nickname_delimiters", _DelimiterManager( {name: name for name in _DELIMITER_SENTINELS}, _on_change=self._bump)) diff --git a/nameparser/_facade.py b/nameparser/_facade.py index c1626f13..c13e2e1b 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -233,6 +233,16 @@ def _resolve(self) -> Parser: self._snapshot_gen = gen return _cached_parser(self._lexicon, self._policy) + def parse_full_name(self) -> None: + """Re-parse the stored ``full_name`` (v1's documented re-parse + trigger, docs/customize.rst): mutate ``name.C`` then call this to + force a re-parse without reassigning ``full_name``. The v1 + parsing INTERNALS this name evokes live in the core ``Parser``, + not here; a subclass overriding this method still triggers the + #280 hook-override warning, and full_name assignment never + consults it.""" + self._apply_full_name(self._full_name) + def _apply_full_name(self, value: str) -> None: if isinstance(value, bytes): raise TypeError( @@ -273,6 +283,28 @@ def original(self) -> str: def C(self) -> Constants: return self._C + @C.setter + def C(self, constants: Constants | None) -> None: + # v1.4 closed #239 by making C a validating setter that ONLY + # stores the new value -- no re-parse (checked against v1.4: + # `git show 2d5d8c2:nameparser/parser.py` lines ~204-206, the C + # setter body is exactly `self._C = self._validate_constants(...)`). + # A caller who wants the new config reflected must still trigger + # a re-parse, e.g. via parse_full_name() or a full_name + # reassignment -- matched here rather than re-parsing eagerly. + if constants is None: + raise TypeError( + "assigning constants=None to C was removed in 2.0 (#261): " + "pass a Constants instance, or use the new Parser/Lexicon/" + "Policy API for per-call configuration" + ) + if not isinstance(constants, Constants): + raise TypeError( + f"constants must be a Constants instance, got {constants!r}" + ) + self._C = constants + self._snapshot_gen = -1 # invalidate: next _resolve() rebuilds + @property def has_own_config(self) -> bool: return self._C is not CONSTANTS @@ -512,6 +544,12 @@ def group(items: list[str]) -> str: def matches(self, other: str | HumanName) -> bool: """Component-wise case-insensitive comparison (v1 parity); a str argument is parsed with this instance's resolved parser.""" + if not isinstance(other, (str, HumanName)): + # pre-check so the error names the facade type a caller + # actually passed HumanName.matches(), not the core + # ParsedName it delegates to below + raise TypeError( + f"matches() takes a str or HumanName, got {other!r}") target = other._parsed if isinstance(other, HumanName) else other return self._parsed.matches(target, parser=self._resolve()) diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index 811f5a87..f32989ba 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -374,3 +374,54 @@ def test_parser_cache_shared_across_equal_snapshots() -> None: la, pa, _ = a._snapshot() lb, pb, _ = b._snapshot() assert _cached_parser(la, pa) is _cached_parser(lb, pb) + + +# -- Gap 1: Constants() constructor kwargs (v1.4 parity, #238/#242/#244) ---- + + +def test_constants_kwarg_replaces_field_others_keep_defaults() -> None: + default = Constants() + c = Constants(titles=["Abc"]) + assert set(c.titles) == {"abc"} # lc()-normalized + assert "abc" not in default.titles # sanity: not a real default + # a kwarg REPLACES that field's vocabulary wholesale (v1 parity); the + # other eight set fields are untouched + for field in ("prefixes", "suffix_acronyms", "suffix_not_acronyms", + "suffix_acronyms_ambiguous", "first_name_titles", + "conjunctions", "bound_first_names", + "non_first_name_prefixes"): + assert set(getattr(c, field)) == set(getattr(default, field)) + + +def test_constants_kwarg_bare_string_raises_typeerror() -> None: + with pytest.raises(TypeError, match="titles"): + Constants(titles="abc") # type: ignore[arg-type] + + +def test_constants_kwarg_non_iterable_raises_typeerror() -> None: + with pytest.raises(TypeError): + Constants(titles=5) # type: ignore[arg-type] + + +def test_constants_kwarg_capitalization_exceptions_wraps_tuple_manager() -> None: + c = Constants(capitalization_exceptions={"mcdonald": "McDonald"}) + assert isinstance(c.capitalization_exceptions, TupleManager) + assert c.capitalization_exceptions["mcdonald"] == "McDonald" + + +def test_constants_kwarg_regexes_raises_typeerror() -> None: + with pytest.raises(TypeError, match="Policy"): + Constants(regexes={}) # type: ignore[call-arg] + + +def test_constants_kwarg_unknown_raises_typeerror() -> None: + with pytest.raises(TypeError): + Constants(bogus=1) # type: ignore[call-arg] + + +def test_constants_kwarg_feeds_facade_parse() -> None: + from nameparser._facade import HumanName + + c = Constants(titles=["zzqtitle"]) + n = HumanName("Zzqtitle Judy Dench", constants=c) + assert n.title == "Zzqtitle" diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index 8b63b844..4ffc9314 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -307,6 +307,61 @@ def test_pickle_round_trip_preserves_components() -> None: assert loaded.C is CONSTANTS # shared sentinel restored +# -- Gap 2: parse_full_name() (v1's documented re-parse idiom) ------------- + + +def test_parse_full_name_reparses_documented_idiom() -> None: + c = Constants() + n = HumanName("Zzqtitle Judy Dench", constants=c) + assert n.title == "" # 'zzqtitle' not a default title + c.titles.add("zzqtitle") # mutate C in place, no full_name reassignment + n.parse_full_name() # documented v1 idiom + assert n.title == "Zzqtitle" + + +def test_subclass_overriding_parse_full_name_warns() -> None: # #280 + class CustomReparse(HumanName): + def parse_full_name(self) -> None: + pass + + with pytest.deprecated_call(match="parse_full_name"): + CustomReparse("John Smith") + + +# -- Gap 3: HumanName.C setter (v1.4 #239) ---------------------------------- + + +def test_c_setter_assigns_and_next_parse_honors_it() -> None: + n = HumanName("John Smith") + c2 = Constants() + c2.titles.add("zzqtitle") + n.C = c2 + assert n.C is c2 + n.full_name = "Zzqtitle Judy Dench" # next parse: v1's setter only stored, no reparse + assert n.title == "Zzqtitle" + + +def test_c_setter_none_raises_migration_hint() -> None: + n = HumanName("John Smith") + with pytest.raises(TypeError, match="constants"): + n.C = None # type: ignore[assignment] + + +def test_c_setter_non_constants_raises() -> None: + n = HumanName("John Smith") + with pytest.raises(TypeError): + n.C = 42 # type: ignore[assignment] + + +# -- Gap 4: matches() error message names the facade type ------------------- + + +def test_matches_type_error_names_facade_type() -> None: + n = HumanName("John Smith") + with pytest.raises(TypeError, match="HumanName"): + n.matches(None) # type: ignore[arg-type] + + def test_v14_humanname_blob_unpickles() -> None: # tests/v2/data/humanname_v14.pickle was produced by a real 1.4.0 # install (`uv run --no-project --with "nameparser==1.4.*" ...`) on From 81f2109009154ca2dd34a2d7649171757cad63fa Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 23:48:30 -0700 Subject: [PATCH 107/275] Reconcile test_python_api against the facade (buckets A, B, E) 158 dual-run cases -> 67 tests (the empty_attribute_default dual-run parametrization died in the previous batch commit; per-run count 79 -> 67). Bucket A (deleted machinery): - test_unpickle_legacy_state_without_derived_sets: pinned __setstate__ backfill of the v1 per-parse _derived_* sets and the is_title predicate, both gone with the v1 parser. - the six is_prefix/is_conjunction/is_suffix list-fallthrough tests: the v1 predicate methods are not part of the facade (#280 hook list). - test_get_full_name_attribute_references_internal_lists: pinned the v1 full_name getter re-rendering from the live *_list state; 2.0 deliberately returns the stored input string (spec section 2 mutation semantics, pinned in tests/v2/test_facade.py::test_field_assignment_str_list_none) and the *_list attributes are snapshots without setters (spec section 2 exc. 1). Bucket B (scheduled removals now raising / silently changed): - #245 bytes: both HumanName(bytes) and the full_name setter now pytest.raises(TypeError, match="decode"). - #223 ==/hash: rewritten to assert the identity semantics (distinct instances unequal, strings never match, hash == object default, no warnings), with matches()/comparison_key() shown as the replacements. - #258 slices/__setitem__: slice access raises TypeError naming the issue; item assignment raises TypeError for every key; str-key reads stay silent. - #255: given_names empty expectation is '' outright. - Constants(regexes=...) raises TypeError pointing at Policy (deliberate 2.0 divergence, migration spec section 3 uniform rule). 3 of 67 tests still fail on one confirmed pipeline bug (SUFFIX_COMMA detection tests all post-comma segments where v1 tested only parts[1]; repro 'Dr. John P. Doe-Ray, CLU, CFP, LUTC'), so the file stays in collect_ignore_glob with a pointer comment; it is mypy-clean and leaves the mypy exclude. test_constants.py stays ignored, blocked on a wider shim-regression inventory (see conftest comment). Co-Authored-By: Claude Fable 5 --- pyproject.toml | 2 +- tests/conftest.py | 25 ++-- tests/test_python_api.py | 250 ++++++++++++++------------------------- 3 files changed, 104 insertions(+), 173 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d7d862cc..467f06cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ packages = ["nameparser", "tests"] # TEMPORARY (M11 -> M12): the v1 suite is being reconciled against the # 2.0 facade; lifted with tests/conftest.py's collect_ignore_glob. Must # be gone before the branch leaves draft. tests/v2/ stays checked. -exclude = '^tests/(test_(bound_first_names|brute_force|capitalization|comma_variants|conjunctions|constants|east_slavic_patronymic_order|first_name|initials|middle_name_as_last|nicknames|output_format|parser_util|prefixes|python_api|suffixes|titles|turkic_patronymic_order|variations)\.py)$' +exclude = '^tests/(test_(bound_first_names|brute_force|capitalization|comma_variants|conjunctions|constants|east_slavic_patronymic_order|first_name|initials|middle_name_as_last|nicknames|output_format|parser_util|prefixes|suffixes|titles|turkic_patronymic_order|variations)\.py)$' [[tool.mypy.overrides]] module = [ diff --git a/tests/conftest.py b/tests/conftest.py index 9889a6bc..b20d0975 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,22 +13,30 @@ # # Explicit list of not-yet-reconciled files. Narrow this as each file is # reconciled; remove entirely once the whole suite runs against the facade. -# -# test_python_api.py and test_constants.py are BACK on this list (not -# reconciled this batch): both are blocked by confirmed facade/pipeline bugs -# (missing Constants(**kwargs) constructor support, missing -# HumanName.parse_full_name()/C setter, a "Doe,, Jr." title/suffix routing -# regression -- see the M12 batch report) that this task must not paper over -# by editing nameparser/ source or weakening the tests. Re-narrow once those -# are fixed upstream. collect_ignore_glob = [ "test_bound_first_names.py", "test_brute_force.py", "test_capitalization.py", "test_comma_variants.py", "test_conjunctions.py", + # test_constants.py is UNTOUCHED and blocked: reconciliation probing found + # a cluster of shim regressions of shipped v1 fixes (SetManager missing + # discard()/clear()/set operators; #238/#241/#242 bare-string and + # shred guards gone; #260 subclass-preserving copy(); #221 repr) plus two + # pipeline bugs (period-joined title/suffix derivation: 'Lt.Gov. John + # Doe' and 'John Doe JD.CPA' misparse). See the M12 batch report for the + # full inventory with repros. Reconcile once those are fixed. "test_constants.py", "test_east_slavic_patronymic_order.py", + # test_python_api.py is fully reconciled (and mypy-clean, so it is NOT in + # the mypy exclude) but 3 of its 67 tests fail on one confirmed pipeline + # bug: _pipeline/_segment.py:85 requires ALL post-first comma segments to + # be suffix-shaped for SUFFIX_COMMA, where v1 tested only parts[1] and + # consumed the rest as suffixes unconditionally. Repro: + # Parser().parse('Dr. John P. Doe-Ray, CLU, CFP, LUTC') + # ('lutc' is not in the suffix vocab) -> family='Dr. John P. Doe-Ray'. + # Remove this line once that is fixed. + "test_python_api.py", "test_first_name.py", "test_initials.py", "test_middle_name_as_last.py", @@ -36,7 +44,6 @@ "test_output_format.py", "test_parser_util.py", "test_prefixes.py", - "test_python_api.py", "test_suffixes.py", "test_titles.py", "test_turkic_patronymic_order.py", diff --git a/tests/test_python_api.py b/tests/test_python_api.py index f0be687e..fb763d9b 100644 --- a/tests/test_python_api.py +++ b/tests/test_python_api.py @@ -28,21 +28,23 @@ def test_string_output(self) -> None: self.m(str(hn), "Jüan de la Véña", hn) def test_escaped_utf8_bytes(self) -> None: - # bytes input is deprecated (#245), still supported until 2.0 - with pytest.deprecated_call(): - hn = HumanName(b'B\xc3\xb6ck, Gerald') + # bytes input was removed in 2.0 (#245, warned since 1.3.0): + # decode first + with pytest.raises(TypeError, match="decode"): + HumanName(b'B\xc3\xb6ck, Gerald') # type: ignore[arg-type] + hn = HumanName(b'B\xc3\xb6ck, Gerald'.decode('utf-8')) self.m(hn.first, "Gerald", hn) self.m(hn.last, "Böck", hn) - def test_bytes_full_name_emits_deprecation_warning(self) -> None: - # bytes input will be removed in 2.0 (#245); the caller should decode - with pytest.deprecated_call(match="decode"): - hn = HumanName(b'John Smith') - self.m(hn.first, "John", hn) - hn2 = HumanName("Jane Doe") - with pytest.deprecated_call(match="decode"): - hn2.full_name = b'John Smith' - self.m(hn2.first, "John", hn2) + def test_bytes_full_name_raises_with_decode_hint(self) -> None: + # bytes input was removed in 2.0 (#245); both entry points raise + with pytest.raises(TypeError, match="decode"): + HumanName(b'John Smith') # type: ignore[arg-type] + hn = HumanName("Jane Doe") + with pytest.raises(TypeError, match="decode"): + hn.full_name = b'John Smith' # type: ignore[assignment] + # the failed assignment must not have corrupted the parse + self.m(hn.first, "Jane", hn) def test_str_full_name_does_not_warn(self) -> None: with warnings.catch_warnings(): @@ -157,27 +159,6 @@ def test_name_instance_deepcopy_isolates_instance_config(self) -> None: self.assertIn('chancellor', dup.C.titles) self.assertNotIn('marker', hn.C.titles) - def test_unpickle_legacy_state_without_derived_sets(self) -> None: - """Pickles from before the per-parse derived sets existed must still work. - - Their state lacks the ``_derived_*`` attributes, which the ``is_*`` - predicates (and through them ``capitalize()``) read directly, so - ``__setstate__`` must backfill them rather than crash with - AttributeError on first use. - """ - hn = HumanName("dr. juan de la vega jr.") - legacy_state = { - k: v for k, v in hn.__getstate__().items() - if not k.startswith('_derived_') - } - - restored = HumanName.__new__(HumanName) - restored.__setstate__(legacy_state) - - self.assertTrue(restored.is_title('dr.')) - restored.capitalize() # reads _derived_prefixes via cap_word/is_prefix - self.assertEqual(str(restored), "Dr. Juan de la Vega Jr.") - def test_pickle_default_name_preserves_singleton_identity(self) -> None: """A default HumanName must re-attach to CONSTANTS after a pickle round-trip. @@ -235,22 +216,24 @@ def test_deepcopy_default_name_preserves_singleton_identity(self) -> None: self.assertEqual(dup.last, hn.last) def test_comparison(self) -> None: - # deprecated behavior (#223/#224), still supported until 2.0: - # deprecated_call() asserts the warning fires while silencing it + # == and hash reverted to object identity in 2.0 (#223, warned since + # 1.3.0): distinct instances never compare equal, however they parse, + # and strings never match. matches()/comparison_key() are the + # replacements (covered below). hn1 = HumanName("Doe-Ray, Dr. John P., CLU, CFP, LUTC") hn2 = HumanName("Dr. John P. Doe-Ray, CLU, CFP, LUTC") - with pytest.deprecated_call(): - self.assertTrue(hn1 == hn2) - self.assertIsNot(hn1, hn2) - self.assertTrue(hn1 == "Dr. John P. Doe-Ray CLU, CFP, LUTC") + self.assertTrue(hn1 == hn1) + self.assertFalse(hn1 == hn2) + self.assertIsNot(hn1, hn2) + self.assertFalse(hn1 == "Dr. John P. Doe-Ray CLU, CFP, LUTC") + self.assertTrue(hn1.matches(hn2)) # the 2.0 spelling of the old == hn1 = HumanName("Doe, Dr. John P., CLU, CFP, LUTC") hn2 = HumanName("Dr. John P. Doe-Ray, CLU, CFP, LUTC") - with pytest.deprecated_call(): - self.assertTrue(not hn1 == hn2) - self.assertTrue(not hn1 == 0) - self.assertTrue(not hn1 == "test") - self.assertTrue(not hn1 == ["test"]) - self.assertTrue(not hn1 == {"test": hn2}) + self.assertTrue(not hn1 == hn2) + self.assertTrue(not hn1 == 0) + self.assertTrue(not hn1 == "test") + self.assertTrue(not hn1 == ["test"]) + self.assertTrue(not hn1 == {"test": hn2}) def test_assignment_to_full_name(self) -> None: hn = HumanName("John A. Kenneth Doe, Jr.") @@ -263,11 +246,6 @@ def test_assignment_to_full_name(self) -> None: self.m(hn.last, "Velasquez y Garcia", hn) self.m(hn.suffix, "III", hn) - def test_get_full_name_attribute_references_internal_lists(self) -> None: - hn = HumanName("John Williams") - hn.first_list = ["Larry"] - self.m(hn.full_name, "Larry Williams", hn) - def test_assignment_to_attribute(self) -> None: hn = HumanName("John A. Kenneth Doe, Jr.") hn.last = "de la Vega" @@ -299,37 +277,38 @@ def test_assign_list_to_attribute(self) -> None: self.m(hn.suffix, "test", hn) def test_comparison_case_insensitive(self) -> None: - # deprecated behavior (#223/#224), still supported until 2.0 + # 2.0 identity semantics (#223): equivalently-parsed instances no + # longer compare ==; matches() carries the case-insensitive + # component comparison instead hn1 = HumanName("Doe-Ray, Dr. John P., CLU, CFP, LUTC") hn2 = HumanName("dr. john p. doe-Ray, CLU, CFP, LUTC") - with pytest.deprecated_call(): - self.assertTrue(hn1 == hn2) - self.assertIsNot(hn1, hn2) - self.assertTrue(hn1 == "Dr. John P. Doe-ray clu, CFP, LUTC") - - def test_hash_matches_case_insensitive_equality(self) -> None: - # deprecated behavior (#223/#224), still supported until 2.0. - # __eq__ compares lowercased strings, so __hash__ must too: - # equal objects are required to have equal hashes. + self.assertFalse(hn1 == hn2) + self.assertIsNot(hn1, hn2) + self.assertFalse(hn1 == "Dr. John P. Doe-ray clu, CFP, LUTC") + self.assertTrue(hn1.matches(hn2)) + self.assertTrue(hn1.matches("Dr. John P. Doe-ray clu, CFP, LUTC")) + + def test_hash_is_identity_based(self) -> None: + # 2.0 identity semantics (#223): hash reverted to the object default, + # consistent with identity ==; equal-parsing instances stay distinct + # in sets/dicts. comparison_key() is the dedup replacement. hn1 = HumanName("Doe-Ray, Dr. John P., CLU, CFP, LUTC") hn2 = HumanName("dr. john p. doe-Ray, CLU, CFP, LUTC") - with pytest.deprecated_call(): - self.assertEqual(hn1, hn2) - self.assertEqual(hash(hn1), hash(hn2)) - self.assertEqual(len({hn1, hn2}), 1) - # __eq__ also accepts plain strings, so hashing str(self).lower() - # specifically (not e.g. an attribute tuple) is what lets strings - # and HumanName instances interoperate in sets and dicts - hn = HumanName("John Smith") - self.assertEqual(hash(hn), hash("john smith")) - self.assertIn("john smith", {hn}) + self.assertEqual(hash(hn1), object.__hash__(hn1)) + self.assertEqual(len({hn1, hn2}), 2) + # strings no longer interoperate with HumanName in sets/dicts + hn = HumanName("John Smith") + self.assertNotEqual(hash(hn), hash("john smith")) + self.assertNotIn("john smith", {hn}) + self.assertEqual(hn1.comparison_key(), hn2.comparison_key()) def test_not_equal_operator(self) -> None: - # deprecated behavior (#223/#224), still supported until 2.0; - # != routes through __eq__, so it warns too - with pytest.deprecated_call(): - self.assertTrue(HumanName("John Smith") != HumanName("Jane Smith")) - self.assertFalse(HumanName("John Smith") != HumanName("john smith")) + # != routes through the 2.0 identity __eq__ (#223): any two distinct + # instances are unequal, however similar their parse + self.assertTrue(HumanName("John Smith") != HumanName("Jane Smith")) + self.assertTrue(HumanName("John Smith") != HumanName("john smith")) + hn = HumanName("John Smith") + self.assertFalse(hn != hn) def test_comparison_key_components(self) -> None: hn = HumanName("Dr. Juan Q. Xavier de la Vega III") @@ -416,19 +395,15 @@ def test_matches_rejects_other_types(self) -> None: with pytest.raises(TypeError, match="str or HumanName"): hn.matches(bad) # type: ignore[arg-type] - def test_eq_emits_deprecation_warning(self) -> None: - # behavior is unchanged until 2.0 (#223); 1.3.0 only warns + def test_eq_and_hash_are_silent_in_2_0(self) -> None: + # the 1.3.0/1.4 DeprecationWarnings on __eq__/__hash__ ended with the + # 2.0 switch to identity semantics (#223): both are now warning-free hn1 = HumanName("John Smith") hn2 = HumanName("john smith") - with pytest.deprecated_call(match="matches"): - result = hn1 == hn2 - self.assertTrue(result) - - def test_hash_emits_deprecation_warning(self) -> None: - hn = HumanName("John Smith") - with pytest.deprecated_call(match="comparison_key"): - result = hash(hn) - self.assertEqual(result, hash("john smith")) + with warnings.catch_warnings(): + warnings.simplefilter("error") + self.assertFalse(hn1 == hn2) + hash(hn1) def test_new_comparison_api_does_not_warn(self) -> None: # the replacements must be adoptable before 2.0 without tripping @@ -458,21 +433,16 @@ def test_repr_blank_name(self) -> None: self.assertIn("first: ''", repr(hn)) self.assertIn(hn.__class__.__name__, repr(hn)) - def test_slice(self) -> None: + def test_slice_removed(self) -> None: + # slice access was removed in 2.0 (#258, warned since 1.4); iteration + # and string-key access are the remaining spellings hn = HumanName("Doe-Ray, Dr. John P., CLU, CFP, LUTC") self.m(list(hn), ['Dr.', 'John', 'P.', 'Doe-Ray', 'CLU, CFP, LUTC'], hn) - with pytest.deprecated_call(match="Slic"): - sliced = hn[1:] - self.m(sliced, ['John', 'P.', 'Doe-Ray', 'CLU, CFP, LUTC', hn.C.empty_attribute_default, hn.C.empty_attribute_default], hn) - with pytest.deprecated_call(match="Slic"): - self.m(hn[1:-3], ['John', 'P.', 'Doe-Ray'], hn) - - def test_slice_getitem_deprecation_names_issue(self) -> None: - # slice access is deprecated for removal in 2.0 (#258); string-key - # access is unaffected and stays silent - hn = HumanName("Doe-Ray, Dr. John P., CLU, CFP, LUTC") - with pytest.deprecated_call(match="258"): - hn[1:] + with pytest.raises(TypeError, match="258"): + hn[1:] # type: ignore[index] + with pytest.raises(TypeError, match="258"): + hn[1:-3] # type: ignore[index] + # string-key access is unaffected and stays silent with warnings.catch_warnings(): warnings.simplefilter("error") hn['first'] @@ -485,30 +455,16 @@ def test_getitem(self) -> None: self.m(hn['middle'], "A. Kenneth", hn) self.m(hn['suffix'], "Jr.", hn) - def test_setitem(self) -> None: - hn = HumanName("Dr. John A. Kenneth Doe, Jr.") - with pytest.deprecated_call(match="258"): - hn['title'] = 'test' - self.m(hn['title'], "test", hn) - with pytest.deprecated_call(match="258"): - hn['last'] = ['test', 'test2'] - self.m(hn['last'], "test test2", hn) - with pytest.raises(TypeError), pytest.deprecated_call(): - hn["suffix"] = [['test']] # type: ignore[list-item] - with pytest.raises(TypeError), pytest.deprecated_call(): - hn["suffix"] = {"test": "test"} # type: ignore[assignment] - - def test_setitem_invalid_key_raises_keyerror(self) -> None: - hn = HumanName("Dr. John A. Kenneth Doe, Jr.") - with pytest.raises(KeyError), pytest.deprecated_call(): - hn["bogus"] = "value" - - def test_setitem_emits_deprecation_warning_naming_attribute_assignment(self) -> None: - # __setitem__ duplicates plain attribute assignment and is deprecated - # for removal in 2.0 (#258); use hn.first = ... instead + def test_setitem_removed(self) -> None: + # __setitem__ was removed in 2.0 (#258, warned since 1.4): item + # assignment raises TypeError for every key, valid or not; plain + # attribute assignment is the replacement hn = HumanName("Dr. John A. Kenneth Doe, Jr.") - with pytest.deprecated_call(match="attribute assignment"): - hn['first'] = 'Jane' + with pytest.raises(TypeError, match="item assignment"): + hn['title'] = 'test' # type: ignore[index] + with pytest.raises(TypeError, match="item assignment"): + hn['bogus'] = 'value' # type: ignore[index] + hn.first = 'Jane' # the replacement spelling self.m(hn.first, "Jane", hn) def test_conjunction_names(self) -> None: @@ -583,55 +539,23 @@ def test_given_names_attribute_first_only(self) -> None: self.m(hn.given_names, "John", hn) def test_given_names_attribute_empty(self) -> None: + # empty attributes are always '' in 2.0 (#255) hn = HumanName("Dr. Williams") - self.m(hn.given_names, hn.C.empty_attribute_default, hn) - - def test_is_prefix_with_list(self) -> None: - hn = HumanName() - items = ['firstname', 'lastname', 'del'] - self.assertTrue(hn.is_prefix(items)) - self.assertTrue(hn.is_prefix(items[1:])) - - def test_is_prefix_with_list_no_match(self) -> None: - hn = HumanName() - self.assertFalse(hn.is_prefix(['firstname', 'lastname'])) - - def test_is_conjunction_with_list(self) -> None: - hn = HumanName() - items = ['firstname', 'lastname', 'and'] - self.assertTrue(hn.is_conjunction(items)) - self.assertTrue(hn.is_conjunction(items[1:])) - - def test_is_conjunction_with_list_no_match(self) -> None: - hn = HumanName() - self.assertFalse(hn.is_conjunction(['firstname', 'lastname'])) - - def test_is_suffix_with_list(self) -> None: - hn = HumanName() - items = ['firstname', 'lastname', 'jr'] - self.assertTrue(hn.is_suffix(items)) - self.assertTrue(hn.is_suffix(items[1:])) - - def test_is_suffix_with_list_no_match(self) -> None: - hn = HumanName() - self.assertFalse(hn.is_suffix(['firstname', 'lastname'])) + self.m(hn.given_names, "", hn) def test_override_constants(self) -> None: C = Constants() hn = HumanName(constants=C) self.assertIs(hn.C, C) - def test_override_regex(self) -> None: - # A regexes dict this sparse omits keys the parser reads - # unconditionally (e.g. squash_bidi's 'bidi'); each miss falls back - # to EMPTY_REGEX but now also warns -- unknown-key access is - # deprecated for removal in 2.0 (#256), which would make this build - # AttributeError-crash the parser instead of degrading silently. + def test_override_regex_raises(self) -> None: + # Custom regexes are not supported in 2.0 (deliberate divergence, + # migration spec §3 uniform rule): the constructor kwarg raises the + # same TypeError as attribute assignment, pointing at named Policy + # flags. Reads of the built-in patterns stay available. var = TupleManager([("spaces", re.compile(r"\s+")),]) - C = Constants(regexes=var) - with pytest.deprecated_call(): - hn = HumanName(constants=C) - self.assertTrue(hn.C.regexes == var) + with pytest.raises(TypeError, match="Policy"): + Constants(regexes=var) # type: ignore[call-arg] def test_override_titles(self) -> None: var = ["abc","def"] From 7bf53f3ce71b072fcc56544616b38d1bc2177c75 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 16 Jul 2026 23:51:35 -0700 Subject: [PATCH 108/275] Port v1's parts[1]-only suffix-comma decision and period-joined vocabulary M12 reconciliation surfaced two more Plan-3 misports, both pinned live against v1.4 (2026-07-16): - the suffix-comma structure is decided by parts[1] ALONE (parser.py:1318); v2 wrongly required every post-first segment to be suffix-shaped, so one unrecognized credential ('LUTC') collapsed the whole parse into FAMILY_COMMA. Non-suffix tails now ride along as suffixes with the COMMA_STRUCTURE flag (v1 consumed them silently; the flag is the v2 value-add). - v1's period-joined derivation (parse_pieces): a token with an interior period whose chunks include ANY title is a title ('Lt.Gov.', 'Mr.Smith'), else ANY suffix chunk makes it a suffix ('JD.CPA'). Title wins. Ported into classify with the ANY rule kept deliberately. Six case rows pin the matrix. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_classify.py | 18 ++++++++++++++++++ nameparser/_pipeline/_segment.py | 13 ++++++++----- tests/v2/cases.py | 23 +++++++++++++++++++++++ 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/nameparser/_pipeline/_classify.py b/nameparser/_pipeline/_classify.py index 94d26bc7..1aba58a5 100644 --- a/nameparser/_pipeline/_classify.py +++ b/nameparser/_pipeline/_classify.py @@ -17,11 +17,17 @@ from __future__ import annotations import dataclasses +import re from nameparser._lexicon import _normalize from nameparser._pipeline._state import ParseState, WorkToken from nameparser._pipeline._vocab import is_initial, suffix_as_written +# Ported verbatim from v1 (nameparser/config/regexes.py +# "period_not_at_end") -- layering forbids the config import; keep in +# sync by hand. +_PERIOD_NOT_AT_END = re.compile(r".*\..+$", re.I) + def _tags_for(token: WorkToken, state: ParseState) -> frozenset[str]: lex = state.lexicon @@ -49,6 +55,18 @@ def _tags_for(token: WorkToken, state: ParseState) -> frozenset[str]: tags.add("vocab:maiden-marker") if is_initial(token.text): tags.add("initial") + # v1's period-joined derivation (parse_pieces): a token with a + # period not at the end, ANY of whose period chunks is a title, is + # a title as a whole ('Lt.Gov.', and by the ANY rule 'Mr.Smith'); + # else ANY suffix chunk makes it a suffix ('JD.CPA'). Title wins + # (v1's continue). Skipped when the whole token already matched. + if ("vocab:title" not in tags and "vocab:suffix" not in tags + and _PERIOD_NOT_AT_END.match(token.text)): + chunks = [_normalize(c) for c in token.text.split(".") if c] + if any(c in lex.titles for c in chunks): + tags.add("vocab:title") + elif any(suffix_as_written(c, c, lex) for c in chunks): + tags.add("vocab:suffix") return frozenset(tags) diff --git a/nameparser/_pipeline/_segment.py b/nameparser/_pipeline/_segment.py index 6372eda4..ce735019 100644 --- a/nameparser/_pipeline/_segment.py +++ b/nameparser/_pipeline/_segment.py @@ -81,10 +81,13 @@ def suffixy(seg: tuple[int, ...]) -> bool: return bool(seg) and all( counts_as_suffix(state.tokens[i].text) for i in seg) - rest = groups[1:] - if all(suffixy(s) for s in rest) and len(groups[0]) > 1: - return dataclasses.replace(state, segments=tuple(groups), - structure=Structure.SUFFIX_COMMA) + # v1 parity: only parts[1] decides the suffix-comma structure + # (parser.py:1318); parts[2:] are consumed as suffixes + # unconditionally either way, so a non-suffix tail segment gets the + # COMMA_STRUCTURE flag, not a structure veto + structure = (Structure.SUFFIX_COMMA + if suffixy(groups[1]) and len(groups[0]) > 1 + else Structure.FAMILY_COMMA) ambiguities = list(state.ambiguities) for seg in groups[2:]: # empty segments are consumed silently (v1 skips them without @@ -97,5 +100,5 @@ def suffixy(seg: tuple[int, ...]) -> bool: f"structures; consumed as suffix best-effort", tuple(seg))) return dataclasses.replace(state, segments=tuple(groups), - structure=Structure.FAMILY_COMMA, + structure=structure, ambiguities=tuple(ambiguities)) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 9f830a83..791d956a 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -41,6 +41,29 @@ class Case: {"given": "John", "family": "Smith"}), Case("suffix_comma", "John Smith, PhD", {"given": "John", "family": "Smith", "suffix": "PhD"}), + Case("suffix_comma_decided_by_first_segment", + "Dr. John P. Doe-Ray, CLU, CFP, LUTC", + {"title": "Dr.", "given": "John", "middle": "P.", + "family": "Doe-Ray", "suffix": "CLU, CFP, LUTC"}, + ambiguities=("comma-structure",), + notes="only parts[1] decides the suffix-comma structure " + "(v1 parser.py:1318); 'lutc' is not in the vocabulary " + "but rides along (v1 parity, pinned live 2026-07-16)"), + Case("suffix_comma_nonsuffix_tail_flagged", "John Smith, MD, Xyzzy", + {"given": "John", "family": "Smith", "suffix": "MD, Xyzzy"}, + ambiguities=("comma-structure",), + notes="the unrecognized tail is consumed best-effort with the " + "flag (v1 consumed it silently)"), + Case("period_joined_titles", "Lt.Gov. John Doe", + {"title": "Lt.Gov.", "given": "John", "family": "Doe"}, + notes="v1 derived-title rule: ANY period chunk being a title " + "makes the token a title (pinned live 2026-07-16)"), + Case("period_joined_suffixes", "John Doe JD.CPA", + {"given": "John", "family": "Doe", "suffix": "JD.CPA"}), + Case("period_joined_any_rule", "Mr.Smith", + {"title": "Mr.Smith"}, + notes="the ANY rule is deliberate v1 parity: one title chunk " + "claims the whole token"), Case("doubled_comma_suffix", "Doe,, Jr.", {"family": "Doe", "suffix": "Jr."}, notes="the EMPTY given segment keeps its position: 'Jr.' is a " From 7e81187e283f392155bf3d6eb2e71b248d1bd306 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 00:02:36 -0700 Subject: [PATCH 109/275] Restore v1.3/1.4 manager hardening in the shim (#221/#238/#241/#242/#260 + operators) Co-Authored-By: Claude Fable 5 --- nameparser/_config_shim.py | 254 ++++++++++++++++++++++++++++++----- tests/v2/test_config_shim.py | 132 ++++++++++++++++++ 2 files changed, 351 insertions(+), 35 deletions(-) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index ebf6bfee..8fb87b58 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -21,7 +21,9 @@ import functools import warnings -from collections.abc import Callable, Iterable, Iterator, KeysView, Mapping +from collections.abc import ( + Callable, Iterable, Iterator, KeysView, Mapping, Set, +) from typing import NamedTuple, Self from nameparser._lexicon import Lexicon @@ -30,6 +32,47 @@ from nameparser.util import lc +def _reject_bare_str_or_bytes(value: object, expected: str) -> None: + # A bare string is an iterable of its characters, so e.g. SetManager('dr') + # would silently shred it into {'d', 'r'} instead of raising -- shared by + # SetManager's constructor/operands (#238/#241) and TupleManager's + # constructor (#242). Ported from v1's `_reject_bare_str_or_bytes`. + if isinstance(value, bytes): + raise TypeError( + f"expected {expected}, got a single bytes; decode it first: " + f"{value!r}.decode()" + ) + if isinstance(value, str): + raise TypeError( + f"expected {expected}, got a single str; wrap it in a list: " + f"[{value!r}]" + ) + + +def _lc_validated(s: object) -> str: + # Validates and lc()-normalizes a single element -- shared by bulk + # iterable normalization (constructor/operands) and add()'s per-string + # loop, so every path raises the same #238/#245-shaped TypeError instead + # of lc() crashing cryptically (bytes) or silently transmuting (None). + if isinstance(s, bytes): + raise TypeError( + f"expected a str, got bytes; decode it first: {s!r}.decode()" + ) + if not isinstance(s, str): + raise TypeError(f"expected a str, got {type(s).__name__}: {s!r}") + return lc(s) + + +def _normalize_iterable_of_strings( + elements: object, expected: str = "an iterable of strings") -> set[str]: + # a SetManager's elements were already validated/normalized when it was + # built, so copy them instead of re-validating (v1's fast path) + if isinstance(elements, SetManager): + return set(elements._elements) + _reject_bare_str_or_bytes(elements, expected) + return {_lc_validated(s) for s in elements} # type: ignore[attr-defined] + + class SetManager: """v1 ``SetManager`` surface over a plain set of ``lc()``-normalized strings. Mutations call ``_on_change`` (the owning Constants' @@ -43,8 +86,16 @@ class SetManager: _on_change: Callable[[], None] | None def __init__(self, elements: Iterable[str] = (), - _on_change: Callable[[], None] | None = None) -> None: - self._elements = {lc(e) for e in elements} + _on_change: Callable[[], None] | None = None, + _field: str | None = None) -> None: + # _field carries a Constants field name (e.g. "titles") through to + # the bare-str/bytes guard's message, when this SetManager is being + # built on behalf of a named Constants field (constructor kwarg or + # __setattr__ auto-wrap); a direct SetManager(...) call gets the + # generic v1 message instead. + expected = f"{_field} to be an iterable of strings" if _field \ + else "an iterable of strings" + self._elements = _normalize_iterable_of_strings(elements, expected) self._on_change = _on_change def _changed(self) -> None: @@ -58,7 +109,7 @@ def add(self, *strings: str) -> SetManager: # bump the owner's generation changed = False for s in strings: - normalized = lc(s) + normalized = _lc_validated(s) # TypeError on bytes (#245) if normalized not in self._elements: self._elements.add(normalized) changed = True @@ -82,6 +133,28 @@ def remove(self, *strings: str) -> SetManager: self._changed() return self + def discard(self, *strings: str) -> SetManager: + """Remove the normalized string arguments from the set if + present; missing members are ignored, like ``set.discard``. + Returns ``self`` for chaining.""" + changed = False + for s in strings: + normalized = lc(s) + if normalized in self._elements: + self._elements.discard(normalized) + changed = True + if changed: + self._changed() + return self + + def clear(self) -> SetManager: + """Remove all entries from the set. Returns ``self`` for + chaining.""" + if self._elements: + self._elements.clear() + self._changed() + return self + def __contains__(self, item: object) -> bool: return isinstance(item, str) and lc(item) in self._elements @@ -100,28 +173,62 @@ def __eq__(self, other: object) -> bool: __hash__ = None # type: ignore[assignment] # mutable; v1 parity - def _as_operand(self, other: object) -> set[str]: - if isinstance(other, SetManager): - return other._elements - if isinstance(other, (set, frozenset)): - return {lc(e) if isinstance(e, str) else e for e in other} - raise TypeError(f"unsupported operand type for SetManager: {other!r}") + # -- set operators: accept ANY iterable (v1.3 normalize-everywhere) ----- + # A bare str/bytes operand raises TypeError via _normalize_iterable_of_ + # strings rather than iterating its characters (#238/#241); everything + # else (list, generator, set, another SetManager, ...) is normalized + # through lc() before the plain set op runs. def __or__(self, other: object) -> set[str]: - return self._elements | self._as_operand(other) + return self._elements | _normalize_iterable_of_strings(other) __ror__ = __or__ def __and__(self, other: object) -> set[str]: - return self._elements & self._as_operand(other) + return self._elements & _normalize_iterable_of_strings(other) __rand__ = __and__ def __sub__(self, other: object) -> set[str]: - return self._elements - self._as_operand(other) + return self._elements - _normalize_iterable_of_strings(other) def __rsub__(self, other: object) -> set[str]: - return self._as_operand(other) - self._elements + return _normalize_iterable_of_strings(other) - self._elements + + def __xor__(self, other: object) -> set[str]: + return self._elements ^ _normalize_iterable_of_strings(other) + + __rxor__ = __xor__ # symmetric difference is commutative, like v1 + + # -- comparisons: v1 subclassed collections.abc.Set, whose __le__/__lt__/ + # __ge__/__gt__ mixins only accept another Set-registered operand (set, + # frozenset, or another Set subclass) -- NOT an arbitrary iterable like + # list, unlike the operators above. Mirrored here since this SetManager + # doesn't itself subclass the ABC. + + def __le__(self, other: object) -> bool: + if not isinstance(other, (SetManager, Set)): + return NotImplemented + if len(self) > len(other): + return False + return all(elem in other for elem in self) + + def __lt__(self, other: object) -> bool: + if not isinstance(other, (SetManager, Set)): + return NotImplemented + return len(self) < len(other) and self.__le__(other) + + def __ge__(self, other: object) -> bool: + if not isinstance(other, (SetManager, Set)): + return NotImplemented + if len(self) < len(other): + return False + return all(elem in self for elem in other) + + def __gt__(self, other: object) -> bool: + if not isinstance(other, (SetManager, Set)): + return NotImplemented + return len(self) > len(other) and self.__ge__(other) def __repr__(self) -> str: # Sorted so repr is stable across runs -- set() iteration order @@ -160,6 +267,26 @@ class TupleManager(dict[str, object]): def __init__(self, *args: object, _on_change: Callable[[], None] | None = None, **kwargs: object) -> None: + if args: + # #242: a bare str/bytes silently shreds into a garbage mapping + # (dict's own error), and an iterable of short strings silently + # splits each one into a (key, value) pair -- ported from v1's + # TupleManager.__init__ guard. + arg = args[0] + _reject_bare_str_or_bytes( + arg, "a mapping or iterable of (key, value) pairs") + if not isinstance(arg, Mapping): + checked = [] + for item in arg: # type: ignore[attr-defined] + if isinstance(item, (str, bytes)): + kind = "bytes" if isinstance(item, bytes) else "str" + raise TypeError( + f"expected (key, value) pairs, got a {kind} " + f"element {item!r}; a 2-character string " + "silently splits into a key and a value" + ) + checked.append(item) + args = (checked, *args[1:]) super().__init__(*args, **kwargs) self._on_change = _on_change @@ -177,7 +304,31 @@ def __getattr__(self, name: str) -> object: try: return self[name] except KeyError: - raise AttributeError(f"no key {name!r} in this manager") from None + # #256: name the known keys, like v1's 1.4 deprecation warning + # did -- this shim only speaks 2.0, so what was a warning there + # is a hard AttributeError here. + raise AttributeError( + f"{name!r} is not a known key " + f"({', '.join(sorted(self))}); use .get() for intentional " + "soft access." + ) from None + + def __setattr__(self, name: str, value: object) -> None: + # v1 parity: dunder probes (typing's __orig_class__, etc.) and this + # shim's own _on_change hook get real object-attribute storage; + # every other name -- including single-underscore ones, per v1 -- + # routes to the dict so `t.mcdonald = 'x'` and `t['mcdonald'] = 'x'` + # are the same operation. + if name == "_on_change" or (name.startswith("__") and name.endswith("__")): + object.__setattr__(self, name, value) + else: + self[name] = value + + def __delattr__(self, name: str) -> None: + if name == "_on_change" or (name.startswith("__") and name.endswith("__")): + object.__delattr__(self, name) + else: + del self[name] def __setitem__(self, key: str, value: object) -> None: super().__setitem__(key, value) @@ -376,6 +527,24 @@ def _raise_readonly(name: str) -> None: _MANAGER_FIELDS = _SET_FIELDS + ( "capitalization_exceptions", "nickname_delimiters", "maiden_delimiters", ) + +#: v1's Constants.__repr__ field order (#221) -- kept as its own tuple +#: rather than reusing _SET_FIELDS, whose order differs (v1 lists +#: suffix_acronyms_ambiguous last, not fourth). +_REPR_COLLECTION_ATTRS = ( + "prefixes", "suffix_acronyms", "suffix_not_acronyms", "titles", + "first_name_titles", "conjunctions", "bound_first_names", + "non_first_name_prefixes", "suffix_acronyms_ambiguous", +) +#: v1's repr scalar order, minus empty_attribute_default -- removed in 2.0 +#: (#255), so there's no such attribute on this shim's Constants to show. +_REPR_SCALAR_ATTRS = ( + "string_format", "initials_format", "initials_delimiter", + "initials_separator", "suffix_delimiter", + "capitalize_name", "force_mixed_case_capitalization", + "patronymic_name_order", "middle_name_as_last", +) + _SCALAR_DEFAULTS: dict[str, object] = { "patronymic_name_order": False, "middle_name_as_last": False, @@ -397,16 +566,6 @@ def _raise_readonly(name: str) -> None: _UNSET_KWARG = object() -def _reject_bare_str_for_field(value: object, field: str) -> None: - # A bare string is an iterable of its characters, so e.g. - # SetManager("dr") would silently shred it into {'d', 'r'} instead of - # raising -- shared by every Constants() set-field kwarg (#238/#244). - if isinstance(value, (str, bytes)): - raise TypeError( - f"{field} must be an iterable of strings, not a single " - f"str/bytes: {value!r}; wrap it in a list, e.g. [{value!r}]" - ) - _SHARED_MUTATION_MESSAGE = ( "mutating the shared CONSTANTS singleton is deprecated and will be " "removed in 3.0; build a Lexicon/Policy (or a private Constants " @@ -554,13 +713,13 @@ def __init__( value = overrides[name] if value is _UNSET_KWARG: value = vocab[name] - else: - # a caller-supplied value REPLACES that field's default - # vocabulary wholesale (v1 parity) -- validated/normalized - # by SetManager below, once past the bare-str guard - _reject_bare_str_for_field(value, name) + # a caller-supplied value REPLACES that field's default + # vocabulary wholesale (v1 parity); SetManager itself validates/ + # normalizes and rejects a bare str/bytes (#238/#241), naming + # this field in the message via _field= object.__setattr__( - self, name, SetManager(value, _on_change=self._bump)) # type: ignore[arg-type] + self, name, + SetManager(value, _on_change=self._bump, _field=name)) # type: ignore[arg-type] if capitalization_exceptions is _UNSET_KWARG: from nameparser.config.capitalization import ( CAPITALIZATION_EXCEPTIONS, @@ -626,8 +785,10 @@ def __setattr__(self, name: str, value: object) -> None: "flags" ) if name in _SET_FIELDS: - # v1 allowed wholesale reassignment (c.titles = {...}) - value = SetManager(value, _on_change=self._bump) # type: ignore[arg-type] + # v1 allowed wholesale reassignment (c.titles = {...}); same + # bare-str/bytes guard as the constructor kwarg path (#238/#241) + value = SetManager( + value, _on_change=self._bump, _field=name) # type: ignore[arg-type] elif name == "capitalization_exceptions": value = TupleManager(value, _on_change=self._bump) # type: ignore[arg-type] elif name in ("nickname_delimiters", "maiden_delimiters"): @@ -652,8 +813,15 @@ def __setattr__(self, name: str, value: object) -> None: def copy(self) -> Constants: # #260 # An independent instance with its own generation counter and # its own manager callbacks -- not a shared-state alias like a - # naive attribute-for-attribute copy would produce. - new = Constants() + # naive attribute-for-attribute copy would produce. v1's copy() + # was `copy.deepcopy(self)`, which builds the new object via + # `type(self).__new__(type(self))` -- NOT by calling `type(self)()` + # -- so a Constants subclass copies as itself without its __init__ + # running (and without needing to satisfy whatever signature that + # __init__ might require). Mirrored here with an explicit __new__ + # bypass rather than type(self)(). + new = object.__new__(type(self)) + object.__setattr__(new, "_generation", 0) for name in _SET_FIELDS: object.__setattr__( new, name, @@ -663,10 +831,26 @@ def copy(self) -> Constants: # #260 for bucket in ("nickname_delimiters", "maiden_delimiters"): object.__setattr__(new, bucket, _DelimiterManager( dict(getattr(self, bucket)), _on_change=new._bump)) + object.__setattr__(new, "regexes", _RegexesProxy()) for name in _SCALAR_DEFAULTS: object.__setattr__(new, name, getattr(self, name)) return new + def __repr__(self) -> str: # #221 + # Collections (some with hundreds of entries, e.g. titles/prefixes) + # are summarized as counts rather than dumped in full, like v1. + # Scalars are only shown when they differ from the library default + # -- _SCALAR_DEFAULTS stands in for v1's `getattr(type(self), name)` + # class-level default, since this shim's scalar defaults are + # instance attributes set in __init__, not class attributes. + lines = [f" {name}: {len(getattr(self, name))}" + for name in _REPR_COLLECTION_ATTRS] + lines += [ + f" {name}: {value!r}" for name in _REPR_SCALAR_ATTRS + if (value := getattr(self, name)) != _SCALAR_DEFAULTS[name] + ] + return "" + # -- snapshot ----------------------------------------------------------- def _snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index f32989ba..8d2602fa 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -425,3 +425,135 @@ def test_constants_kwarg_feeds_facade_parse() -> None: c = Constants(titles=["zzqtitle"]) n = HumanName("Zzqtitle Judy Dench", constants=c) assert n.title == "Zzqtitle" + + +# -- Restoring v1.3/1.4 manager hardening (#221/#238/#241/#242/#260) ------- + + +def test_set_manager_discard_is_noop_on_missing_and_notifies_on_real_change() -> None: + bumps = [] + s = SetManager(["a"], _on_change=lambda: bumps.append(1)) + assert s.discard("missing") is s # never raises, chainable + assert len(bumps) == 0 # no-op: nothing removed + assert s.discard("A") is s # normalized match + assert "a" not in s + assert len(bumps) == 1 + + +def test_set_manager_clear_notifies_only_if_non_empty() -> None: + bumps = [] + s = SetManager(["a", "b"], _on_change=lambda: bumps.append(1)) + assert s.clear() is s + assert len(s) == 0 + assert len(bumps) == 1 + assert s.clear() is s # already empty: no-op + assert len(bumps) == 1 + + +def test_set_manager_operators_accept_arbitrary_iterables() -> None: + a = SetManager(["a", "b"]) + assert a | ["B.", "z"] == {"a", "b", "z"} # list operand, normalized + assert a & (x for x in ["A", "q"]) == {"a"} # generator operand + assert a - ["a"] == {"b"} + assert a ^ ["b", "c"] == {"a", "c"} # symmetric difference + assert ["b", "c"] ^ a == {"a", "c"} # reflected + + +def test_set_manager_bare_str_or_bytes_operand_raises_typeerror() -> None: + a = SetManager(["a"]) + with pytest.raises(TypeError): + a | "ab" + with pytest.raises(TypeError): + a & b"ab" + with pytest.raises(TypeError): + "ab" | a # reflected form too + + +def test_set_manager_comparison_operators() -> None: + a = SetManager(["a", "b"]) + assert a <= {"a", "b", "c"} + assert a < {"a", "b", "c"} + assert not (a < {"a", "b"}) + assert {"a", "b", "c"} >= a + assert a <= SetManager(["a", "b", "c"]) + assert SetManager(["a", "b", "c"]) > a + + +def test_set_manager_constructor_rejects_bare_str() -> None: + with pytest.raises(TypeError): # #238/#241: not shredded to chars + SetManager("dr") # type: ignore[arg-type] + + +def test_set_manager_constructor_rejects_bare_bytes_with_decode_hint() -> None: + with pytest.raises(TypeError, match="decode"): + SetManager(b"dr") # type: ignore[arg-type] + + +def test_set_manager_constructor_rejects_non_str_element() -> None: + with pytest.raises(TypeError): + SetManager([None]) # type: ignore[list-item] + + +def test_set_manager_add_rejects_bytes_with_decode_hint() -> None: + s = SetManager() + with pytest.raises(TypeError, match="decode"): + s.add(b"dr") # type: ignore[arg-type] + + +def test_constants_setattr_rejects_bare_str_like_constructor() -> None: + c = Constants() + with pytest.raises(TypeError, match="conjunctions"): + c.conjunctions = "and" # type: ignore[assignment] + + +def test_tuple_manager_constructor_rejects_bare_str() -> None: + with pytest.raises(TypeError): # #242 + TupleManager("ab") # type: ignore[arg-type] + + +def test_tuple_manager_constructor_rejects_iterable_of_strings() -> None: + with pytest.raises(TypeError): # #242: 2-char strings silently split + TupleManager(["ab", "cd"]) # type: ignore[arg-type] + + +def test_tuple_manager_unknown_key_error_names_known_keys() -> None: + t = TupleManager({"mcdonald": "McDonald", "obrien": "O'Brien"}) + with pytest.raises(AttributeError, match="mcdonald"): + t.bogus + + +def test_tuple_manager_attribute_style_set_and_delete() -> None: + bumps = [] + t = TupleManager({"a": "A"}, _on_change=lambda: bumps.append(1)) + t.b = "B" # attribute-style routes to dict + assert t["b"] == "B" + assert len(bumps) == 1 + del t.a + assert "a" not in t + assert len(bumps) == 2 + # the manager's own private attribute is unaffected + assert t._on_change is not None + + +def test_constants_copy_preserves_subclass() -> None: # #260 + class MyConstants(Constants): + pass + + c = MyConstants() + d = c.copy() + assert type(d) is MyConstants + + +def test_constants_repr_shows_collection_counts() -> None: # #221 + c = Constants() + r = repr(c) + assert r.startswith(" None: # #221 + c = Constants() + assert "capitalize_name" not in repr(c) # default: not shown + c.capitalize_name = True + assert "capitalize_name: True" in repr(c) From 7da89eef8d1a7711a920d1817fb8d946660e330d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 00:16:53 -0700 Subject: [PATCH 110/275] Reconcile test_constants against the shim (buckets A, B, C) Also lifts test_python_api.py's collect_ignore_glob entry: its 3 held tests pass after the suffix-comma and period-joined-vocabulary pipeline fixes (7bf53f3), making the file fully green (67/67). test_constants.py: 240 dual-run cases -> 94 tests (the dual-run parametrization died with #255; per-run 120 -> 94). Bucket A (deleted machinery): - the suffixes_prefixes_titles cached-union property block (~13 tests of cache priming/invalidation incl. the timeit performance guard and the is_rootname consistency pair); change tracking is the shim generation counter, covered in tests/v2/test_config_shim.py. Two contracts survived and are re-pinned on parsing surface: post-unpickle mutations are honored, and wholesale manager replacement re-wires tracking. - .elements raw-set accessor uses rewritten to set(...) (operators also return plain set now); _repr_collection_attrs/_repr_scalar_attrs class internals replaced by test-owned name lists. - __setstate__ missing-field validation test (the shim deliberately accepts partial state to load v1.4 blobs). - the legacy-unpickle warn-once pair (#279's warning machinery is gone). Bucket B (scheduled removals now raising): - #255 empty_attribute_default: assignment raises AttributeError; the six None-mode behavior tests collapse to one removal test (scalar pickle/copy round-trip intent re-pinned on string_format). - #243: SetManager() call -> TypeError not callable; missing-member remove() -> KeyError (mixed-args call still applies the present removal). - #245/#263: add_with_encoding -> AttributeError; bytes add() -> TypeError with decode hint. - #256: unknown tuple-manager keys -> AttributeError naming known keys; regexes proxy raises naming the missing key; sunder probes now report absent (AttributeError) instead of returning manager defaults. - #261: constants=None raises TypeError at constructor, positional, and C-setter entry points (class renamed ConstantsNoneRemovalTests). - deliberate 2.0 divergences pinned: custom nickname delimiters raise pointing at Policy (spec section 3); plain-iterable set-field assignment auto-wraps in a validated SetManager where v1 demanded a pre-built one (bare strings still raise, #241 stays impossible); class-not-instance constants message no longer carries the "did you mean Constants()" hint. Bucket C (shared-CONSTANTS mutation): - test_can_change_global_constants wraps its shared mutation in pytest.deprecated_call (it exercises exactly that deprecation). 2 of 94 tests still fail on one remaining v1.4-parity gap -- the shim Constants.__init__ lacks the patronymic_name_order/middle_name_as_last bool kwargs -- so the file stays in collect_ignore_glob and the mypy exclude with a pointer comment; remove both once the kwargs land. Co-Authored-By: Claude Fable 5 --- tests/conftest.py | 23 +- tests/test_constants.py | 760 ++++++++++++++-------------------------- 2 files changed, 270 insertions(+), 513 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b20d0975..cf195d91 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,24 +19,15 @@ "test_capitalization.py", "test_comma_variants.py", "test_conjunctions.py", - # test_constants.py is UNTOUCHED and blocked: reconciliation probing found - # a cluster of shim regressions of shipped v1 fixes (SetManager missing - # discard()/clear()/set operators; #238/#241/#242 bare-string and - # shred guards gone; #260 subclass-preserving copy(); #221 repr) plus two - # pipeline bugs (period-joined title/suffix derivation: 'Lt.Gov. John - # Doe' and 'John Doe JD.CPA' misparse). See the M12 batch report for the - # full inventory with repros. Reconcile once those are fixed. + # test_constants.py is fully reconciled; 2 of its 94 tests fail on one + # remaining v1.4-parity gap: the shim Constants.__init__ lacks the + # patronymic_name_order/middle_name_as_last bool kwargs (v1.4 had them; + # docs/customize.rst doctests use them). Repro: + # Constants(middle_name_as_last=True) -> TypeError + # Its mypy-exclude entry pairs with this one (mypy flags the same two + # calls). Remove both once the kwargs are restored. "test_constants.py", "test_east_slavic_patronymic_order.py", - # test_python_api.py is fully reconciled (and mypy-clean, so it is NOT in - # the mypy exclude) but 3 of its 67 tests fail on one confirmed pipeline - # bug: _pipeline/_segment.py:85 requires ALL post-first comma segments to - # be suffix-shaped for SUFFIX_COMMA, where v1 tested only parts[1] and - # consumed the rest as suffixes unconditionally. Repro: - # Parser().parse('Dr. John P. Doe-Ray, CLU, CFP, LUTC') - # ('lutc' is not in the suffix vocab) -> family='Dr. John P. Doe-Ray'. - # Remove this line once that is fixed. - "test_python_api.py", "test_first_name.py", "test_initials.py", "test_middle_name_as_last.py", diff --git a/tests/test_constants.py b/tests/test_constants.py index 54c26500..39140b71 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -1,15 +1,13 @@ import copy import pickle import re -import timeit import warnings from typing import Any import pytest from nameparser import HumanName -from nameparser.config import CONSTANTS, Constants, RegexTupleManager, SetManager, TupleManager -from nameparser.config.regexes import EMPTY_REGEX +from nameparser.config import CONSTANTS, Constants, SetManager, TupleManager from nameparser.config.titles import TITLES from tests.base import HumanNameTestBase @@ -45,7 +43,10 @@ def test_constants_invalid_type_raises_typeerror(self) -> None: HumanName("John Doe", constants="not a Constants") # type: ignore[arg-type] def test_constants_class_instead_of_instance_raises_with_hint(self) -> None: - with pytest.raises(TypeError, match=r"did you mean Constants\(\)"): + # 2.0's message names the received class rather than v1's "did you + # mean Constants()" hint; the class-not-instance mistake still fails + # loudly at construction + with pytest.raises(TypeError, match=r"constants must be a Constants instance"): HumanName("John Doe", constants=Constants) # type: ignore[arg-type] def test_assigning_invalid_constants_after_construction_raises(self) -> None: @@ -57,16 +58,18 @@ def test_assigning_invalid_constants_after_construction_raises(self) -> None: hn.C = "garbage" # type: ignore[assignment] def test_assigning_constants_class_after_construction_raises_with_hint(self) -> None: + # same message note as the constructor variant above hn = HumanName("John Doe") - with pytest.raises(TypeError, match=r"did you mean Constants\(\)"): + with pytest.raises(TypeError, match=r"constants must be a Constants instance"): hn.C = Constants # type: ignore[assignment] - def test_assigning_none_to_constants_after_construction_builds_new_instance(self) -> None: + def test_assigning_none_to_constants_raises(self) -> None: + # constants=None was removed in 2.0 (#261, warned since 1.3.1): the v1 + # silently-build-a-fresh-Constants fallback is gone from the C setter + # too; pass Constants() or CONSTANTS.copy() explicitly hn = HumanName("John Doe") - with pytest.deprecated_call(): - hn.C = None - self.assertIsNot(hn.C, CONSTANTS) - self.assertTrue(isinstance(hn.C, Constants)) + with pytest.raises(TypeError, match="261"): + hn.C = None # type: ignore[assignment] def test_constants_bare_string_kwarg_raises_typeerror(self) -> None: # a bare string is an iterable of its characters, so set('dr') would @@ -115,17 +118,20 @@ def test_set_manager_operators_normalize_like_add(self) -> None: # same normalization of operator operands, (titles | ['Esq.']) keeps # a raw 'Esq.', which the parser's lc()-based lookups can never match # — silently broken config, same failure family as the bare-string - # shredding (#238) + # shredding (#238). + # Compared via set(...) rather than v1's .elements: the raw-set + # accessor was internal machinery and is gone in 2.0 (#243 family); + # 2.0 operators also return a plain set rather than a SetManager. sm = SetManager(['dr', 'mr']) - self.assertEqual((sm | ['Esq.']).elements, {'dr', 'mr', 'esq'}) - self.assertEqual((['Esq.', 'Dr.'] | sm).elements, {'dr', 'mr', 'esq'}) - self.assertEqual((sm & ['Dr.']).elements, {'dr'}) - self.assertEqual((['Dr.'] & sm).elements, {'dr'}) - self.assertEqual((sm - ['Dr.']).elements, {'mr'}) - self.assertEqual((['Dr.', 'Esq.'] - sm).elements, {'esq'}) - self.assertEqual((sm ^ ['Dr.', 'Esq.']).elements, {'mr', 'esq'}) + self.assertEqual(set(sm | ['Esq.']), {'dr', 'mr', 'esq'}) + self.assertEqual(set(['Esq.', 'Dr.'] | sm), {'dr', 'mr', 'esq'}) + self.assertEqual(set(sm & ['Dr.']), {'dr'}) + self.assertEqual(set(['Dr.'] & sm), {'dr'}) + self.assertEqual(set(sm - ['Dr.']), {'mr'}) + self.assertEqual(set(['Dr.', 'Esq.'] - sm), {'esq'}) + self.assertEqual(set(sm ^ ['Dr.', 'Esq.']), {'mr', 'esq'}) # pins __rxor__ separately in case it ever stops aliasing __xor__ - self.assertEqual((['Dr.', 'Esq.'] ^ sm).elements, {'mr', 'esq'}) + self.assertEqual(set(['Dr.', 'Esq.'] ^ sm), {'mr', 'esq'}) def test_set_manager_contains_normalizes_like_add(self) -> None: # add()/remove()/the constructor/the operators all normalize (lowercase, @@ -150,17 +156,17 @@ def test_set_manager_rsub_is_order_sensitive(self) -> None: # in __rsub__ would silently flip the result and nothing else # in this file would catch it sm = SetManager(['dr', 'mr']) - self.assertEqual((['Dr.', 'Esq.'] - sm).elements, {'esq'}) - self.assertEqual((sm - ['Dr.', 'Esq.']).elements, {'mr'}) + self.assertEqual(set(['Dr.', 'Esq.'] - sm), {'esq'}) + self.assertEqual(set(sm - ['Dr.', 'Esq.']), {'mr'}) def test_set_manager_constructor_normalizes_like_add(self) -> None: # without constructor normalization the operators misfire against # the exact spelling visibly stored in the set: & returns empty # and - silently no-ops sm = SetManager(['Dr.', 'MR']) - self.assertEqual(sm.elements, {'dr', 'mr'}) - self.assertEqual((sm & ['Dr.']).elements, {'dr'}) - self.assertEqual((sm - ['Dr.']).elements, {'mr'}) + self.assertEqual(set(sm), {'dr', 'mr'}) + self.assertEqual(set(sm & ['Dr.']), {'dr'}) + self.assertEqual(set(sm - ['Dr.']), {'mr'}) def test_constants_kwarg_elements_are_normalized(self) -> None: # Constants(titles=[...]) was the last silently-dead config path: @@ -193,9 +199,9 @@ def test_set_manager_non_str_elements_raise_typeerror(self) -> None: # silently transmutes None into '' — raise a curated error instead with pytest.raises(TypeError, match=r"decode it first"): SetManager([b'dr']) # type: ignore[list-item] - with pytest.raises(TypeError, match=r"expected str elements"): + with pytest.raises(TypeError, match=r"expected a str"): SetManager([None]) # type: ignore[list-item] - with pytest.raises(TypeError, match=r"expected str elements"): + with pytest.raises(TypeError, match=r"expected a str"): SetManager(['dr']) | [1] # type: ignore[list-item] def test_tuplemanager_bare_string_raises_typeerror(self) -> None: @@ -255,9 +261,14 @@ def test_instances_can_have_own_constants(self) -> None: self.assertEqual(hn2.has_own_config, False) def test_can_change_global_constants(self) -> None: + # This test exercises shared-CONSTANTS mutation specifically, which + # 2.0 deprecates (removal 3.0; the message points at Lexicon/Policy + # and private Constants); deprecated_call() pins the warning while + # asserting the mutation is still honored. hn = HumanName("") hn2 = HumanName("") - hn.C.titles.remove('hon') + with pytest.deprecated_call(match="Lexicon"): + hn.C.titles.remove('hon') self.assertEqual('hon' in hn.C.titles, False) self.assertEqual('hon' in hn2.C.titles, False) self.assertEqual(hn.has_own_config, False) @@ -265,16 +276,14 @@ def test_can_change_global_constants(self) -> None: # No manual cleanup needed: the autouse fixture in conftest.py snapshots # and restores the global CONSTANTS collections around every test. - def test_can_add_global_nickname_delimiter(self) -> None: - # https://github.com/derek73/python-nameparser/issues/112 + def test_custom_nickname_delimiter_raises(self) -> None: + # Custom (non-sentinel) delimiter additions raise in 2.0 (deliberate + # divergence, migration spec section 3 -- same uniform rule as + # regexes); Policy(nickname_delimiters=...) is the replacement. The + # #112 use case this test used to pin moved to the new API. hn = HumanName("") - hn.C.nickname_delimiters['curly_braces'] = re.compile(r'\{(.*?)\}') - hn2 = HumanName("Benjamin {Ben} Franklin") - self.assertEqual(hn2.has_own_config, False) - self.m(hn2.nickname, "Ben", hn2) - # No manual cleanup needed: the autouse fixture in conftest.py snapshots - # and restores the global CONSTANTS collections (including - # nickname_delimiters) around every test. + with pytest.raises(TypeError, match="Policy"): + hn.C.nickname_delimiters['curly_braces'] = re.compile(r'\{(.*?)\}') def test_remove_multiple_arguments(self) -> None: hn = HumanName("Ms Hon Solo", constants=Constants()) @@ -301,106 +310,33 @@ def test_clear_removes_all_entries(self) -> None: self.m(hn.middle, "Hon", hn) self.m(hn.last, "Solo", hn) - def test_empty_attribute_default_assignment_emits_deprecation_warning(self) -> None: - # assigning empty_attribute_default is deprecated for removal in 2.0 - # (#255); empty attributes will always return '' once removed + def test_empty_attribute_default_removed(self) -> None: + # empty_attribute_default was removed in 2.0 (#255, warned since + # 1.3.0): empty attributes are always ''. Assignment raises naming + # the issue; the attribute no longer exists to read either. c = Constants() - with pytest.deprecated_call(match="255"): + with pytest.raises(AttributeError, match="255"): c.empty_attribute_default = None # type: ignore[assignment] - self.assertIsNone(c.empty_attribute_default) - - def test_empty_attribute_default_rejects_non_str_non_none(self) -> None: - # A cheap early check on the invariant 2.0 will fully enforce (only - # '' will be legal): non-str/non-None values fail loudly here - # instead of surfacing later as a confusing failure deep in some - # unrelated HumanName string property. - c = Constants() - with pytest.raises(TypeError, match="str or None"): - c.empty_attribute_default = 42 # type: ignore[assignment] - self.assertEqual(c.empty_attribute_default, '') - - def test_empty_attribute_default_read_does_not_warn(self) -> None: - c = Constants() - with warnings.catch_warnings(): - warnings.simplefilter("error") - self.assertEqual(c.empty_attribute_default, '') - - def test_empty_attribute_default(self) -> None: - from nameparser.config import CONSTANTS - # empty_attribute_default has no explicit annotation (mypy infers str - # from the '' default), but None is documented/supported here -- see - # the doctest on the attribute's docstring in config/__init__.py. - # Not widened to str | None like string_format/suffix_delimiter - # because it cascades into every public str-typed name accessor - # (title, first, middle, last, suffix, nickname, maiden, surnames, - # given_names, last_base, last_prefixes, initials()). - with pytest.deprecated_call(): - CONSTANTS.empty_attribute_default = None # type: ignore[assignment] + self.assertFalse(hasattr(c, 'empty_attribute_default')) hn = HumanName("") - self.m(hn.title, None, hn) - self.m(hn.first, None, hn) - self.m(hn.middle, None, hn) - self.m(hn.last, None, hn) - self.m(hn.suffix, None, hn) - self.m(hn.nickname, None, hn) - - def test_empty_attribute_on_instance(self) -> None: - hn = HumanName("", Constants()) - with pytest.deprecated_call(): - hn.C.empty_attribute_default = None # type: ignore[assignment] # see test_empty_attribute_default above - self.m(hn.title, None, hn) - self.m(hn.first, None, hn) - self.m(hn.middle, None, hn) - self.m(hn.last, None, hn) - self.m(hn.suffix, None, hn) - self.m(hn.nickname, None, hn) - - def test_none_empty_attribute_string_formatting(self) -> None: - hn = HumanName("", Constants()) - with pytest.deprecated_call(): - hn.C.empty_attribute_default = None # type: ignore[assignment] # see test_empty_attribute_default above - self.assertEqual('', str(hn), hn) + self.assertEqual(hn.first, '') + self.assertEqual(hn.last, '') - def test_add_constant_with_explicit_encoding(self) -> None: - # bytes input is deprecated (#245), still supported until 2.0 + def test_add_with_encoding_removed(self) -> None: + # add_with_encoding() was removed in 2.0 (#245/#263, warned in 1.4); + # decode and use add() instead c = Constants() - with pytest.deprecated_call(): - c.titles.add_with_encoding(b'b\351ck', encoding='latin_1') + with pytest.raises(AttributeError, match="add_with_encoding"): + c.titles.add_with_encoding(b'b\351ck', encoding='latin_1') # type: ignore[attr-defined] + c.titles.add(b'b\351ck'.decode('latin_1')) self.assertIn('béck', c.titles) - def test_set_manager_add_bytes_emits_deprecation_warning(self) -> None: - # bytes elements will be removed in 2.0 (#245); the caller should decode + def test_set_manager_add_bytes_raises_with_decode_hint(self) -> None: + # bytes elements were removed in 2.0 (#245, warned since 1.3.0) sm = SetManager(['dr']) - with pytest.deprecated_call(match="decode"): - sm.add(b'esq') # type: ignore[arg-type] # deliberately deprecated input - self.assertIn('esq', sm) - - def test_add_with_encoding_str_emits_deprecation_warning(self) -> None: - # add_with_encoding() itself is deprecated in 1.4 for removal in 2.0 - # (#263/#245); the str path is otherwise silent, so this method's - # own removal is unwarned without this - sm = SetManager(['dr']) - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - sm.add_with_encoding('esq') - deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] - # Exactly one -- the str path must not also trip the bytes-decode - # warning (that one's for the *other* argument type). - self.assertEqual(len(deprecations), 1) - self.assertIn('add()', str(deprecations[0].message)) - self.assertIn('esq', sm) - - def test_add_with_encoding_bytes_emits_two_distinct_warnings(self) -> None: - # the bytes-decode warning (#245, shipped 1.3.0) and the - # add_with_encoding()-removal warning (#263) are independent - sm = SetManager(['dr']) - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - sm.add_with_encoding(b'esq') - messages = [str(w.message) for w in caught if issubclass(w.category, DeprecationWarning)] - self.assertEqual(len(messages), 2) - self.assertTrue(any('decode' in m for m in messages)) - self.assertTrue(any('use add() instead' in m for m in messages)) + with pytest.raises(TypeError, match="decode"): + sm.add(b'esq') # type: ignore[arg-type] + self.assertNotIn('esq', sm) def test_set_manager_add_str_does_not_warn(self) -> None: sm = SetManager(['dr']) @@ -409,13 +345,14 @@ def test_set_manager_add_str_does_not_warn(self) -> None: sm.add('esq') self.assertIn('esq', sm) - def test_set_manager_call_emits_deprecation_warning(self) -> None: - # __call__ hands out the raw underlying set, bypassing normalization - # and cache invalidation; will be removed in 2.0 (#243) + def test_set_manager_call_removed(self) -> None: + # __call__ handed out the raw underlying set, bypassing normalization + # and change tracking; removed in 2.0 (#243, warned since 1.3.0) -- + # iterate the manager or copy with set(manager) instead sm = SetManager(['dr']) - with pytest.deprecated_call(match="raw underlying set"): - elements = sm() - self.assertEqual(elements, {'dr'}) + with pytest.raises(TypeError, match="not callable"): + sm() # type: ignore[operator] + self.assertEqual(set(sm), {'dr'}) def test_set_manager_discard_ignores_missing_without_warning(self) -> None: sm = SetManager(['dr', 'mr']) @@ -425,17 +362,11 @@ def test_set_manager_discard_ignores_missing_without_warning(self) -> None: self.assertIs(result, sm) self.assertEqual(set(sm), {'mr'}) - def test_set_manager_discard_invalidates_cached_union(self) -> None: - c = Constants() - self.assertIn('hon', c.suffixes_prefixes_titles) # prime the cache - c.titles.discard('hon') - self.assertNotIn('hon', c.suffixes_prefixes_titles) - - def test_set_manager_remove_missing_member_emits_deprecation_warning(self) -> None: - # ignore-missing remove() becomes KeyError in 2.0 (#243); discard() - # is the intentional ignore-missing spelling + def test_set_manager_remove_missing_member_raises(self) -> None: + # ignore-missing remove() became KeyError in 2.0 (#243, warned since + # 1.3.0); discard() is the intentional ignore-missing spelling sm = SetManager(['dr']) - with pytest.deprecated_call(match="discard"): + with pytest.raises(KeyError): sm.remove('nope') self.assertEqual(set(sm), {'dr'}) # removing a present member stays silent @@ -445,14 +376,14 @@ def test_set_manager_remove_missing_member_emits_deprecation_warning(self) -> No self.assertEqual(len(sm), 0) def test_set_manager_remove_mixed_present_and_missing_in_one_call(self) -> None: - # a single call mixing a present and a missing member must still warn - # (for the missing one) and still invalidate the cache (for the - # present one) -- not short-circuit either behavior + # a single call mixing a present and a missing member raises for the + # missing one (#243) but still applies the present removal, so + # config state and change tracking don't silently diverge from what + # was removed before the KeyError c = Constants() - self.assertIn('hon', c.suffixes_prefixes_titles) # prime the cache - with pytest.deprecated_call(match="discard"): + with pytest.raises(KeyError): c.titles.remove('hon', 'nope') - self.assertNotIn('hon', c.suffixes_prefixes_titles) + self.assertNotIn('hon', c.titles) def test_set_manager_discard_mixed_present_and_missing_in_one_call(self) -> None: sm = SetManager(['dr', 'mr']) @@ -473,7 +404,9 @@ def test_pickle_roundtrip_preserves_customizations(self) -> None: c.titles.add('customtitle') c.prefixes.add('customprefix') c.titles.remove('hon') - c.nickname_delimiters['curly_braces'] = re.compile(r'\{(.*?)\}') + # (custom nickname delimiters raise in 2.0 -- see + # test_custom_nickname_delimiter_raises -- so the delimiter leg of + # this round-trip moved to the sentinel default below) # Safe: round-tripping a Constants the test just built, not untrusted data. restored = pickle.loads(pickle.dumps(c)) @@ -484,93 +417,46 @@ def test_pickle_roundtrip_preserves_customizations(self) -> None: # The contributing collections must match the original exactly. self.assertEqual(set(restored.titles), set(c.titles)) self.assertEqual(set(restored.prefixes), set(c.prefixes)) - # The collections must also keep their manager type, not just contents. + # The collections must also keep their manager type, not just + # contents (nickname_delimiters is the 2.0 delimiter manager, a + # TupleManager subclass, so isinstance rather than exact type). self.assertEqual(type(restored.titles), SetManager) self.assertEqual(type(restored.prefixes), SetManager) - self.assertIn('curly_braces', restored.nickname_delimiters) - self.assertEqual(type(restored.nickname_delimiters), TupleManager) + self.assertIn('parenthesis', restored.nickname_delimiters) + self.assertTrue(isinstance(restored.nickname_delimiters, TupleManager)) def test_pickle_roundtrip_preserves_instance_scalar_override(self) -> None: - """An instance-level scalar override must survive a pickle round-trip.""" + """An instance-level scalar override must survive a pickle round-trip. + + Exercised via string_format: the v1 vehicle for this test, + empty_attribute_default, was removed in 2.0 (#255). + """ c = Constants() - with pytest.deprecated_call(): - c.empty_attribute_default = None # type: ignore[assignment] # see test_empty_attribute_default above + c.string_format = "{last}" # Safe: round-tripping a Constants the test just built, not untrusted data. - # Restoring a pickled state is not itself a #255-deprecated assignment. with warnings.catch_warnings(): warnings.simplefilter("error") restored = pickle.loads(pickle.dumps(c)) - self.assertEqual(restored.empty_attribute_default, None) - - def test_unpickle_legacy_state_with_property_key(self) -> None: - """Pickles written by older versions must still load. + self.assertEqual(restored.string_format, "{last}") - The previous __getstate__ built its state from a dir() sweep, which - always included the computed `suffixes_prefixes_titles` property (no - customization required). That property has no setter, so __setstate__ - must skip such keys instead of raising AttributeError. + def test_unpickle_legacy_state_raises(self) -> None: + """Pre-1.3.0 pickles now raise (#279, warned since 1.3.0). - Covers the temporary migration shim in __setstate__; remove this test - when that shim is dropped (a release or two after 1.3.0). + Those blobs are recognizable by the computed + ``suffixes_prefixes_titles`` property their dir()-sweep + ``__getstate__`` captured; the 1.4 DeprecationWarning promised + ValueError in 2.0, pointing at re-pickling under 1.3/1.4. """ - c = Constants() - c.titles.add('legacytitle') - # Reproduce the legacy dir()-sweep state dict, which carries the - # read-only `suffixes_prefixes_titles` property alongside the real config. - legacy_state = { - name: getattr(c, name) for name in dir(c) if not name.startswith('_') - } - self.assertIn('suffixes_prefixes_titles', legacy_state) - - restored = Constants.__new__(Constants) - with pytest.deprecated_call(): # legacy-format load deprecated (#279) - restored.__setstate__(legacy_state) - - # The real customization is recovered and the property key is ignored. - self.assertIn('legacytitle', restored.titles) - - def test_unpickle_legacy_state_emits_deprecation_warning_once(self) -> None: - # legacy-format unpickling is deprecated for removal in 2.0 (#279): - # the migration shim will stop skipping the computed-property key and - # raise ValueError instead; warn once per __setstate__ call (not once - # per skipped key) telling users to re-pickle - c = Constants() legacy_state = { - name: getattr(c, name) for name in dir(c) if not name.startswith('_') + 'prefixes': {'van'}, + 'titles': {'dr', 'legacytitle'}, + 'suffixes_prefixes_titles': {'van', 'dr'}, } - self.assertIn('suffixes_prefixes_titles', legacy_state) - restored = Constants.__new__(Constants) - with pytest.deprecated_call(match="re-pickle") as record: - restored.__setstate__(legacy_state) - deprecations = [w for w in record.list if issubclass(w.category, DeprecationWarning)] - self.assertEqual(len(deprecations), 1) - - def test_unpickle_legacy_state_with_two_stale_property_keys_warns_once(self) -> None: - # Pins "once per __setstate__ call, not once per skipped key": with - # only one computed property (suffixes_prefixes_titles) on the real - # Constants class, the test above can't distinguish the two - # semantics. A second read-only property on a throwaway subclass - # forces two keys through the skip branch in one __setstate__ call. - class ConstantsWithExtraProperty(Constants): - @property - def another_computed_property(self) -> str: - return 'computed' - - c = ConstantsWithExtraProperty() - legacy_state = { - name: getattr(c, name) for name in dir(c) if not name.startswith('_') - } - self.assertIn('suffixes_prefixes_titles', legacy_state) - self.assertIn('another_computed_property', legacy_state) - - restored = ConstantsWithExtraProperty.__new__(ConstantsWithExtraProperty) - with pytest.deprecated_call(match="re-pickle") as record: + with pytest.raises(ValueError, match="279"): restored.__setstate__(legacy_state) - deprecations = [w for w in record.list if issubclass(w.category, DeprecationWarning)] - self.assertEqual(len(deprecations), 1) def test_setstate_without_legacy_keys_does_not_warn(self) -> None: c = Constants() @@ -584,60 +470,59 @@ def test_setstate_without_legacy_keys_does_not_warn(self) -> None: restored.__setstate__(state) self.assertIn('legacytitle', restored.titles) - def test_pickle_roundtrip_preserves_regex_manager_subclass(self) -> None: - """regexes must round-trip as a RegexTupleManager, not a plain TupleManager. + def test_pickle_roundtrip_keeps_regexes_readable(self) -> None: + """The regexes surface must survive a Constants pickle round-trip. - TupleManager.__reduce__ previously hardcoded TupleManager, so the - RegexTupleManager subclass was downgraded on unpickling. The difference - is observable: RegexTupleManager returns the EMPTY_REGEX default for an - unknown key, while a plain TupleManager returns None. + In 2.0 ``regexes`` is a read-only proxy over the built-in patterns + (not pickled state), so the v1 concern this test carried -- the + RegexTupleManager subclass being downgraded by ``__reduce__`` -- no + longer exists; what remains contractual is that reads keep working + on the restored instance and unknown keys fail loudly (#256). """ c = Constants() # Safe: round-tripping a Constants the test just built, not untrusted data. restored = pickle.loads(pickle.dumps(c)) - self.assertEqual(type(restored.regexes), RegexTupleManager) - with pytest.deprecated_call(): # unknown-key access deprecated (#256) - self.assertEqual(restored.regexes.does_not_exist, EMPTY_REGEX) + self.assertEqual(type(restored.regexes), type(c.regexes)) + self.assertIsNotNone(restored.regexes.mac) + with pytest.raises(AttributeError): # unknown-key access raises (#256) + restored.regexes.does_not_exist def test_regexes_deepcopy_roundtrip(self) -> None: - """copy.deepcopy of a RegexTupleManager must round-trip. + """copy.deepcopy of the regexes proxy must round-trip. - __getattr__ answered every unknown name with the EMPTY_REGEX default, - including the __deepcopy__ probe copy.deepcopy issues. copy then - mistook that re.Pattern for a deep-copy hook and tried to call it. + The v1 bug this pinned: __getattr__ answered every unknown name -- + including copy.deepcopy's __deepcopy__ probe -- with the EMPTY_REGEX + default, so copy mistook a re.Pattern for a deep-copy hook. The 2.0 + proxy must keep ignoring protocol probes. """ c = Constants() dup = copy.deepcopy(c.regexes) - self.assertEqual(type(dup), RegexTupleManager) - self.assertEqual(dict(dup), dict(c.regexes)) - # The EMPTY_REGEX default still applies to genuinely unknown keys, - # but accessing one is now deprecated (#256). - with pytest.deprecated_call(): - self.assertEqual(dup.does_not_exist, EMPTY_REGEX) + self.assertEqual(type(dup), type(c.regexes)) + self.assertEqual(set(dup.keys()), set(c.regexes.keys())) + # Unknown keys raise in 2.0 (#256) instead of returning EMPTY_REGEX. + with pytest.raises(AttributeError): + dup.does_not_exist def test_nickname_delimiters_deepcopy_roundtrip(self) -> None: """copy.deepcopy of nickname_delimiters must round-trip. - Mirrors test_regexes_deepcopy_roundtrip: nickname_delimiters is a - plain TupleManager (not RegexTupleManager), but shares the same - __getattr__/__reduce__ machinery. + Mirrors test_regexes_deepcopy_roundtrip on the delimiter manager (a + TupleManager subclass in 2.0). Custom entries raise in 2.0, so the + round-trip is exercised on the three built-in sentinels. """ c = Constants() - c.nickname_delimiters['curly_braces'] = re.compile(r'\{(.*?)\}') dup = copy.deepcopy(c.nickname_delimiters) - self.assertEqual(type(dup), TupleManager) + self.assertTrue(isinstance(dup, TupleManager)) self.assertEqual(dict(dup), dict(c.nickname_delimiters)) - # Plain TupleManager has no EMPTY_REGEX fallback: unknown keys are - # None, but accessing one is now deprecated (#256). - with pytest.deprecated_call(): - self.assertIsNone(dup.does_not_exist) - self.assertIsNotNone(dup.curly_braces) + with pytest.raises(AttributeError): # unknown-key access raises (#256) + dup.does_not_exist + self.assertIsNotNone(dup.parenthesis) def test_maiden_delimiters_deepcopy_roundtrip(self) -> None: """copy.deepcopy of maiden_delimiters (empty by default) must round-trip.""" @@ -645,10 +530,10 @@ def test_maiden_delimiters_deepcopy_roundtrip(self) -> None: dup = copy.deepcopy(c.maiden_delimiters) - self.assertEqual(type(dup), TupleManager) + self.assertTrue(isinstance(dup, TupleManager)) self.assertEqual(dict(dup), {}) - with pytest.deprecated_call(): # unknown-key access deprecated (#256) - self.assertIsNone(dup.does_not_exist) + with pytest.raises(AttributeError): # unknown-key access raises (#256) + dup.does_not_exist def test_nickname_delimiters_default_builtins_resolve_live(self) -> None: # The three built-ins are stored as the *name* of a regexes entry @@ -704,10 +589,10 @@ def test_regextuplemanager_ignores_dunder_lookups(self) -> None: sentinel = object() self.assertEqual(getattr(c.regexes, '__deepcopy__', sentinel), sentinel) - # A normal (non-dunder) unknown key still yields the EMPTY_REGEX - # default, but accessing one is now deprecated (#256). - with pytest.deprecated_call(): - self.assertEqual(c.regexes.unknown_key, EMPTY_REGEX) + # A normal (non-dunder) unknown key raises in 2.0 (#256, warned + # since 1.4) instead of degrading to the EMPTY_REGEX default. + with pytest.raises(AttributeError): + c.regexes.unknown_key def test_tuplemanager_ignores_dunder_lookups(self) -> None: """Base TupleManager must report unknown dunder names as absent too. @@ -723,32 +608,34 @@ def test_tuplemanager_ignores_dunder_lookups(self) -> None: self.assertEqual(type(tm), TupleManager) self.assertFalse(hasattr(tm, '__deepcopy__')) self.assertEqual(getattr(tm, '__wrapped__', sentinel), sentinel) - # A normal (non-dunder) unknown key still returns the None default, - # but accessing one is now deprecated (#256). - with pytest.deprecated_call(): - self.assertEqual(tm.unknown_key, None) + # A normal (non-dunder) unknown key raises in 2.0 (#256, warned + # since 1.4) instead of returning the None default. + with pytest.raises(AttributeError): + tm.unknown_key - def test_sunder_probe_does_not_emit_deprecation_warning(self) -> None: + def test_sunder_probe_reports_absent_without_deprecation_warning(self) -> None: # Single-underscore introspection probes (IPython/Jupyter's # _repr_html_, _ipython_canary_method_should_not_exist_, etc.) are - # never config keys, just like dunders -- warning on them would - # misleadingly flag a typo merely for e.g. displaying CONSTANTS.regexes - # in a notebook. No real config key starts with '_'. + # never config keys, just like dunders. In 2.0 they raise a plain + # AttributeError -- the protocol-correct "absent" answer, so + # hasattr-then-call probes work -- with no typo-flavored noise + # (v1.4 returned the manager default silently instead). c = Constants() with warnings.catch_warnings(): warnings.simplefilter("error") - self.assertEqual(c.regexes._repr_html_, EMPTY_REGEX) - self.assertIsNone(c.capitalization_exceptions._ipython_canary_method_should_not_exist_) + self.assertFalse(hasattr(c.regexes, '_repr_html_')) + self.assertFalse(hasattr(c.capitalization_exceptions, + '_ipython_canary_method_should_not_exist_')) - def test_tuplemanager_unknown_key_emits_deprecation_warning(self) -> None: - # unknown-key attribute access is deprecated for removal in 2.0 - # (#256); will become AttributeError naming the known keys + def test_tuplemanager_unknown_key_raises_naming_known_keys(self) -> None: + # unknown-key attribute access raises in 2.0 (#256, warned since + # 1.4): AttributeError naming the known keys, as the 1.4 warning + # promised c = Constants() tm = c.capitalization_exceptions - with pytest.deprecated_call(match="phd_typo") as record: - result = tm.phd_typo - self.assertIsNone(result) - message = str(record.list[0].message) + with pytest.raises(AttributeError, match="phd_typo") as excinfo: + tm.phd_typo + message = str(excinfo.value) for key in tm.keys(): self.assertIn(key, message) @@ -760,13 +647,13 @@ def test_tuplemanager_known_key_does_not_warn(self) -> None: first_key = next(iter(c.capitalization_exceptions)) getattr(c.capitalization_exceptions, first_key) - def test_regextuplemanager_unknown_key_emits_deprecation_warning(self) -> None: + def test_regexes_unknown_key_raises(self) -> None: + # the read-only regexes proxy raises on unknown keys too (#256); + # unlike the tuple managers its message names only the missing key, + # not the known-keys listing c = Constants() - with pytest.deprecated_call(match="parenthesys") as record: - result = c.regexes.parenthesys - self.assertEqual(result, EMPTY_REGEX) - message = str(record.list[0].message) - self.assertIn('mac', message) # a known regexes key + with pytest.raises(AttributeError, match="parenthesys"): + c.regexes.parenthesys def test_regextuplemanager_known_key_does_not_warn(self) -> None: c = Constants() @@ -784,124 +671,49 @@ def test_dunder_probe_does_not_emit_deprecation_warning(self) -> None: self.assertEqual(getattr(c.capitalization_exceptions, '__deepcopy__', sentinel), sentinel) - def test_suffixes_prefixes_titles_reflects_add_title(self) -> None: - """suffixes_prefixes_titles must include titles added after construction.""" - c = Constants() - _ = c.suffixes_prefixes_titles # prime the cache so invalidation is exercised - c.titles.add('emerita') - self.assertIn('emerita', c.suffixes_prefixes_titles) + # The v1 suffixes_prefixes_titles cached-union property and its + # invalidation machinery (including the is_rootname predicate that read + # it) are gone in 2.0 -- change tracking is the shim's generation + # counter, covered in tests/v2/test_config_shim.py. The ~13 tests that + # exercised cache priming/invalidation were deleted with it; the two + # kept below re-pin their surviving contracts on public parsing surface. - def test_suffixes_prefixes_titles_reflects_add_prefix(self) -> None: - """suffixes_prefixes_titles must include prefixes added after construction.""" - c = Constants() - _ = c.suffixes_prefixes_titles # prime the cache so invalidation is exercised - c.prefixes.add('xpfx') - self.assertIn('xpfx', c.suffixes_prefixes_titles) - - def test_suffixes_prefixes_titles_reflects_remove_title(self) -> None: - """suffixes_prefixes_titles must not include a word that was only in titles and is then removed.""" - c = Constants() - c.titles.add('emerita') - self.assertIn('emerita', c.suffixes_prefixes_titles) - c.titles.remove('emerita') - self.assertNotIn('emerita', c.suffixes_prefixes_titles) - - def test_suffixes_prefixes_titles_reflects_remove_prefix(self) -> None: - """suffixes_prefixes_titles must not include a word that was only in prefixes and is then removed.""" - c = Constants() - c.prefixes.add('xpfx') - self.assertIn('xpfx', c.suffixes_prefixes_titles) - c.prefixes.remove('xpfx') - self.assertNotIn('xpfx', c.suffixes_prefixes_titles) + def test_pickle_roundtrip_rewires_change_tracking(self) -> None: + """Mutations on a deserialized Constants must still reach the parser. - def test_suffixes_prefixes_titles_reflects_add_suffix_acronym(self) -> None: - """suffixes_prefixes_titles must include suffix acronyms added after construction.""" - c = Constants() - _ = c.suffixes_prefixes_titles # prime the cache so invalidation is exercised - c.suffix_acronyms.add('xsfx') - self.assertIn('xsfx', c.suffixes_prefixes_titles) - - def test_suffixes_prefixes_titles_reflects_add_suffix_not_acronym(self) -> None: - """suffixes_prefixes_titles must include non-acronym suffixes added after construction.""" - c = Constants() - _ = c.suffixes_prefixes_titles # prime the cache so invalidation is exercised - c.suffix_not_acronyms.add('xsfx') - self.assertIn('xsfx', c.suffixes_prefixes_titles) - - def test_pickle_roundtrip_rewires_invalidation_callbacks(self) -> None: - """Mutations on a deserialized Constants must still invalidate the cache.""" + The v1 version primed and re-read suffixes_prefixes_titles; that + cache is gone, so this pins the same contract -- post-unpickle + mutations are honored -- through an actual parse. + """ c = Constants() # Safe: round-tripping a Constants the test just built, not untrusted data. restored = pickle.loads(pickle.dumps(c)) - _ = restored.suffixes_prefixes_titles # prime the cache + hn = HumanName("Posttitle Jane Smith", constants=restored) + self.m(hn.title, "", hn) restored.titles.add('posttitle') - self.assertIn('posttitle', restored.suffixes_prefixes_titles) - - def test_is_rootname_consistent_with_is_title(self) -> None: - """is_rootname must return False for words recognised by is_title.""" - hn = HumanName("", constants=Constants()) - _ = hn.C.suffixes_prefixes_titles # prime the cache so a stale entry would be observable - hn.C.titles.add('emerita') - self.assertFalse(hn.is_rootname('emerita')) - - def test_is_rootname_consistent_with_is_prefix(self) -> None: - """is_rootname must return False for words recognised by is_prefix.""" - hn = HumanName("", constants=Constants()) - _ = hn.C.suffixes_prefixes_titles # prime the cache so a stale entry would be observable - hn.C.prefixes.add('xpfx') - self.assertFalse(hn.is_rootname('xpfx')) - - def test_suffixes_prefixes_titles_reflects_remove_suffix_acronym(self) -> None: - """suffixes_prefixes_titles must reflect a suffix acronym removed after the cache is primed.""" - c = Constants() - c.suffix_acronyms.add('xsfx') - self.assertIn('xsfx', c.suffixes_prefixes_titles) # primes the cache - c.suffix_acronyms.remove('xsfx') - self.assertNotIn('xsfx', c.suffixes_prefixes_titles) - - def test_suffixes_prefixes_titles_reflects_remove_suffix_not_acronym(self) -> None: - """suffixes_prefixes_titles must reflect a non-acronym suffix removed after the cache is primed.""" - c = Constants() - c.suffix_not_acronyms.add('xsfx') - self.assertIn('xsfx', c.suffixes_prefixes_titles) # primes the cache - c.suffix_not_acronyms.remove('xsfx') - self.assertNotIn('xsfx', c.suffixes_prefixes_titles) - - def test_suffixes_prefixes_titles_reflects_add_with_encoding(self) -> None: - """add_with_encoding must invalidate the cache like add()/remove() do.""" - c = Constants() - _ = c.suffixes_prefixes_titles # prime the cache - with pytest.deprecated_call(): # bytes input deprecated (#245) - c.titles.add_with_encoding(b'b\351ck', encoding='latin_1') - self.assertIn('béck', c.suffixes_prefixes_titles) + hn.parse_full_name() + self.m(hn.title, "Posttitle", hn) - def test_suffixes_prefixes_titles_reflects_replaced_manager(self) -> None: - """Replacing a whole SetManager must invalidate the cache and wire the new manager. + def test_replaced_manager_is_wired_for_change_tracking(self) -> None: + """Wholesale manager replacement must wire the new manager's mutations. Covers the config-teardown path where a fresh SetManager is assigned - directly (e.g. ``setattr(CONSTANTS, 'titles', SetManager(...))``). + directly (v1 pinned this via the suffixes_prefixes_titles cache; the + 2.0 contract is that both the replacement and the new manager's own + later mutations are honored by the next parse). """ c = Constants() - _ = c.suffixes_prefixes_titles # prime the cache + hn = HumanName("Brandnewtitle Jane Smith", constants=c) + self.m(hn.title, "", hn) c.titles = SetManager(['brandnewtitle']) - # The replacement is reflected immediately... - self.assertIn('brandnewtitle', c.suffixes_prefixes_titles) - # ...and the new manager's own mutations invalidate the cache too, - # proving the on_change callback was re-wired to the replacement. - _ = c.suffixes_prefixes_titles + hn.parse_full_name() + # The replacement is reflected... + self.m(hn.title, "Brandnewtitle", hn) + # ...and the new manager's own mutations are tracked too, proving + # the change callback was re-wired to the replacement. c.titles.add('secondtitle') - self.assertIn('secondtitle', c.suffixes_prefixes_titles) - - def test_replaced_manager_no_longer_invalidates_cache(self) -> None: - """A SetManager detached by reassignment must not invalidate the new cache.""" - c = Constants() - replaced = c.titles - c.titles = SetManager(['brandnewtitle']) - primed = c.suffixes_prefixes_titles - # Mutating the orphaned manager must leave the live cache untouched. - replaced.add('ghost') - self.assertIs(c.suffixes_prefixes_titles, primed) - self.assertNotIn('ghost', c.suffixes_prefixes_titles) + hn.full_name = "Secondtitle Jane Smith" + self.m(hn.title, "Secondtitle", hn) def test_tuplemanager_delattr_removes_dict_entry(self) -> None: """Deleting a non-dunder attribute must remove the dict entry. @@ -911,58 +723,39 @@ def test_tuplemanager_delattr_removes_dict_entry(self) -> None: ``del tm['key']`` are the same operation. """ c = Constants() - c.nickname_delimiters['curly_braces'] = re.compile(r'\{(.*?)\}') - self.assertIn('curly_braces', c.nickname_delimiters) - del c.nickname_delimiters.curly_braces # type: ignore[attr-defined] - self.assertNotIn('curly_braces', c.nickname_delimiters) - - def test_assigning_non_setmanager_to_cached_union_member_raises(self) -> None: - """A cached-union attribute rejects a plain iterable, demanding a SetManager. - - The four ``_CachedUnionMember`` attributes wire an on-change callback into - the assigned manager; a bare list would silently break cache invalidation, - so __set__ fails loud with a message telling the caller to wrap it. - """ - c = Constants() - with pytest.raises(TypeError, match='SetManager'): - c.titles = ['mr', 'ms'] # type: ignore[assignment] + self.assertIn('ii', c.capitalization_exceptions) + del c.capitalization_exceptions.ii # type: ignore[attr-defined] + self.assertNotIn('ii', c.capitalization_exceptions) - def test_assigning_non_setmanager_to_plain_set_attr_raises(self) -> None: - """The five non-cached-union SetManager attributes reject bare assignment too. + def test_assigning_iterable_to_set_attr_wraps_and_normalizes(self) -> None: + """Assigning a plain iterable to a set field wraps it in a SetManager. - Only the four ``_CachedUnionMember`` attributes were guarded; these five - were plain instance attributes, so ``c.conjunctions = 'and'`` silently - replaced the SetManager with a str, turning later ``in`` membership - checks into substring tests (#241). + 2.0 divergence from v1's guard: v1 demanded a pre-built SetManager + (raising TypeError otherwise) because a bare collection would have + broken its cache-invalidation wiring; the shim wraps the value itself + (with full element validation and normalization), which protects the + same invariant -- change tracking stays wired -- without the ceremony. + Bare strings still raise (below), so #241's silent substring-test + corruption stays impossible. """ c = Constants() + c.titles = ['Mr.', 'ms'] # type: ignore[assignment] + self.assertEqual(type(c.titles), SetManager) + self.assertEqual(set(c.titles), {'mr', 'ms'}) for name in ('first_name_titles', 'conjunctions', 'bound_first_names', 'non_first_name_prefixes', 'suffix_acronyms_ambiguous'): - with pytest.raises(TypeError, match='SetManager'): - setattr(c, name, ['x']) + setattr(c, name, ['X.']) + self.assertEqual(type(getattr(c, name)), SetManager) + self.assertIn('x', getattr(c, name)) def test_bare_string_assignment_to_conjunctions_raises(self) -> None: - # the original #241 repro: 'and' assigned as a bare str silently - # degrades `piece.lower() in self.C.conjunctions` into a substring test + # the original #241 repro: 'and' assigned as a bare str would + # silently degrade `piece.lower() in self.C.conjunctions` into a + # substring test (or, wrapped naively, shred into {'a','n','d'}) c = Constants() - with pytest.raises(TypeError, match='SetManager'): + with pytest.raises(TypeError, match='wrap it in a list'): c.conjunctions = 'and' # type: ignore[assignment] - def test_setstate_raises_on_missing_descriptor_field(self) -> None: - """Unpickling a state blob missing a cached-union collection must fail loudly. - - __setstate__ verifies every ``_CachedUnionMember``-backed attribute was - restored; a missing one would otherwise surface much later as an - AttributeError on the private mangled name (e.g. ``_titles``), which is - far harder to diagnose than a named ValueError here. - """ - c = Constants() - state = dict(c.__getstate__()) - del state['titles'] - restored = Constants.__new__(Constants) - with pytest.raises(ValueError, match='titles'): - restored.__setstate__(state) - class ParsingDoesNotMutateConfigTests(HumanNameTestBase): """Parsing a name must never write back into the Constants it reads. @@ -1014,7 +807,6 @@ def _assert_config_unchanged(self, constants: Constants, before: dict, parsed: s self.assertEqual(diffs, [], f"parsing {parsed!r} changed the config: {diffs}") def _assert_parse_leaves_config_unchanged(self, name: str) -> HumanName: - from nameparser.config import CONSTANTS before = self._config_snapshot(CONSTANTS) hn = HumanName(name) self._assert_config_unchanged(CONSTANTS, before, name) @@ -1065,55 +857,34 @@ def test_derivations_reset_between_parses_of_same_instance(self) -> None: self.m(hn.last, "Roe", hn) -class SuffixesPrefixesTitlesPerformanceTests(HumanNameTestBase): - """Guard against accidental cache removal on suffixes_prefixes_titles. - - This library is commonly used to parse large batches of names, so - suffixes_prefixes_titles must remain cached. Without the cache, each call - rebuilds the union from ~700 strings; with it, repeated access is - orders of magnitude faster. Rather than compare against a fixed - wall-clock budget (which flakes on slower/noisier CI runners), this - measures the uncached build cost on the same machine and asserts cached - access is much faster than that baseline. - """ - - def test_repeated_access_is_cached(self) -> None: - c = Constants() - first = c.suffixes_prefixes_titles - second = c.suffixes_prefixes_titles - assert first is second, "suffixes_prefixes_titles should return the same cached object on repeated access" - - # Baseline: cost of an uncached build, measured via fresh instances - # (each instance's first access rebuilds the union) on this machine. - m = 50 - uncached_per_call = timeit.timeit(lambda: Constants().suffixes_prefixes_titles, number=m) / m - - n = 10_000 - cached_per_call = timeit.timeit(lambda: c.suffixes_prefixes_titles, number=n) / n - - # If caching is broken, cached_per_call would be roughly the same as - # uncached_per_call. With caching intact it should be at least an - # order of magnitude faster. - limit = uncached_per_call / 10 - assert cached_per_call < limit, ( - f"suffixes_prefixes_titles appears uncached: cached access averaged " - f"{cached_per_call * 1e6:.1f} us/call vs an uncached build cost of " - f"{uncached_per_call * 1e6:.1f} us/call. Was _pst caching removed?" - ) - - class ConstantsReprTests(HumanNameTestBase): + # The name lists live here rather than reading v1's + # Constants._repr_collection_attrs/_repr_scalar_attrs class attributes, + # which were internals and are gone in 2.0 (the shim keeps them as + # module-level tuples). empty_attribute_default left the scalar list + # with #255. + collection_attrs = ( + 'prefixes', 'suffix_acronyms', 'suffix_not_acronyms', 'titles', + 'first_name_titles', 'conjunctions', 'bound_first_names', + 'non_first_name_prefixes', 'suffix_acronyms_ambiguous', + ) + scalar_attrs = ( + 'string_format', 'initials_format', 'initials_delimiter', + 'initials_separator', 'suffix_delimiter', 'capitalize_name', + 'force_mixed_case_capitalization', 'patronymic_name_order', + 'middle_name_as_last', + ) def test_repr_reports_actual_collection_sizes(self) -> None: c = Constants() repr_str = repr(c) - for name in Constants._repr_collection_attrs: + for name in self.collection_attrs: self.assertIn(f"{name}: {len(getattr(c, name))}", repr_str) def test_repr_omits_scalars_at_default_value(self) -> None: c = Constants() repr_str = repr(c) - for name in Constants._repr_scalar_attrs: + for name in self.scalar_attrs: self.assertNotIn(name, repr_str) def test_repr_shows_scalar_override_via_constructor(self) -> None: @@ -1175,19 +946,17 @@ class CustomConstants(Constants): dup = c.copy() self.assertTrue(isinstance(dup, CustomConstants)) - def test_copy_preserves_empty_attribute_default_without_warning(self) -> None: - # copy() round-trips through __getstate__/__setstate__ like pickle - # does; restoring saved state isn't a user assignment, so it must not - # emit #255's deprecation warning (the __setstate__ bypass exists - # specifically for this call path, not just pickle.loads). + def test_copy_preserves_scalar_override_without_warning(self) -> None: + # copy() restores saved state rather than replaying user + # assignments, so it must carry scalar overrides across silently. + # (The v1 vehicle for this test, empty_attribute_default, was + # removed in 2.0 -- #255.) c = Constants() - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - c.empty_attribute_default = None # type: ignore[assignment] + c.string_format = "{last}" with warnings.catch_warnings(): warnings.simplefilter("error") dup = c.copy() - self.assertIsNone(dup.empty_attribute_default) + self.assertEqual(dup.string_format, "{last}") def test_copy_snapshots_current_customizations(self) -> None: # Unlike Constants(), which always starts from library defaults, @@ -1209,28 +978,31 @@ def test_fresh_constants_does_not_include_source_customizations(self) -> None: self.assertNotIn('zephyrmark', fresh.titles) -class ConstantsNoneDeprecationTests(HumanNameTestBase): - """constants=None is deprecated in favor of Constants() or CONSTANTS.copy() (#260).""" +class ConstantsNoneRemovalTests(HumanNameTestBase): + """constants=None was removed in 2.0 (#261, warned since 1.3.1). - def test_explicit_none_warns_on_construction(self) -> None: - with pytest.deprecated_call(match="Constants()"): - HumanName("John Doe", constants=None) + The v1 fallback silently built a fresh Constants(), discarding any + customizations the caller may have expected to carry over; 2.0 raises + TypeError at every entry point instead. + """ - def test_explicit_none_warns_on_positional_argument(self) -> None: - with pytest.deprecated_call(match="Constants()"): - HumanName("John Doe", None) + def test_explicit_none_raises_on_construction(self) -> None: + with pytest.raises(TypeError, match="261"): + HumanName("John Doe", constants=None) # type: ignore[arg-type] - def test_explicit_none_warning_names_both_replacements(self) -> None: - with pytest.warns(DeprecationWarning) as record: - HumanName("John Doe", constants=None) - message = str(record[0].message) - self.assertIn("Constants()", message) - self.assertIn("CONSTANTS.copy()", message) + def test_explicit_none_raises_on_positional_argument(self) -> None: + with pytest.raises(TypeError, match="261"): + HumanName("John Doe", None) # type: ignore[arg-type] - def test_explicit_none_warns_on_c_setter(self) -> None: + def test_explicit_none_raises_on_c_setter(self) -> None: hn = HumanName("John Doe") - with pytest.deprecated_call(match="Constants()"): - hn.C = None + with pytest.raises(TypeError, match="261"): + hn.C = None # type: ignore[assignment] + + def test_none_error_names_the_replacement(self) -> None: + with pytest.raises(TypeError) as excinfo: + HumanName("John Doe", constants=None) # type: ignore[arg-type] + self.assertIn("Constants", str(excinfo.value)) def test_omitted_constants_argument_does_not_warn(self) -> None: with warnings.catch_warnings(): @@ -1241,9 +1013,3 @@ def test_explicit_own_constants_instance_does_not_warn(self) -> None: with warnings.catch_warnings(): warnings.simplefilter("error") HumanName("John Doe", constants=Constants()) - - def test_explicit_none_still_produces_a_working_private_config(self) -> None: - # Behavior is unchanged, only newly warned about. - with pytest.deprecated_call(): - hn = HumanName("John Doe", constants=None) - self.assertTrue(hn.has_own_config) From c1b5730533d9032811dc45bf783c98bca2ca99ac Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 00:19:02 -0700 Subject: [PATCH 111/275] Add the two v1.4 bool constructor kwargs to the shim Constants patronymic_name_order and middle_name_as_last were v1.4 constructor kwargs (docs/customize.rst doctests use them); the M12 reconciliation of test_constants.py caught the gap. Co-Authored-By: Claude Fable 5 --- nameparser/_config_shim.py | 9 +++++++++ tests/v2/test_config_shim.py | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index 8fb87b58..52f83ce6 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -679,6 +679,8 @@ def __init__( Mapping[str, str] | Iterable[tuple[str, str]] | object = _UNSET_KWARG, regexes: object = _UNSET_KWARG, + patronymic_name_order: bool = False, + middle_name_as_last: bool = False, ) -> None: # v1.4 parity constructor kwargs (#238/#242/#244 hardening); the # signature is spelled out rather than **kwargs so an unknown @@ -736,6 +738,13 @@ def __init__( object.__setattr__(self, "regexes", _RegexesProxy()) for name, value in _SCALAR_DEFAULTS.items(): object.__setattr__(self, name, value) + # the two behavior bools were v1.4 constructor kwargs too + # (docs/customize.rst doctests use them); truthiness matches v1, + # storage goes through the plain scalar slot + if patronymic_name_order: + object.__setattr__(self, "patronymic_name_order", True) + if middle_name_as_last: + object.__setattr__(self, "middle_name_as_last", True) def _invalidate_pst(self) -> None: """Pickle-compat alias only, never called at runtime: v1's four diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index 8d2602fa..0df949fc 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -557,3 +557,13 @@ def test_constants_repr_shows_customized_scalars_only() -> None: # #221 assert "capitalize_name" not in repr(c) # default: not shown c.capitalize_name = True assert "capitalize_name: True" in repr(c) + + +def test_constants_bool_kwargs() -> None: + # v1.4 constructor kwargs used by the customize.rst doctests + c = Constants(middle_name_as_last=True) + assert c.middle_name_as_last is True + assert c.patronymic_name_order is False + d = Constants(patronymic_name_order=True) + _, policy, _ = d._snapshot() + assert len(policy.patronymic_rules) == 2 From eb4b36281be6b4b415e65e0073925ae1ed523b73 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 00:21:13 -0700 Subject: [PATCH 112/275] Lift test_constants from the skip lists (bool kwargs landed) Constants(patronymic_name_order=True) / Constants(middle_name_as_last= True) work again (c1b5730), so the two held ConstantsReprTests pass: 94/94 green. Removes the file's collect_ignore_glob and mypy-exclude entries and settles the three mypy items its un-excluding surfaced (|= re-wrap annotations, a dict[str, object] state annotation, and a note that the 2.0 TupleManager subscripts via dict's __class_getitem__ rather than v1's Generic). Co-Authored-By: Claude Fable 5 --- pyproject.toml | 2 +- tests/conftest.py | 8 -------- tests/test_constants.py | 15 +++++++++++---- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 467f06cb..27a72a36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ packages = ["nameparser", "tests"] # TEMPORARY (M11 -> M12): the v1 suite is being reconciled against the # 2.0 facade; lifted with tests/conftest.py's collect_ignore_glob. Must # be gone before the branch leaves draft. tests/v2/ stays checked. -exclude = '^tests/(test_(bound_first_names|brute_force|capitalization|comma_variants|conjunctions|constants|east_slavic_patronymic_order|first_name|initials|middle_name_as_last|nicknames|output_format|parser_util|prefixes|suffixes|titles|turkic_patronymic_order|variations)\.py)$' +exclude = '^tests/(test_(bound_first_names|brute_force|capitalization|comma_variants|conjunctions|east_slavic_patronymic_order|first_name|initials|middle_name_as_last|nicknames|output_format|parser_util|prefixes|suffixes|titles|turkic_patronymic_order|variations)\.py)$' [[tool.mypy.overrides]] module = [ diff --git a/tests/conftest.py b/tests/conftest.py index cf195d91..508aa566 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,14 +19,6 @@ "test_capitalization.py", "test_comma_variants.py", "test_conjunctions.py", - # test_constants.py is fully reconciled; 2 of its 94 tests fail on one - # remaining v1.4-parity gap: the shim Constants.__init__ lacks the - # patronymic_name_order/middle_name_as_last bool kwargs (v1.4 had them; - # docs/customize.rst doctests use them). Repro: - # Constants(middle_name_as_last=True) -> TypeError - # Its mypy-exclude entry pairs with this one (mypy flags the same two - # calls). Remove both once the kwargs are restored. - "test_constants.py", "test_east_slavic_patronymic_order.py", "test_first_name.py", "test_initials.py", diff --git a/tests/test_constants.py b/tests/test_constants.py index 39140b71..233860f9 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -107,8 +107,11 @@ def test_set_manager_operators_accept_lists(self) -> None: # behave like an add()-built one, normalization included c = Constants() with pytest.raises(TypeError, match=r"wrap it in a list"): - c.titles |= 'esq' - c.titles |= ['Esq.'] + c.titles |= 'esq' # type: ignore[assignment] + # |= produces a plain set that Constants.__setattr__ re-wraps in a + # SetManager (2.0's auto-wrap; see the plain-iterable assignment + # test below) -- mypy sees only the set-for-SetManager assignment + c.titles |= ['Esq.'] # type: ignore[assignment] self.assertIn('esq', c.titles) hn = HumanName("Esq Jane Smith", constants=c) self.m(hn.title, "Esq", hn) @@ -449,7 +452,7 @@ def test_unpickle_legacy_state_raises(self) -> None: ``__getstate__`` captured; the 1.4 DeprecationWarning promised ValueError in 2.0, pointing at re-pickling under 1.3/1.4. """ - legacy_state = { + legacy_state: dict[str, object] = { 'prefixes': {'van'}, 'titles': {'dr', 'legacytitle'}, 'suffixes_prefixes_titles': {'van', 'dr'}, @@ -567,7 +570,11 @@ def test_tuplemanager_setattr_delattr_ignore_dunder_names(self) -> None: parse_nicknames() iterates over. This bit nickname_delimiters' construction (#22) before the guard was added. """ - tm = TupleManager[re.Pattern[str] | str]({'a': re.compile('x')}) + # The 2.0 TupleManager is a plain dict subclass, not Generic like + # v1's -- but dict's inherited __class_getitem__ still makes the + # subscription work at runtime, which is exactly the GenericAlias + # __orig_class__ probe this test exists to exercise. + tm = TupleManager[re.Pattern[str] | str]({'a': re.compile('x')}) # type: ignore[misc] self.assertNotIn('__orig_class__', tm) self.assertEqual(dict(tm), {'a': re.compile('x')}) # Dunder assignment/deletion still work as normal object attributes, From be4e106d96d0c7cbfdb68c7ad44db74e2368faf2 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 00:23:19 -0700 Subject: [PATCH 113/275] Reconcile test_output_format against the facade (buckets B, C) 23 tests -> 22, all green. Bucket B (scheduled removals / deliberate divergences): - the two #254 None-mode scrubbing regressions merge into one residual test: the None display mode is gone (#255), but literal 'None'/'Nonez' name text must still survive formatting unscathed. - regexes.emoji/.bidi = False opt-outs raise TypeError in 2.0 (migration spec section 3 uniform rule); the tests now pin the error naming the Policy(strip_emoji/strip_bidi=False) replacements. Bucket C (shared-CONSTANTS mutation): - the four *_constants_attribute tests (string_format, capitalize_name, force_mixed_case_capitalization, both) move onto a private Constants passed as constants= -- the migration-guide idiom -- instead of mutating the shared singleton. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 2 +- tests/conftest.py | 1 - tests/test_output_format.py | 82 ++++++++++++++++++------------------- 3 files changed, 40 insertions(+), 45 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 27a72a36..cac4bdc0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ packages = ["nameparser", "tests"] # TEMPORARY (M11 -> M12): the v1 suite is being reconciled against the # 2.0 facade; lifted with tests/conftest.py's collect_ignore_glob. Must # be gone before the branch leaves draft. tests/v2/ stays checked. -exclude = '^tests/(test_(bound_first_names|brute_force|capitalization|comma_variants|conjunctions|east_slavic_patronymic_order|first_name|initials|middle_name_as_last|nicknames|output_format|parser_util|prefixes|suffixes|titles|turkic_patronymic_order|variations)\.py)$' +exclude = '^tests/(test_(bound_first_names|brute_force|capitalization|comma_variants|conjunctions|east_slavic_patronymic_order|first_name|initials|middle_name_as_last|nicknames|parser_util|prefixes|suffixes|titles|turkic_patronymic_order|variations)\.py)$' [[tool.mypy.overrides]] module = [ diff --git a/tests/conftest.py b/tests/conftest.py index 508aa566..5408fa88 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,7 +24,6 @@ "test_initials.py", "test_middle_name_as_last.py", "test_nicknames.py", - "test_output_format.py", "test_parser_util.py", "test_prefixes.py", "test_suffixes.py", diff --git a/tests/test_output_format.py b/tests/test_output_format.py index d7c4cfcd..08e37a27 100644 --- a/tests/test_output_format.py +++ b/tests/test_output_format.py @@ -13,30 +13,35 @@ def test_formatting_init_argument(self) -> None: string_format="TEST1") self.assertEqual(str(hn), "TEST1") + # The four *_constants_attribute tests below ran on the shared CONSTANTS + # singleton in v1; 2.0 deprecates shared mutation, so they use a private + # Constants passed as constants= (the migration-guide idiom) -- the + # config attribute under test is the same either way. + def test_formatting_constants_attribute(self) -> None: - from nameparser.config import CONSTANTS - CONSTANTS.string_format = "TEST2" - hn = HumanName("Rev John A. Kenneth Doe III (Kenny)") + c = Constants() + c.string_format = "TEST2" + hn = HumanName("Rev John A. Kenneth Doe III (Kenny)", constants=c) self.assertEqual(str(hn), "TEST2") def test_capitalize_name_constants_attribute(self) -> None: - from nameparser.config import CONSTANTS - CONSTANTS.capitalize_name = True - hn = HumanName("bob v. de la macdole-eisenhower phd") + c = Constants() + c.capitalize_name = True + hn = HumanName("bob v. de la macdole-eisenhower phd", constants=c) self.assertEqual(str(hn), "Bob V. de la MacDole-Eisenhower Ph.D.") def test_force_mixed_case_capitalization_constants_attribute(self) -> None: - from nameparser.config import CONSTANTS - CONSTANTS.force_mixed_case_capitalization = True - hn = HumanName('Shirley Maclaine') + c = Constants() + c.force_mixed_case_capitalization = True + hn = HumanName('Shirley Maclaine', constants=c) hn.capitalize() self.assertEqual(str(hn), "Shirley MacLaine") def test_capitalize_name_and_force_mixed_case_capitalization_constants_attributes(self) -> None: - from nameparser.config import CONSTANTS - CONSTANTS.capitalize_name = True - CONSTANTS.force_mixed_case_capitalization = True - hn = HumanName('Shirley Maclaine') + c = Constants() + c.capitalize_name = True + c.force_mixed_case_capitalization = True + hn = HumanName('Shirley Maclaine', constants=c) self.assertEqual(str(hn), "Shirley MacLaine") def test_quote_nickname_formating(self) -> None: @@ -102,22 +107,13 @@ def test_formating_of_nicknames_in_middle(self) -> None: hn.nickname = '' self.assertEqual(str(hn), "Rev John A. Kenneth Doe III") - def test_name_containing_none_substring_with_none_empty_attribute_default(self) -> None: - # Regression for #254: with empty_attribute_default = None, __str__ - # scrubbed the literal string 'None' from the formatted output, - # corrupting real name text containing that substring. - hn = HumanName("Nonez Smith", Constants()) - with pytest.deprecated_call(): - hn.C.empty_attribute_default = None # type: ignore[assignment] # see test_constants.test_empty_attribute_default - self.assertEqual(str(hn), "Nonez Smith") - - def test_name_none_as_literal_name_with_none_empty_attribute_default(self) -> None: - # Companion to the #254 regression: a name piece that is exactly - # 'None' must survive formatting in None-mode. - hn = HumanName("None Smith", Constants()) - with pytest.deprecated_call(): - hn.C.empty_attribute_default = None # type: ignore[assignment] # see test_constants.test_empty_attribute_default - self.assertEqual(str(hn), "None Smith") + def test_name_containing_none_substring_survives_formatting(self) -> None: + # Residue of the #254 regression: v1's None-mode __str__ once + # scrubbed the literal string 'None' from formatted output. The + # None mode is gone in 2.0 (#255), but real name text spelled + # 'None'/'Nonez' must still never be scrubbed by formatting. + self.assertEqual(str(HumanName("Nonez Smith")), "Nonez Smith") + self.assertEqual(str(HumanName("None Smith")), "None Smith") def test_empty_field_drops_surrounding_whitespace(self) -> None: # issue #139: adjacent whitespace/punctuation should be dropped when a field is empty @@ -147,15 +143,14 @@ def test_keep_non_emojis(self) -> None: self.m(hn.last, "Smith", hn) self.assertEqual(str(hn), "∫≜⩕ Smith") - def test_keep_emojis(self) -> None: - from nameparser.config import Constants + def test_keep_emojis_opt_out_moved_to_policy(self) -> None: + # v1's regexes.emoji = False opt-out is not supported in 2.0 + # (deliberate divergence, migration spec section 3 uniform rule): + # the assignment raises, and the error names the replacement, + # Policy(strip_emoji=False). constants = Constants() - constants.regexes.emoji = False # type: ignore[assignment] - hn = HumanName("∫≜⩕ Smith😊", constants) - self.m(hn.first, "∫≜⩕", hn) - self.m(hn.last, "Smith😊", hn) - self.assertEqual(str(hn), "∫≜⩕ Smith😊") - # test cleanup + with pytest.raises(TypeError, match="strip_emoji"): + constants.regexes.emoji = False # type: ignore[assignment] def test_remove_bidi_control_chars(self) -> None: # LRM/RLM and friends ride along with copy-pasted names and stick to @@ -173,10 +168,11 @@ def test_bidi_stripped_name_compares_equal(self) -> None: hn = HumanName("\u200fمحمد بن سلمان\u200f") self.assertEqual(hn.first, "محمد") - def test_keep_bidi_control_chars(self) -> None: - from nameparser.config import Constants + def test_keep_bidi_opt_out_moved_to_policy(self) -> None: + # v1's regexes.bidi = False opt-out is not supported in 2.0 + # (deliberate divergence, migration spec section 3 uniform rule): + # the assignment raises, and the error names the replacement, + # Policy(strip_bidi=False). constants = Constants() - constants.regexes.bidi = False # type: ignore[assignment] - hn = HumanName("\u200fJohn\u200f Smith", constants) - self.m(hn.first, "\u200fJohn\u200f", hn) - self.m(hn.last, "Smith", hn) + with pytest.raises(TypeError, match="strip_bidi"): + constants.regexes.bidi = False # type: ignore[assignment] From 036808115d203ec9344fb6fedac388dc8d427983 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 00:25:33 -0700 Subject: [PATCH 114/275] Reconcile test_initials against the facade (buckets B, C, E) 33 tests -> 31, all green. Bucket B: the two #255 None-mode regressions (literal-"None" interpolation, all-empty fallback) collapse to one test pinning the 2.0 invariant -- a fully-empty initials() renders ''. Bucket C: the three *_constants tests (initials_format, initials_delimiter, delimiter+separator+format combo) move onto a private Constants passed as constants= instead of mutating the shared singleton; the fixture-restore sentinel test keeps asserting the shared default is ' '. Bucket E: the #232 doubled-space test's direct middle_list assignment becomes field assignment; the v1 hole it pinned (unnormalized *_list elements crashing initials) is unreachable in 2.0 -- list snapshots plus setter whitespace normalization -- so it now pins the normalized result. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 2 +- tests/conftest.py | 1 - tests/test_initials.py | 83 ++++++++++++++++++------------------------ 3 files changed, 36 insertions(+), 50 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cac4bdc0..bb46a7dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ packages = ["nameparser", "tests"] # TEMPORARY (M11 -> M12): the v1 suite is being reconciled against the # 2.0 facade; lifted with tests/conftest.py's collect_ignore_glob. Must # be gone before the branch leaves draft. tests/v2/ stays checked. -exclude = '^tests/(test_(bound_first_names|brute_force|capitalization|comma_variants|conjunctions|east_slavic_patronymic_order|first_name|initials|middle_name_as_last|nicknames|parser_util|prefixes|suffixes|titles|turkic_patronymic_order|variations)\.py)$' +exclude = '^tests/(test_(bound_first_names|brute_force|capitalization|comma_variants|conjunctions|east_slavic_patronymic_order|first_name|middle_name_as_last|nicknames|parser_util|prefixes|suffixes|titles|turkic_patronymic_order|variations)\.py)$' [[tool.mypy.overrides]] module = [ diff --git a/tests/conftest.py b/tests/conftest.py index 5408fa88..07cc76b1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,7 +21,6 @@ "test_conjunctions.py", "test_east_slavic_patronymic_order.py", "test_first_name.py", - "test_initials.py", "test_middle_name_as_last.py", "test_nicknames.py", "test_parser_util.py", diff --git a/tests/test_initials.py b/tests/test_initials.py index c50879ff..c6ab90a5 100644 --- a/tests/test_initials.py +++ b/tests/test_initials.py @@ -1,5 +1,3 @@ -import pytest - from nameparser import HumanName from nameparser.config import Constants @@ -23,31 +21,12 @@ def test_initials_simple_name(self) -> None: hn = HumanName("John Doe", initials_format="{middle}") self.m(hn.initials(), "", hn) - def test_initials_empty_part_with_none_default_not_literal_none(self) -> None: - # Regression: when empty_attribute_default is None, an empty name part - # used to be interpolated by str.format as the literal "None" (e.g. - # "John Doe" -> "J. None D."). Empty parts must render as ''. - hn = HumanName("John Doe", constants=Constants()) - # empty_attribute_default has no explicit annotation (mypy infers str - # from the '' default), but None is documented/supported here -- see - # the doctest on the attribute's docstring in config/__init__.py. Not - # widened to str | None like string_format/suffix_delimiter because - # it cascades into every public str-typed name accessor (title, - # first, middle, last, suffix, nickname, maiden, surnames, - # given_names, last_base, last_prefixes, initials()). - with pytest.deprecated_call(): - hn.C.empty_attribute_default = None # type: ignore[assignment] - self.assertEqual(hn.initials(), "J. D.") - self.assertTrue("None" not in hn.initials()) - - def test_initials_all_empty_returns_empty_attribute_default(self) -> None: - # Regression: a fully-empty result must fall back to - # empty_attribute_default (here None), matching the first/last accessors, - # rather than rendering the literal "None None None". + def test_initials_all_empty_renders_empty_string(self) -> None: + # The v1 None display mode (and its literal-"None" interpolation + # regressions) died with #255: a fully-empty result is now always + # '', matching the first/last accessors. hn = HumanName("", constants=Constants()) - with pytest.deprecated_call(): - hn.C.empty_attribute_default = None # type: ignore[assignment] # see test above - self.assertEqual(hn.initials(), None) + self.assertEqual(hn.initials(), "") def test_initials_middle_name_all_prefixes(self) -> None: # "Vega, Juan de la" parses with middle name "de la", which contains @@ -73,13 +52,18 @@ def test_initials_format(self) -> None: hn = HumanName("Doe, John A. Kenneth, Jr.", initials_format="{first}, {last}") self.m(hn.initials(), "J., D.", hn) + # The *_constants tests below ran on the shared CONSTANTS singleton in + # v1; 2.0 deprecates shared mutation, so they use a private Constants + # passed as constants= (the migration-guide idiom) -- the config + # attribute under test is the same either way. + def test_initials_format_constants(self) -> None: - from nameparser.config import CONSTANTS - CONSTANTS.initials_format = "{first} {last}" - hn = HumanName("Doe, John A. Kenneth, Jr.") + c = Constants() + c.initials_format = "{first} {last}" + hn = HumanName("Doe, John A. Kenneth, Jr.", constants=c) self.m(hn.initials(), "J. D.", hn) - CONSTANTS.initials_format = "{first} {last}" - hn = HumanName("Doe, John A. Kenneth, Jr.") + c.initials_format = "{first} {last}" + hn = HumanName("Doe, John A. Kenneth, Jr.", constants=c) self.m(hn.initials(), "J. D.", hn) def test_initials_delimiter(self) -> None: @@ -87,9 +71,9 @@ def test_initials_delimiter(self) -> None: self.m(hn.initials(), "J; A; K; D;", hn) def test_initials_delimiter_constants(self) -> None: - from nameparser.config import CONSTANTS - CONSTANTS.initials_delimiter = ";" - hn = HumanName("Doe, John A. Kenneth, Jr.") + c = Constants() + c.initials_delimiter = ";" + hn = HumanName("Doe, John A. Kenneth, Jr.", constants=c) self.m(hn.initials(), "J; A; K; D;", hn) def test_initials_list(self) -> None: @@ -147,13 +131,16 @@ def test_str_default_behavior_unchanged(self) -> None: self.assertEqual(str(hn), "John Doe") def test_initials_with_doubled_space_in_list_element(self) -> None: - # direct *_list assignment bypasses parse_pieces whitespace - # normalization, so initials must tolerate unnormalized elements - # instead of raising IndexError (#232) + # v1's #232 hole -- direct *_list assignment injecting unnormalized + # elements that made initials raise IndexError -- is unreachable in + # 2.0: *_list attributes are read-only snapshots (spec section 2 + # exc. 1) and the field setter normalizes whitespace, splitting a + # doubled-space element into clean members. hn = HumanName(first="John") - hn.middle_list = ["Q R"] - self.assertEqual(hn.initials_list(), ["J", "Q R"]) - self.assertEqual(hn.initials(), "J. Q R.") + hn.middle = ["Q R"] + self.assertEqual(hn.middle_list, ["Q", "R"]) + self.assertEqual(hn.initials_list(), ["J", "Q", "R"]) + self.assertEqual(hn.initials(), "J. Q. R.") def test_constructor_first(self) -> None: hn = HumanName(first="TheName") @@ -210,16 +197,16 @@ def test_initials_separator_empty_multi_part_middle(self) -> None: self.m(hn.initials(), "JAKD", hn) def test_initials_separator_constants_multi_part_middle(self) -> None: - from nameparser.config import CONSTANTS - CONSTANTS.initials_delimiter = "" - CONSTANTS.initials_separator = "" - CONSTANTS.initials_format = "{first}{middle}{last}" - hn = HumanName("Doe, John A. Kenneth") + c = Constants() + c.initials_delimiter = "" + c.initials_separator = "" + c.initials_format = "{first}{middle}{last}" + hn = HumanName("Doe, John A. Kenneth", constants=c) self.m(hn.initials(), "JAKD", hn) def test_initials_separator_default_on_constants(self) -> None: - # Runs after test_initials_separator_constants_multi_part_middle so that, - # in file/definition order, it verifies the autouse fixture restored - # CONSTANTS.initials_separator rather than leaking the "" set above. + # The shared singleton's default must be untouched by the private- + # Constants tests above (and, in file order, by anything the autouse + # fixture restores). from nameparser.config import CONSTANTS self.assertEqual(CONSTANTS.initials_separator, " ") From 91dad40e271b1ad3472d72a94423f3615caecd45 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 00:27:59 -0700 Subject: [PATCH 115/275] Hold test_capitalization on a confirmed _cap_word conjunction/initial bug The file needs no reconciliation edits (20 pass, 1 pre-existing v1 xfail), but one test fails on a render-layer parity bug: _render.py's _cap_word lowercases every conjunction-set word, missing v1 is_conjunction's is_an_initial exclusion, so an initial that doubles as a conjunction stays lowercase: HumanName('scott e. werner').capitalize() -> 'Scott e. Werner' (v1: 'Scott E. Werner'; 'e' is in CONJUNCTIONS) The file stays in collect_ignore_glob with a pointer comment; it is mypy-clean and leaves the mypy exclude. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 2 +- tests/conftest.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bb46a7dc..c7357b5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ packages = ["nameparser", "tests"] # TEMPORARY (M11 -> M12): the v1 suite is being reconciled against the # 2.0 facade; lifted with tests/conftest.py's collect_ignore_glob. Must # be gone before the branch leaves draft. tests/v2/ stays checked. -exclude = '^tests/(test_(bound_first_names|brute_force|capitalization|comma_variants|conjunctions|east_slavic_patronymic_order|first_name|middle_name_as_last|nicknames|parser_util|prefixes|suffixes|titles|turkic_patronymic_order|variations)\.py)$' +exclude = '^tests/(test_(bound_first_names|brute_force|comma_variants|conjunctions|east_slavic_patronymic_order|first_name|middle_name_as_last|nicknames|parser_util|prefixes|suffixes|titles|turkic_patronymic_order|variations)\.py)$' [[tool.mypy.overrides]] module = [ diff --git a/tests/conftest.py b/tests/conftest.py index 07cc76b1..67875c40 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,6 +16,13 @@ collect_ignore_glob = [ "test_bound_first_names.py", "test_brute_force.py", + # test_capitalization.py needs NO reconciliation edits (20 pass, 1 v1 + # xfail) but 1 test fails on a confirmed render bug: _render.py's + # _cap_word lowercases every conjunction-set word, missing v1 + # is_conjunction's is_an_initial exclusion (75e3219^ parser.py:761), so + # initial-shaped words that double as conjunctions stay lowercase. Repro: + # HumanName('scott e. werner').capitalize() -> 'Scott e. Werner' + # (v1: 'Scott E. Werner'; 'e' is in CONJUNCTIONS). Remove once fixed. "test_capitalization.py", "test_comma_variants.py", "test_conjunctions.py", From e8dd59fb3ef0452f384daf0bb601cd4750b96a61 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 00:28:51 -0700 Subject: [PATCH 116/275] Hold test_first_name on a family-segment trailing-suffix pipeline bug The file needs no reconciliation edits (8 pass, 2 pre-existing v1 xfails), but one test fails on a confirmed pipeline parity bug: in the family-comma format, a trailing suffix inside the family segment is not split out of it. Parser().parse('Smith Jr., John') -> family='Smith Jr.', suffix='' (v1: last='Smith', first='John', suffix='Jr.') Stays in collect_ignore_glob with a pointer comment; mypy-clean, so it leaves the mypy exclude. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 2 +- tests/conftest.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c7357b5c..db98ffd0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ packages = ["nameparser", "tests"] # TEMPORARY (M11 -> M12): the v1 suite is being reconciled against the # 2.0 facade; lifted with tests/conftest.py's collect_ignore_glob. Must # be gone before the branch leaves draft. tests/v2/ stays checked. -exclude = '^tests/(test_(bound_first_names|brute_force|comma_variants|conjunctions|east_slavic_patronymic_order|first_name|middle_name_as_last|nicknames|parser_util|prefixes|suffixes|titles|turkic_patronymic_order|variations)\.py)$' +exclude = '^tests/(test_(bound_first_names|brute_force|comma_variants|conjunctions|east_slavic_patronymic_order|middle_name_as_last|nicknames|parser_util|prefixes|suffixes|titles|turkic_patronymic_order|variations)\.py)$' [[tool.mypy.overrides]] module = [ diff --git a/tests/conftest.py b/tests/conftest.py index 67875c40..88fab765 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,6 +27,11 @@ "test_comma_variants.py", "test_conjunctions.py", "test_east_slavic_patronymic_order.py", + # test_first_name.py needs NO reconciliation edits (8 pass, 2 v1 xfails) + # but 1 test fails on a confirmed pipeline bug: in the family-comma + # format a trailing suffix inside the FAMILY segment is not split out. + # Repro: Parser().parse('Smith Jr., John') -> family='Smith Jr.', + # suffix='' (v1: last='Smith', suffix='Jr.'). Remove once fixed. "test_first_name.py", "test_middle_name_as_last.py", "test_nicknames.py", From dac3f22d497fc060dbd7164f94f2f82065d3de96 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 00:36:39 -0700 Subject: [PATCH 117/275] Reconcile test_titles; hold on two comma/nickname parity bugs (bucket D) 57 tests: the stale 'U.S. District Judge' xfail is removed -- the 2.0 period-joined title derivation makes the long-desired expectation hold (logged in docs/superpowers/plans/notes-m12-diffs.md for the M14 expected-changes list). 54 pass, 1 pre-existing v1 xfail remains. 2 tests fail on confirmed pipeline parity bugs, so the file stays in collect_ignore_glob (mypy-clean, leaves the mypy exclude): 1. _lexicon._normalize strips INTERIOR periods, so 'J.R.' normalizes to 'jr' -- a TITLES vocabulary entry; v1's lc() kept 'j.r' unmatched. HumanName('Smith, J.R.') -> title='J.R.' (v1: first='J.R.', last='Smith') 2. _assign's nickname rule (plan deviation #2) counts name pieces AFTER title peeling; v1's p_len==1 counted the whole segment before it. HumanName('Xyz. (Bud) Smith') -> last='Smith' (v1: title='Xyz.', first='Smith', nickname='Bud') Co-Authored-By: Claude Fable 5 --- pyproject.toml | 2 +- tests/conftest.py | 12 ++++++++++++ tests/test_titles.py | 5 +++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index db98ffd0..7742070b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ packages = ["nameparser", "tests"] # TEMPORARY (M11 -> M12): the v1 suite is being reconciled against the # 2.0 facade; lifted with tests/conftest.py's collect_ignore_glob. Must # be gone before the branch leaves draft. tests/v2/ stays checked. -exclude = '^tests/(test_(bound_first_names|brute_force|comma_variants|conjunctions|east_slavic_patronymic_order|middle_name_as_last|nicknames|parser_util|prefixes|suffixes|titles|turkic_patronymic_order|variations)\.py)$' +exclude = '^tests/(test_(bound_first_names|brute_force|comma_variants|conjunctions|east_slavic_patronymic_order|middle_name_as_last|nicknames|parser_util|prefixes|suffixes|turkic_patronymic_order|variations)\.py)$' [[tool.mypy.overrides]] module = [ diff --git a/tests/conftest.py b/tests/conftest.py index 88fab765..cc826ea6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -38,6 +38,18 @@ "test_parser_util.py", "test_prefixes.py", "test_suffixes.py", + # test_titles.py is reconciled (stale 'U.S.' xfail removed -- the 2.0 + # period-joined derivation makes it pass) but 2 of its 57 tests fail on + # confirmed pipeline parity bugs: + # (1) _lexicon._normalize strips INTERIOR periods, so 'J.R.' -> 'jr', + # which is in the TITLES vocabulary; v1's lc() kept 'j.r' out of it. + # Repro: HumanName('Smith, J.R.') -> title='J.R.' + # (v1: first='J.R.', last='Smith'). + # (2) _assign's nickname rule (plan deviation #2) counts pieces AFTER + # title peeling; v1's p_len==1 counted the whole segment before it. + # Repro: HumanName('Xyz. (Bud) Smith') -> last='Smith' + # (v1: first='Smith', title='Xyz.', nickname='Bud'). + # Remove once fixed. "test_titles.py", "test_turkic_patronymic_order.py", "test_variations.py", diff --git a/tests/test_titles.py b/tests/test_titles.py index 56c066c4..700c83e1 100644 --- a/tests/test_titles.py +++ b/tests/test_titles.py @@ -56,9 +56,10 @@ def test_title_is_title(self) -> None: hn = HumanName("Coach") self.m(hn.title, "Coach", hn) - # TODO: fix handling of U.S. - @pytest.mark.xfail def test_chained_title_first_name_title_is_initials(self) -> None: + # xfail in v1 ("TODO: fix handling of U.S."); the 2.0 pipeline's + # period-joined title derivation chains 'U.S.' into the title, so + # the long-desired expectation now holds. hn = HumanName("U.S. District Judge Marc Thomas Treadwell") self.m(hn.title, "U.S. District Judge", hn) self.m(hn.first, "Marc", hn) From 2c451505c2308d30a844735cdd301b423b8b883a Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 00:43:40 -0700 Subject: [PATCH 118/275] Fix four v1-parity bugs from the M12 batch-2 queue All pinned live against v1.4 (2026-07-17): - _normalize now matches v1's lc(): casefold + EDGE periods only. 'J.R.' no longer collapses to 'jr' and hits the periodless titles vocabulary. Suffix-ACRONYM membership alone uses the period-free form ('M.D.' -> 'md'), mirroring v1's is_suffix, which removed periods only for the acronym test. The capitalization-exception lookup tries both forms (v1 cap_word parity), keeping 'phd' -> 'Ph.D.' idempotent. - _cap_word's conjunction lowering excludes initial-shaped words (v1 is_conjunction): 'scott e. werner' capitalizes to 'Scott E. Werner'; an uppercase lone 'Y' stays 'Y'. - FAMILY_COMMA's family segment peels per-piece suffixes after the first piece ('Smith Jr., John' -> family=Smith, suffix=Jr.). - the lone-piece nickname rule counts the segment BEFORE title peeling ('Xyz. (Bud) Smith' -> title, given, nickname). Six case rows pin the matrix; the three stale v2 unit tests that pinned the pre-fix semantics are updated with v1 citations. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 12 +++++++----- nameparser/_pipeline/_assign.py | 20 ++++++++++++++++---- nameparser/_pipeline/_vocab.py | 14 ++++++++++---- nameparser/_render.py | 19 +++++++++++++++---- tests/v2/cases.py | 22 ++++++++++++++++++++++ tests/v2/test_lexicon.py | 17 ++++++++++------- tests/v2/test_render.py | 17 ++++++++++++++--- 7 files changed, 94 insertions(+), 27 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index dea3ed5f..0cb4e34e 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -25,11 +25,13 @@ def _normalize(word: str) -> str: - """Casefold, remove ALL periods, strip whitespace -- v2's stricter - analogue of v1's lc(), which lower()s and trims only edge periods. - Membership tests never re-normalize because construction already - did.""" - return word.casefold().replace(".", "").strip() + """Casefold, strip whitespace and EDGE periods -- v1's lc() + semantics. Interior periods survive on purpose: 'J.R.' must not + collapse to 'jr' and hit the periodless vocabulary (v1 parity, + pinned live 2026-07-17). Suffix-ACRONYM membership alone uses the + period-free form (see _vocab.suffix_as_written), mirroring v1's + is_suffix, which removed periods only for the acronym test.""" + return word.casefold().strip().strip(".") def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 4ff3d7ab..57f8f948 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -112,8 +112,11 @@ def _assign_main(seg_idx: int, state: ParseState, rest = [k for k in rest if "suffix" not in ptags[k]] if not rest: return - # v1 nickname rule (plan deviation #2) - if len(rest) == 1 and has_nickname: + # v1 nickname rule (plan deviation #2): v1's p_len == 1 counted + # the WHOLE segment before any title peeling (parser.py:1285) -- + # 'Xyz. (Bud) Smith' has two pieces, so the title peel wins and + # Smith stays the given name (pinned live 2026-07-17) + if len(pieces) == 1 and len(rest) == 1 and has_nickname: _set_roles(tokens, pieces[rest[0]], Role.FAMILY) return # peel the trailing suffix run: k = first index in rest from which @@ -169,8 +172,17 @@ def assign(state: ParseState) -> ParseState: # PARTICLE_OR_GIVEN is deliberately not emitted here: after a # comma the family is already fixed, so a leading given-position # particle is not meaningfully ambiguous. - for piece in state.pieces[0]: - _set_roles(tokens, piece, Role.FAMILY) + # v1: "lastname part may have suffixes in it" (parser.py:1370) + # -- the first piece is always the family even if suffix-shaped; + # any later strict-suffix piece goes to SUFFIX per piece + # ('Smith Jr., John' -> family=Smith, suffix=Jr.) + fam_pieces = state.pieces[0] + fam_tags = state.piece_tags[0] + for k, piece in enumerate(fam_pieces): + if k > 0 and _is_suffix_piece(piece, fam_tags[k], tokens): + _set_roles(tokens, piece, Role.SUFFIX) + else: + _set_roles(tokens, piece, Role.FAMILY) if len(state.segments) > 1: pieces = state.pieces[1] ptags = state.piece_tags[1] diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index dfc8cd8b..3ceae869 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -35,17 +35,23 @@ def suffix_as_written(n: str, text: str, lexicon: Lexicon) -> bool: subset of suffix_acronyms, and without the exclusion the period gate is dead code (bare 'Ed'/'Jd' would silently become suffixes). """ - if "." in text and n in lexicon.suffix_acronyms_ambiguous: + # acronyms may be written with periods ('M.B.A.'): the ACRONYM + # membership alone uses the period-free form (v1's is_suffix + # removed periods only for the suffix_acronyms test); suffix WORDS + # match on the plain normalized form + a = n.replace(".", "") + if "." in text and a in lexicon.suffix_acronyms_ambiguous: return True - return (n in lexicon.suffix_acronyms - and n not in lexicon.suffix_acronyms_ambiguous) \ + return (a in lexicon.suffix_acronyms + and a not in lexicon.suffix_acronyms_ambiguous) \ or n in lexicon.suffix_words def _is_suffix_strict_n(n: str, text: str, lexicon: Lexicon) -> bool: if is_initial(text): # period-written ambiguous acronyms are exempt from the veto - return "." in text and n in lexicon.suffix_acronyms_ambiguous + return "." in text and \ + n.replace(".", "") in lexicon.suffix_acronyms_ambiguous return suffix_as_written(n, text, lexicon) diff --git a/nameparser/_render.py b/nameparser/_render.py index 3ce93c66..18399b71 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -35,6 +35,11 @@ #: Not STABLE_TAGS -- that also contains "initial", which must contribute. _SKIP_TAGS = frozenset({"particle", "conjunction"}) +# Ported verbatim from v1 (nameparser/config/regexes.py "initial", +# minus the empty alternative) -- layering forbids importing the +# pipeline here; keep in sync with _pipeline/_vocab.py by hand. +_INITIAL = re.compile(r"^(\w\.|[A-Z])$") + def _collapse(rendered: str) -> str: """The #254 collapse, normative (core spec §5b): empty fields @@ -103,12 +108,18 @@ def _cap_word(word: str, role: Role, lex: Lexicon) -> str: # v1 cap_word order: particle/conjunction rule first, then the # exceptions map, then Mac/Mc, then str.capitalize normalized = _normalize(word) + # v1's is_conjunction excludes initials: 'E.' in 'Scott E. Werner' + # is an initial, not the conjunction 'e' (pinned live 2026-07-17) if ((normalized in lex.particles and role in (Role.MIDDLE, Role.FAMILY)) - or normalized in lex.conjunctions): + or (normalized in lex.conjunctions + and not _INITIAL.fullmatch(word))): return word.lower() - exception = lex.capitalization_exceptions_map.get(normalized) - if exception is not None: - return exception + # v1 cap_word tries the edge-stripped form, then the period-free + # form ('Ph.D.' -> 'ph.d' -> 'phd' hits the exceptions map) + for key in (normalized, normalized.replace(".", "")): + exception = lex.capitalization_exceptions_map.get(key) + if exception is not None: + return exception if _MAC.match(word): return _MAC.sub( lambda m: m.group(1).capitalize() + m.group(2).capitalize(), diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 791d956a..b4f66a9f 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -41,6 +41,28 @@ class Case: {"given": "John", "family": "Smith"}), Case("suffix_comma", "John Smith, PhD", {"given": "John", "family": "Smith", "suffix": "PhD"}), + Case("family_segment_trailing_suffix", "Smith Jr., John", + {"given": "John", "family": "Smith", "suffix": "Jr."}, + notes="v1: the family part may have suffixes in it " + "(parser.py:1370); the first piece is always the family " + "(pinned live 2026-07-17)"), + Case("family_segment_multiple_suffixes", "Smith Jr. MD, John", + {"given": "John", "family": "Smith", "suffix": "Jr., MD"}), + Case("family_segment_particle_chain_suffix", "de la Vega III, Juan", + {"given": "Juan", "family": "de la Vega", "suffix": "III"}), + Case("interior_periods_block_vocab", "Smith, J.R.", + {"given": "J.R.", "family": "Smith"}, + notes="v1's lc() keeps interior periods: 'J.R.' is not the " + "title 'jr' (pinned live 2026-07-17)"), + Case("dotted_acronym_suffix", "John Smith M.D.", + {"given": "John", "family": "Smith", "suffix": "M.D."}, + notes="suffix-ACRONYM membership alone strips periods (v1 " + "is_suffix parity)"), + Case("nickname_rule_counts_whole_segment", "Xyz. (Bud) Smith", + {"title": "Xyz.", "given": "Smith", "nickname": "Bud"}, + notes="v1's lone-piece nickname rule counts the segment " + "BEFORE title peeling (parser.py:1285, pinned live " + "2026-07-17)"), Case("suffix_comma_decided_by_first_segment", "Dr. John P. Doe-Ray, CLU, CFP, LUTC", {"title": "Dr.", "given": "John", "middle": "P.", diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 690fc927..e0365e38 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -77,7 +77,8 @@ def test_entries_normalizing_to_empty_raise() -> None: def test_colliding_exception_keys_dedupe_last_wins() -> None: - lex = Lexicon(capitalization_exceptions=(("Ph.D.", "A"), ("phd", "B"))) + # 'Phd.' and 'phd' collide under edge-period normalization + lex = Lexicon(capitalization_exceptions=(("Phd.", "A"), ("phd", "B"))) assert lex.capitalization_exceptions == (("phd", "B"),) rebuilt = Lexicon(capitalization_exceptions=lex.capitalization_exceptions_map) # type: ignore[arg-type] assert rebuilt == lex and hash(rebuilt) == hash(lex) @@ -180,12 +181,14 @@ def test_setstate_rejects_mismatched_field_layout() -> None: Lexicon.__new__(Lexicon).__setstate__(extra) -def test_normalization_casefolds_and_strips_interior_periods() -> None: - # Stricter than v1's lc(), which lower()s and trims only EDGE - # periods: casefold handles ß, and interior periods are removed too. - # A "simplify to .lower()/.strip('.')" regression must fail here. - lex = Lexicon(titles=frozenset({"STRAßE", "Ph.D"})) - assert lex.titles == frozenset({"strasse", "phd"}) +def test_normalization_casefolds_and_keeps_interior_periods() -> None: + # v1's lc() semantics plus casefold: EDGE periods trimmed, interior + # periods KEPT ('J.R.' must not collapse to 'jr' and hit the + # periodless vocabulary -- v1 parity, pinned live 2026-07-17). + # Suffix-ACRONYM membership alone strips periods (see + # _vocab.suffix_as_written). + lex = Lexicon(titles=frozenset({"STRAßE", "Ph.D", "Dr."})) + assert lex.titles == frozenset({"strasse", "ph.d", "dr"}) def test_suffix_ambiguous_must_be_subset_of_acronyms() -> None: diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 249e3d47..aedaaff5 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -192,16 +192,27 @@ def test_capitalized_with_explicit_lexicon() -> None: assert out.suffix == "Phd" -def test_capitalized_lowers_conjunctions() -> None: +def test_capitalized_lowers_conjunctions_but_not_initial_shapes() -> None: + # v1's is_conjunction excludes initial-shaped words: a lowercase + # 'y' lowers, but an uppercase 'Y' looks like an initial and + # capitalizes ('JOSE ORTEGA Y GASSET' -> 'Jose Ortega Y Gasset', + # pinned live against v1.4 2026-07-17) # 01234567890123456789 - pn = _pn("juan ortega Y gasset", [ + pn = _pn("juan ortega y gasset", [ Token("juan", Span(0, 4), Role.GIVEN), Token("ortega", Span(5, 11), Role.FAMILY), - Token("Y", Span(12, 13), Role.FAMILY, frozenset({"conjunction"})), + Token("y", Span(12, 13), Role.FAMILY, frozenset({"conjunction"})), Token("gasset", Span(14, 20), Role.FAMILY), ]) out = pn.capitalized(force=True) assert out.family == "Ortega y Gasset" + upper = _pn("JUAN ORTEGA Y GASSET", [ + Token("JUAN", Span(0, 4), Role.GIVEN), + Token("ORTEGA", Span(5, 11), Role.FAMILY), + Token("Y", Span(12, 13), Role.FAMILY, frozenset({"conjunction"})), + Token("GASSET", Span(14, 20), Role.FAMILY), + ]) + assert upper.capitalized(force=True).family == "Ortega Y Gasset" def test_capitalized_rebuilds_ambiguity_tokens() -> None: From 8ea1b36069b62f65d500bc75607b25e1a3093528 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 00:46:05 -0700 Subject: [PATCH 119/275] Lift capitalization, first_name, and titles from the skip list All four queued parity bugs are fixed (2c45150); the three held files run green (85 pass, 5 xfail). Two xfail flips settled in the process: - 'Andrews, M.D.': the family-comma suffix peel makes the v1 xfail's long-desired expectation hold (last='Andrews', suffix='M.D.'; v1 parsed first='M.D.') -- marker removed, diff logged in docs/superpowers/plans/notes-m12-diffs.md as fix(comma-family/suffix-routing). - 'U.S. District Judge ...': the interim interior-period normalization that briefly made this chain was itself the 'Smith, J.R.' bug; with v1's lc() restored the name matches v1 again, so the xfail marker is restored with a comment recording why. Co-Authored-By: Claude Fable 5 --- tests/conftest.py | 27 --------------------------- tests/test_first_name.py | 8 ++++---- tests/test_titles.py | 9 ++++++--- 3 files changed, 10 insertions(+), 34 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index cc826ea6..367d5e4e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,41 +16,14 @@ collect_ignore_glob = [ "test_bound_first_names.py", "test_brute_force.py", - # test_capitalization.py needs NO reconciliation edits (20 pass, 1 v1 - # xfail) but 1 test fails on a confirmed render bug: _render.py's - # _cap_word lowercases every conjunction-set word, missing v1 - # is_conjunction's is_an_initial exclusion (75e3219^ parser.py:761), so - # initial-shaped words that double as conjunctions stay lowercase. Repro: - # HumanName('scott e. werner').capitalize() -> 'Scott e. Werner' - # (v1: 'Scott E. Werner'; 'e' is in CONJUNCTIONS). Remove once fixed. - "test_capitalization.py", "test_comma_variants.py", "test_conjunctions.py", "test_east_slavic_patronymic_order.py", - # test_first_name.py needs NO reconciliation edits (8 pass, 2 v1 xfails) - # but 1 test fails on a confirmed pipeline bug: in the family-comma - # format a trailing suffix inside the FAMILY segment is not split out. - # Repro: Parser().parse('Smith Jr., John') -> family='Smith Jr.', - # suffix='' (v1: last='Smith', suffix='Jr.'). Remove once fixed. - "test_first_name.py", "test_middle_name_as_last.py", "test_nicknames.py", "test_parser_util.py", "test_prefixes.py", "test_suffixes.py", - # test_titles.py is reconciled (stale 'U.S.' xfail removed -- the 2.0 - # period-joined derivation makes it pass) but 2 of its 57 tests fail on - # confirmed pipeline parity bugs: - # (1) _lexicon._normalize strips INTERIOR periods, so 'J.R.' -> 'jr', - # which is in the TITLES vocabulary; v1's lc() kept 'j.r' out of it. - # Repro: HumanName('Smith, J.R.') -> title='J.R.' - # (v1: first='J.R.', last='Smith'). - # (2) _assign's nickname rule (plan deviation #2) counts pieces AFTER - # title peeling; v1's p_len==1 counted the whole segment before it. - # Repro: HumanName('Xyz. (Bud) Smith') -> last='Smith' - # (v1: first='Smith', title='Xyz.', nickname='Bud'). - # Remove once fixed. - "test_titles.py", "test_turkic_patronymic_order.py", "test_variations.py", ] diff --git a/tests/test_first_name.py b/tests/test_first_name.py index 0058acc0..38ffa4bd 100644 --- a/tests/test_first_name.py +++ b/tests/test_first_name.py @@ -15,11 +15,11 @@ def test_assume_title_and_one_other_name_is_last_name(self) -> None: self.m(hn.title, "Rev", hn) self.m(hn.last, "Andrews", hn) - # TODO: Seems "Andrews, M.D.", Andrews should be treated as a last name - # but other suffixes like "George Jr." should be first names. Might be - # related to https://github.com/derek73/python-nameparser/issues/2 - @pytest.mark.xfail def test_assume_suffix_title_and_one_other_name_is_last_name(self) -> None: + # xfail in v1 (which parsed first='M.D.'); 2.0's family-comma + # suffix peel routes the post-comma suffix piece to suffix and + # keeps the pre-comma piece in last, so the long-desired + # expectation now holds. 2.0: fix(comma-family/suffix-routing). hn = HumanName("Andrews, M.D.") self.m(hn.suffix, "M.D.", hn) self.m(hn.last, "Andrews", hn) diff --git a/tests/test_titles.py b/tests/test_titles.py index 700c83e1..d01ae0bd 100644 --- a/tests/test_titles.py +++ b/tests/test_titles.py @@ -56,10 +56,13 @@ def test_title_is_title(self) -> None: hn = HumanName("Coach") self.m(hn.title, "Coach", hn) + # TODO: fix handling of U.S. -- 2.0 matches v1 here: lc()-style + # normalization keeps 'u.s' out of the titles vocabulary, so the + # chain never starts (an interim 2.0 build that stripped interior + # periods made this pass, but that normalization wrongly turned + # 'J.R.' into the title 'jr'; v1 parity won) + @pytest.mark.xfail def test_chained_title_first_name_title_is_initials(self) -> None: - # xfail in v1 ("TODO: fix handling of U.S."); the 2.0 pipeline's - # period-joined title derivation chains 'U.S.' into the title, so - # the long-desired expectation now holds. hn = HumanName("U.S. District Judge Marc Thomas Treadwell") self.m(hn.title, "U.S. District Judge", hn) self.m(hn.first, "Marc", hn) From 8784070bfb9f201f297d5be0c0f38744388763d6 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 00:49:39 -0700 Subject: [PATCH 120/275] Hold test_suffixes on six suspected pipeline gaps (bucket D, reported) The file needs no bucket A/B/C edits (30 pass, 2 pre-existing v1 xfails); all 24 failures are component-value diffs traced to six root causes, each reproduced on the core -- reported for fixing, not classified: B1 two-word trailing suffix: 'Jack Ma' -> given='Jack', suffix='Ma', family='' (v1: last='Ma'). Same shape the fix(suffix-routing) case row pins intentionally for 'Johnson PhD'; needs a decision on suffix-shaped words that are also common surnames. B2 suffix-comma segments split per word: 'John Smith, V MD' -> suffix='V, MD' (v1 kept the whole segment: 'V MD'); also 'Jr. MD', 'Q.C. M.P.'. B3 split/period-joined suffix segments fail SUFFIX_COMMA detection: 'John Smith, Ph. D.' and 'John Doe, Msc.Ed.' -> family swallows the whole head, given='' (v1: first/last split + suffix). B4 #144 family-comma trailing suffix_not_acronyms rule missing: 'Smith, John V' -> middle='V'; 'Maier, Amy Lauren I' -> middle='Lauren I' (v1: suffix). B5 suffix-in-parens/quotes routing (#111, shipped 1.3) absent: 'Andrew Perkins (MBA)' / 'MBA' / "MBA" -> nickname (v1: suffix); 'Lon (Jr.) Williams', 'Col. (Ret.) Smith', '(Mgr.)', and the suffix_acronyms_ambiguous customization all diverge with it. B6 suffix_delimiter edges: a trailing delimiter ('John Doe, MD-PhD-', delimiter '-') kills detection entirely; a multi-word side leaks the delimiter token into suffix ('MD, PhD, -, FACS, Fellow' vs the pinned drop rule); the delimiter-unset degenerate 'Steven Hardman, RN - CRNA' diverges from v1's documented limitation (first='-' vs first='RN'). Stays in collect_ignore_glob with a summary comment; mypy-clean, so it leaves the mypy exclude. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 2 +- tests/conftest.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7742070b..da31c1ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ packages = ["nameparser", "tests"] # TEMPORARY (M11 -> M12): the v1 suite is being reconciled against the # 2.0 facade; lifted with tests/conftest.py's collect_ignore_glob. Must # be gone before the branch leaves draft. tests/v2/ stays checked. -exclude = '^tests/(test_(bound_first_names|brute_force|comma_variants|conjunctions|east_slavic_patronymic_order|middle_name_as_last|nicknames|parser_util|prefixes|suffixes|turkic_patronymic_order|variations)\.py)$' +exclude = '^tests/(test_(bound_first_names|brute_force|comma_variants|conjunctions|east_slavic_patronymic_order|middle_name_as_last|nicknames|parser_util|prefixes|turkic_patronymic_order|variations)\.py)$' [[tool.mypy.overrides]] module = [ diff --git a/tests/conftest.py b/tests/conftest.py index 367d5e4e..0dfb63ae 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,6 +23,23 @@ "test_nicknames.py", "test_parser_util.py", "test_prefixes.py", + # test_suffixes.py needs NO bucket edits (30 pass, 2 v1 xfails) but 24 + # tests fail across six suspected pipeline gaps -- see the M12 batch-3 + # report for full repros. Headlines: + # B1 'Jack Ma' -> suffix='Ma', family='' (v1: last='Ma'; the + # fix(suffix-routing) row pins the same shape for 'Johnson PhD' -- + # decision needed on ambiguous-surname suffixes) + # B2 'John Smith, V MD' -> suffix='V, MD' (v1 kept the segment whole: + # 'V MD') + # B3 'John Smith, Ph. D.' -> family='John Smith', given='' (split/ + # period-joined suffixes fail the segment-stage suffixy test) + # B4 '#144: 'Smith, John V' -> middle='V' (v1: suffix='V'; family- + # comma given-segment trailing suffix_not_acronyms rule missing) + # B5 suffix-in-parens/quotes (#111) missing: 'Andrew Perkins (MBA)' + # -> nickname='MBA' (v1: suffix='MBA'); 9 tests + # B6 suffix_delimiter edges: trailing delimiter kills detection; + # multi-word sides leak the '-' token into suffix + # Remove once fixed. "test_suffixes.py", "test_turkic_patronymic_order.py", "test_variations.py", From 35f6280f246d79a96a236315e089210dcdad735b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 00:50:21 -0700 Subject: [PATCH 121/275] Route suffix-shaped delimited content past nickname extraction (v1 parity) CI's sphinx doctests caught this unported v1 rule: parse_nicknames leaves suffix-shaped delimited content -- an unambiguous suffix word, an unambiguous period-free acronym, or anything ending in a period -- in place for normal parsing ('Andrew Perkins (MBA)' -> suffix=MBA, not nickname). The v2 spelling masks only the two delimiter spans so the inner content joins the main token stream (spans keep indexing the original, anti-#100). Two case rows pin it. The three customize.rst doctest blocks that taught the removed regexes.emoji/bidi and custom-delimiter-pattern idioms now teach the 2.0 replacements (Policy strip flags; Policy delimiter pairs). Co-Authored-By: Claude Fable 5 --- docs/customize.rst | 49 ++++++++++++++------------------ nameparser/_pipeline/_extract.py | 33 +++++++++++++++++++-- tests/v2/cases.py | 7 +++++ 3 files changed, 58 insertions(+), 31 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index da1b8dcd..9814f95b 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -80,21 +80,17 @@ overriding e.g. ``CONSTANTS.regexes.parenthesis`` still works exactly as before) and :py:obj:`~nameparser.config.Constants.maiden_delimiters` (empty by default -- see "Routing to Maiden Name" below). -To recognize an *additional* delimiter, add a compiled pattern to -``nickname_delimiters`` under any key, then re-run -:py:meth:`~nameparser.parser.HumanName.parse_full_name` to pick it up: +In 2.0, custom delimiter patterns on ``Constants`` were removed; adding a +non-built-in key raises ``TypeError``. Custom delimiter *pairs* are a +first-class feature of the new API instead -- pass them to +:py:class:`~nameparser.Policy`: .. doctest:: - >>> import re - >>> from nameparser import HumanName - >>> from nameparser.config import Constants - >>> hn = HumanName("Benjamin {Ben} Franklin", constants=Constants()) - >>> hn.nickname - '' - >>> hn.C.nickname_delimiters['curly_braces'] = re.compile(r'\{(.*?)\}', re.U) - >>> hn.parse_full_name() - >>> hn.nickname + >>> from nameparser import Parser, Policy + >>> policy = Policy( + ... nickname_delimiters=Policy().nickname_delimiters | {("{", "}")}) + >>> Parser(policy=policy).parse("Benjamin {Ben} Franklin").nickname 'Ben' Routing to Maiden Name @@ -521,17 +517,16 @@ values with the behavior described above. Don't Remove Emojis ~~~~~~~~~~~~~~~~~~~ -By default, all emojis are removed from the input string before the name is parsed. -You can turn this off by setting the ``emoji`` regex to ``False``. +By default, all emojis are removed from the input string before the name is +parsed. In 2.0 the ``regexes.emoji`` override was removed (assignment raises +``TypeError``); the switch is the :py:class:`~nameparser.Policy` flag +``strip_emoji``: .. doctest:: - >>> from nameparser import HumanName - >>> from nameparser.config import Constants - >>> constants = Constants() - >>> constants.regexes.emoji = False - >>> hn = HumanName("Sam 😊 Smith", constants=constants) - >>> str(hn) + >>> from nameparser import Parser, Policy + >>> parser = Parser(policy=Policy(strip_emoji=False)) + >>> str(parser.parse("Sam 😊 Smith")) 'Sam 😊 Smith' Don't Remove Bidi Control Characters @@ -539,17 +534,15 @@ Don't Remove Bidi Control Characters By default, invisible bidirectional control characters (the left-to-right and right-to-left marks and friends, common in copy-pasted right-to-left names) are -removed from the input string before the name is parsed. You can turn this off -by setting the ``bidi`` regex to ``False``. +removed from the input string before the name is parsed. In 2.0 the +``regexes.bidi`` override was removed (assignment raises ``TypeError``); the +switch is the :py:class:`~nameparser.Policy` flag ``strip_bidi``: .. doctest:: - >>> from nameparser import HumanName - >>> from nameparser.config import Constants - >>> constants = Constants() - >>> constants.regexes.bidi = False - >>> hn = HumanName("\u200fJohn\u200f Smith", constants=constants) - >>> hn.first == "\u200fJohn\u200f" + >>> from nameparser import Parser, Policy + >>> parser = Parser(policy=Policy(strip_bidi=False)) + >>> parser.parse("\u200fJohn\u200f Smith").given == "\u200fJohn\u200f" True Config Changes May Need Parse Refresh diff --git a/nameparser/_pipeline/_extract.py b/nameparser/_pipeline/_extract.py index 4ac1ca66..d52896bc 100644 --- a/nameparser/_pipeline/_extract.py +++ b/nameparser/_pipeline/_extract.py @@ -22,12 +22,26 @@ import dataclasses +from nameparser._lexicon import Lexicon, _normalize from nameparser._pipeline._state import ( COMMA_CHARS, ParseState, PendingAmbiguity, ) from nameparser._types import AmbiguityKind, Role, Span +def _suffix_shaped(content: str, lexicon: Lexicon) -> bool: + """v1 parse_nicknames' escape (parser.py:1125-1141): an unambiguous + suffix_words member (edge-normalized), an unambiguous acronym + (period-free form), or anything ending in a period. No initial + veto -- v1 deliberately skipped it here.""" + stripped = _normalize(content) + acronym = stripped.replace(".", "") + return (stripped in lexicon.suffix_words + or (acronym in lexicon.suffix_acronyms + and acronym not in lexicon.suffix_acronyms_ambiguous) + or content.endswith(".")) + + def _open_ok(text: str, i: int) -> bool: return i == 0 or text[i - 1].isspace() @@ -88,9 +102,22 @@ def extract_delimited(state: ParseState) -> ParseState: full = Span(i, j + len(close)) if not _overlaps(full, masked): inner = Span(i + len(open_), j) - if inner.start < inner.end: - extracted.append((role, inner)) - masked.append(full) + if inner.start < inner.end and _suffix_shaped( + text[inner.start:inner.end], state.lexicon): + # v1 parse_nicknames: suffix-shaped delimited + # content is left IN PLACE (undelimited) for + # normal downstream parsing -- 'Andrew Perkins + # (MBA)' keeps MBA a suffix, not a nickname. + # Spans index the original (anti-#100), so the + # v2 spelling masks only the two delimiter + # spans and lets the inner content join the + # main token stream. + masked.append(Span(i, i + len(open_))) + masked.append(Span(j, j + len(close))) + else: + if inner.start < inner.end: + extracted.append((role, inner)) + masked.append(full) pos = j + len(close) extracted.sort(key=lambda pair: pair[1]) masked.sort() diff --git a/tests/v2/cases.py b/tests/v2/cases.py index b4f66a9f..27ff0a5c 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -156,6 +156,13 @@ class Case: notes="v2 surfaces #121's irreducible ambiguity"), Case("family_comma_particles", "de la Vega, Juan", {"given": "Juan", "family": "de la Vega"}), + Case("paren_suffix_escapes_nickname", "Andrew Perkins (MBA)", + {"given": "Andrew", "family": "Perkins", "suffix": "MBA"}, + notes="v1 parse_nicknames: suffix-shaped delimited content is " + "left in place for normal parsing (pinned live " + "2026-07-17)"), + Case("paren_period_escapes_nickname", "Andrew Perkins (Ret.)", + {"given": "Andrew", "family": "Perkins", "suffix": "Ret."}), Case("nickname_quotes", 'John "Jack" Kennedy', {"given": "John", "family": "Kennedy", "nickname": "Jack"}), Case("nickname_parens", "John (Jack) Kennedy", From 21795cc026800e686656191f56462214f50b2a74 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 00:51:44 -0700 Subject: [PATCH 122/275] Reconcile test_prefixes (bucket C); hold on the B1 trailing-suffix gap 58 tests: one bucket-C rewrite (the custom-title fold test moves its 'gunny' title onto a private Constants instead of mutating the shared singleton); 57 pass. 1 test fails on the same B1 root cause already reported for test_suffixes -- two-word trailing-suffix routing: HumanName('Anh Do') -> suffix='Do', family='' (v1: first='Anh', last='Do'; 'do' is the D.O. suffix acronym) Stays in collect_ignore_glob next to test_suffixes' entry; mypy-clean, so it leaves the mypy exclude. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 2 +- tests/conftest.py | 5 +++++ tests/test_prefixes.py | 8 +++++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index da31c1ee..6ac826ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ packages = ["nameparser", "tests"] # TEMPORARY (M11 -> M12): the v1 suite is being reconciled against the # 2.0 facade; lifted with tests/conftest.py's collect_ignore_glob. Must # be gone before the branch leaves draft. tests/v2/ stays checked. -exclude = '^tests/(test_(bound_first_names|brute_force|comma_variants|conjunctions|east_slavic_patronymic_order|middle_name_as_last|nicknames|parser_util|prefixes|turkic_patronymic_order|variations)\.py)$' +exclude = '^tests/(test_(bound_first_names|brute_force|comma_variants|conjunctions|east_slavic_patronymic_order|middle_name_as_last|nicknames|parser_util|turkic_patronymic_order|variations)\.py)$' [[tool.mypy.overrides]] module = [ diff --git a/tests/conftest.py b/tests/conftest.py index 0dfb63ae..62a05f62 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,6 +22,11 @@ "test_middle_name_as_last.py", "test_nicknames.py", "test_parser_util.py", + # test_prefixes.py is reconciled (one bucket-C rewrite; 57 pass) but 1 + # test fails on the same B1 gap held for test_suffixes: the two-word + # trailing-suffix routing. Repro: HumanName('Anh Do') -> suffix='Do', + # family='' (v1: last='Do'; 'do' is the D.O. suffix acronym). Remove + # with test_suffixes' entry once B1 is settled. "test_prefixes.py", # test_suffixes.py needs NO bucket edits (30 pass, 2 v1 xfails) but 24 # tests fail across six suspected pipeline gaps -- see the M12 batch-3 diff --git a/tests/test_prefixes.py b/tests/test_prefixes.py index e21e3af7..26b2d75d 100644 --- a/tests/test_prefixes.py +++ b/tests/test_prefixes.py @@ -291,9 +291,11 @@ def test_non_first_name_prefix_with_custom_title(self) -> None: # 'Gunny' is NOT a default title -> genuinely exercises the custom-title # path. Title is consumed first (first_list == ['']), so the fold does # not fire and the surname is already correct; asserts we don't corrupt - # the title case. - CONSTANTS.titles.add('gunny') - hn = HumanName("Gunny de Mesnil") + # the title case. Runs on a private Constants (2.0 deprecates shared- + # CONSTANTS mutation; the custom-title path under test is identical). + c = Constants() + c.titles.add('gunny') + hn = HumanName("Gunny de Mesnil", constants=c) self.m(hn.title, "Gunny", hn) self.m(hn.first, "", hn) self.m(hn.last, "de Mesnil", hn) From 20c2829e1bf98e85305b8c575d4ad112a94d3321 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 00:53:08 -0700 Subject: [PATCH 123/275] Hold test_conjunctions on the join-stage conjunction/initial bug The file needs no bucket edits (37 pass, 2 pre-existing v1 xfails); one test fails on the join-stage sibling of the already-fixed _cap_word bug: the classify-stage "conjunction" tag lacks v1 is_conjunction's is_an_initial exclusion, so an initial that doubles as a conjunction is joined into its neighbors. The group-stage #11 carve-out only exempts the bare single letter ('e'), not the initial form ('e.'). HumanName('john e. smith') -> first='john e. smith' (v1: first='john', middle='e.', last='smith') Stays in collect_ignore_glob with a pointer comment; mypy-clean, so it leaves the mypy exclude. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 2 +- tests/conftest.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6ac826ca..a718115d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ packages = ["nameparser", "tests"] # TEMPORARY (M11 -> M12): the v1 suite is being reconciled against the # 2.0 facade; lifted with tests/conftest.py's collect_ignore_glob. Must # be gone before the branch leaves draft. tests/v2/ stays checked. -exclude = '^tests/(test_(bound_first_names|brute_force|comma_variants|conjunctions|east_slavic_patronymic_order|middle_name_as_last|nicknames|parser_util|turkic_patronymic_order|variations)\.py)$' +exclude = '^tests/(test_(bound_first_names|brute_force|comma_variants|east_slavic_patronymic_order|middle_name_as_last|nicknames|parser_util|turkic_patronymic_order|variations)\.py)$' [[tool.mypy.overrides]] module = [ diff --git a/tests/conftest.py b/tests/conftest.py index 62a05f62..7ef0b98a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,6 +17,14 @@ "test_bound_first_names.py", "test_brute_force.py", "test_comma_variants.py", + # test_conjunctions.py needs NO bucket edits (37 pass, 2 v1 xfails) but + # 1 test fails on the join-stage sibling of the fixed _cap_word bug: + # the classify-stage "conjunction" tag (_classify.py:51) lacks v1 + # is_conjunction's is_an_initial exclusion (75e3219^ parser.py:761), so + # the initial 'e.' is joined as a conjunction. The group-stage #11 + # carve-out only covers the bare single letter 'e', not 'e.'. Repro: + # HumanName('john e. smith') -> first='john e. smith' + # (v1: first='john', middle='e.', last='smith'). Remove once fixed. "test_conjunctions.py", "test_east_slavic_patronymic_order.py", "test_middle_name_as_last.py", From 2c1745425d196ef7b79c7cbd9e0c065fcaed8a00 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 00:56:10 -0700 Subject: [PATCH 124/275] Reconcile test_nicknames (buckets A, B); hold on a bucket-precedence flip 39 tests -> 35; 33 pass, 1 v1 xfail. Bucket A/B (custom-delimiter mechanism removed, spec section 3): - test_add_custom_nickname_delimiter becomes the raises pin (TypeError naming Policy); the remove/multiple/ambiguous-through-custom variants die with the mechanism they exercised. - the regexes-override live-resolution test becomes a raises pin too (regexes item assignment raises, same uniform rule). - the unresolvable-string-sentinel ValueError-at-parse test becomes a TypeError-at-assignment pin (2.0 enforces the invariant earlier). - as_dict()['maiden'] empty expectation is '' outright (#255). Held (bucket D, reported): the documented both-buckets misuse case flipped precedence -- with 'parenthesis' in BOTH delimiter buckets, v1 processed nickname first (nickname='Johnson'); 2.0 gives maiden='Johnson'. The v1 test exists precisely to catch this silent change; needs fix-or-classify. File stays in collect_ignore_glob; mypy-clean, leaves the mypy exclude. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 2 +- tests/conftest.py | 11 ++++++ tests/test_nicknames.py | 86 +++++++++++++---------------------------- 3 files changed, 39 insertions(+), 60 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a718115d..19962f0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ packages = ["nameparser", "tests"] # TEMPORARY (M11 -> M12): the v1 suite is being reconciled against the # 2.0 facade; lifted with tests/conftest.py's collect_ignore_glob. Must # be gone before the branch leaves draft. tests/v2/ stays checked. -exclude = '^tests/(test_(bound_first_names|brute_force|comma_variants|east_slavic_patronymic_order|middle_name_as_last|nicknames|parser_util|turkic_patronymic_order|variations)\.py)$' +exclude = '^tests/(test_(bound_first_names|brute_force|comma_variants|east_slavic_patronymic_order|middle_name_as_last|parser_util|turkic_patronymic_order|variations)\.py)$' [[tool.mypy.overrides]] module = [ diff --git a/tests/conftest.py b/tests/conftest.py index 7ef0b98a..df86716b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,6 +28,17 @@ "test_conjunctions.py", "test_east_slavic_patronymic_order.py", "test_middle_name_as_last.py", + # test_nicknames.py is reconciled (buckets A/B: custom-delimiter and + # regexes-override mechanisms now pin their Policy-pointing TypeErrors; + # #255 as_dict expectation) but 1 test fails on a precedence flip in the + # documented both-buckets misuse case: with 'parenthesis' routed to BOTH + # nickname_delimiters and maiden_delimiters, v1 processed nickname + # first (nickname='Johnson', maiden=''); 2.0 gives maiden='Johnson', + # nickname=''. Repro: C.maiden_delimiters['parenthesis'] = + # C.regexes['parenthesis']; HumanName('Baker (Johnson), Jenny', + # constants=C). The v1 test exists precisely to catch this silent + # change -- needs a decision (fix precedence or classify). Remove once + # settled. "test_nicknames.py", "test_parser_util.py", # test_prefixes.py is reconciled (one bucket-C rewrite; 57 pass) but 1 diff --git a/tests/test_nicknames.py b/tests/test_nicknames.py index 423f5b13..2034a04d 100644 --- a/tests/test_nicknames.py +++ b/tests/test_nicknames.py @@ -18,48 +18,28 @@ def test_nickname_in_parenthesis(self) -> None: self.m(hn.nickname, "Ben", hn) # https://github.com/derek73/python-nameparser/issues/112 - def test_add_custom_nickname_delimiter(self) -> None: + def test_add_custom_nickname_delimiter_raises(self) -> None: + # Custom (non-sentinel) delimiter additions raise in 2.0 (deliberate + # divergence, migration spec section 3 -- same uniform rule as + # regexes); Policy(nickname_delimiters=...) is the replacement. The + # v1 tests for removing custom delimiters, registering several at + # once, and the ambiguous-acronym carve-out through a custom + # delimiter died with the mechanism. hn = HumanName("Benjamin {Ben} Franklin", constants=Constants()) # curly braces aren't a recognized delimiter by default self.m(hn.nickname, "", hn) - hn.C.nickname_delimiters['curly_braces'] = re.compile(r'\{(.*?)\}') - hn.parse_full_name() - self.m(hn.first, "Benjamin", hn) - self.m(hn.last, "Franklin", hn) - self.m(hn.nickname, "Ben", hn) - - def test_remove_custom_nickname_delimiter(self) -> None: - hn = HumanName("Benjamin {Ben} Franklin", constants=Constants()) - hn.C.nickname_delimiters['curly_braces'] = re.compile(r'\{(.*?)\}') - hn.parse_full_name() - self.m(hn.nickname, "Ben", hn) - del hn.C.nickname_delimiters['curly_braces'] - hn.parse_full_name() - self.m(hn.nickname, "", hn) - - def test_multiple_custom_nickname_delimiters_together(self) -> None: - # Two extras registered at once must both be recognized in a single - # parse, independent of insertion order. - hn = HumanName("Benjamin {Ben} Franklin", constants=Constants()) - hn.C.nickname_delimiters['curly_braces'] = re.compile(r'\{(.*?)\}') - hn.C.nickname_delimiters['angle_brackets'] = re.compile(r'<(.*?)>') - hn.parse_full_name() - self.m(hn.first, "Benjamin", hn) - self.m(hn.last, "Franklin", hn) - self.m(hn.nickname, "Ben Benny", hn) - - def test_overriding_builtin_regex_still_affects_nickname_parsing(self) -> None: - # The pre-existing customization path (overriding self.C.regexes - # directly) must keep working: nickname_delimiters' three built-in - # entries resolve self.C.regexes. live at parse time rather than - # storing a snapshotted pattern. + with pytest.raises(TypeError, match="Policy"): + hn.C.nickname_delimiters['curly_braces'] = re.compile(r'\{(.*?)\}') + + def test_overriding_builtin_regex_raises(self) -> None: + # v1's other customization path -- replacing a regexes entry so the + # built-in delimiter sentinels resolve to it live -- is the same + # uniform 2.0 removal: any regexes assignment raises, pointing at + # Policy. hn = HumanName("Benjamin [Ben] Franklin", constants=Constants()) self.m(hn.nickname, "", hn) - hn.C.regexes['parenthesis'] = re.compile(r'\[(.*?)\]') - hn.parse_full_name() - self.m(hn.first, "Benjamin", hn) - self.m(hn.last, "Franklin", hn) - self.m(hn.nickname, "Ben", hn) + with pytest.raises(TypeError, match="Policy"): + hn.C.regexes['parenthesis'] = re.compile(r'\[(.*?)\]') def test_two_word_nickname_in_parenthesis(self) -> None: hn = HumanName("Benjamin (Big Ben) Franklin") @@ -189,18 +169,6 @@ def test_ambiguous_suffix_acronym_in_parenthesis_stays_nickname(self) -> None: self.m(hn.nickname, "JD", hn) self.m(hn.suffix, "", hn) - def test_ambiguous_suffix_acronym_in_custom_delimiter_stays_nickname(self) -> None: - # Same suffix-vs-nickname disambiguation as above, but through a - # custom delimiter added via nickname_delimiters -- confirms - # handle_match() is applied uniformly regardless of which delimiter - # matched, not just the three built-ins. - hn = HumanName("JEFFREY {JD} BRICKEN", constants=Constants()) - hn.C.nickname_delimiters['curly_braces'] = re.compile(r'\{(.*?)\}') - hn.parse_full_name() - self.m(hn.nickname, "JD", hn) - self.m(hn.suffix, "", hn) - - class MaidenNameTestCase(HumanNameTestBase): def test_maiden_assignment_and_property(self) -> None: hn = HumanName("Jenny Baker") @@ -212,8 +180,9 @@ def test_maiden_defaults_empty(self) -> None: self.m(hn.maiden, "", hn) def test_maiden_key_always_in_as_dict(self) -> None: + # empty attributes are always '' in 2.0 (#255) hn = HumanName("Bob Dole") - self.assertEqual(hn.as_dict()['maiden'], hn.C.empty_attribute_default) + self.assertEqual(hn.as_dict()['maiden'], '') self.assertNotIn('maiden', hn.as_dict(False)) def test_maiden_appears_in_as_dict_when_populated(self) -> None: @@ -284,16 +253,15 @@ def test_maiden_appears_in_as_dict_via_routing(self) -> None: self.assertEqual(hn.as_dict()['maiden'], "Johnson") def test_unresolvable_string_sentinel_raises(self) -> None: - # A string value in nickname_delimiters/maiden_delimiters that - # doesn't name a real regexes key used to silently fall back to - # EMPTY_REGEX, which matches at every position and corrupts parsing - # (appends '' into the bucket repeatedly, and leaves the intended - # delimiter's content unstripped elsewhere in the name). It must - # raise instead. + # A string value that doesn't name a real delimiter used to silently + # fall back to EMPTY_REGEX in v1 (matching at every position and + # corrupting parsing) until 1.3 made the parse raise ValueError. 2.0 + # enforces the invariant one step earlier: the delimiter managers + # accept only the named sentinels, so the typo fails loudly at + # assignment instead of at the next parse. C = Constants() - C.nickname_delimiters['typo'] = 'parenthesus' - with pytest.raises(ValueError): - HumanName("Jenny (Johnson) Baker", constants=C) + with pytest.raises(TypeError, match="sentinel"): + C.nickname_delimiters['typo'] = 'parenthesus' def test_routing_same_delimiter_to_both_buckets_nickname_wins(self) -> None: # Misuse case: assigning the same key into both dicts instead of From 7e044dd606df4319525a50ddd1c28660285696d5 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 00:57:13 -0700 Subject: [PATCH 125/275] Reconcile test_comma_variants against the facade (bucket B) 5 tests -> 4, all green; lifted from both skip lists. Bucket B: the custom-regexes-without-commas shatter guard died with its mechanism -- Constants(regexes=...) raises TypeError in 2.0 (migration spec section 3 uniform rule), already pinned in test_python_api.py's test_override_regex_raises. The four non-ASCII comma tests (#265) pass unchanged. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 2 +- tests/conftest.py | 1 - tests/test_comma_variants.py | 29 +++++------------------------ 3 files changed, 6 insertions(+), 26 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 19962f0f..e3bb8e75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ packages = ["nameparser", "tests"] # TEMPORARY (M11 -> M12): the v1 suite is being reconciled against the # 2.0 facade; lifted with tests/conftest.py's collect_ignore_glob. Must # be gone before the branch leaves draft. tests/v2/ stays checked. -exclude = '^tests/(test_(bound_first_names|brute_force|comma_variants|east_slavic_patronymic_order|middle_name_as_last|parser_util|turkic_patronymic_order|variations)\.py)$' +exclude = '^tests/(test_(bound_first_names|brute_force|east_slavic_patronymic_order|middle_name_as_last|parser_util|turkic_patronymic_order|variations)\.py)$' [[tool.mypy.overrides]] module = [ diff --git a/tests/conftest.py b/tests/conftest.py index df86716b..b7489373 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,7 +16,6 @@ collect_ignore_glob = [ "test_bound_first_names.py", "test_brute_force.py", - "test_comma_variants.py", # test_conjunctions.py needs NO bucket edits (37 pass, 2 v1 xfails) but # 1 test fails on the join-stage sibling of the fixed _cap_word bug: # the classify-stage "conjunction" tag (_classify.py:51) lacks v1 diff --git a/tests/test_comma_variants.py b/tests/test_comma_variants.py index ca53581f..17597487 100644 --- a/tests/test_comma_variants.py +++ b/tests/test_comma_variants.py @@ -1,8 +1,4 @@ -import pytest - from nameparser import HumanName -from nameparser.config import Constants -from nameparser.config.regexes import REGEXES from tests.base import HumanNameTestBase @@ -31,23 +27,8 @@ def test_trailing_arabic_comma_stripped(self) -> None: hn = HumanName("سلمان،") self.m(hn.first, "سلمان", hn) - def test_custom_regexes_without_commas_key_does_not_shatter_name(self) -> None: - # A custom regexes dict that omits "commas" entirely must not fall - # back to RegexTupleManager's EMPTY_REGEX default for splitting -- - # re.compile('').split(...) matches between every character, which - # explodes any name into single-char pieces instead of leaving it - # unsplit (the EMPTY_REGEX convention elsewhere in this codebase - # means "feature disabled", not "split on every character"). - # With comma splitting disabled, "Smith, John" is tokenized like any - # other no-comma input (word tokenizing drops the punctuation), - # yielding a plain first/last pair -- not the inverted "Last, First" - # reading, and definitely not single-character pieces. - # Unknown-key access (here, the deliberately-omitted 'commas') is - # deprecated for removal in 2.0 (#256); this fallback pattern will - # AttributeError-crash the parser instead of degrading silently. - custom = {k: v for k, v in REGEXES.items() if k != 'commas'} - c = Constants(regexes=custom) - with pytest.deprecated_call(): - hn = HumanName("Smith, John", constants=c) - self.m(hn.first, "Smith", hn) - self.m(hn.last, "John", hn) + # The v1 test for a custom regexes dict omitting the 'commas' key (the + # EMPTY_REGEX shatter guard) died with the mechanism: Constants(regexes= + # ...) raises TypeError in 2.0 (deliberate divergence, migration spec + # section 3 uniform rule) -- pinned in test_python_api.py's + # test_override_regex_raises. From c131dde4002ac4589001a0a01f75c4319b5432c5 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 06:44:51 -0700 Subject: [PATCH 126/275] Fix the M12 batch-3 bug queue: walk order, chunk rules, tail entries All pinned live against v1.4 (2026-07-17) unless classified: - 'ma'/'do' join suffix_acronyms_ambiguous (data): common surnames need periods to read as credentials -- 'Jack Ma' keeps its family name while the approved fix(suffix-routing) rows keep theirs. The set's doc comment gains the surname criterion and the 2.0 gating note. Release-log item. - classify's conjunction tag excludes initial-shaped tokens (v1 is_conjunction): 'john e. smith' keeps its middle initial. - the period-joined derivation moves to _vocab.period_joined_vocab, shared by classify AND segment's suffix-comma detection ('John Doe, Msc.Ed.'); chunk-level membership is v1's is_suffix (bare ambiguous acronyms count within period-joined tokens). - segment's detection collapses an adjacent Ph./D. pair into one suffix unit (v1 extracted the credential pre-parse via fix_phd). - extract iterates the nickname bucket first: when one delimiter pair sits in both buckets, the nickname reading wins (v1 order). - the FAMILY_COMMA walk restores v1's order -- the first post-comma piece is the given BEFORE any suffix check ('Hardman, RN - CRNA') -- with one classified fix(comma-family) deviation: a LONE unambiguous suffix piece routes to suffix ('Andrews, M.D.'). The trailing piece of a two-part name takes the lenient test (v1 #144: 'Smith, John V' -> suffix=V). - tail comma segments render as ONE suffix entry each: continuation tokens take the 'joined' tag (space-join), a delimiter core inside a segment separates entries and is dropped, and a lone-core segment stays whole (v1 expand semantics). Ten case rows pin the matrix. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_assign.py | 30 ++++++++++++++++++-- nameparser/_pipeline/_classify.py | 23 ++++++++-------- nameparser/_pipeline/_extract.py | 6 +++- nameparser/_pipeline/_group.py | 30 ++++++++++++++++---- nameparser/_pipeline/_segment.py | 32 ++++++++++++++++----- nameparser/_pipeline/_vocab.py | 25 +++++++++++++++++ nameparser/config/suffixes.py | 12 ++++++-- tests/v2/cases.py | 46 +++++++++++++++++++++++++++++++ 8 files changed, 172 insertions(+), 32 deletions(-) diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 57f8f948..21fb4a2e 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -27,6 +27,7 @@ import dataclasses import re +from nameparser._pipeline._vocab import is_suffix_lenient from nameparser._pipeline._group import ( _is_suffix_piece, _is_title_piece, ) @@ -189,11 +190,34 @@ def assign(state: ParseState) -> ParseState: n = _peel_leading_titles(pieces, ptags, tokens) given_done = False for m in range(n, len(pieces)): - if _is_suffix_piece(pieces[m], ptags[m], tokens): - _set_roles(tokens, pieces[m], Role.SUFFIX) - elif not given_done: + # v1 walk order (parser.py:1390): the first non-title + # piece is ALWAYS the given, before any suffix check -- + # 'Hardman, RN - CRNA' keeps first='RN'. One deliberate + # 2.0 deviation, classified fix(comma-family): when that + # piece is the segment's ONLY piece and unambiguously + # suffix-shaped ('Andrews, M.D.'), it is a suffix -- v1 + # made it the given. + if not given_done: + if (m == len(pieces) - 1 + and _is_suffix_piece(pieces[m], ptags[m], + tokens)): + _set_roles(tokens, pieces[m], Role.SUFFIX) + continue _set_roles(tokens, pieces[m], Role.GIVEN) given_done = True + continue + # trailing piece of a two-part name is unambiguously + # positioned: v1 accepts the lenient test there + # ('Smith, John V' -> suffix='V', #144); with a third + # comma part the trailing token is more likely a middle + # initial, so strict only + last_of_two = (m == len(pieces) - 1 + and len(state.segments) == 2) + if _is_suffix_piece(pieces[m], ptags[m], tokens) or ( + last_of_two and len(pieces[m]) == 1 + and is_suffix_lenient( + tokens[pieces[m][0]].text, state.lexicon)): + _set_roles(tokens, pieces[m], Role.SUFFIX) else: _set_roles(tokens, pieces[m], Role.MIDDLE) tail = 2 diff --git a/nameparser/_pipeline/_classify.py b/nameparser/_pipeline/_classify.py index 1aba58a5..317be8f4 100644 --- a/nameparser/_pipeline/_classify.py +++ b/nameparser/_pipeline/_classify.py @@ -17,16 +17,14 @@ from __future__ import annotations import dataclasses -import re from nameparser._lexicon import _normalize from nameparser._pipeline._state import ParseState, WorkToken -from nameparser._pipeline._vocab import is_initial, suffix_as_written +from nameparser._pipeline._vocab import ( + is_initial, period_joined_vocab, suffix_as_written, +) + -# Ported verbatim from v1 (nameparser/config/regexes.py -# "period_not_at_end") -- layering forbids the config import; keep in -# sync by hand. -_PERIOD_NOT_AT_END = re.compile(r".*\..+$", re.I) def _tags_for(token: WorkToken, state: ParseState) -> frozenset[str]: @@ -47,7 +45,9 @@ def _tags_for(token: WorkToken, state: ParseState) -> frozenset[str]: tags.add("particle") if n in lex.particles_ambiguous: tags.add("vocab:particle-ambiguous") - if n in lex.conjunctions: + if n in lex.conjunctions and not is_initial(token.text): + # v1's is_conjunction excludes initials: 'e.' in 'john e. smith' + # is a middle initial, not the Spanish conjunction 'e' tags.add("conjunction") if n in lex.bound_given_names: tags.add("vocab:bound-given") @@ -60,12 +60,11 @@ def _tags_for(token: WorkToken, state: ParseState) -> frozenset[str]: # a title as a whole ('Lt.Gov.', and by the ANY rule 'Mr.Smith'); # else ANY suffix chunk makes it a suffix ('JD.CPA'). Title wins # (v1's continue). Skipped when the whole token already matched. - if ("vocab:title" not in tags and "vocab:suffix" not in tags - and _PERIOD_NOT_AT_END.match(token.text)): - chunks = [_normalize(c) for c in token.text.split(".") if c] - if any(c in lex.titles for c in chunks): + if "vocab:title" not in tags and "vocab:suffix" not in tags: + derived = period_joined_vocab(token.text, lex) + if derived == "title": tags.add("vocab:title") - elif any(suffix_as_written(c, c, lex) for c in chunks): + elif derived == "suffix": tags.add("vocab:suffix") return frozenset(tags) diff --git a/nameparser/_pipeline/_extract.py b/nameparser/_pipeline/_extract.py index d52896bc..db31c4b9 100644 --- a/nameparser/_pipeline/_extract.py +++ b/nameparser/_pipeline/_extract.py @@ -60,9 +60,13 @@ def extract_delimited(state: ParseState) -> ParseState: extracted: list[tuple[Role, Span]] = [] masked: list[Span] = [] ambiguities: list[PendingAmbiguity] = [] + # nickname first (v1 parse_nicknames order): when the same + # delimiter pair sits in BOTH buckets, the nickname reading wins; + # the documented bucket-move idiom removes it from nickname, so + # maiden still gets it after a move for role, pairs in ( - (Role.MAIDEN, state.policy.maiden_delimiters), (Role.NICKNAME, state.policy.nickname_delimiters), + (Role.MAIDEN, state.policy.maiden_delimiters), ): for open_, close in sorted(pairs): pos = 0 diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index c747c396..7d70040d 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -188,13 +188,31 @@ def group(state: ParseState) -> ParseState: Structure.FAMILY_COMMA: 2}.get(state.structure) for seg_idx, seg in enumerate(state.segments): pieces, ptags = _group_segment(seg, additional, tokens) - if cores and tail_start is not None and seg_idx >= tail_start: - kept = [k for k in range(len(pieces)) - if not (len(pieces[k]) == 1 - and tokens[pieces[k][0]].text in cores)] + if tail_start is not None and seg_idx >= tail_start: + # v1 renders each tail COMMA SEGMENT as one suffix entry + # ('Smith, V MD' -> suffix 'V MD'); a delimiter core inside + # a segment separates entries and is dropped, but a segment + # that IS only the core stays whole (v1 expand() splits + # within a part, never erases a lone part). Continuation + # tokens within an entry take the stable "joined" tag so + # the suffix view space-joins them (the fix_phd mechanism). + entry_open = False + kept: list[int] = [] + for k in range(len(pieces)): + is_core = (len(pieces[k]) == 1 + and tokens[pieces[k][0]].text in cores + and len(pieces) > 1) + if is_core: + dropped.extend(pieces[k]) + entry_open = False + continue + kept.append(k) + for pos, i in enumerate(pieces[k]): + if entry_open or pos > 0: + tokens[i] = dataclasses.replace( + tokens[i], tags=tokens[i].tags | {"joined"}) + entry_open = True if len(kept) != len(pieces): - dropped.extend(i for k in range(len(pieces)) - if k not in kept for i in pieces[k]) pieces = [pieces[k] for k in kept] ptags = [ptags[k] for k in kept] # continuation tokens of a suffix-merged piece (the ph-d merge) diff --git a/nameparser/_pipeline/_segment.py b/nameparser/_pipeline/_segment.py index ce735019..df7ab767 100644 --- a/nameparser/_pipeline/_segment.py +++ b/nameparser/_pipeline/_segment.py @@ -21,15 +21,21 @@ import bisect import dataclasses +import re from nameparser._pipeline._state import ParseState, PendingAmbiguity, Structure from nameparser._pipeline._vocab import ( delimiter_cores, is_suffix_lenient, is_suffix_strict, - splits_into_suffixes, + period_joined_vocab, splits_into_suffixes, ) from nameparser._types import AmbiguityKind +# keep in sync with _group.py's merge pair (the fix_phd port) +_PH = re.compile(r"^ph\.?$", re.IGNORECASE) +_D = re.compile(r"^d\.?$", re.IGNORECASE) + + def segment(state: ParseState) -> ParseState: main = [i for i, t in enumerate(state.tokens) if t.role is None] if not main: @@ -70,16 +76,28 @@ def segment(state: ParseState) -> ParseState: def counts_as_suffix(text: str) -> bool: if text in cores: return True - return predicate(text, state.lexicon) or ( - bool(cores) - and splits_into_suffixes(text, cores, state.lexicon)) + return (predicate(text, state.lexicon) + or period_joined_vocab(text, state.lexicon) == "suffix" + or (bool(cores) + and splits_into_suffixes(text, cores, state.lexicon))) def suffixy(seg: tuple[int, ...]) -> bool: # an EMPTY segment is not suffix-shaped: v1's suffix-comma # detection fails on an empty parts[1] ('John Smith,, MD' is a - # family-comma parse) - return bool(seg) and all( - counts_as_suffix(state.tokens[i].text) for i in seg) + # family-comma parse). An adjacent Ph./D. pair counts as ONE + # suffix unit (v1's fix_phd extracted the credential pre-parse, + # so 'Smith, Ph. D.' read as suffix-comma; keep in sync with + # group's _PH/_D merge). + if not seg: + return False + texts = [state.tokens[i].text for i in seg] + k = 0 + while k < len(texts) - 1: + if _PH.fullmatch(texts[k]) and _D.fullmatch(texts[k + 1]): + texts[k:k + 2] = ["phd"] + else: + k += 1 + return all(counts_as_suffix(t) for t in texts) # v1 parity: only parts[1] decides the suffix-comma structure # (parser.py:1318); parts[2:] are consumed as suffixes diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 3ceae869..b2243374 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -92,3 +92,28 @@ def splits_into_suffixes(text: str, cores: frozenset[str], is_suffix_lenient(part, lexicon) for part in parts): return True return False + + +# Ported verbatim from v1 (nameparser/config/regexes.py +# "period_not_at_end") -- layering forbids the config import; keep in +# sync by hand. +_PERIOD_NOT_AT_END = re.compile(r".*\..+$", re.I) + + +def period_joined_vocab(text: str, lexicon: Lexicon) -> str | None: + """v1's parse_pieces derivation for interior-period tokens + ('Lt.Gov.', 'Msc.Ed.', and by the ANY rule 'Mr.Smith'): ANY title + chunk makes the token a title (checked first, v1's continue); else + ANY suffix chunk makes it a suffix. Chunk-level suffix membership + is v1's is_suffix: bare ambiguous acronyms COUNT ('Msc.Ed.' + derives via 'ed') -- the ambiguous period-gate applies to whole + tokens only. Returns "title", "suffix", or None.""" + if not _PERIOD_NOT_AT_END.match(text): + return None + chunks = [_normalize(c) for c in text.split(".") if c] + if any(c in lexicon.titles for c in chunks): + return "title" + if any(c in lexicon.suffix_acronyms or c in lexicon.suffix_words + for c in chunks): + return "suffix" + return None diff --git a/nameparser/config/suffixes.py b/nameparser/config/suffixes.py index 41bb6fdf..88962836 100644 --- a/nameparser/config/suffixes.py +++ b/nameparser/config/suffixes.py @@ -37,11 +37,17 @@ # in ambiguous, delimiter-only context. # # When adding a new entry to SUFFIX_ACRONYMS, also add it here only if - # the exact letter sequence could plausibly be someone's given name or - # common nickname on its own (e.g. 'jd', 'ed'). Unambiguous - # certifications/degrees (e.g. 'mba', 'cpa', 'phd') don't need an entry. + # the exact letter sequence could plausibly be someone's name on its + # own -- a given name or nickname (e.g. 'jd', 'ed') or a common + # surname (e.g. 'ma', 'do'). Unambiguous certifications/degrees + # (e.g. 'mba', 'cpa', 'phd') don't need an entry. In 2.0 this set + # also gates bare recognition: an ambiguous acronym counts as a + # suffix only when written with periods ('M.A.' yes, 'Ma' no), so + # 'Jack Ma' keeps its family name. + 'do', 'ed', 'jd', + 'ma', } """ diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 27ff0a5c..394aa8a4 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -41,6 +41,52 @@ class Case: {"given": "John", "family": "Smith"}), Case("suffix_comma", "John Smith, PhD", {"given": "John", "family": "Smith", "suffix": "PhD"}), + Case("ambiguous_surname_acronyms", "Jack Ma", + {"given": "Jack", "family": "Ma"}, + notes="'ma'/'do' joined suffix_acronyms_ambiguous: common " + "surnames need periods to read as credentials (v1 " + "parity restored; data change, flag for release log)"), + Case("ambiguous_surname_acronym_with_suffix", "Jack Ma Jr", + {"given": "Jack", "family": "Ma", "suffix": "Jr"}), + Case("initial_shaped_not_conjunction", "john e. smith", + {"given": "john", "middle": "e.", "family": "smith"}, + notes="v1 is_conjunction excludes initials at classify too"), + Case("family_comma_lenient_trailing", "Smith, John V", + {"given": "John", "family": "Smith", "suffix": "V"}, + notes="v1 #144: the trailing piece of a two-part comma name " + "takes the lenient suffix test"), + Case("family_comma_first_piece_is_given", "Steven Hardman, RN - CRNA", + {"given": "RN", "middle": "-", "family": "Steven Hardman", + "suffix": "CRNA"}, + notes="v1 walk order: the first post-comma piece is the given " + "before any suffix check (the delimiter is UNSET here " + "-- v1's documented limitation, kept)"), + Case("family_comma_lone_suffix_piece", "Andrews, M.D.", + {"family": "Andrews", "suffix": "M.D."}, + classification="fix(comma-family)", + notes="v1 made the lone post-comma strict-suffix piece the " + "given; 2.0 routes it to suffix (same family as the " + "'Smith, Dr.' row)"), + Case("period_joined_ambiguous_chunk", "John Doe, Msc.Ed.", + {"given": "John", "family": "Doe", "suffix": "Msc.Ed."}, + notes="chunk-level suffix membership is v1's is_suffix: bare " + "ambiguous acronyms count within period-joined tokens"), + Case("suffix_comma_split_phd", "John Smith, Ph. D.", + {"given": "John", "family": "Smith", "suffix": "Ph. D."}, + notes="the adjacent Ph./D. pair counts as one suffix unit in " + "the suffix-comma detection (v1 fix_phd parity)"), + Case("tail_segment_entry_space_joined", "John Smith, V MD", + {"given": "John", "family": "Smith", "suffix": "V MD"}, + notes="v1 renders each tail comma segment as ONE suffix " + "entry; words within an entry space-join via the " + "'joined' tag"), + Case("nickname_bucket_wins_when_shared", + 'Baker (Johnson), Jenny', + {"given": "Jenny", "family": "Baker", "nickname": "Johnson"}, + policy=Policy(maiden_delimiters=frozenset({("(", ")")})), + notes="when the same delimiter pair sits in both buckets the " + "nickname reading wins (v1 parse_nicknames order); the " + "bucket-move idiom removes it from nickname first"), Case("family_segment_trailing_suffix", "Smith Jr., John", {"given": "John", "family": "Smith", "suffix": "Jr."}, notes="v1: the family part may have suffixes in it " From 95b148a700f8e82ca151e2292afe85acd79cfb1f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 06:47:01 -0700 Subject: [PATCH 127/275] Lift suffixes, prefixes, conjunctions, nicknames from the skip list The batch-3 bug queue is fixed (c131dde): all four held files run green (184 pass, 5 xfail). Two last touches in test_suffixes: - Bucket D with classification: 'John Doe, MD-PhD-' (delimiter '-') keeps the no-space delimiter-core token whole, suffix='MD-PhD-' -- fix(suffix-delimiter-rendering), same rule as the pinned 'RN/CRNA' case row; logged in docs/superpowers/plans/notes-m12-diffs.md. - Bucket C: the constants-level suffix_delimiter test moves onto a private Constants instead of mutating the shared singleton. Co-Authored-By: Claude Fable 5 --- tests/conftest.py | 45 ------------------------------------------ tests/test_suffixes.py | 19 ++++++++++++------ 2 files changed, 13 insertions(+), 51 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b7489373..15cd7c86 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,54 +16,9 @@ collect_ignore_glob = [ "test_bound_first_names.py", "test_brute_force.py", - # test_conjunctions.py needs NO bucket edits (37 pass, 2 v1 xfails) but - # 1 test fails on the join-stage sibling of the fixed _cap_word bug: - # the classify-stage "conjunction" tag (_classify.py:51) lacks v1 - # is_conjunction's is_an_initial exclusion (75e3219^ parser.py:761), so - # the initial 'e.' is joined as a conjunction. The group-stage #11 - # carve-out only covers the bare single letter 'e', not 'e.'. Repro: - # HumanName('john e. smith') -> first='john e. smith' - # (v1: first='john', middle='e.', last='smith'). Remove once fixed. - "test_conjunctions.py", "test_east_slavic_patronymic_order.py", "test_middle_name_as_last.py", - # test_nicknames.py is reconciled (buckets A/B: custom-delimiter and - # regexes-override mechanisms now pin their Policy-pointing TypeErrors; - # #255 as_dict expectation) but 1 test fails on a precedence flip in the - # documented both-buckets misuse case: with 'parenthesis' routed to BOTH - # nickname_delimiters and maiden_delimiters, v1 processed nickname - # first (nickname='Johnson', maiden=''); 2.0 gives maiden='Johnson', - # nickname=''. Repro: C.maiden_delimiters['parenthesis'] = - # C.regexes['parenthesis']; HumanName('Baker (Johnson), Jenny', - # constants=C). The v1 test exists precisely to catch this silent - # change -- needs a decision (fix precedence or classify). Remove once - # settled. - "test_nicknames.py", "test_parser_util.py", - # test_prefixes.py is reconciled (one bucket-C rewrite; 57 pass) but 1 - # test fails on the same B1 gap held for test_suffixes: the two-word - # trailing-suffix routing. Repro: HumanName('Anh Do') -> suffix='Do', - # family='' (v1: last='Do'; 'do' is the D.O. suffix acronym). Remove - # with test_suffixes' entry once B1 is settled. - "test_prefixes.py", - # test_suffixes.py needs NO bucket edits (30 pass, 2 v1 xfails) but 24 - # tests fail across six suspected pipeline gaps -- see the M12 batch-3 - # report for full repros. Headlines: - # B1 'Jack Ma' -> suffix='Ma', family='' (v1: last='Ma'; the - # fix(suffix-routing) row pins the same shape for 'Johnson PhD' -- - # decision needed on ambiguous-surname suffixes) - # B2 'John Smith, V MD' -> suffix='V, MD' (v1 kept the segment whole: - # 'V MD') - # B3 'John Smith, Ph. D.' -> family='John Smith', given='' (split/ - # period-joined suffixes fail the segment-stage suffixy test) - # B4 '#144: 'Smith, John V' -> middle='V' (v1: suffix='V'; family- - # comma given-segment trailing suffix_not_acronyms rule missing) - # B5 suffix-in-parens/quotes (#111) missing: 'Andrew Perkins (MBA)' - # -> nickname='MBA' (v1: suffix='MBA'); 9 tests - # B6 suffix_delimiter edges: trailing delimiter kills detection; - # multi-word sides leak the '-' token into suffix - # Remove once fixed. - "test_suffixes.py", "test_turkic_patronymic_order.py", "test_variations.py", ] diff --git a/tests/test_suffixes.py b/tests/test_suffixes.py index 04d91f56..35661547 100644 --- a/tests/test_suffixes.py +++ b/tests/test_suffixes.py @@ -238,9 +238,13 @@ def test_suffix_delimiter_no_effect_without_comma(self) -> None: self.m(hn.suffix, "MD, PhD", hn) def test_suffix_delimiter_constants_level(self) -> None: - from nameparser.config import CONSTANTS - CONSTANTS.suffix_delimiter = " - " - hn = HumanName("Steven Hardman, RN - CRNA") + # ran on the shared CONSTANTS in v1; 2.0 deprecates shared mutation, + # so the config-level (non-kwarg) path is exercised on a private + # Constants passed as constants= + from nameparser.config import Constants + c = Constants() + c.suffix_delimiter = " - " + hn = HumanName("Steven Hardman, RN - CRNA", constants=c) self.m(hn.first, "Steven", hn) self.m(hn.last, "Hardman", hn) self.m(hn.suffix, "RN, CRNA", hn) @@ -254,12 +258,15 @@ def test_suffix_delimiter_none_by_default_known_limitation(self) -> None: self.m(hn.suffix, "CRNA", hn) def test_suffix_delimiter_trailing_delimiter_ignored(self) -> None: - # Trailing delimiter produces an empty token that must be filtered out. - # Using a non-whitespace-terminated delimiter so stripping doesn't consume it. + # Trailing delimiter must not defeat suffix detection. Using a + # non-whitespace-terminated delimiter so stripping doesn't consume it. hn = HumanName("John Doe, MD-PhD-", suffix_delimiter="-") self.m(hn.first, "John", hn) self.m(hn.last, "Doe", hn) - self.m(hn.suffix, "MD, PhD", hn) + # 2.0: fix(suffix-delimiter-rendering) -- v1 split the token and + # rendered 'MD, PhD'; v2 keeps a no-space delimiter-core token whole + # (anti-#100), same rule as the pinned 'RN/CRNA' case row. + self.m(hn.suffix, "MD-PhD-", hn) def test_suffix_delimiter_comma_space_is_noop(self) -> None: hn = HumanName("John Doe, MD, PhD", suffix_delimiter=", ") From bf060454cda2978443cf55d1414dbf17b9c662f5 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 06:50:11 -0700 Subject: [PATCH 128/275] Reconcile patronymic + bound-given files; hold two on parity bugs - test_east_slavic_patronymic_order (26) and test_turkic_patronymic_order (39): all green; the only edits are typing -- the regex-inspection tests read patterns through the 2.0 regexes proxy (typed object), now cast once at module level. Lifted from both skip lists. - test_bound_first_names: bucket A -- the two is_bound_first_name predicate tests died with the v1 parsing hooks (#280); vocabulary behavior stays pinned through the parsing tests. 1 test held on a join-guard bug: the FAMILY_COMMA bound-given join does not fire when it would consume the whole post-comma segment ('salem, abdul salam' -> first='abdul'; v1: 'abdul salam'). - test_middle_name_as_last: no bucket edits; 4 tests held on a fold-order bug -- v1 PREPENDED middle_list to last_list so comma and rotated forms converge ('Hassan, Mohamad Ahmad Ali' -> last='Ahmad Ali Hassan'); v2 renders the folded family in token order ('Hassan Ahmad Ali'). Both held files stay in collect_ignore_glob with repro comments; mypy-clean, so they leave the mypy exclude. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 2 +- tests/conftest.py | 19 +++++++++-- tests/test_bound_first_names.py | 11 ++---- tests/test_east_slavic_patronymic_order.py | 39 ++++++++++++---------- tests/test_turkic_patronymic_order.py | 38 ++++++++++++--------- 5 files changed, 65 insertions(+), 44 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e3bb8e75..50522348 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ packages = ["nameparser", "tests"] # TEMPORARY (M11 -> M12): the v1 suite is being reconciled against the # 2.0 facade; lifted with tests/conftest.py's collect_ignore_glob. Must # be gone before the branch leaves draft. tests/v2/ stays checked. -exclude = '^tests/(test_(bound_first_names|brute_force|east_slavic_patronymic_order|middle_name_as_last|parser_util|turkic_patronymic_order|variations)\.py)$' +exclude = '^tests/(test_(brute_force|parser_util|variations)\.py)$' [[tool.mypy.overrides]] module = [ diff --git a/tests/conftest.py b/tests/conftest.py index 15cd7c86..00674681 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,12 +14,27 @@ # Explicit list of not-yet-reconciled files. Narrow this as each file is # reconciled; remove entirely once the whole suite runs against the facade. collect_ignore_glob = [ + # test_bound_first_names.py is reconciled (bucket A: the two + # is_bound_first_name predicate tests died with the v1 hooks) but 1 test + # fails on a join-guard bug: the FAMILY_COMMA bound-given join does not + # fire when the join would consume the whole post-comma segment. Repro: + # HumanName('salem, abdul salam') -> first='abdul', middle='salam' + # (v1: first='abdul salam'; three-token 'salem, abdul salam ahmed' + # joins fine). Remove once fixed. "test_bound_first_names.py", "test_brute_force.py", - "test_east_slavic_patronymic_order.py", + # test_middle_name_as_last.py needs NO bucket edits but 4 tests fail on + # a fold-order bug: v1's middle_name_as_last fold PREPENDED middle_list + # to last_list, so comma and rotated forms converge on the no-comma + # result; v2 renders the folded family in token order. Repros: + # HumanName('Hassan, Mohamad Ahmad Ali', constants= + # Constants(middle_name_as_last=True)) -> last='Hassan Ahmad Ali' + # (v1: 'Ahmad Ali Hassan'); + # with patronymic_name_order too: 'Ivanov Petr Sergeyevich' -> + # last='Ivanov Sergeyevich' (v1: 'Sergeyevich Ivanov'). + # Remove once fixed. "test_middle_name_as_last.py", "test_parser_util.py", - "test_turkic_patronymic_order.py", "test_variations.py", ] diff --git a/tests/test_bound_first_names.py b/tests/test_bound_first_names.py index cb5aae95..3c1fb475 100644 --- a/tests/test_bound_first_names.py +++ b/tests/test_bound_first_names.py @@ -3,14 +3,9 @@ class BoundFirstNamesTestCase(HumanNameTestBase): - - def test_is_bound_first_name_true(self) -> None: - hn = HumanName("test") - assert hn.is_bound_first_name("Abdul") - - def test_is_bound_first_name_false(self) -> None: - hn = HumanName("test") - assert not hn.is_bound_first_name("Ahmed") + # The v1 is_bound_first_name predicate is gone with the other v1 parsing + # hooks (#280); the vocabulary's behavior is pinned through the parsing + # tests below. # --- no-comma: basic joining --- def test_no_comma_basic_join(self) -> None: diff --git a/tests/test_east_slavic_patronymic_order.py b/tests/test_east_slavic_patronymic_order.py index 314592a5..fb1eb11c 100644 --- a/tests/test_east_slavic_patronymic_order.py +++ b/tests/test_east_slavic_patronymic_order.py @@ -1,34 +1,39 @@ +import re +from typing import cast + from nameparser import HumanName from nameparser.config import Constants from tests.base import FlaggedConstantsTestBase, HumanNameTestBase +# The 2.0 regexes proxy types its attributes as ``object`` (reads are +# informational); the pattern-inspection tests below cast once here. +_LATIN = cast('re.Pattern[str]', Constants().regexes.east_slavic_patronymic) +_CYRILLIC = cast('re.Pattern[str]', + Constants().regexes.east_slavic_patronymic_cyrillic) + def test_latin_patronymic_matches() -> None: # One common suffix and one irregular — the integration tests cover the rest. - C = Constants() - assert C.regexes.east_slavic_patronymic.search("Ivanovich") - assert C.regexes.east_slavic_patronymic.search("Ilyich") + assert _LATIN.search("Ivanovich") + assert _LATIN.search("Ilyich") def test_latin_patronymic_rejects_non_patronymic() -> None: # EMPTY_REGEX (the default for missing keys) matches everything, # so this test is red until the real pattern is in place. - C = Constants() - assert not C.regexes.east_slavic_patronymic.search("Smith") + assert not _LATIN.search("Smith") def test_latin_patronymic_end_anchored() -> None: # A surname ending in a patronymic suffix matches; the end-anchor does not # prevent this. The parser guard tests verify reordering is suppressed. - C = Constants() - assert C.regexes.east_slavic_patronymic.search("Abramovich") + assert _LATIN.search("Abramovich") def test_cyrillic_patronymic_matches() -> None: # One common suffix and one irregular. - C = Constants() - assert C.regexes.east_slavic_patronymic_cyrillic.search("Иванович") - assert C.regexes.east_slavic_patronymic_cyrillic.search("ильич") + assert _CYRILLIC.search("Иванович") + assert _CYRILLIC.search("ильич") def test_cyrillic_patronymic_matches_capitalized_irregular_forms() -> None: @@ -36,17 +41,15 @@ def test_cyrillic_patronymic_matches_capitalized_irregular_forms() -> None: # capitalized first letter falls within the matched suffix itself, unlike # the common suffixes (-ович, -евна, ...) where only the surname root is # capitalized. Case-insensitivity is required for these to match. - C = Constants() - assert C.regexes.east_slavic_patronymic_cyrillic.search("Ильич") - assert C.regexes.east_slavic_patronymic_cyrillic.search("Кузьмич") - assert C.regexes.east_slavic_patronymic_cyrillic.search("Лукич") - assert C.regexes.east_slavic_patronymic_cyrillic.search("Фомич") - assert C.regexes.east_slavic_patronymic_cyrillic.search("Фокич") + assert _CYRILLIC.search("Ильич") + assert _CYRILLIC.search("Кузьмич") + assert _CYRILLIC.search("Лукич") + assert _CYRILLIC.search("Фомич") + assert _CYRILLIC.search("Фокич") def test_cyrillic_patronymic_rejects_non_patronymic() -> None: - C = Constants() - assert not C.regexes.east_slavic_patronymic_cyrillic.search("Иванов") + assert not _CYRILLIC.search("Иванов") class PatronymicNameOrderReorderTests(FlaggedConstantsTestBase): diff --git a/tests/test_turkic_patronymic_order.py b/tests/test_turkic_patronymic_order.py index 9b026600..ad275fcb 100644 --- a/tests/test_turkic_patronymic_order.py +++ b/tests/test_turkic_patronymic_order.py @@ -1,18 +1,28 @@ +import re +from typing import cast + from nameparser import HumanName from nameparser.config import Constants from tests.base import FlaggedConstantsTestBase, HumanNameTestBase +# The 2.0 regexes proxy types its attributes as ``object`` (reads are +# informational); the pattern-inspection tests below cast once here. +_R = Constants().regexes +_TURKIC = cast('re.Pattern[str]', _R.turkic_patronymic_marker) +_TURKIC_CYR = cast('re.Pattern[str]', _R.turkic_patronymic_marker_cyrillic) +_EAST_SLAVIC = cast('re.Pattern[str]', _R.east_slavic_patronymic) +_EAST_SLAVIC_CYR = cast('re.Pattern[str]', _R.east_slavic_patronymic_cyrillic) + def test_marker_is_whole_word_not_substring() -> None: # is_turkic_patronymic_marker() deliberately uses whole-word .match(), # not suffix .search() (unlike is_east_slavic_patronymic()) — pin this # so a future .match()->.search() slip doesn't silently start matching # surnames/given-names that merely contain a marker as a substring. - C = Constants() - assert not C.regexes.turkic_patronymic_marker.match("ogluu") - assert not C.regexes.turkic_patronymic_marker.match("Bogluchik") - assert not C.regexes.turkic_patronymic_marker_cyrillic.match("оглуш") - assert not C.regexes.turkic_patronymic_marker_cyrillic.match("Оглуев") + assert not _TURKIC.match("ogluu") + assert not _TURKIC.match("Bogluchik") + assert not _TURKIC_CYR.match("оглуш") + assert not _TURKIC_CYR.match("Оглуев") class TurkicPatronymicNameOrderReorderTests(FlaggedConstantsTestBase): @@ -264,7 +274,6 @@ def test_turkic_shape_unaffected_by_east_slavic_handler(self) -> None: assert n.last == "Aliyev" def test_no_regex_collision_latin(self) -> None: - C = Constants() east_slavic_examples = [ "Ivanovich", "Ivanovna", "Sergeevich", "Sergeevna", "Nikitichna", "Ilyich", "Kuzmich", "Lukich", "Fomich", "Fokich", @@ -276,14 +285,13 @@ def test_no_regex_collision_latin(self) -> None: for word in east_slavic_examples: # Sanity check: each word actually matches its own family's # regex, so the non-collision assertion below is non-vacuous. - assert C.regexes.east_slavic_patronymic.search(word), word - assert not C.regexes.turkic_patronymic_marker.match(word), word + assert _EAST_SLAVIC.search(word), word + assert not _TURKIC.match(word), word for word in turkic_examples: - assert C.regexes.turkic_patronymic_marker.match(word), word - assert not C.regexes.east_slavic_patronymic.search(word), word + assert _TURKIC.match(word), word + assert not _EAST_SLAVIC.search(word), word def test_no_regex_collision_cyrillic(self) -> None: - C = Constants() east_slavic_examples = [ "Иванович", "Ивановна", "Сергеевич", "Сергеевна", "Никитична", "Ильич", "Кузьмич", "Лукич", "Фомич", "Фокич", @@ -295,8 +303,8 @@ def test_no_regex_collision_cyrillic(self) -> None: for word in east_slavic_examples: # Sanity check: each word actually matches its own family's # regex, so the non-collision assertion below is non-vacuous. - assert C.regexes.east_slavic_patronymic_cyrillic.search(word), word - assert not C.regexes.turkic_patronymic_marker_cyrillic.match(word), word + assert _EAST_SLAVIC_CYR.search(word), word + assert not _TURKIC_CYR.match(word), word for word in turkic_examples: - assert C.regexes.turkic_patronymic_marker_cyrillic.match(word), word - assert not C.regexes.east_slavic_patronymic_cyrillic.search(word), word + assert _TURKIC_CYR.match(word), word + assert not _EAST_SLAVIC_CYR.search(word), word From d8e304292d08a89e228a481ce6c8130313705b48 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 06:50:44 -0700 Subject: [PATCH 129/275] Delete test_parser_util.py (bucket A) Its sole subject, nameparser.parser.group_contiguous_integers, was a v1 join_on_conjunctions helper; the 2.0 pipeline's group stage merges by piece index with no such utility, and the swap left no export. Nothing survives to port. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 2 +- tests/conftest.py | 1 - tests/test_parser_util.py | 39 --------------------------------------- 3 files changed, 1 insertion(+), 41 deletions(-) delete mode 100644 tests/test_parser_util.py diff --git a/pyproject.toml b/pyproject.toml index 50522348..c806b2bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ packages = ["nameparser", "tests"] # TEMPORARY (M11 -> M12): the v1 suite is being reconciled against the # 2.0 facade; lifted with tests/conftest.py's collect_ignore_glob. Must # be gone before the branch leaves draft. tests/v2/ stays checked. -exclude = '^tests/(test_(brute_force|parser_util|variations)\.py)$' +exclude = '^tests/(test_(brute_force|variations)\.py)$' [[tool.mypy.overrides]] module = [ diff --git a/tests/conftest.py b/tests/conftest.py index 00674681..3812d81c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,7 +34,6 @@ # last='Ivanov Sergeyevich' (v1: 'Sergeyevich Ivanov'). # Remove once fixed. "test_middle_name_as_last.py", - "test_parser_util.py", "test_variations.py", ] diff --git a/tests/test_parser_util.py b/tests/test_parser_util.py deleted file mode 100644 index a2b74f5d..00000000 --- a/tests/test_parser_util.py +++ /dev/null @@ -1,39 +0,0 @@ -from nameparser.parser import group_contiguous_integers - - -def test_empty_list_returns_no_ranges() -> None: - assert group_contiguous_integers([]) == [] - - -def test_all_isolated_values_returns_no_ranges() -> None: - # no two values are adjacent, so nothing counts as a "run" - assert group_contiguous_integers([1, 3, 5]) == [] - - -def test_single_value_returns_no_ranges() -> None: - # a run of length 1 isn't a contiguous "run" by this function's definition - assert group_contiguous_integers([5]) == [] - - -def test_pair_of_adjacent_values_is_smallest_valid_run() -> None: - assert group_contiguous_integers([4, 5]) == [(4, 5)] - - -def test_single_contiguous_run() -> None: - assert group_contiguous_integers([1, 2, 3]) == [(1, 3)] - - -def test_multiple_separate_contiguous_runs() -> None: - assert group_contiguous_integers([1, 2, 5, 6, 7, 9]) == [(1, 2), (5, 7)] - - -def test_isolated_values_between_runs_are_excluded() -> None: - assert group_contiguous_integers([1, 2, 4, 6, 7]) == [(1, 2), (6, 7)] - - -def test_unsorted_input_is_not_treated_as_contiguous() -> None: - # the grouping key (enumerate index - value) only repeats for ascending, - # strictly increasing runs (as produced by an `enumerate`-based index - # scan, which is how join_on_conjunctions uses it); a descending - # sequence changes the key at every step, so no run is ever found - assert group_contiguous_integers([3, 2, 1]) == [] From 11d9e3749f7515aca64b739992621c06c4af2f6f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 06:51:52 -0700 Subject: [PATCH 130/275] Reconcile test_variations against the facade (buckets A, B) The three-code-paths variation bank (180+ names, cross-checked across no-comma / lastname-comma / suffix-comma spellings) passes wholesale; only two v1-internals touches were needed: - Bucket B (#255): the forced empty_attribute_default='' step is gone -- empty attributes are always '' in 2.0, which is exactly what the format strings needed. - Bucket A: the private hn._members iteration becomes the public as_dict() key set (same seven fields). Lifted from both skip lists. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 2 +- tests/conftest.py | 1 - tests/test_variations.py | 14 ++++++-------- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c806b2bf..c7edd730 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ packages = ["nameparser", "tests"] # TEMPORARY (M11 -> M12): the v1 suite is being reconciled against the # 2.0 facade; lifted with tests/conftest.py's collect_ignore_glob. Must # be gone before the branch leaves draft. tests/v2/ stays checked. -exclude = '^tests/(test_(brute_force|variations)\.py)$' +exclude = '^tests/(test_(brute_force)\.py)$' [[tool.mypy.overrides]] module = [ diff --git a/tests/conftest.py b/tests/conftest.py index 3812d81c..f502871d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,7 +34,6 @@ # last='Ivanov Sergeyevich' (v1: 'Sergeyevich Ivanov'). # Remove once fixed. "test_middle_name_as_last.py", - "test_variations.py", ] # Scalar (non-collection) config attributes that individual tests mutate on the diff --git a/tests/test_variations.py b/tests/test_variations.py index 249afd84..6e7c4154 100644 --- a/tests/test_variations.py +++ b/tests/test_variations.py @@ -1,5 +1,3 @@ -import warnings - from nameparser import HumanName from tests.base import HumanNameTestBase @@ -205,11 +203,9 @@ def test_variations_of_TEST_NAMES(self) -> None: if len(hn.suffix_list) > 1: d = SimpleNamespace(**hn.as_dict()) hn = HumanName(f"{d.title} {d.first} {d.middle} {d.last} {d.suffix}".split(',')[0]) - with warnings.catch_warnings(): - # format strings below require empty string; #255 deprecation - # noise isn't the point of this broad parsing-variation test - warnings.simplefilter("ignore", DeprecationWarning) - hn.C.empty_attribute_default = '' + # v1 forced empty_attribute_default='' here so the format + # strings below got '' for empty parts; 2.0 removed the setting + # (#255) and empty attributes are always '' -- nothing to force. d = SimpleNamespace(**hn.as_dict()) nocomma = HumanName(f"{d.title} {d.first} {d.middle} {d.last} {d.suffix}") lastnamecomma = HumanName(f"{d.last}, {d.title} {d.first} {d.middle} {d.suffix}") @@ -220,7 +216,9 @@ def test_variations_of_TEST_NAMES(self) -> None: lastnamecomma = HumanName(f"{d.last}, {d.title} {d.first} {d.middle} {d.suffix} ({d.nickname})") if d.suffix: suffixcomma = HumanName(f"{d.title} {d.first} {d.middle} {d.last}, {d.suffix} ({d.nickname})") - for attr in hn._members: + # v1 iterated the private hn._members; 2.0's public equivalent + # is the as_dict() key set (same seven fields) + for attr in hn.as_dict(): self.m(getattr(hn, attr), getattr(nocomma, attr), hn) self.m(getattr(hn, attr), getattr(lastnamecomma, attr), hn) if hn.suffix: From fb524d88410af5851764632952467abbf1e1cee5 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 06:52:55 -0700 Subject: [PATCH 131/275] Reconcile test_brute_force (no edits needed); drop the mypy exclude The 117-case brute-force bank passes wholesale against the facade with zero edits. With it, every v1 test file is reconciled and mypy-clean, so the TEMPORARY [tool.mypy] exclude is deleted entirely; the conftest collect_ignore_glob shrinks to its final remnant -- the two files each held on one confirmed parity bug (bound-given whole-segment join; middle_name_as_last fold order), to be deleted when those land. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 4 ---- tests/conftest.py | 14 ++++++-------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c7edd730..6a86e4b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,10 +49,6 @@ dev = [ [tool.mypy] packages = ["nameparser", "tests"] -# TEMPORARY (M11 -> M12): the v1 suite is being reconciled against the -# 2.0 facade; lifted with tests/conftest.py's collect_ignore_glob. Must -# be gone before the branch leaves draft. tests/v2/ stays checked. -exclude = '^tests/(test_(brute_force)\.py)$' [[tool.mypy.overrides]] module = [ diff --git a/tests/conftest.py b/tests/conftest.py index f502871d..acfd01a0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,13 +6,12 @@ from nameparser.config import CONSTANTS -# TEMPORARY (Task M11 -> M12): the v1 suite is being reconciled against -# the 2.0 facade file by file. Lifted in Task M12 together with its mypy -# twin (the `exclude` under [tool.mypy] in pyproject.toml); both must be -# GONE before this branch leaves draft. -# -# Explicit list of not-yet-reconciled files. Narrow this as each file is -# reconciled; remove entirely once the whole suite runs against the facade. +# TEMPORARY (Task M12, final remnant): the whole v1 suite is reconciled +# against the 2.0 facade and mypy-clean (the [tool.mypy] exclude twin in +# pyproject.toml is gone). Only the two files below remain skipped, each +# held on one confirmed, repro'd parity bug; delete this list (and the +# entries' tests will run) once those land. Must be GONE before this +# branch leaves draft. collect_ignore_glob = [ # test_bound_first_names.py is reconciled (bucket A: the two # is_bound_first_name predicate tests died with the v1 hooks) but 1 test @@ -22,7 +21,6 @@ # (v1: first='abdul salam'; three-token 'salem, abdul salam ahmed' # joins fine). Remove once fixed. "test_bound_first_names.py", - "test_brute_force.py", # test_middle_name_as_last.py needs NO bucket edits but 4 tests fail on # a fold-order bug: v1's middle_name_as_last fold PREPENDED middle_list # to last_list, so comma and rotated forms converge on the no-comma From 06bab725ebb5deabf1ef9abd1fbc5f3ddb071fb9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 06:56:25 -0700 Subject: [PATCH 132/275] Fix the last two M12 bugs and lift the reconciliation skip list - bound given names join freely in FAMILY_COMMA's post-comma segment (v1 reserve_last=False, parser.py:1366): 'salem, abdul salam' -> given 'abdul salam'. _group_segment takes the per-segment bound_required (3 main / 2 post-comma / 0 family segment, which v1 never bound-joined). - middle_as_family renders in v1's prepend order: folded tokens carry vocab:folded-middle and the family view orders them before the original family tokens (spans cannot reorder, anti-#100) -- 'Hassan, Mohamad Ahmad Ali' -> family 'Ahmad Ali Hassan', matching the no-comma and rotated forms. The stage-ownership contract gains tags for post_rules accordingly. With these, every v1 test file is reconciled: the conftest collect_ignore_glob is GONE (the mypy exclude went earlier). The full v1 suite runs against the facade: 1160 passed, 12 xfailed. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_group.py | 22 +++++++++++++++++----- nameparser/_pipeline/_post_rules.py | 8 ++++++-- nameparser/_types.py | 7 ++++++- tests/conftest.py | 27 --------------------------- tests/v2/cases.py | 11 +++++++++++ tests/v2/pipeline/test_state.py | 4 +++- 6 files changed, 43 insertions(+), 36 deletions(-) diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 7d70040d..4bc3ab86 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -74,6 +74,7 @@ def _is_rootname(piece: Sequence[int], ptags: Set[str], def _group_segment(seg: tuple[int, ...], additional: int, tokens: Sequence[WorkToken], + bound_required: int = 3, ) -> tuple[list[Piece], list[set[str]]]: pieces: list[Piece] = [[i] for i in seg] ptags: list[set[str]] = [set() for _ in seg] @@ -156,18 +157,23 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), j += 1 merge(k, j, drop={"prefix"}) k += 1 - # bound given names: first non-title piece joins the next when - # enough rootname pieces remain (v1 reserve_last) + # bound given names: first non-title piece joins the next. + # bound_required is v1's reserve_last: 3 keeps a family piece in + # reserve (main segments); 2 joins freely (FAMILY_COMMA's + # post-comma segment, v1 reserve_last=False -- 'salem, abdul + # salam' -> given 'abdul salam'); 0 disables (family segment, + # which v1 never bound-joined). first_name_k = next( (k for k in range(len(pieces)) if not title(k)), None) - if (first_name_k is not None + if (bound_required + and first_name_k is not None and first_name_k + 1 < len(pieces) and len(pieces[first_name_k]) == 1 and "vocab:bound-given" in tokens[pieces[first_name_k][0]].tags): non_suffix = sum(1 for k in range(len(pieces)) if not title(k) and not suffix(k)) - if non_suffix >= 3: + if non_suffix >= bound_required: merge(first_name_k, first_name_k + 2) return pieces, ptags @@ -186,8 +192,14 @@ def group(state: ParseState) -> ParseState: cores = delimiter_cores(state.policy.extra_suffix_delimiters) tail_start = {Structure.SUFFIX_COMMA: 1, Structure.FAMILY_COMMA: 2}.get(state.structure) + family_comma = state.structure is Structure.FAMILY_COMMA for seg_idx, seg in enumerate(state.segments): - pieces, ptags = _group_segment(seg, additional, tokens) + if family_comma: + bound_required = 2 if seg_idx == 1 else 0 + else: + bound_required = 3 + pieces, ptags = _group_segment(seg, additional, tokens, + bound_required) if tail_start is not None and seg_idx >= tail_start: # v1 renders each tail COMMA SEGMENT as one suffix entry # ('Smith, V MD' -> suffix 'V MD'); a delimiter core inside diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index 70fba69e..5f3975eb 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -118,8 +118,12 @@ def post_rules(state: ParseState) -> ParseState: _retag(tokens, f, Role.MIDDLE) _retag(tokens, g, Role.FAMILY) # rule 4: opt-in fold of middles into family (v1 - # handle_middle_name_as_last; span order reproduces v1's prepend) + # handle_middle_name_as_last). v1 PREPENDED middle_list to + # last_list; spans cannot reorder (anti-#100), so folded tokens + # carry a tag and the family views order them first. if state.policy.middle_as_family: for i in _idx(tokens, Role.MIDDLE): - _retag(tokens, i, Role.FAMILY) + tokens[i] = dataclasses.replace( + tokens[i], role=Role.FAMILY, + tags=tokens[i].tags | {"vocab:folded-middle"}) return dataclasses.replace(state, tokens=tuple(tokens)) diff --git a/nameparser/_types.py b/nameparser/_types.py index a04aca4d..dde82e4a 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -306,6 +306,7 @@ def _text_for(self, *roles: Role, tag: str | None = None, without_tag: str | None = None) -> str: suffix_join = roles == (Role.SUFFIX,) parts: list[str] = [] + folded: list[str] = [] for tok in self.tokens: if tok.role not in roles: continue @@ -318,9 +319,13 @@ def _text_for(self, *roles: Role, tag: str | None = None, # view's ", " join does not split one credential in two if suffix_join and "joined" in tok.tags and parts: parts[-1] += " " + tok.text + elif "vocab:folded-middle" in tok.tags: + # middle_as_family fold: v1 PREPENDED middle_list to + # last_list; spans cannot reorder, so the view does + folded.append(tok.text) else: parts.append(tok.text) - return (", " if suffix_join else " ").join(parts) + return (", " if suffix_join else " ").join(folded + parts) @property def title(self) -> str: diff --git a/tests/conftest.py b/tests/conftest.py index acfd01a0..f6e5e487 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,33 +6,6 @@ from nameparser.config import CONSTANTS -# TEMPORARY (Task M12, final remnant): the whole v1 suite is reconciled -# against the 2.0 facade and mypy-clean (the [tool.mypy] exclude twin in -# pyproject.toml is gone). Only the two files below remain skipped, each -# held on one confirmed, repro'd parity bug; delete this list (and the -# entries' tests will run) once those land. Must be GONE before this -# branch leaves draft. -collect_ignore_glob = [ - # test_bound_first_names.py is reconciled (bucket A: the two - # is_bound_first_name predicate tests died with the v1 hooks) but 1 test - # fails on a join-guard bug: the FAMILY_COMMA bound-given join does not - # fire when the join would consume the whole post-comma segment. Repro: - # HumanName('salem, abdul salam') -> first='abdul', middle='salam' - # (v1: first='abdul salam'; three-token 'salem, abdul salam ahmed' - # joins fine). Remove once fixed. - "test_bound_first_names.py", - # test_middle_name_as_last.py needs NO bucket edits but 4 tests fail on - # a fold-order bug: v1's middle_name_as_last fold PREPENDED middle_list - # to last_list, so comma and rotated forms converge on the no-comma - # result; v2 renders the folded family in token order. Repros: - # HumanName('Hassan, Mohamad Ahmad Ali', constants= - # Constants(middle_name_as_last=True)) -> last='Hassan Ahmad Ali' - # (v1: 'Ahmad Ali Hassan'); - # with patronymic_name_order too: 'Ivanov Petr Sergeyevich' -> - # last='Ivanov Sergeyevich' (v1: 'Sergeyevich Ivanov'). - # Remove once fixed. - "test_middle_name_as_last.py", -] # Scalar (non-collection) config attributes that individual tests mutate on the # global CONSTANTS singleton. Several tests change these without restoring them; diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 394aa8a4..95348473 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -41,6 +41,17 @@ class Case: {"given": "John", "family": "Smith"}), Case("suffix_comma", "John Smith, PhD", {"given": "John", "family": "Smith", "suffix": "PhD"}), + Case("bound_given_whole_segment", "salem, abdul salam", + {"given": "abdul salam", "family": "salem"}, + notes="v1 joins bound given names freely in the post-comma " + "segment (reserve_last=False, parser.py:1366) -- even " + "when the join consumes the whole segment"), + Case("middle_as_family_fold_order", "Hassan, Mohamad Ahmad Ali", + {"given": "Mohamad", "family": "Ahmad Ali Hassan"}, + policy=Policy(middle_as_family=True), + notes="v1 PREPENDED middle_list to last_list; folded tokens " + "carry vocab:folded-middle and the family view orders " + "them first (spans cannot reorder)"), Case("ambiguous_surname_acronyms", "Jack Ma", {"given": "Jack", "family": "Ma"}, notes="'ma'/'do' joined suffix_acronyms_ambiguous: common " diff --git a/tests/v2/pipeline/test_state.py b/tests/v2/pipeline/test_state.py index 4ceff2bc..eed8d005 100644 --- a/tests/v2/pipeline/test_state.py +++ b/tests/v2/pipeline/test_state.py @@ -63,7 +63,9 @@ def test_stage_field_ownership() -> None: "classify": {"tags"}, "group": {"tags", "role"}, "assign": {"role"}, - "post_rules": {"role"}, + # post_rules also tags: the middle_as_family fold marks folded + # tokens vocab:folded-middle for the family view's prepend order + "post_rules": {"role", "tags"}, } for case in CASES: state = ParseState(original=case.text, lexicon=_Lexicon.default(), From 785c3e5b2a88afe3e5b78975c49b14ef7b931cc4 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 06:59:59 -0700 Subject: [PATCH 133/275] Add the facade runner over the shared case table Co-Authored-By: Claude Fable 5 --- tests/v2/test_facade_cases.py | 69 +++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 tests/v2/test_facade_cases.py diff --git a/tests/v2/test_facade_cases.py b/tests/v2/test_facade_cases.py new file mode 100644 index 00000000..58457755 --- /dev/null +++ b/tests/v2/test_facade_cases.py @@ -0,0 +1,69 @@ +"""Facade runner (migration spec §5): the shared case table asserted +through HumanName. Deleted wholesale in 3.0 with the facade.""" +import pytest + +from nameparser import Policy +from nameparser._config_shim import Constants +from nameparser._facade import HumanName + +from .cases import CASES, Case + +_V1_KEY = {"given": "first", "family": "last"} # identity for the rest + +#: the one non-default maiden_delimiters shape the table exercises +#: ("nickname_bucket_wins_when_shared"): expressible in v1 Constants via +#: the delimiter-manager's parenthesis sentinel added to the maiden +#: bucket, leaving the nickname bucket at its (also-parenthesis-holding) +#: default -- see _config_shim._DelimiterManager. +_MAIDEN_PARENS = frozenset({("(", ")")}) + + +def _constants_for(case: Case) -> Constants | None: + """Translate the row's Policy to a Constants, or None if the policy + has no v1 spelling (those rows are core-only).""" + policy = case.policy or Policy() + default = Policy() + c = Constants() + maiden_via_sentinel = ( + policy.maiden_delimiters == _MAIDEN_PARENS + and policy.nickname_delimiters == default.nickname_delimiters + ) + unexpressible = ( + policy.name_order != default.name_order + or policy.lenient_comma_suffixes != default.lenient_comma_suffixes + or policy.strip_emoji != default.strip_emoji + or policy.strip_bidi != default.strip_bidi + or policy.nickname_delimiters != default.nickname_delimiters + or (policy.maiden_delimiters != default.maiden_delimiters + and not maiden_via_sentinel) + ) + if unexpressible: + return None + if policy.patronymic_rules: + c.patronymic_name_order = True + if policy.middle_as_family: + c.middle_name_as_last = True + if policy.extra_suffix_delimiters: + c.suffix_delimiter = next(iter(policy.extra_suffix_delimiters)) + if maiden_via_sentinel: + # bucket-move idiom (see _DelimiterManager docstring): adds + # parenthesis to maiden while nickname keeps its own default + # (which already includes parenthesis) -- the nickname reading + # still wins on a shared delimiter, matching the row's Policy. + c.maiden_delimiters["parenthesis"] = "parenthesis" + return c + + +@pytest.mark.parametrize("case", CASES, ids=lambda c: c.id) +def test_facade_case(case: Case) -> None: + constants = _constants_for(case) + if constants is None: + pytest.skip("policy not expressible through v1 Constants") + name = HumanName(case.text, constants=constants) + for field, expected in case.expect.items(): + assert getattr(name, _V1_KEY.get(field, field)) == expected, ( + f"{case.id}: {field}") + for field in {"title", "given", "middle", "family", "suffix", + "nickname", "maiden"} - set(case.expect): + assert getattr(name, _V1_KEY.get(field, field)) == "", ( + f"{case.id}: {field} expected empty") From 174849306c60d569e04462a4b8bbd2014542c29f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 07:07:34 -0700 Subject: [PATCH 134/275] Add the v1-vs-2.0 differential harness with a classified allowlist Co-Authored-By: Claude Fable 5 --- docs/release_log.rst | 24 ++ tools/differential/README.md | 87 ++++ tools/differential/build_corpus.py | 107 +++++ tools/differential/compare.py | 83 ++++ tools/differential/corpus.jsonl | 486 +++++++++++++++++++++++ tools/differential/expected_changes.toml | 57 +++ tools/differential/worker_v1.py | 25 ++ 7 files changed, 869 insertions(+) create mode 100644 tools/differential/README.md create mode 100644 tools/differential/build_corpus.py create mode 100644 tools/differential/compare.py create mode 100644 tools/differential/corpus.jsonl create mode 100644 tools/differential/expected_changes.toml create mode 100644 tools/differential/worker_v1.py diff --git a/docs/release_log.rst b/docs/release_log.rst index 7ad2fc2e..a390be0d 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -1,5 +1,29 @@ Release Log =========== +* 2.0.0 - unreleased + + **Behavior Changes (draft)** + + .. note:: + This section is a **draft**, generated from the v1-vs-2.0 + differential harness (``tools/differential/``) run against the + pre-M12 v1 test corpus (486 name strings). It will be replaced + by hand-written release notes before 2.0 ships; entries below + are grouped by the harness's classification and reference the + ``tests/v2/cases.py`` row (if any) that pins the new behavior. + + - Fix ``"Andrews, M.D."``-shaped input: the lone strict-suffix-or-title piece after a comma (e.g. ``"Smith, Dr."``, ``"Andrews, M.D."``) was routed to ``first`` in 1.x; 2.0 routes it to ``suffix``/``title`` instead, since the pre-comma piece is definitionally the family name (``tests/v2/cases.py`` rows ``family_comma_lone_suffix_piece``, ``family_comma_lone_title``; verified live against 1.4.0 -- 1/486 corpus names diff, all others parity) + - Fix a lone recognized trailing suffix (e.g. ``"Johnson PhD"``, ``"Mr. Johnson PhD"``) being routed to ``first``/``last`` in 1.x when no comma is present; 2.0 keeps a recognized suffix in ``suffix`` (``tests/v2/cases.py`` rows ``suffix_stays_suffix``, ``suffix_stays_suffix_title``; not present in the differential corpus -- no v1 test string exercises the bare two-token shape, so this is unverified against 1.4 live but pinned by the case table) + - Fix maiden-name markers (``née``/``nee``/``born``/``geb.``/``roz.``) being folded into ``middle``/``last`` in 1.x (e.g. ``"Jane Smith née Jones"`` → ``middle="Smith née"``); 2.0 recognizes the marker and routes the following name to the new ``maiden`` field (closes #274; ``tests/v2/cases.py`` row ``maiden_marker``; not present in the differential corpus) + - Data change: ``ma``/``do`` added to ``suffix_acronyms_ambiguous`` so a bare common surname (``"Jack Ma"``) is no longer misread as a suffix acronym, restoring v1's older (pre-regression) parity (``tests/v2/cases.py`` row ``ambiguous_surname_acronyms``). Side effect: parenthesized/quoted ``"(MA)"``/``"(DO)"`` (no periods) no longer escape to ``suffix`` the way 1.x did -- they now fall through to nickname parsing like any other ambiguous-acronym delimited content. Not present in the differential corpus + - Change suffix-delimiter rendering: with a custom ``Policy``/``Constants`` suffix delimiter configured (e.g. ``suffix_delimiter="/"``, ``"John Smith, RN/CRNA"``), 1.x split the token and rendered ``suffix="RN, CRNA"``; 2.0 keeps the no-space delimiter-core token whole (``suffix="RN/CRNA"``) -- role assignment is unchanged, only rendering differs (anti-#100, migration plan deviation 5; ``tests/v2/cases.py`` row ``suffix_delimiter_no_space_core``). Only fires with a non-default policy, so it does not appear in the (default-policy) differential corpus + + Everything else in the 486-name differential corpus (built from the + v1 test banks as of commit ``2d5d8c2``, pre-dating the M12 test + reconciliation) parses identically between nameparser 1.4.0 and + this working tree; see ``tools/differential/README.md`` for how to + reproduce the comparison. + * 1.4.0 - July 12, 2026 - Add ``Constants.copy()``, a detached deep copy that preserves the source instance's current customizations (unlike ``Constants()``, which always starts from library defaults) -- useful as ``CONSTANTS.copy()`` for a private snapshot of the shared config (#260) diff --git a/tools/differential/README.md b/tools/differential/README.md new file mode 100644 index 00000000..783f9c58 --- /dev/null +++ b/tools/differential/README.md @@ -0,0 +1,87 @@ +# Differential harness (v1 vs 2.0) + +Dev-only tooling for the 2.0 migration (migration plan S5). Not +shipped (excluded from the wheel by the packaging config -- only +`nameparser/` is packaged) and not CI-gated. Run it by hand when +touching parsing behavior, and before cutting a 2.0 release. + +Two processes, two environments: + +- `worker_v1.py` runs under a **pinned nameparser 1.4** installed fresh + from PyPI via a PEP 723 inline script. It must be invoked with + `uv run --no-project` -- **without `--no-project`, `uv` installs the + working tree as an editable dependency and the 1.4 pin never takes + effect**, silently comparing 2.0 against itself. +- `compare.py` runs in the project's own dev environment and imports + `nameparser` normally (the 2.0 facade, which still speaks the v1 + component names). + +## Running it + +``` +uv run python tools/differential/build_corpus.py --ref > tools/differential/corpus.jsonl # only when regenerating +uv run python tools/differential/compare.py +``` + +`compare.py` spawns the worker as a subprocess, feeds it every corpus +name as a line of JSON, and diffs the two component dicts on the seven +v1 field names (`title`, `first`, `middle`, `last`, `suffix`, +`nickname`, `maiden` -- both sides use these keys, so no field mapping +is needed). Every diff is checked against `expected_changes.toml`: + +- Matches a rule -> counted as an intentional, classified change. +- Matches no rule -> printed under `UNEXPLAINED` and the run exits 1. + +An unexplained diff means either a real 2.0 parity bug (fix it, don't +allowlist it) or a known change whose `expected_changes.toml` rule +needs widening. The run must exit 0 before a 2.0 release; the classified +summary it prints is the source for the "Behavior Changes" section of +`docs/release_log.rst`. + +## Corpus provenance + +`corpus.jsonl` is checked in as a test fixture. It was built by +`build_corpus.py`'s AST walk over every top-level `tests/test_*.py` +file (the v1-style test banks; `tests/v2/` is a separate 2.0-only +harness and is deliberately not scanned), reading each file via +`git show :` rather than the working tree: + +``` +uv run python tools/differential/build_corpus.py --ref 2d5d8c2 > tools/differential/corpus.jsonl +``` + +`2d5d8c2` ("Trim constant-factor waste on the tokenize hot path") is +the last commit before M12 reconciled/edited the v1 test banks against +the 2.0 facade -- M12 changed some expectations in place and deleted +bucket-A tests outright, which would have shrunk and skewed the +corpus. Reading history at an old ref via `git show` is a read-only +operation on this worktree's own log; it does not check out, stash, or +otherwise mutate anything. + +The AST extraction over-collects on purpose (string literals passed as +`HumanName(...)`'s first argument, plus string members of module-level +list/dict/tuple banks that contain a space) -- more candidate strings +is more coverage, and the corpus is deduplicated. Obvious non-names +(strings containing `{`, `@`, or a backslash -- format placeholders, +decorator/email-shaped fixtures, escape sequences) are dropped. + +Regenerate the corpus only if the v1 test banks are revisited again at +a still-earlier point in history; otherwise leave the checked-in file +alone so the harness stays comparable run to run. + +## `expected_changes.toml` + +Each `[[change]]` entry needs `issue` (a short label, ideally an +issue number or `fix()` matching a `tests/v2/cases.py` +classification) and may narrow its match with `name_regex` (searched +against the raw input string) and/or `fields` (the diffing rule +matches only if the observed diff fields are a subset of this list). +Keep both as tight as the actual diff allows -- a loose rule can mask +a real regression. + +Some entries in the seed list are for behavior families that this +particular corpus (pre-M12 v1 test strings) happens not to contain any +example of (e.g. custom suffix-delimiter rendering, which only fires +under a non-default `Policy`). They're kept in the file anyway, +matching the family documented in `tests/v2/cases.py`, so the rule is +ready the moment a matching string is added to the corpus. diff --git a/tools/differential/build_corpus.py b/tools/differential/build_corpus.py new file mode 100644 index 00000000..2c2613b8 --- /dev/null +++ b/tools/differential/build_corpus.py @@ -0,0 +1,107 @@ +"""Extract every plausible test-name string from the v1 test suite: +string literals passed as the first argument to HumanName(...) calls, +plus string elements/keys of module-level list/dict banks. Dev tooling +-- over-collection is fine, the comparator just parses more names. + +PROVENANCE: the checked-in tools/differential/corpus.jsonl was built +against commit 2d5d8c2 ("Trim constant-factor waste on the tokenize +hot path"), the last commit before M12 reconciled/edited the v1 test +banks (M12 rewrote expectations and deleted the bucket-A tests, which +would have shrunk the corpus). Reading historical test content is +fine; per the migration rules, no git *operations* are ever run against +the original (non-worktree) checkout -- this script only shells out to +`git show :` inside the current worktree's own history, +which contains that commit as an ancestor. + +Regenerate with: + uv run python tools/differential/build_corpus.py --ref 2d5d8c2 \\ + > tools/differential/corpus.jsonl +""" +import argparse +import ast +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +TESTS = ROOT / "tests" + +# Obvious non-names the AST heuristic sweeps up along with real names: +# format-string placeholders, decorators/emails, escape sequences. +_NOISE_CHARS = ("{", "@", "\\") + + +def _is_name_like(value: str) -> bool: + return not any(ch in value for ch in _NOISE_CHARS) + + +def _strings_from(tree: ast.Module) -> set[str]: + found: set[str] = set() + for node in ast.walk(tree): + if (isinstance(node, ast.Call) + and getattr(node.func, "id", getattr( + node.func, "attr", "")) == "HumanName" + and node.args + and isinstance(node.args[0], ast.Constant) + and isinstance(node.args[0].value, str)): + found.add(node.args[0].value) + if isinstance(node, ast.Assign) and isinstance( + node.value, (ast.List, ast.Tuple, ast.Dict)): + for c in ast.walk(node.value): + if isinstance(c, ast.Constant) and isinstance(c.value, str) \ + and " " in c.value and "\n" not in c.value: + found.add(c.value) + return {n for n in found if _is_name_like(n)} + + +def _working_tree_sources() -> dict[str, str]: + return {path.name: path.read_text() + for path in sorted(TESTS.glob("test_*.py"))} + + +def _ref_sources(ref: str) -> dict[str, str]: + """Read every top-level tests/test_*.py file as it existed at `ref`, + via `git show`, without touching the working tree or index.""" + listing = subprocess.run( + ["git", "-C", str(ROOT), "ls-tree", "-r", "--name-only", ref, + "--", "tests"], + capture_output=True, text=True, check=True).stdout + paths = sorted( + p for p in listing.splitlines() + if p.startswith("tests/") and Path(p).parent == Path("tests") + and Path(p).name.startswith("test_") and p.endswith(".py")) + sources = {} + for path in paths: + show = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{ref}:{path}"], + capture_output=True, text=True, check=True) + sources[Path(path).name] = show.stdout + return sources + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument( + "--ref", default=None, + help="git ref to read tests/test_*.py from (via `git show`) " + "instead of the working tree, e.g. 2d5d8c2 (pre-M12).") + args = ap.parse_args() + + sources = _ref_sources(args.ref) if args.ref else _working_tree_sources() + names: set[str] = set() + for filename, text in sources.items(): + try: + tree = ast.parse(text, filename=filename) + except SyntaxError as e: + print(f"skipping {filename}: {e}", file=sys.stderr) + continue + names |= _strings_from(tree) + + for name in sorted(names): + print(json.dumps(name, ensure_ascii=False)) + print(f"{len(names)} names", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/tools/differential/compare.py b/tools/differential/compare.py new file mode 100644 index 00000000..f0576899 --- /dev/null +++ b/tools/differential/compare.py @@ -0,0 +1,83 @@ +"""Differential harness (migration spec S5): 1.4-on-PyPI vs the working +tree over the corpus. Every diff must classify against +expected_changes.toml or the run fails. + + uv run python tools/differential/compare.py [--corpus corpus.jsonl] +""" +import argparse +import json +import re +import subprocess +import tomllib +from pathlib import Path + +HERE = Path(__file__).resolve().parent +FIELDS = ("title", "first", "middle", "last", "suffix", "nickname", + "maiden") + + +def classify(name: str, diff_fields: set[str], + rules: list[dict[str, object]]) -> str | None: + for rule in rules: + name_regex = rule.get("name_regex") + if isinstance(name_regex, str) and not re.search(name_regex, name): + continue + fields = rule.get("fields") + if isinstance(fields, list) and not diff_fields <= set(fields): + continue + issue = rule["issue"] + assert isinstance(issue, str) + return issue + return None + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--corpus", default=str(HERE / "corpus.jsonl")) + args = ap.parse_args() + rules = tomllib.loads( + (HERE / "expected_changes.toml").read_text()).get("change", []) + corpus = [json.loads(line) for line in + Path(args.corpus).read_text().splitlines() if line.strip()] + + proc = subprocess.Popen( + ["uv", "run", "--no-project", str(HERE / "worker_v1.py")], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + v1_input = "".join(json.dumps(n, ensure_ascii=False) + "\n" + for n in corpus) + v1_lines, _ = proc.communicate(v1_input) + v1_results = [json.loads(line) for line in v1_lines.splitlines()] + assert len(v1_results) == len(corpus), "worker line count mismatch" + + from nameparser import HumanName # the working tree (2.0 facade) + by_issue: dict[str, list[str]] = {} + unexplained: list[tuple[str, dict[str, str], dict[str, str]]] = [] + for name, old in zip(corpus, v1_results): + new = {k: v or "" for k, v in HumanName(name).as_dict().items()} + diff = {f for f in FIELDS if old.get(f, "") != new.get(f, "")} + if not diff: + continue + issue = classify(name, diff, rules) + if issue is None: + unexplained.append((name, old, new)) + else: + by_issue.setdefault(issue, []).append(name) + + print(f"corpus: {len(corpus)} names; " + f"intentional diffs: {sum(map(len, by_issue.values()))}; " + f"unexplained: {len(unexplained)}\n") + for issue, names in sorted(by_issue.items()): + print(f"## {issue} ({len(names)})") + for n in names[:10]: + print(f" {n!r}") + print() + for name, old, new in unexplained: + print(f"UNEXPLAINED {name!r}") + for f in FIELDS: + if old.get(f, "") != new.get(f, ""): + print(f" {f}: {old.get(f, '')!r} -> {new.get(f, '')!r}") + return 1 if unexplained else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/differential/corpus.jsonl b/tools/differential/corpus.jsonl new file mode 100644 index 00000000..b057779b --- /dev/null +++ b/tools/differential/corpus.jsonl @@ -0,0 +1,486 @@ +"" +"\"Rick\" Edmonds" +"()" +"," +", John" +"A.B. Vajpayee" +"Abdul Salam Ahmed Salem, MD" +"Abdul Salam Hassan, MD" +"Abdul Salam, MD" +"Abu Bakr Al Baghdadi, MD" +"Adolph D" +"Ahmad ben Husain" +"Al Arnold Gore, Jr." +"Alex Ben Johnson" +"Alex van Johnson" +"Aliyev Vusal Said oglu" +"Alois von und zu Liechtenstein" +"Alois von und zu und von Liechtenstein" +"Amy E Maid" +"Amy E. Maid" +"Andrew" +"Andrew Boris Petersen" +"Andrew Perkins \"MBA\"" +"Andrew Perkins 'MBA'" +"Andrew Perkins (JD)" +"Andrew Perkins (M.D)" +"Andrew Perkins (MBA)" +"Andrew Perkins (Mgr.)" +"Andrew Perkins (XYZ)" +"Andrew Perkins, Jr., Col. (Ret)" +"Andrews, M.D." +"Anh Do" +"Annette Charlotte Freiherrin von und zu der Tann-Rathsamhausen" +"Assoc Dean of Chemistry Robert Johns" +"Baker (Johnson), Jenny" +"Baker (Jr.), Jenny" +"Ben Alex Johnson" +"Ben Johnson" +"Benjamin \"Ben\" Franklin" +"Benjamin \"Big Ben\" Franklin" +"Benjamin 'Ben' Franklin" +"Benjamin (Ben) Franklin" +"Benjamin (Big Ben) Franklin" +"Benjamin [Ben] Franklin" +"Bob Dole" +"Brian Andrew O'connor" +"Brian O'connor" +"Buca di Beppo" +"Cardinal Secretary of State Hillary Clinton" +"Chancellor Jane Smith" +"Chang, Andy C I" +"Charles van der van der Berg" +"Chemistry Jane Smith" +"Chief Judge J. Leon Holmes" +"Chief Judge Sharon Lovelace Blackburn" +"Clarke, Kenneth, Q.C. M.P." +"Coach" +"Col. (Ret.) Smith" +"DE MESNIL" +"DR DOE" +"Dame Mary" +"Dean Ms Hon Solo" +"Del Toro" +"Della Reese" +"Designated Judge David A. Ezra" +"Di Caprio" +"Doctor, Jane E." +"Doe, Dr. John" +"Doe, Dr. John A." +"Doe, Dr. John A. III" +"Doe, Dr. John A. Jr." +"Doe, Dr. John A. Kenneth" +"Doe, Dr. John A. Kenneth III" +"Doe, Dr. John A. Kenneth Jr." +"Doe, Dr. John III" +"Doe, Dr. John P., CLU, CFP, LUTC" +"Doe, Dr. John, Jr." +"Doe, John" +"Doe, John A." +"Doe, John A. III" +"Doe, John A. Kenneth" +"Doe, John A. Kenneth III" +"Doe, John A. Kenneth, Jr." +"Doe, John A., III" +"Doe, John A., Jr." +"Doe, John III" +"Doe, John Msc.Ed." +"Doe, John jr., MD" +"Doe, John, Jr." +"Doe, John, MD PhD - FACS Fellow" +"Doe, John,, Jr." +"Doe, John,, Jr.,, III" +"Doe, John. A. Kenneth" +"Doe, John. A. Kenneth III" +"Doe, John. A. Kenneth, Jr." +"Doe, Lt. Gen. John A. Kenneth IV" +"Doe, Lt.Gov. John" +"Doe, Mary - Kate, RN" +"Doe, Rev. John A. Jr." +"Doe, Rev. John A., V, Jr." +"Doe, Rev. John V, Jr." +"Doe,, Jr." +"Doe-Ray, Dr. John P., CLU, CFP, LUTC" +"Doe-Ray, Hon. Barrington P. Jr." +"Doe-Ray, Hon. Barrington P. Jr., CFP, LUTC" +"Donovan McNabb-Smith" +"Dr King Jr" +"Dr Martin Luther King, Jr." +"Dr. Abdul Salam Hassan, MD" +"Dr. John A. Doe" +"Dr. John A. Doe III" +"Dr. John A. Doe, Jr." +"Dr. John A. Kenneth Doe" +"Dr. John A. Kenneth Doe III" +"Dr. John A. Kenneth Doe, Jr." +"Dr. John Doe" +"Dr. John Doe III" +"Dr. John Doe, Jr." +"Dr. John P. Doe-Ray, CLU" +"Dr. John P. Doe-Ray, CLU, CFP, LUTC" +"Dr. John Smith" +"Dr. Juan Q. Velasquez y Garcia" +"Dr. Juan Q. Velasquez y Garcia III" +"Dr. Juan Q. Velasquez y Garcia, Jr." +"Dr. Juan Q. Xavier Velasquez y Garcia" +"Dr. Juan Q. Xavier Velasquez y Garcia III" +"Dr. Juan Q. Xavier Velasquez y Garcia, Jr." +"Dr. Juan Q. Xavier de la Vega" +"Dr. Juan Q. Xavier de la Vega III" +"Dr. Juan Q. Xavier de la Vega, Jr." +"Dr. Juan Q. Xavier de la dos Vega III" +"Dr. Juan Q. Xavier de la dos Vega, III" +"Dr. Juan Q. de la Vega" +"Dr. Juan Q. de la Vega III" +"Dr. Juan Q. de la Vega, Jr." +"Dr. Juan Velasquez y Garcia" +"Dr. Juan Velasquez y Garcia III" +"Dr. Juan Velasquez y Garcia, Jr." +"Dr. Juan de la Vega" +"Dr. Juan de la Vega III" +"Dr. Juan de la Vega, Jr." +"Dr. Martin Luther King Jr." +"Dr. Williams" +"Dr. abdul salam ahmed salem" +"Dra. Andréia da Silva" +"E.T. Smith" +"E.T. Smith, II" +"Esq Jane Smith" +"Foo. John Smith" +"Foo. Xyz. John Smith" +"Franklin Washington, Jr. MD" +"Franklin, Benjamin (Ben)" +"Franklin, Benjamin (Ben), Jr." +"Frau Anna Müller" +"Fritz Freiherr und von Bar" +"Frøken Jensen" +"GREGORY HOUSE M.D." +"Gunny de Mesnil" +"Harietta Keopuolani Nahi'ena'ena" +"Harrieta Keōpūolani Nāhiʻenaʻena" +"Her Majesty Queen Elizabeth" +"Herr Klaus Schmidt" +"Herr Schmidt" +"His Excellency Lord Duncan" +"Hon Solo" +"Hon. Barrington P. Doe-Ray, Jr." +"Hon. Charles J. Siragusa" +"Hon. Marian W. Payson" +"Honorable Judge Susan Russ Walker" +"Honorable Judge Terry F. Moorer" +"Honorable Judge W. Harold Albritton, III" +"Honorable Terry F. Moorer" +"Honorable W. Harold Albritton, III" +"Ivanov Ivan Ivanovich" +"J. Smith" +"JEFFREY (JD) BRICKEN" +"JOHN DOE" +"JOHN DOE PHD" +"JOHN DOE PHD MD" +"JOHN SMITH" +"JOSÉ GARCÍA" +"Jack Ma" +"Jack Ma Jr" +"Jane Doctor" +"Jane Doe" +"Jane Mac Beth" +"Jane Smith" +"Jean de Mesnil" +"Jenny \"JJ\" Baker (Johnson)" +"Jenny (Johnson) Baker" +"Jenny Baker" +"Jenny Baker (Johnson)" +"Jill St. John" +"Joao da Silva do Amaral de Souza" +"Joe Dentist D.D.S." +"Joe Franklin Jr" +"John & Jane" +"John A. Doe" +"John A. Doe III" +"John A. Doe, Jr" +"John A. Doe, Jr." +"John A. Kenneth Doe" +"John A. Kenneth Doe III" +"John A. Kenneth Doe, Jr." +"John Doe" +"John Doe III" +"John Doe MD PhD" +"John Doe Msc.Ed." +"John Doe jr., MD" +"John Doe, CLU, CFP, LUTC" +"John Doe, Jr." +"John Doe, Jr.,," +"John Doe, MD - PhD - FACS" +"John Doe, MD - PhD, FACS" +"John Doe, MD, PhD" +"John Doe, MD-PhD-" +"John Doe, Msc.Ed." +"John E Smith" +"John Edgar Casey Williams III" +"John Jones (Google Docs)" +"John Jones (Google Docs), Jr. (Unknown)" +"John Jones (Unknown)" +"John King" +"John Major. Smith" +"John P. Doe, CLU, CFP, LUTC" +"John Q. Smith" +"John Smith" +"John Smith Jr" +"John Smith Ph. D." +"John Smith VI" +"John Smith, Ph. D." +"John Smith, V Jr." +"John Smith, V MD" +"John W. Ingram, V" +"John W. Smith, I" +"John Williams" +"John and Jane Aznar y Lopez" +"John and Jane Smith" +"John e Smith" +"John e Smith III" +"John e Smith, III" +"John y Jane" +"Jon Dough and" +"Jon Dough and of" +"Jose Aznar y Lopez" +"Juan Q. Velasquez y Garcia" +"Juan Q. Velasquez y Garcia III" +"Juan Q. Velasquez y Garcia, Jr." +"Juan Q. Xavier Velasquez y Garcia" +"Juan Q. Xavier Velasquez y Garcia III" +"Juan Q. Xavier Velasquez y Garcia, Jr." +"Juan Q. Xavier de la Vega" +"Juan Q. Xavier de la Vega III" +"Juan Q. Xavier de la Vega, Jr." +"Juan Q. de la Vega" +"Juan Q. de la Vega III" +"Juan Q. de la Vega, Jr." +"Juan Velasquez y Garcia" +"Juan Velasquez y Garcia III" +"Juan Velasquez y Garcia, Jr." +"Juan de la Vega" +"Juan de la Vega III" +"Juan de la Vega, Jr." +"Juan de la de la Vega" +"Juan de la de la Vega Jr." +"Juan de la de la de la Vega" +"Juan del Sur" +"Judge C Lynwood Smith, Jr" +"Judge G. Thomas Eisele" +"Judge James M. Moody" +"Kenneth Clarke Q.C." +"Kenneth Clarke Q.C., M.P." +"Kenneth Clarke QC MP" +"King Henry" +"King John Alexander V" +"King John V." +"LT. GEN. JOHN A. KENNETH DOE IV" +"La'tanya O'connor" +"Larry James Johnson I" +"Larry V I" +"Lon (Jr.) Williams" +"Lord God Almighty" +"Lord and of the Universe" +"Lord of the Universe" +"Lord of the Universe and Associate Supreme Queen of the World Lisa Simpson" +"Lord of the Universe and Supreme King of the World Lisa Simpson" +"Lt. Gen. John A. Kenneth Doe IV" +"Lt. Gen. John A. Kenneth Doe, Jr." +"Lt.Gen. John A. Kenneth Doe IV" +"Lt.Gov. John Doe" +"Lt.Gov. juan e garcia" +"Ma III, Jack Jr" +"Ma, Jack" +"Mac Miller" +"Mag-Judge Harwell G Davis, III" +"Mag. Judge Byron G. Cudmore" +"Magistrate Judge John F. Forster, Jr" +"Magistrate-Judge Elizabeth Todd Campbell" +"Maid Marion" +"Maier, Amy I, Jr." +"Maier, Amy Lauren I" +"Major. Dona Smith" +"Major. John Smith, Jr." +"Mari' Aube'" +"Mevrouw Anna de Vries" +"Mike van der Velt" +"Mohamad Ahmad Ali Hassan" +"Mohamad Ali Khalil" +"Monsieur Jean Dupont" +"Mr. & Mrs. John Smith" +"Mr. Van Nguyen" +"Mr. and Mrs. John Smith" +"Mr. and Mrs. John and Jane Smith" +"Ms Hon Solo" +"Naomi Wambui Ng'ang'a" +"Nguyen, Van" +"No1. John Smith" +"None Smith" +"Nonez Smith" +"O'B. John Smith" +"Q R" +"Queen Elizabeth" +"RONALD MACDONALD" +"RONALD MCDONALD" +"Rafael Sousa dos Anjos" +"Rev Andrews" +"Rev John A. Kenneth Doe" +"Rev John A. Kenneth Doe III (Kenny)" +"Rev. John A. Kenneth Doe" +"Rt. Hon. Paul E. Mary" +"Sam Smith 😊" +"Secretary of State Hillary Clinton" +"Senator \"Rick\" Edmonds" +"Senior Judge Charles R. Butler, Jr" +"Senior Judge Harold D. Vietor" +"Senior Judge Virgil Pittman" +"Señor Carlos García" +"Señora María García" +"Shirley Maclaine" +"Signor Marco Rossi" +"Sir Gerald" +"Smith Jr., John" +"Smith van der" +"Smith, Dr. John" +"Smith, E.T., Jr." +"Smith, J.R." +"Smith, John" +"Smith, John I" +"Smith, John V" +"Smith, John e, III, Jr" +"Smith, MD - PhD - FACS" +"Smith, Major. John" +"Smith，John" +"Sr US District Judge Richard G Kopf" +"Srta. Andréia da Silva" +"Steven Hardman, RN - CRNA" +"Te Awanui-a-Rangi Black" +"The Lord of the Universe" +"The Right Hon. the President of the Queen's Bench Division" +"The Rt Hon John Jones" +"Title First Middle Middle Last, Jr." +"U. S. Grant" +"U.S. District Judge Marc Thomas Treadwell" +"US Magistrate Judge T Michael Putnam" +"US Magistrate-Judge Elizabeth E Campbell" +"VINCENT VAN GOGH" +"Va'apu'u Vitale" +"Van Jeremy Johnson" +"Van Johnson" +"Van Nguyen" +"Vega, Juan de la" +"Velasquez y Garcia, Dr. Juan" +"Velasquez y Garcia, Dr. Juan III" +"Velasquez y Garcia, Dr. Juan Q." +"Velasquez y Garcia, Dr. Juan Q. III" +"Velasquez y Garcia, Dr. Juan Q. Xavier" +"Velasquez y Garcia, Dr. Juan Q. Xavier III" +"Velasquez y Garcia, Dr. Juan Q. Xavier, Jr." +"Velasquez y Garcia, Dr. Juan Q., Jr." +"Velasquez y Garcia, Dr. Juan, Jr." +"Velasquez y Garcia, Juan" +"Velasquez y Garcia, Juan III" +"Velasquez y Garcia, Juan Q." +"Velasquez y Garcia, Juan Q. III" +"Velasquez y Garcia, Juan Q. Xavier" +"Velasquez y Garcia, Juan Q. Xavier III" +"Velasquez y Garcia, Juan Q. Xavier, Jr." +"Velasquez y Garcia, Juan Q., Jr." +"Velasquez y Garcia, Juan, Jr." +"Vincent van Gogh" +"Vincent van Gogh van Beethoven" +"Washington Jr. MD, Franklin" +"Xyz. (Bud) Smith" +"Yin Le" +"Yin a Le" +"Zephyrmark Jane Smith" +"abdul" +"abdul salam" +"abdul salam ahmed salem" +"abdul salam ahmed salem jr" +"abdul salam jr" +"abdul salam salem" +"abdulsalam ahmed salem" +"abu bakr al baghdadi" +"ahmed abu bakr" +"and Jon Dough" +"and van Buren" +"bob v. de la macdole-eisenhower phd" +"de" +"de Mesnil" +"de Mesnil Garcia" +"de Mesnil Jr." +"de la Vega" +"de la Vega, Dr. Juan" +"de la Vega, Dr. Juan III" +"de la Vega, Dr. Juan Q." +"de la Vega, Dr. Juan Q. III" +"de la Vega, Dr. Juan Q. Xavier" +"de la Vega, Dr. Juan Q. Xavier III" +"de la Vega, Dr. Juan Q. Xavier, Jr." +"de la Vega, Dr. Juan Q., Jr." +"de la Vega, Dr. Juan, Jr." +"de la Vega, Juan" +"de la Vega, Juan III" +"de la Vega, Juan Q." +"de la Vega, Juan Q. III" +"de la Vega, Juan Q. Xavier" +"de la Vega, Juan Q. Xavier III" +"de la Vega, Juan Q. Xavier, Jr." +"de la Vega, Juan Q., Jr." +"de la Vega, Juan, Jr." +"de la Véña, Jüan" +"de la dos Vega, Dr. Juan Q. Xavier III" +"de la vega, dr. juan Q. xavier III" +"donovan mcnabb-smith" +"dos Santos" +"dr Vincent James van Gogh dr" +"dr Vincent van Gogh dr" +"dr Vincent van der Gogh dr" +"dr. ben alex johnson III" +"dr. john p. doe-Ray, CLU, CFP, LUTC" +"dr. juan de la vega jr." +"e and e" +"e j smith" +"joao da silva do amaral de souza" +"john doe" +"john e jones" +"john e jones, III" +"john e. smith" +"john smith" +"johnny y" +"jones, john e" +"juan garcia III" +"juan q. xavier velasquez y garcia iii" +"larry james edward johnson v" +"lt. gen. john a. kenneth doe iv" +"mack johnson" +"matthëus schmidt" +"part1 of The part2 of the part3 and part4" +"part1 of and The part2 of the part3 And part4" +"pennie von bergen wessels" +"pennie von bergen wessels III" +"pennie von bergen wessels M.D." +"pennie von bergen wessels MD, III" +"pennie von bergen wessels, III" +"salem, abdul" +"salem, abdul salam" +"salem, abdul salam ahmed" +"scott e. werner" +"señora María García" +"test" +"the and Jon Dough" +"vai la" +"van nguyen" +"von Braun" +"von bergen wessels MD, pennie" +"von bergen wessels MD, pennie III" +"von bergen wessels, pennie III" +"von bergen wessels, pennie MD" +"xyz. John Smith" +"سلمان،" +"سلمان، محمد" +"‏John‏ Smith" +"‏محمد بن سلمان‏" +"∫≜⩕ Smith 😊" +"∫≜⩕ Smith😊" diff --git a/tools/differential/expected_changes.toml b/tools/differential/expected_changes.toml new file mode 100644 index 00000000..4e98c16a --- /dev/null +++ b/tools/differential/expected_changes.toml @@ -0,0 +1,57 @@ +# Every rule needs `issue`; optional `name_regex` and `fields` narrow +# it. An unexplained diff is a release blocker until classified (spec +# S5). Rules are seeded from docs/superpowers/plans/notes-m12-diffs.md +# and the `classification="fix(...)"` rows in tests/v2/cases.py; keep +# each entry's `name_regex`/`fields` as tight as the diff allows. + +[[change]] +issue = "fix(#274) maiden markers consumed" +name_regex = "(?i)\\b(n[ée]e|born|geb\\.?|roz\\.?)\\b" +fields = ["maiden", "middle", "last"] + +[[change]] +issue = "fix(comma-family) lone post-comma piece routes to suffix/title, not first" +# 'Smith, Dr.' / 'Andrews, M.D.': v1 put the lone strict-suffix-or-title +# post-comma piece in `first`; 2.0 routes it to `suffix`/`title` instead +# (family/`last` is unchanged either way -- "pre-comma is definitionally +# family"). +name_regex = "," +fields = ["first", "title", "suffix"] + +[[change]] +issue = "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix" +# 'Johnson PhD' / 'Mr. Johnson PhD': v1 routed a lone trailing suffix +# to family/first (no comma present); 2.0 keeps recognized suffixes in +# `suffix`. +fields = ["first", "last", "suffix"] + +[[change]] +issue = "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole" +# Only fires when a custom suffix delimiter is configured (Policy / +# Constants.suffix_delimiters); the corpus runs with default policy, so +# this rule is expected to match nothing here. Kept for documentation +# parity with tests/v2/cases.py's 'suffix_delimiter_no_space_core' row +# (anti-#100, migration plan deviation 5). +name_regex = "/" +fields = ["suffix"] + +[[change]] +issue = "ambiguous-surname-acronym data change: parenthesized (MA)/(DO) now stays nickname" +# suffix_acronyms_ambiguous gained 'ma'/'do' so bare 'Jack Ma' keeps its +# surname (v1 parity restored); side effect: parenthesized/quoted "MA" +# or "DO" no longer escape to suffix (v1 did, since v1 treated them as +# unambiguous there) -- they now fall through to nickname parsing. +# Not expected to fire against this corpus (no such strings survived +# into the v1 banks); kept for documentation completeness. +name_regex = "(?i)[(\"'](m\\.?a\\.?|d\\.?o\\.?)[)\"']" +fields = ["suffix", "nickname"] + +# Deliberately NOT a [[change]] rule: 'Ph. D.' split-token healing +# ('John Ph. D.', 'John Smith, Ph. D.') is PARITY, not a 2.0 behavior +# change -- v1's fix_phd healing of the adjacent 'Ph.'/'D.' pair into +# one suffix unit is replicated exactly by 2.0's 'joined' vocab tag +# (tests/v2/cases.py classification="parity" on both 'phd_split' and +# 'suffix_comma_split_phd'; verified empirically, zero diff against the +# 1.4 worker). Adding a suppression rule for it would risk masking a +# real regression in this exact shape, so it is intentionally left +# unclassified: if it ever starts diffing, the harness must fail. diff --git a/tools/differential/worker_v1.py b/tools/differential/worker_v1.py new file mode 100644 index 00000000..f854bd0e --- /dev/null +++ b/tools/differential/worker_v1.py @@ -0,0 +1,25 @@ +# /// script +# requires-python = ">=3.9" +# dependencies = ["nameparser==1.4.*"] +# /// +"""v1 worker: reads JSON name strings on stdin (one per line), writes +the 1.4 component dict per line. Run ONLY via: + + uv run --no-project tools/differential/worker_v1.py + +--no-project is load-bearing: without it uv installs the working tree +and shadows the 1.4 pin from PyPI. +""" +import json +import sys + +from nameparser import HumanName + +for line in sys.stdin: + line = line.strip() + if not line: + continue + name = json.loads(line) + n = HumanName(name) + print(json.dumps({k: v or "" for k, v in n.as_dict().items()}, + ensure_ascii=False), flush=True) From cd94143a9fdbc3205ab0cc97a7c59941b723feda Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 07:09:57 -0700 Subject: [PATCH 135/275] Reimplement the CLI over the new API with --json Co-Authored-By: Claude Fable 5 --- nameparser/__main__.py | 45 +++++++++++++++++++-------------------- tests/v2/test_cli.py | 26 ++++++++++++++++++++++ tests/v2/test_layering.py | 3 +++ 3 files changed, 51 insertions(+), 23 deletions(-) create mode 100644 tests/v2/test_cli.py diff --git a/nameparser/__main__.py b/nameparser/__main__.py index 5d96ec8f..eec2503c 100644 --- a/nameparser/__main__.py +++ b/nameparser/__main__.py @@ -1,31 +1,30 @@ -"""Command-line debug helper: parse a name and print the result. - -Usage: +"""Command-line debug helper over the 2.0 API (migration spec §6). python -m nameparser "Dr. Juan Q. Xavier de la Vega III" + python -m nameparser --json "Doe, John" """ -import logging -import sys +import argparse +import json -from nameparser import HumanName +from nameparser import parse -def main() -> None: - if len(sys.argv) <= 1: - print('Usage: python -m nameparser "Name String"') - raise SystemExit(1) - log = logging.getLogger('HumanName') - log.setLevel(logging.ERROR) - log.addHandler(logging.StreamHandler()) - name_string = sys.argv[1] - hn = HumanName(name_string) - print(repr(hn)) - hn.capitalize() - print(repr(hn)) - # Use comma rather than concatenation: initials() returns - # empty_attribute_default (possibly None) when there are no initials. - print("Initials:", hn.initials()) +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser( + prog="nameparser", description="Parse a personal name.") + ap.add_argument("name", help="the name string to parse") + ap.add_argument("--json", action="store_true", + help="print the component dict as JSON") + args = ap.parse_args(argv) + n = parse(args.name) + if args.json: + print(json.dumps(n.as_dict(), ensure_ascii=False)) + return 0 + print(repr(n)) + print(repr(n.capitalized())) + print("Initials:", n.initials()) + return 0 -if __name__ == '__main__': - main() +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/v2/test_cli.py b/tests/v2/test_cli.py new file mode 100644 index 00000000..36b85773 --- /dev/null +++ b/tests/v2/test_cli.py @@ -0,0 +1,26 @@ +import json +import subprocess +import sys + + +def _run(*args: str) -> subprocess.CompletedProcess: + return subprocess.run([sys.executable, "-m", "nameparser", *args], + capture_output=True, text=True) + + +def test_cli_prints_repr_capitalized_and_initials() -> None: + out = _run("dr. juan de la vega iii").stdout + assert "juan" in out # raw repr first + assert "Juan" in out # capitalized repr second + assert "Initials:" in out + + +def test_cli_json() -> None: + proc = _run("John Smith", "--json") + data = json.loads(proc.stdout) + assert data["given"] == "John" and data["family"] == "Smith" + + +def test_cli_no_args_usage() -> None: + proc = _run() + assert proc.returncode != 0 diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 3be5d8a0..5896fbc3 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -78,6 +78,9 @@ # v1 import-path preservation: thin re-exports of the facade/shim "parser.py": ("nameparser._facade",), "config/__init__.py": ("nameparser._config_shim",), + # CLI (migration spec §6): imports only the public package, same as + # any other consumer -- no access to internal modules. + "__main__.py": ("nameparser",), } From fa95cd1aebc95bdd3a4dea0aa7e92ad462de4bb7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 07:11:10 -0700 Subject: [PATCH 136/275] Record the migration layer in AGENTS.md The 2.0 modules section gains the facade layer: the re-export swap of nameparser.parser/config, the _facade/_config_shim mechanisms (dirty-tracking, v1-shaped pickle, #280 hook warning), the layering rule, and the differential-harness pointer. The v1 #256 extension-pattern paragraph is marked historical. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 84cd03dc..4bd476dd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -111,8 +111,9 @@ Each named attribute (`title`, `first`, etc.) is a `@property` that joins its co The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These conventions apply to all new-API code and are stricter than the v1 sections above. The full design record (rationale, settled-decision logs, dated amendments) lives in untracked `docs/superpowers/specs/`; this section is the enforceable subset. **A commit that establishes or amends one of these conventions must update this section in the same commit** — grep-driven staleness sweeps miss paraphrased prose (see Workflow above), so write-time maintenance is the mechanism, audits are the backstop. -- **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`, `_pipeline/`, `_parser.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Old paths (`nameparser.parser`, `nameparser.config`) stay v1-shaped. -- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers; `_parser` sits on everything except `_render`. Extend the test's `ALLOWED` table when adding a module. +- **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`, `_pipeline/`, `_parser.py`, plus the facade layer: `_facade.py`, `_config_shim.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Since the M11 swap, the old paths are import-path-preserving re-exports: `nameparser.parser` re-exports the `_facade` `HumanName`, `nameparser.config` re-exports the `_config_shim` names (`Constants`, `CONSTANTS`, `SetManager`, `TupleManager`, `RegexTupleManager`); the `config/` DATA modules stay the vocabulary source through 2.x. The whole facade layer is deleted in 3.0. +- **Facade layer** (`_facade.py`, `_config_shim.py`): the v1-compat `HumanName`/`Constants` over the core. Key mechanisms: `Constants._generation` dirty-tracking (every mutation bumps; facades resolve their `Parser` lazily via `_cached_parser(lexicon, policy)`); `Constants._snapshot()` mirrors `_lexicon._default_lexicon()` (equality-pinned); the facade pickles v1-SHAPED state (component lists, one `__setstate__` path for 1.4 and 2.x blobs; components rebuild via `replace()`, never a re-parse); `_V1_HOOKS` overrides warn once per subclass (#280). The compat contract is the migration spec's promise: warning-free 1.4 code behaves identically except release-log-classified fixes — `tools/differential/` (dev-only, not shipped) verifies this against 1.4-on-PyPI over a 486-name corpus. +- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers; `_parser` sits on everything except `_render`; the facade layer (`_facade`, `_config_shim`, `parser`, `config/__init__`, `__main__`) may import anything public plus `_render`. Extend the test's `ALLOWED` table when adding a module. - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). @@ -135,7 +136,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_deepcopy_roundtrip`/`test_nickname_delimiters_deepcopy_roundtrip` in `tests/test_constants.py`), not just reliance on conftest's autouse snapshot/restore exercising it incidentally. `TupleManager`/`RegexTupleManager.__getattr__` answer *any* unknown attribute lookup — dunder probes like `__deepcopy__` raise `AttributeError` (never answered as config keys), everything else still returns `self.get(attr)`/`EMPTY_REGEX` — so a new manager subtype or a `__getattr__` tweak can silently break `copy.deepcopy` (this bit `RegexTupleManager` before the dunder-lookup guard was added). A direct test on the new attribute's own manager instance catches that where the conftest fixture, which never asserts on the copy, would not. As with the scalar-attribute pattern above, also check `AGENTS.md` itself for now-stale references when you touch this. -**Unknown-key attribute access on `TupleManager`/`RegexTupleManager` warns (1.4, #256)** — a key not currently in the dict emits `DeprecationWarning` naming the miss and the known keys (`_warn_unknown_key`), before falling back to the same `None`/`EMPTY_REGEX` default as before; `.get()` stays silent. Dunder probes (`__deepcopy__`) still raise `AttributeError` outright, and single-underscore probes (`_repr_html_`, IPython's `_ipython_canary_method_should_not_exist_`, etc.) are excluded from the warning too — no real config key starts with `_`, so both guards just keep protocol/introspection probes from misfiring as "typo" warnings. This means internal parser code that reads `self.C.regexes.` unconditionally (e.g. `squash_bidi`'s `bidi`) now warns if a caller's custom `regexes` dict omits that key — a previously-silent partial-override pattern is on the same deprecation path as an actual typo. +**Unknown-key attribute access on `TupleManager`/`RegexTupleManager` warns (1.4, #256) — 2.0 note: the warning became a hard `AttributeError` naming the known keys (shim `TupleManager`); the paragraph below describes the deleted v1 machinery and applies only when reading v1 history.** — a key not currently in the dict emits `DeprecationWarning` naming the miss and the known keys (`_warn_unknown_key`), before falling back to the same `None`/`EMPTY_REGEX` default as before; `.get()` stays silent. Dunder probes (`__deepcopy__`) still raise `AttributeError` outright, and single-underscore probes (`_repr_html_`, IPython's `_ipython_canary_method_should_not_exist_`, etc.) are excluded from the warning too — no real config key starts with `_`, so both guards just keep protocol/introspection probes from misfiring as "typo" warnings. This means internal parser code that reads `self.C.regexes.` unconditionally (e.g. `squash_bidi`'s `bidi`) now warns if a caller's custom `regexes` dict omits that key — a previously-silent partial-override pattern is on the same deprecation path as an actual typo. **Adding a word to a config set** — first check the *other* sets for the same word (grep `nameparser/config/` or intersect the sets in a `python3 -c`). Real overlaps exist: `do`/`st`/`mc` ∈ `PREFIXES` ∩ `TITLES`/`SUFFIX_ACRONYMS`; `abd` = "ABD" ∈ `SUFFIX_ACRONYMS`; `abu` ∈ `PREFIXES` ∩ `bound_first_names` (position-dependent: leading token → first-name join, mid-name → last-name join). Usually position-dependent and harmless, but can force a guard or an exclusion (the `last_base` all-particles guard; dropping `abd` from `bound_first_names`). From 912e608decc82a71a2a8389419f3a81a54de24cf Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 19:15:45 -0700 Subject: [PATCH 137/275] Harden shim notification/validation paths and facade failure ordering /pr-review-toolkit findings (Critical + Important): - SetManager.add() wraps its loop in try/finally: a TypeError on a later argument left earlier additions applied WITHOUT the owner's generation bump -- every bound facade kept parsing with a stale cached parser, silently. Same rule remove() already had. - remove()/discard() bytes arguments get the same decode-hint TypeError as add() (#245) instead of lc()'s cryptic one. - _DelimiterManager's constructor routes through the parent's #242 guard (shared _validated_mapping_args helper): a bare-str arg gets the friendly TypeError, not dict's ValueError. - Constants._shared is read-only (the flag the shared-mutation DeprecationWarning hinges on); the singleton flip uses object.__setattr__. - _apply_full_name parses BEFORE assigning _full_name: a snapshot failure no longer leaves full_name disagreeing with the parsed fields. - RegexTupleManager carries a grep-able do-NOT-delete marker; docstrings for has_own_config/comparison_key/as_dict. Tests: partial-add notification, bytes hints, delimiter #242 path, read-only _shared, post-construction suffix_delimiter reassignment, C-swap + delimiter interplay, failed-reparse consistency. Co-Authored-By: Claude Fable 5 --- nameparser/_config_shim.py | 90 +++++++++++++++++++++++------------- nameparser/_facade.py | 11 ++++- tests/v2/test_config_shim.py | 36 +++++++++++++++ tests/v2/test_facade.py | 41 ++++++++++++++++ 4 files changed, 145 insertions(+), 33 deletions(-) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index 52f83ce6..812be2af 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -108,13 +108,18 @@ def add(self, *strings: str) -> SetManager: # notify only on real change (v1 parity): a no-op add must not # bump the owner's generation changed = False - for s in strings: - normalized = _lc_validated(s) # TypeError on bytes (#245) - if normalized not in self._elements: - self._elements.add(normalized) - changed = True - if changed: - self._changed() + try: + for s in strings: + normalized = _lc_validated(s) # TypeError on bytes (#245) + if normalized not in self._elements: + self._elements.add(normalized) + changed = True + finally: + # a TypeError mid-list still leaves earlier additions + # applied, so the owner must hear about them or its cache + # goes stale (same rule as remove() below) + if changed: + self._changed() return self def remove(self, *strings: str) -> SetManager: @@ -124,7 +129,10 @@ def remove(self, *strings: str) -> SetManager: changed = False try: for s in strings: - self._elements.remove(lc(s)) # KeyError on missing (#243) + # _lc_validated: bytes get the same decode-hint TypeError + # as add() (#245) + normalized = _lc_validated(s) + self._elements.remove(normalized) # KeyError on missing (#243) changed = True finally: # a KeyError mid-list still leaves earlier removals applied, @@ -139,7 +147,7 @@ def discard(self, *strings: str) -> SetManager: Returns ``self`` for chaining.""" changed = False for s in strings: - normalized = lc(s) + normalized = _lc_validated(s) # bytes decode hint (#245) if normalized in self._elements: self._elements.discard(normalized) changed = True @@ -253,6 +261,32 @@ def __setstate__(self, state: dict[str, object]) -> None: self._on_change = None # rewired by the owning Constants +def _validated_mapping_args(args: tuple[object, ...]) -> tuple[object, ...]: + """The #242 constructor guard, shared by TupleManager and + _DelimiterManager: a bare str/bytes silently shreds into a garbage + mapping (dict's own error), and an iterable of short strings + silently splits each one into a (key, value) pair -- ported from + v1's TupleManager.__init__ guard.""" + if not args: + return args + arg = args[0] + _reject_bare_str_or_bytes( + arg, "a mapping or iterable of (key, value) pairs") + if not isinstance(arg, Mapping): + checked = [] + for item in arg: # type: ignore[attr-defined] + if isinstance(item, (str, bytes)): + kind = "bytes" if isinstance(item, bytes) else "str" + raise TypeError( + f"expected (key, value) pairs, got a {kind} " + f"element {item!r}; a 2-character string " + "silently splits into a key and a value" + ) + checked.append(item) + args = (checked, *args[1:]) + return args + + class TupleManager(dict[str, object]): """v1 ``TupleManager``: a dict with dot-notation access. Backs ``capitalization_exceptions``. Unknown-key attribute access raises @@ -267,26 +301,7 @@ class TupleManager(dict[str, object]): def __init__(self, *args: object, _on_change: Callable[[], None] | None = None, **kwargs: object) -> None: - if args: - # #242: a bare str/bytes silently shreds into a garbage mapping - # (dict's own error), and an iterable of short strings silently - # splits each one into a (key, value) pair -- ported from v1's - # TupleManager.__init__ guard. - arg = args[0] - _reject_bare_str_or_bytes( - arg, "a mapping or iterable of (key, value) pairs") - if not isinstance(arg, Mapping): - checked = [] - for item in arg: # type: ignore[attr-defined] - if isinstance(item, (str, bytes)): - kind = "bytes" if isinstance(item, bytes) else "str" - raise TypeError( - f"expected (key, value) pairs, got a {kind} " - f"element {item!r}; a 2-character string " - "silently splits into a key and a value" - ) - checked.append(item) - args = (checked, *args[1:]) + args = _validated_mapping_args(args) super().__init__(*args, **kwargs) self._on_change = _on_change @@ -405,7 +420,7 @@ def __setstate__(self, state: dict[str, object]) -> None: _DELIMITER_SENTINELS = tuple(_SENTINEL_PAIRS) -class RegexTupleManager(TupleManager): +class RegexTupleManager(TupleManager): # pickle-compat: do NOT delete """Pickle-compat alias only: v1.4's ``Constants.regexes`` field was a ``nameparser.config.RegexTupleManager`` instance (a ``TupleManager`` subclass whose ``__getattr__`` fell back to ``EMPTY_REGEX`` for an @@ -436,7 +451,10 @@ def __init__(self, *args: object, **kwargs: object) -> None: # dict's C-level __init__ never calls a subclass __setitem__, so # collect and validate the initial items here -- BEFORE any item - # lands -- or the sentinel rule silently misses the constructor + # lands -- or the sentinel rule silently misses the constructor. + # The parent's #242 guard runs first: bare str/bytes gets the + # friendly TypeError, not dict's cryptic one. + args = _validated_mapping_args(args) items: dict[str, object] = dict(*args, **kwargs) for key in items: self._reject_non_sentinel(key) @@ -782,6 +800,14 @@ def _bump(self) -> None: object.__setattr__(self, "_generation", self._generation + 1) def __setattr__(self, name: str, value: object) -> None: + if name == "_shared": + # the flag the whole shared-mutation DeprecationWarning + # mechanism hinges on: only the module-level singleton flip + # (object.__setattr__ below) may set it + raise AttributeError( + "Constants._shared is read-only; it marks the module " + "singleton and is set once at import" + ) if name == "empty_attribute_default": raise AttributeError( "empty_attribute_default was removed in 2.0 (#255): " @@ -972,4 +998,4 @@ def __setstate__(self, state: dict[str, object]) -> None: CONSTANTS = Constants() -CONSTANTS._shared = True # type: ignore[attr-defined] +object.__setattr__(CONSTANTS, "_shared", True) diff --git a/nameparser/_facade.py b/nameparser/_facade.py index c13e2e1b..a738253d 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -251,8 +251,11 @@ def _apply_full_name(self, value: str) -> None: ) if not isinstance(value, str): raise TypeError(f"full_name must be a str, got {value!r}") + # parse FIRST: if snapshot resolution raises, the instance must + # not be left with a new full_name over the old parsed fields + parsed = self._resolve().parse(value) self._full_name = value - self._parsed = self._resolve().parse(value) + self._parsed = parsed if self._C.capitalize_name: self.capitalize() # v1 parser.py:1653 parity @@ -307,6 +310,8 @@ def C(self, constants: Constants | None) -> None: @property def has_own_config(self) -> bool: + """True when this instance is not using the shared module-level + CONSTANTS.""" return self._C is not CONSTANTS # -- fields --------------------------------------------------------- @@ -554,6 +559,8 @@ def matches(self, other: str | HumanName) -> bool: return self._parsed.matches(target, parser=self._resolve()) def comparison_key(self) -> tuple[str, ...]: + """One casefolded component per field in canonical order -- the + v1 replacement for ==/hash (#223); see ParsedName.comparison_key.""" return self._parsed.comparison_key() # -- dunders ------------------------------------------------------------ @@ -602,6 +609,8 @@ def __getitem__(self, key: str) -> str: return getattr(self, key) def as_dict(self, include_empty: bool = True) -> dict[str, str]: + """The seven v1-named components as a dict; include_empty=False + drops empty fields.""" d = {member: getattr(self, member) for member in _MEMBERS} if include_empty: return d diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index 0df949fc..7cee0d66 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -567,3 +567,39 @@ def test_constants_bool_kwargs() -> None: d = Constants(patronymic_name_order=True) _, policy, _ = d._snapshot() assert len(policy.patronymic_rules) == 2 + + +def test_set_manager_partial_add_still_notifies_owner() -> None: + # a TypeError mid-argument-list leaves earlier additions applied; + # the owner must hear about them or its cached parser goes stale + bumps: list[int] = [] + s = SetManager(["a"], _on_change=lambda: bumps.append(1)) + with pytest.raises(TypeError, match="decode"): + s.add("b", b"bad") # type: ignore[arg-type] + assert "b" in s + assert len(bumps) == 1 + + +def test_set_manager_remove_discard_bytes_decode_hint() -> None: + s = SetManager(["dr"]) + with pytest.raises(TypeError, match="decode"): + s.remove(b"dr") # type: ignore[arg-type] + with pytest.raises(TypeError, match="decode"): + s.discard(b"dr") # type: ignore[arg-type] + + +def test_delimiter_manager_constructor_bare_str_gets_242_error() -> None: + # the parent's #242 guard, not dict's cryptic ValueError + with pytest.raises(TypeError, match="mapping or iterable"): + _DelimiterManager("ab") # type: ignore[arg-type] + with pytest.raises(TypeError, match="pairs"): + _DelimiterManager(["ab", "cd"]) # type: ignore[list-item] + + +def test_constants_shared_flag_is_read_only() -> None: + c = Constants() + with pytest.raises(AttributeError, match="read-only"): + c._shared = True + with pytest.raises(AttributeError, match="read-only"): + CONSTANTS._shared = False + assert CONSTANTS._shared is True and c._shared is False diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index 4ffc9314..298577b0 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -380,3 +380,44 @@ def test_v14_humanname_blob_unpickles() -> None: assert loaded.title == "Dr." and loaded.suffix == "III" assert loaded.original == "Dr. Juan de la Vega III" assert loaded.C is CONSTANTS # shared sentinel restored (v1.4's C: None) + + +def test_suffix_delimiter_reassignment_after_construction() -> None: + n = HumanName("Doe, John, RN - CRNA") + assert n.suffix == "RN - CRNA" # no delimiter: one entry + n.suffix_delimiter = " - " + n.parse_full_name() + assert n.suffix == "RN, CRNA" # setter invalidated the Policy + n.suffix_delimiter = None + n.parse_full_name() + assert n.suffix == "RN - CRNA" # and back + + +def test_c_swap_keeps_instance_suffix_delimiter() -> None: + # both invalidation paths together: swapping C must not lose the + # instance-level delimiter layered onto the new snapshot's Policy + n = HumanName("Doe, John, RN - CRNA", suffix_delimiter=" - ") + assert n.suffix == "RN, CRNA" + n.C = Constants() + n.parse_full_name() + assert n.suffix == "RN, CRNA" + + +def test_failed_reparse_leaves_state_consistent() -> None: + # _resolve() raising must not leave full_name pointing at the new + # value over the OLD parsed fields + n = HumanName("John Smith") + + class _Boom(Exception): + pass + + def explode() -> None: + raise _Boom + + n._C = Constants() + n._C._snapshot = explode # type: ignore[method-assign, assignment] + n._snapshot_gen = -1 + with pytest.raises(_Boom): + n.full_name = "Jane Doe" + assert n.full_name == "John Smith" + assert n.first == "John" From c39ef491e0113ccca3111de622e4281c95e8cab2 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 19:16:09 -0700 Subject: [PATCH 138/275] Harden the differential comparator against silent false confidence /pr-review-toolkit Critical: a malformed allowlist rule with neither name_regex nor fields matched every diff and shadowed every later rule; rules are now validated loudly at startup. The worker integrity checks are hard SystemExits instead of asserts (-O must not turn a crashed worker into a truncated-but-green run) and the subprocess return code is checked. Co-Authored-By: Claude Fable 5 --- tools/differential/compare.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/tools/differential/compare.py b/tools/differential/compare.py index f0576899..f6edef8f 100644 --- a/tools/differential/compare.py +++ b/tools/differential/compare.py @@ -16,6 +16,25 @@ "maiden") +def validate_rules(rules: list[dict[str, object]]) -> None: + """Reject malformed allowlist rules LOUDLY at startup. A rule with + neither name_regex nor fields would match every diff and shadow + every later rule -- the harness would report false confidence, + the exact failure it exists to prevent.""" + for i, rule in enumerate(rules): + issue = rule.get("issue") + if not isinstance(issue, str) or not issue: + raise SystemExit( + f"expected_changes.toml rule #{i + 1} has no string " + f"'issue': {rule!r}") + if not isinstance(rule.get("name_regex"), str) \ + and not isinstance(rule.get("fields"), list): + raise SystemExit( + f"expected_changes.toml rule #{i + 1} ({issue!r}) has " + f"neither 'name_regex' nor 'fields' -- it would match " + f"every diff and shadow every later rule") + + def classify(name: str, diff_fields: set[str], rules: list[dict[str, object]]) -> str | None: for rule in rules: @@ -25,9 +44,7 @@ def classify(name: str, diff_fields: set[str], fields = rule.get("fields") if isinstance(fields, list) and not diff_fields <= set(fields): continue - issue = rule["issue"] - assert isinstance(issue, str) - return issue + return rule["issue"] # type: ignore[return-value] return None @@ -37,6 +54,7 @@ def main() -> int: args = ap.parse_args() rules = tomllib.loads( (HERE / "expected_changes.toml").read_text()).get("change", []) + validate_rules(rules) corpus = [json.loads(line) for line in Path(args.corpus).read_text().splitlines() if line.strip()] @@ -47,7 +65,15 @@ def main() -> int: for n in corpus) v1_lines, _ = proc.communicate(v1_input) v1_results = [json.loads(line) for line in v1_lines.splitlines()] - assert len(v1_results) == len(corpus), "worker line count mismatch" + # hard checks, not asserts: -O must not turn a crashed worker into + # a truncated-but-green comparison + if proc.returncode != 0: + raise SystemExit( + f"worker_v1.py exited {proc.returncode}; comparison aborted") + if len(v1_results) != len(corpus): + raise SystemExit( + f"worker returned {len(v1_results)} results for " + f"{len(corpus)} corpus names; comparison aborted") from nameparser import HumanName # the working tree (2.0 facade) by_issue: dict[str, list[str]] = {} From b0099a999f49937469d41c28454552ae983d640a Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 19:17:31 -0700 Subject: [PATCH 139/275] Type polish and coverage from the toolkit review - bound_required becomes the three-member BoundJoin IntEnum (the int encoded threshold-and-switch through truthiness; a stray value would have invented a fourth behavior silently) - the folded-middle view marker is single-sourced as _types.FOLDED_TAG; emitter and consumers cannot drift - the facade list view orders folded tokens first, matching the string view (last_list was still in token order -- caught while applying the review) - four case rows close the flagged sub-branch gaps: pairwise-only bound-given join, the three-part strict-trailing distinction, triple trailing commas, the suffix_words paren escape (all pinned live against v1.4, 2026-07-17) Co-Authored-By: Claude Fable 5 --- nameparser/_facade.py | 10 +++++++-- nameparser/_pipeline/_group.py | 33 ++++++++++++++++++----------- nameparser/_pipeline/_post_rules.py | 4 ++-- nameparser/_types.py | 10 ++++++++- tests/v2/cases.py | 18 ++++++++++++++++ tests/v2/test_facade.py | 10 +++++++++ 6 files changed, 68 insertions(+), 17 deletions(-) diff --git a/nameparser/_facade.py b/nameparser/_facade.py index a738253d..47143d55 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -31,7 +31,7 @@ from nameparser._config_shim import CONSTANTS, Constants, _cached_parser from nameparser._lexicon import _normalize from nameparser._parser import Parser -from nameparser._types import ParsedName, Role +from nameparser._types import FOLDED_TAG, ParsedName, Role _V2_FIELD = {"first": "given", "last": "family"} # v1 name -> v2 name _MEMBERS = ("title", "first", "middle", "last", "suffix", "nickname", @@ -344,12 +344,18 @@ def _list_for(self, member: str) -> list[str]: # every role -- a continuation is never its own list element. role = Role(_V2_FIELD.get(member, member)) parts: list[str] = [] + folded: list[str] = [] for tok in self._parsed.tokens_for(role): if "joined" in tok.tags and parts: parts[-1] += " " + tok.text + elif FOLDED_TAG in tok.tags: + # middle_as_family fold: v1 PREPENDED middle_list to + # last_list -- keep the list view consistent with the + # string view (_text_for orders folded-first too) + folded.append(tok.text) else: parts.append(tok.text) - return parts + return folded + parts @property def title(self) -> str: diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 4bc3ab86..c2096d8a 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -21,6 +21,7 @@ import dataclasses import re from collections.abc import Sequence, Set +from enum import IntEnum from nameparser._pipeline._state import ParseState, Structure, WorkToken from nameparser._pipeline._vocab import delimiter_cores @@ -32,6 +33,16 @@ Piece = list[int] +class BoundJoin(IntEnum): + """v1 _join_bound_first_name's reserve_last, as the three states it + actually has. IntEnum: the value IS the non_suffix threshold, so + the >= comparison below reads unchanged.""" + + DISABLED = 0 # the FAMILY_COMMA family segment (v1 never joined it) + LENIENT = 2 # FAMILY_COMMA's post-comma segment (reserve_last=False) + STRICT = 3 # main segments (reserve_last=True: keep a family piece) + + def _is_title_piece(piece: Sequence[int], ptags: Set[str], tokens: Sequence[WorkToken]) -> bool: if "title" in ptags: @@ -74,7 +85,7 @@ def _is_rootname(piece: Sequence[int], ptags: Set[str], def _group_segment(seg: tuple[int, ...], additional: int, tokens: Sequence[WorkToken], - bound_required: int = 3, + bound_join: BoundJoin = BoundJoin.STRICT, ) -> tuple[list[Piece], list[set[str]]]: pieces: list[Piece] = [[i] for i in seg] ptags: list[set[str]] = [set() for _ in seg] @@ -157,15 +168,12 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), j += 1 merge(k, j, drop={"prefix"}) k += 1 - # bound given names: first non-title piece joins the next. - # bound_required is v1's reserve_last: 3 keeps a family piece in - # reserve (main segments); 2 joins freely (FAMILY_COMMA's - # post-comma segment, v1 reserve_last=False -- 'salem, abdul - # salam' -> given 'abdul salam'); 0 disables (family segment, - # which v1 never bound-joined). + # bound given names: the first non-title piece joins the next + # ONCE (pairwise, v1 parity: 'Salem, Abdul Rahman Ahmed' keeps + # Ahmed a middle name). BoundJoin encodes v1's reserve_last. first_name_k = next( (k for k in range(len(pieces)) if not title(k)), None) - if (bound_required + if (bound_join is not BoundJoin.DISABLED and first_name_k is not None and first_name_k + 1 < len(pieces) and len(pieces[first_name_k]) == 1 @@ -173,7 +181,7 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), in tokens[pieces[first_name_k][0]].tags): non_suffix = sum(1 for k in range(len(pieces)) if not title(k) and not suffix(k)) - if non_suffix >= bound_required: + if non_suffix >= bound_join: merge(first_name_k, first_name_k + 2) return pieces, ptags @@ -195,11 +203,12 @@ def group(state: ParseState) -> ParseState: family_comma = state.structure is Structure.FAMILY_COMMA for seg_idx, seg in enumerate(state.segments): if family_comma: - bound_required = 2 if seg_idx == 1 else 0 + bound_join = (BoundJoin.LENIENT if seg_idx == 1 + else BoundJoin.DISABLED) else: - bound_required = 3 + bound_join = BoundJoin.STRICT pieces, ptags = _group_segment(seg, additional, tokens, - bound_required) + bound_join) if tail_start is not None and seg_idx >= tail_start: # v1 renders each tail COMMA SEGMENT as one suffix entry # ('Smith, V MD' -> suffix 'V MD'); a delimiter core inside diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index 5f3975eb..ace67b19 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -35,7 +35,7 @@ from nameparser._lexicon import _normalize from nameparser._pipeline._state import ParseState, Structure, WorkToken from nameparser._policy import PatronymicRule -from nameparser._types import Role +from nameparser._types import FOLDED_TAG, Role # Ported verbatim from v1 (nameparser/config/regexes.py) -- layering # forbids the config import; keep in sync by hand. @@ -125,5 +125,5 @@ def post_rules(state: ParseState) -> ParseState: for i in _idx(tokens, Role.MIDDLE): tokens[i] = dataclasses.replace( tokens[i], role=Role.FAMILY, - tags=tokens[i].tags | {"vocab:folded-middle"}) + tags=tokens[i].tags | {FOLDED_TAG}) return dataclasses.replace(state, tokens=tuple(tokens)) diff --git a/nameparser/_types.py b/nameparser/_types.py index dde82e4a..6dd26cc2 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -59,6 +59,14 @@ def __add__(self, other: object) -> NoReturn: # type: ignore[override] #: drives the suffix view's space-vs-comma join. STABLE_TAGS = frozenset({"particle", "conjunction", "initial", "joined"}) +#: The one sanctioned view-reorder marker (namespaced = unstable API). +#: Tokens cannot reorder (span order is validated), so a role fold that +#: must render BEFORE the role's original tokens tags them with this; +#: _text_for and the facade lists prepend carriers. Single-sourced here +#: so the emitter (_pipeline/_post_rules) and the consumers cannot +#: drift. +FOLDED_TAG = "vocab:folded-middle" + _E = TypeVar("_E", bound=Enum) @@ -319,7 +327,7 @@ def _text_for(self, *roles: Role, tag: str | None = None, # view's ", " join does not split one credential in two if suffix_join and "joined" in tok.tags and parts: parts[-1] += " " + tok.text - elif "vocab:folded-middle" in tok.tags: + elif FOLDED_TAG in tok.tags: # middle_as_family fold: v1 PREPENDED middle_list to # last_list; spans cannot reorder, so the view does folded.append(tok.text) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 95348473..db44eb5d 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -41,6 +41,24 @@ class Case: {"given": "John", "family": "Smith"}), Case("suffix_comma", "John Smith, PhD", {"given": "John", "family": "Smith", "suffix": "PhD"}), + Case("bound_given_pairwise_only", "Salem, Abdul Rahman Ahmed", + {"given": "Abdul Rahman", "middle": "Ahmed", "family": "Salem"}, + notes="the bound-given join is PAIRWISE (one merge, v1 " + "parity): the third piece stays a middle name"), + Case("family_comma_three_part_trailing_strict", "Smith, John V, Jr.", + {"given": "John", "middle": "V", "family": "Smith", + "suffix": "Jr."}, + notes="the lenient trailing test applies only to TWO-part " + "names; a third comma part makes the trailing token a " + "middle initial (v1 parity, pinned live 2026-07-17)"), + Case("triple_trailing_commas", "Doe,,,", + {"family": "Doe"}, + notes="one trailing comma is cosmetic; the rest are " + "structural empties (pinned live 2026-07-17)"), + Case("paren_suffix_word_escapes_nickname", "John Smith (Esq)", + {"given": "John", "family": "Smith", "suffix": "Esq"}, + notes="the suffix_words branch of the delimited-content " + "escape (v1 parity, pinned live 2026-07-17)"), Case("bound_given_whole_segment", "salem, abdul salam", {"given": "abdul salam", "family": "salem"}, notes="v1 joins bound given names freely in the post-comma " diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index 298577b0..31e3468b 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -421,3 +421,13 @@ def explode() -> None: n.full_name = "Jane Doe" assert n.full_name == "John Smith" assert n.first == "John" + + +def test_middle_as_family_list_view_matches_string_view() -> None: + # v1 PREPENDED middle_list to last_list; the list view must agree + # with the string view, not with raw token order + c = Constants() + c.middle_name_as_last = True + n = HumanName("Hassan, Mohamad Ahmad Ali", constants=c) + assert n.last == "Ahmad Ali Hassan" + assert n.last_list == ["Ahmad", "Ali", "Hassan"] From 05329f88b4cb720013cc5b0ff89b88cf4fee71cd Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 19:17:57 -0700 Subject: [PATCH 140/275] Fix comment rot and citation precision from the toolkit review - _group.py's docstring no longer claims Policy is not consulted (stale since the M5 delimiter wiring) - the additional_parts_count citation attaches to the clauses it supports (1313/1369 for the =1 sites, 1333 for the 0 site); the 'lastname part may have suffixes' quote cites 1368 - the EAST_SLAVIC pinned-live note carries its date (2026-07-12) - AGENTS.md: the CLI line describes the 2.0 parse() output + --json, and the facade-layer paragraph records that parser.py:NNNN citations resolve via git show 2d5d8c2:nameparser/parser.py Co-Authored-By: Claude Fable 5 --- AGENTS.md | 6 ++++-- nameparser/_config_shim.py | 7 ++++++- nameparser/_pipeline/_assign.py | 2 +- nameparser/_pipeline/_group.py | 6 ++++-- nameparser/_pipeline/_post_rules.py | 2 +- tests/v2/cases.py | 2 +- 6 files changed, 17 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4bd476dd..e5b3b22f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,7 +29,9 @@ uv run mypy # Lint uv run ruff check nameparser/ -# Debug how a specific name string is parsed (prints HumanName repr) +# Debug how a specific name string is parsed: 2.0 core parse() -- +# prints the ParsedName repr raw and capitalized, then initials; +# --json emits the component dict uv run python -m nameparser "Dr. Juan Q. Xavier de la Vega III" # Build docs @@ -112,7 +114,7 @@ Each named attribute (`title`, `first`, etc.) is a `@property` that joins its co The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These conventions apply to all new-API code and are stricter than the v1 sections above. The full design record (rationale, settled-decision logs, dated amendments) lives in untracked `docs/superpowers/specs/`; this section is the enforceable subset. **A commit that establishes or amends one of these conventions must update this section in the same commit** — grep-driven staleness sweeps miss paraphrased prose (see Workflow above), so write-time maintenance is the mechanism, audits are the backstop. - **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`, `_pipeline/`, `_parser.py`, plus the facade layer: `_facade.py`, `_config_shim.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Since the M11 swap, the old paths are import-path-preserving re-exports: `nameparser.parser` re-exports the `_facade` `HumanName`, `nameparser.config` re-exports the `_config_shim` names (`Constants`, `CONSTANTS`, `SetManager`, `TupleManager`, `RegexTupleManager`); the `config/` DATA modules stay the vocabulary source through 2.x. The whole facade layer is deleted in 3.0. -- **Facade layer** (`_facade.py`, `_config_shim.py`): the v1-compat `HumanName`/`Constants` over the core. Key mechanisms: `Constants._generation` dirty-tracking (every mutation bumps; facades resolve their `Parser` lazily via `_cached_parser(lexicon, policy)`); `Constants._snapshot()` mirrors `_lexicon._default_lexicon()` (equality-pinned); the facade pickles v1-SHAPED state (component lists, one `__setstate__` path for 1.4 and 2.x blobs; components rebuild via `replace()`, never a re-parse); `_V1_HOOKS` overrides warn once per subclass (#280). The compat contract is the migration spec's promise: warning-free 1.4 code behaves identically except release-log-classified fixes — `tools/differential/` (dev-only, not shipped) verifies this against 1.4-on-PyPI over a 486-name corpus. +- **Facade layer** (`_facade.py`, `_config_shim.py`): the v1-compat `HumanName`/`Constants` over the core. Key mechanisms: `Constants._generation` dirty-tracking (every mutation bumps; facades resolve their `Parser` lazily via `_cached_parser(lexicon, policy)`); `Constants._snapshot()` mirrors `_lexicon._default_lexicon()` (equality-pinned); the facade pickles v1-SHAPED state (component lists, one `__setstate__` path for 1.4 and 2.x blobs; components rebuild via `replace()`, never a re-parse); `_V1_HOOKS` overrides warn once per subclass (#280). The compat contract is the migration spec's promise: warning-free 1.4 code behaves identically except release-log-classified fixes — `tools/differential/` (dev-only, not shipped) verifies this against 1.4-on-PyPI over a 486-name corpus. `parser.py:NNNN` citations throughout the 2.0 code refer to the PRE-swap v1 file, deleted at the M11 swap; resolve them with `git show 2d5d8c2:nameparser/parser.py`. - **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers; `_parser` sits on everything except `_render`; the facade layer (`_facade`, `_config_shim`, `parser`, `config/__init__`, `__main__`) may import anything public plus `_render`. Extend the test's `ALLOWED` table when adding a module. - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index 812be2af..c56bfbd8 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -845,7 +845,12 @@ def __setattr__(self, name: str, value: object) -> None: if name in _MANAGER_FIELDS or name in _SCALAR_DEFAULTS: self._bump() - def copy(self) -> Constants: # #260 + def copy(self) -> Constants: + """Independent copy (#260), subclass-preserving. Divergence from + v1 (which deepcopied __dict__): attributes OUTSIDE the known + field surface -- e.g. ad-hoc names stashed on the instance -- + are not carried by copy() or pickling; only the enumerated + fields survive.""" # #260 # An independent instance with its own generation counter and # its own manager callbacks -- not a shared-state alias like a # naive attribute-for-attribute copy would produce. v1's copy() diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 21fb4a2e..1a3d95c7 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -173,7 +173,7 @@ def assign(state: ParseState) -> ParseState: # PARTICLE_OR_GIVEN is deliberately not emitted here: after a # comma the family is already fixed, so a leading given-position # particle is not meaningfully ambiguous. - # v1: "lastname part may have suffixes in it" (parser.py:1370) + # v1: "lastname part may have suffixes in it" (parser.py:1368) # -- the first piece is always the family even if suffix-shaped; # any later strict-suffix piece goes to SUFFIX per piece # ('Smith Jr., John' -> family=Smith, suffix=Jr.) diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index c2096d8a..6f84d2aa 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -4,7 +4,8 @@ Produces: pieces + piece_tags per segment (runs of token indices -- tokens are NEVER joined into strings: the anti-#100 invariant); maiden tail tokens get role=MAIDEN; marker tokens land in dropped. -Reads: token tags (from classify); Policy is not consulted. The v1 +Reads: token tags (from classify), and Policy.extra_suffix_delimiters +for the tail-segment handling below -- no other Policy field. The v1 "derived titles/prefixes" registration becomes piece_tags entries -- per-parse state that dissolves with the state (v1 kept per-parse sets for the same reason). Reads Policy.extra_suffix_delimiters: tail @@ -192,7 +193,8 @@ def group(state: ParseState) -> ParseState: all_pieces: list[tuple[tuple[int, ...], ...]] = [] all_ptags: list[tuple[frozenset[str], ...]] = [] # v1 parity: additional_parts_count=1 applies only to FAMILY_COMMA - # parts (parser.py:1333); the SUFFIX_COMMA pre-comma segment gets 0. + # parts (parser.py:1313, 1369); the SUFFIX_COMMA pre-comma segment + # gets 0 (parser.py:1333). additional = 1 if state.structure is Structure.FAMILY_COMMA else 0 # v1 expand_suffix_delimiter parity (#191): tail segments (wholly # consumed as suffixes by assign) drop delimiter-core tokens, the diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index ace67b19..cca58918 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -13,7 +13,7 @@ patronymic ending, and the MIDDLE-position token does NOT (given + patronymic + patronymic-derived surname like Abramovich must not rotate) -> rotate: given<-old MIDDLE, middle<-old FAMILY (the - patronymic), family<-old GIVEN (v1 parity, pinned live). + patronymic), family<-old GIVEN (v1 parity, pinned live 2026-07-12). 3. TURKIC (opt-in): exactly 1 GIVEN + 2 MIDDLE + 1 FAMILY tokens and the FAMILY-position token is a standalone Turkic marker -> given<-first MIDDLE, middle<-(second MIDDLE, marker), family<-old diff --git a/tests/v2/cases.py b/tests/v2/cases.py index db44eb5d..d12b2aa2 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -119,7 +119,7 @@ class Case: Case("family_segment_trailing_suffix", "Smith Jr., John", {"given": "John", "family": "Smith", "suffix": "Jr."}, notes="v1: the family part may have suffixes in it " - "(parser.py:1370); the first piece is always the family " + "(parser.py:1368); the first piece is always the family " "(pinned live 2026-07-17)"), Case("family_segment_multiple_suffixes", "Smith Jr. MD, John", {"given": "John", "family": "Smith", "suffix": "Jr., MD"}), From 43969ad25a2d77c48f7cf4d52b44d434507256c4 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 19:24:27 -0700 Subject: [PATCH 141/275] Document the ma/do ambiguous-acronym data change Derek approved the data change; this closes the documentation loop: customize.rst's SUFFIX_ACRONYMS_AMBIGUOUS entry carries the broadened criterion (given-name OR common-surname collision) and the 2.0 periods-gate rule, and the release-log entry drops the misleading 'pre-regression' framing -- the change preserves 1.4's output for bare surnames under 2.0's keep-recognized-suffixes routing. Co-Authored-By: Claude Fable 5 --- docs/customize.rst | 2 +- docs/release_log.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index 9814f95b..399042af 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -56,7 +56,7 @@ Editable attributes of nameparser.config.CONSTANTS * :py:data:`~nameparser.config.FIRST_NAME_TITLES` - Titles that, when followed by a single name, that name is a first name, e.g. "King David". * :py:data:`~nameparser.config.SUFFIX_ACRONYMS` - Pieces that come at the end of the name that may or may not have periods separating the letters, e.g. "m.d.". * :py:data:`~nameparser.config.SUFFIX_NOT_ACRONYMS` - Pieces that come at the end of the name that never have periods separating the letters, e.g. "Jr.". -* :py:data:`~nameparser.config.SUFFIX_ACRONYMS_AMBIGUOUS` - Acronym suffixes from ``SUFFIX_ACRONYMS`` that also plausibly work as a given-name nickname on their own, e.g. "JD", "Ed". When one of these appears alone in parenthesis or quotes (e.g. ``'JEFFREY (JD) BRICKEN'``), it's kept as a nickname rather than reclassified as a suffix, since that's the more common reading in ambiguous, delimiter-only context (see the "Nickname Handling" section in the usage guide). +* :py:data:`~nameparser.config.SUFFIX_ACRONYMS_AMBIGUOUS` - Acronym suffixes from ``SUFFIX_ACRONYMS`` that also plausibly work as a name on their own -- a given-name nickname (e.g. "JD", "Ed") or a common surname (e.g. "Ma", "Do"). An ambiguous acronym counts as a suffix only when written with periods: ``"M.A."`` is a credential, ``"Jack Ma"`` keeps its family name. When one appears alone in parenthesis or quotes (e.g. ``'JEFFREY (JD) BRICKEN'``), it's kept as a nickname rather than reclassified as a suffix, since that's the more common reading in ambiguous, delimiter-only context (see the "Nickname Handling" section in the usage guide). * :py:data:`~nameparser.config.CONJUNCTIONS` - Connectors like "and" that join the preceding piece to the following piece. * :py:data:`~nameparser.config.PREFIXES` - Connectors like "del" and "bin" that join to the following piece but not the preceding, similar to titles but can appear anywhere in the name. * :py:data:`~nameparser.config.CAPITALIZATION_EXCEPTIONS` - Dictionary of pieces that do not capitalize the first letter, e.g. "Ph.D". diff --git a/docs/release_log.rst b/docs/release_log.rst index a390be0d..63396aad 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -15,7 +15,7 @@ Release Log - Fix ``"Andrews, M.D."``-shaped input: the lone strict-suffix-or-title piece after a comma (e.g. ``"Smith, Dr."``, ``"Andrews, M.D."``) was routed to ``first`` in 1.x; 2.0 routes it to ``suffix``/``title`` instead, since the pre-comma piece is definitionally the family name (``tests/v2/cases.py`` rows ``family_comma_lone_suffix_piece``, ``family_comma_lone_title``; verified live against 1.4.0 -- 1/486 corpus names diff, all others parity) - Fix a lone recognized trailing suffix (e.g. ``"Johnson PhD"``, ``"Mr. Johnson PhD"``) being routed to ``first``/``last`` in 1.x when no comma is present; 2.0 keeps a recognized suffix in ``suffix`` (``tests/v2/cases.py`` rows ``suffix_stays_suffix``, ``suffix_stays_suffix_title``; not present in the differential corpus -- no v1 test string exercises the bare two-token shape, so this is unverified against 1.4 live but pinned by the case table) - Fix maiden-name markers (``née``/``nee``/``born``/``geb.``/``roz.``) being folded into ``middle``/``last`` in 1.x (e.g. ``"Jane Smith née Jones"`` → ``middle="Smith née"``); 2.0 recognizes the marker and routes the following name to the new ``maiden`` field (closes #274; ``tests/v2/cases.py`` row ``maiden_marker``; not present in the differential corpus) - - Data change: ``ma``/``do`` added to ``suffix_acronyms_ambiguous`` so a bare common surname (``"Jack Ma"``) is no longer misread as a suffix acronym, restoring v1's older (pre-regression) parity (``tests/v2/cases.py`` row ``ambiguous_surname_acronyms``). Side effect: parenthesized/quoted ``"(MA)"``/``"(DO)"`` (no periods) no longer escape to ``suffix`` the way 1.x did -- they now fall through to nickname parsing like any other ambiguous-acronym delimited content. Not present in the differential corpus + - Data change: ``ma``/``do`` added to ``suffix_acronyms_ambiguous`` -- ambiguous acronyms count as suffixes only when written with periods, so a bare common surname (``"Jack Ma"``, ``"Anh Do"``) keeps its family name under 2.0's keep-recognized-suffixes routing, matching 1.4's output (``tests/v2/cases.py`` row ``ambiguous_surname_acronyms``). Side effect: parenthesized/quoted ``"(MA)"``/``"(DO)"`` (no periods) no longer escape to ``suffix`` the way 1.x did -- they now fall through to nickname parsing like any other ambiguous-acronym delimited content. Not present in the differential corpus - Change suffix-delimiter rendering: with a custom ``Policy``/``Constants`` suffix delimiter configured (e.g. ``suffix_delimiter="/"``, ``"John Smith, RN/CRNA"``), 1.x split the token and rendered ``suffix="RN, CRNA"``; 2.0 keeps the no-space delimiter-core token whole (``suffix="RN/CRNA"``) -- role assignment is unchanged, only rendering differs (anti-#100, migration plan deviation 5; ``tests/v2/cases.py`` row ``suffix_delimiter_no_space_core``). Only fires with a non-default policy, so it does not appear in the (default-policy) differential corpus Everything else in the 486-name differential corpus (built from the From 1eca44b8650ca25a35b393a969adc583e817b2cc Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 19:34:54 -0700 Subject: [PATCH 142/275] Apply the /simplify findings: snapshot caching, shared collapse, one PH/D - Constants._snapshot() is generation-cached: rebuilding the Lexicon re-normalized ~1400 vocabulary entries (~185us) TWICE per HumanName() construction -- the legacy path ran ~428us/name against the core's ~35us. With the cache (plus _resolve() storing the resolved Parser instead of re-hashing the lru key per call) the facade is ~66us/name, a ~5us wrapper over the core. A facade-path benchmark test now exists so this regression class stays visible. - HumanName.__str__ delegates its cleanup chain to _render._collapse (the #254 collapse has ONE owner; the facade layering row always allowed the import); collapse_whitespace keeps v1's narrower two-step contract over _render's regexes; the facade's three duplicate regex constants are gone. - _MEMBERS derives from Role in the v1 spellings (declaration order is canonical, never restated). - the fix_phd credential-pair regexes live once in _vocab, imported by segment and group (both are pipeline-internal; the keep-in-sync comment pair is gone). - the inert per-token entry_open rewrite in group's tail loop is hoisted out (piece-level state, now visibly so). - a cold-import regression test pins BOTH import orders around the _facade cycle-breaker (the load-bearing line an import-sort pass could otherwise silently reorder). Co-Authored-By: Claude Fable 5 --- nameparser/_config_shim.py | 13 ++++++++++++ nameparser/_facade.py | 34 ++++++++++++++++++-------------- nameparser/_pipeline/_group.py | 9 +++++---- nameparser/_pipeline/_segment.py | 7 +++---- nameparser/_pipeline/_vocab.py | 7 +++++++ tests/v2/test_benchmark.py | 13 ++++++++++++ tests/v2/test_facade.py | 17 ++++++++++++++++ 7 files changed, 77 insertions(+), 23 deletions(-) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index c56bfbd8..3513b027 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -894,6 +894,19 @@ def __repr__(self) -> str: # #221 # -- snapshot ----------------------------------------------------------- def _snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: + # generation-keyed cache: rebuilding the Lexicon re-normalizes + # ~1400 vocabulary entries (~185us) -- the same dirty-tracking + # the facade uses, applied one level up. A pure read either way + # (no bump, no warning). + cached = getattr(self, "_snapshot_cache", None) + if cached is not None and cached[0] == self._generation: + return cached[1] # type: ignore[no-any-return] + snapshot = self._build_snapshot() + object.__setattr__( + self, "_snapshot_cache", (self._generation, snapshot)) + return snapshot + + def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: """Resolve this v1-shaped, mutable Constants into the frozen 2.0 value objects it corresponds to (spec §3). A pure read: no generation bump, no deprecation warning even on the shared diff --git a/nameparser/_facade.py b/nameparser/_facade.py index 47143d55..0ea0083b 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -7,7 +7,6 @@ from __future__ import annotations import dataclasses -import re import warnings from collections.abc import Iterator from typing import Any @@ -28,19 +27,19 @@ # still-executing __init__. import nameparser.config # noqa: F401 +import nameparser._render as _render from nameparser._config_shim import CONSTANTS, Constants, _cached_parser from nameparser._lexicon import _normalize from nameparser._parser import Parser from nameparser._types import FOLDED_TAG, ParsedName, Role _V2_FIELD = {"first": "given", "last": "family"} # v1 name -> v2 name -_MEMBERS = ("title", "first", "middle", "last", "suffix", "nickname", - "maiden") +_V1_SPELLING = {v2: v1 for v1, v2 in _V2_FIELD.items()} +# derived from Role: declaration order IS the canonical field order +# (never restated), rendered in the v1 spellings +_MEMBERS = tuple(_V1_SPELLING.get(r.value, r.value) for r in Role) + -_SPACES = re.compile(r"\s+") -_SPACE_BEFORE_COMMA = re.compile(r"\s+,") -# the three comma characters, same set as the pipeline's COMMA_CHARS -_TRAILING_COMMA = re.compile(r"[,،，]$") #: v1 parsing hooks the facade never calls (spec §2 exception 2 / #280). _V1_HOOKS = ( @@ -230,8 +229,11 @@ def _resolve(self) -> Parser: extra_suffix_delimiters=frozenset( {self.suffix_delimiter})) self._lexicon, self._policy = lexicon, policy + self._parser = _cached_parser(lexicon, policy) self._snapshot_gen = gen - return _cached_parser(self._lexicon, self._policy) + # the fast path is a plain attribute return: hashing the two + # value objects for the lru lookup is the whole fast-path cost + return self._parser def parse_full_name(self) -> None: """Re-parse the stored ``full_name`` (v1's documented re-parse @@ -572,19 +574,21 @@ def comparison_key(self) -> tuple[str, ...]: # -- dunders ------------------------------------------------------------ def collapse_whitespace(self, string: str) -> str: - # v1 parser.py:976 verbatim (regexes.spaces / regexes.commas) - string = _SPACES.sub(" ", string.strip()) - if string and _TRAILING_COMMA.search(string): + # v1 parser.py:976 verbatim, over _render's regexes (the #254 + # collapse owns them; this public method keeps v1's narrower + # two-step contract for initials() and direct callers) + string = _render._SPACES.sub(" ", string.strip()) + if string and _render._COMMA_CHAR.fullmatch(string[-1]): string = string[:-1] return string def __str__(self) -> str: if self.string_format is not None: - _s = self.string_format.format( + rendered = self.string_format.format( **{k: v or "" for k, v in self.as_dict().items()}) - _s = _s.replace(" ()", "").replace(" ''", "").replace(' ""', "") - _s = _SPACE_BEFORE_COMMA.sub(",", _s) - return self.collapse_whitespace(_s).strip(", ") + # the full #254 collapse is _render._collapse -- one owner + # for the cleanup chain the v1 __str__ spelled inline + return _render._collapse(rendered) return " ".join(self) def __repr__(self) -> str: diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 6f84d2aa..b7643b6a 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -20,16 +20,16 @@ from __future__ import annotations import dataclasses -import re from collections.abc import Sequence, Set from enum import IntEnum from nameparser._pipeline._state import ParseState, Structure, WorkToken +from nameparser._pipeline._vocab import D as _D +from nameparser._pipeline._vocab import PH as _PH from nameparser._pipeline._vocab import delimiter_cores from nameparser._types import Role -_PH = re.compile(r"^ph\.?$", re.IGNORECASE) -_D = re.compile(r"^d\.?$", re.IGNORECASE) +# the credential-pair regexes live in _vocab (shared with segment) Piece = list[int] @@ -234,7 +234,8 @@ def group(state: ParseState) -> ParseState: if entry_open or pos > 0: tokens[i] = dataclasses.replace( tokens[i], tags=tokens[i].tags | {"joined"}) - entry_open = True + # piece-level state: the NEXT piece continues this entry + entry_open = True if len(kept) != len(pieces): pieces = [pieces[k] for k in kept] ptags = [ptags[k] for k in kept] diff --git a/nameparser/_pipeline/_segment.py b/nameparser/_pipeline/_segment.py index df7ab767..c9762d16 100644 --- a/nameparser/_pipeline/_segment.py +++ b/nameparser/_pipeline/_segment.py @@ -21,19 +21,18 @@ import bisect import dataclasses -import re from nameparser._pipeline._state import ParseState, PendingAmbiguity, Structure from nameparser._pipeline._vocab import ( + D as _D, + PH as _PH, delimiter_cores, is_suffix_lenient, is_suffix_strict, period_joined_vocab, splits_into_suffixes, ) from nameparser._types import AmbiguityKind -# keep in sync with _group.py's merge pair (the fix_phd port) -_PH = re.compile(r"^ph\.?$", re.IGNORECASE) -_D = re.compile(r"^d\.?$", re.IGNORECASE) + def segment(state: ParseState) -> ParseState: diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index b2243374..be80ac66 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -117,3 +117,10 @@ def period_joined_vocab(text: str, lexicon: Lexicon) -> str | None: for c in chunks): return "suffix" return None + + +# The fix_phd credential pair ('Ph.' + 'D.' as adjacent tokens), shared +# by segment's suffix-comma detection and group's merge (v1 extracted +# the credential pre-parse; the two stages must agree on the pattern). +PH = re.compile(r"^ph\.?$", re.IGNORECASE) +D = re.compile(r"^d\.?$", re.IGNORECASE) diff --git a/tests/v2/test_benchmark.py b/tests/v2/test_benchmark.py index f3716991..c3f77509 100644 --- a/tests/v2/test_benchmark.py +++ b/tests/v2/test_benchmark.py @@ -13,3 +13,16 @@ def test_parse_thousand_names_under_a_second() -> None: parse(f"Dr. Juan{i} de la Vega III") elapsed = time.perf_counter() - start assert elapsed < 1.0, f"1000 parses took {elapsed:.2f}s" + + +def test_facade_thousand_names_under_a_second() -> None: + # the legacy-API path (what all existing users call): snapshot + # resolution must stay generation-cached, not rebuilt per instance + from nameparser import HumanName + + HumanName("warm up the caches") + start = time.perf_counter() + for i in range(1000): + HumanName(f"Dr. Juan{i} de la Vega III") + elapsed = time.perf_counter() - start + assert elapsed < 1.0, f"1000 facade parses took {elapsed:.2f}s" diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index 31e3468b..0873d529 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -431,3 +431,20 @@ def test_middle_as_family_list_view_matches_string_view() -> None: n = HumanName("Hassan, Mohamad Ahmad Ali", constants=c) assert n.last == "Ahmad Ali Hassan" assert n.last_list == ["Ahmad", "Ali", "Hassan"] + + +def test_cold_import_order_config_first() -> None: + # the _facade cycle-breaker (import nameparser.config first) is + # order-dependent; this pins BOTH cold-start directions in fresh + # interpreters so an import-sorting pass cannot silently break one + import subprocess + import sys + + for first in ("nameparser.config", "nameparser._facade"): + proc = subprocess.run( + [sys.executable, "-c", + f"import {first}; import nameparser; " + f"print(nameparser.HumanName('John Smith').last)"], + capture_output=True, text=True) + assert proc.returncode == 0, (first, proc.stderr) + assert proc.stdout.strip() == "Smith" From 9673f005cc63f75e039d8a8edf7d82f9806e0b67 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 19:45:26 -0700 Subject: [PATCH 143/275] Lift _vocab's regex constants to the module top Successive parity fixes appended _PERIOD_NOT_AT_END mid-file and the PH/D credential pair after the last function; every pipeline module keeps its module-level regexes in one block after the imports, as this file's own _INITIAL already did. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_vocab.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index be80ac66..699da052 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -17,6 +17,17 @@ # sync by hand; layering forbids importing the config package here. _INITIAL = re.compile(r"^(\w\.|[A-Z])$") +# Ported verbatim from v1 (nameparser/config/regexes.py +# "period_not_at_end") -- layering forbids the config import; keep in +# sync by hand. +_PERIOD_NOT_AT_END = re.compile(r".*\..+$", re.I) + +# The fix_phd credential pair ('Ph.' + 'D.' as adjacent tokens), shared +# by segment's suffix-comma detection and group's merge (v1 extracted +# the credential pre-parse; the two stages must agree on the pattern). +PH = re.compile(r"^ph\.?$", re.IGNORECASE) +D = re.compile(r"^d\.?$", re.IGNORECASE) + def is_initial(text: str) -> bool: """'A.' / 'j.' / bare capital -- v1's is_an_initial.""" @@ -94,11 +105,6 @@ def splits_into_suffixes(text: str, cores: frozenset[str], return False -# Ported verbatim from v1 (nameparser/config/regexes.py -# "period_not_at_end") -- layering forbids the config import; keep in -# sync by hand. -_PERIOD_NOT_AT_END = re.compile(r".*\..+$", re.I) - def period_joined_vocab(text: str, lexicon: Lexicon) -> str | None: """v1's parse_pieces derivation for interior-period tokens @@ -118,9 +124,3 @@ def period_joined_vocab(text: str, lexicon: Lexicon) -> str | None: return "suffix" return None - -# The fix_phd credential pair ('Ph.' + 'D.' as adjacent tokens), shared -# by segment's suffix-comma detection and group's merge (v1 extracted -# the credential pre-parse; the two stages must agree on the pattern). -PH = re.compile(r"^ph\.?$", re.IGNORECASE) -D = re.compile(r"^d\.?$", re.IGNORECASE) From 2776fa28ac054fad4468ffa20bda3808e57d1db3 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 19:47:56 -0700 Subject: [PATCH 144/275] Record the facade layer's concern-grouped class organization A conventions sweep (after the _vocab constants fix) found HumanName and the shim Constants organize by v1 concern groups rather than the canonical section order the core types follow. Restructuring ~700 lines of 3.0-doomed compat code would trade its v1-mirroring readability for rule conformance; recorded as a sanctioned facade-layer deviation instead, same as the mutable-state exceptions. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index e5b3b22f..83775acb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,7 +117,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Facade layer** (`_facade.py`, `_config_shim.py`): the v1-compat `HumanName`/`Constants` over the core. Key mechanisms: `Constants._generation` dirty-tracking (every mutation bumps; facades resolve their `Parser` lazily via `_cached_parser(lexicon, policy)`); `Constants._snapshot()` mirrors `_lexicon._default_lexicon()` (equality-pinned); the facade pickles v1-SHAPED state (component lists, one `__setstate__` path for 1.4 and 2.x blobs; components rebuild via `replace()`, never a re-parse); `_V1_HOOKS` overrides warn once per subclass (#280). The compat contract is the migration spec's promise: warning-free 1.4 code behaves identically except release-log-classified fixes — `tools/differential/` (dev-only, not shipped) verifies this against 1.4-on-PyPI over a 486-name corpus. `parser.py:NNNN` citations throughout the 2.0 code refer to the PRE-swap v1 file, deleted at the M11 swap; resolve them with `git show 2d5d8c2:nameparser/parser.py`. - **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers; `_parser` sits on everything except `_render`; the facade layer (`_facade`, `_config_shim`, `parser`, `config/__init__`, `__main__`) may import anything public plus `_render`. Extend the test's `ALLOWED` table when adding a module. - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. -- **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. +- **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. Sanctioned deviation, facade layer only: `HumanName` and the shim `Constants` organize by v1 concern groups (`# -- render defaults --`, `# -- config / parsing --`, `# -- fields --`, ..., dunders and pickle last) — the classes mirror v1's own surface and die in 3.0; the canonical order still binds every core type. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). - **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. From 6ee468193e0d0a98378a85855874ce542774bbd0 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 19:52:14 -0700 Subject: [PATCH 145/275] Anchor the emoji-range CodeQL suppression on the flagged line py/overly-large-range anchors at the first astral character-range line of the multi-line pattern, not the re.compile line the lgtm[...] comment sat on -- so the suppression never applied. Moved one line down in both keep-in-sync copies (config/regexes.py and _pipeline/_tokenize.py). Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_tokenize.py | 4 ++-- nameparser/config/regexes.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nameparser/_pipeline/_tokenize.py b/nameparser/_pipeline/_tokenize.py index ab0390dd..9c61bbbb 100644 --- a/nameparser/_pipeline/_tokenize.py +++ b/nameparser/_pipeline/_tokenize.py @@ -30,8 +30,8 @@ # "bidi") -- layering forbids importing the config package here, so the # patterns are duplicated by design with this provenance note. When # editing, keep both copies in sync. -_EMOJI = re.compile('[' # lgtm[py/overly-large-range] - '\U0001F300-\U0001F64F' +_EMOJI = re.compile('[' + '\U0001F300-\U0001F64F' # lgtm[py/overly-large-range] '\U0001F680-\U0001F6FF' '\u2600-\u26FF\u2700-\u27BF]+') _BIDI = re.compile('[\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+') diff --git a/nameparser/config/regexes.py b/nameparser/config/regexes.py index 792e826e..a8116cf6 100644 --- a/nameparser/config/regexes.py +++ b/nameparser/config/regexes.py @@ -1,8 +1,8 @@ import re # emoji regex from https://stackoverflow.com/questions/26568722/remove-unicode-emoji-using-re-in-python -re_emoji = re.compile('[' # lgtm[py/overly-large-range] - '\U0001F300-\U0001F64F' +re_emoji = re.compile('[' + '\U0001F300-\U0001F64F' # lgtm[py/overly-large-range] '\U0001F680-\U0001F6FF' '\u2600-\u26FF\u2700-\u27BF]+') From 697dbe3ec66b5a7cf41f480b21efcd556b71d5c4 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 19:56:01 -0700 Subject: [PATCH 146/275] Eliminate the py/overly-large-range false positive at the source The lgtm[...] suppression comment is not honored by the repo's CodeQL setup -- moving it only changed alert fingerprints and resurfaced the alerts. Root fix instead: py/overly-large-range fires on LITERAL astral character-class ranges (it decomposes them into overlapping surrogate pairs), so neither file has one anymore. _tokenize's per-char test becomes plain integer codepoint ranges (no regex at all, cheaper); regexes.py keeps its public compiled re_emoji surface but builds the class from the same codepoint pairs -- verified byte-identical to the previous pattern. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_tokenize.py | 22 +++++++++++++--------- nameparser/config/regexes.py | 13 ++++++++----- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/nameparser/_pipeline/_tokenize.py b/nameparser/_pipeline/_tokenize.py index 9c61bbbb..85fcbcfa 100644 --- a/nameparser/_pipeline/_tokenize.py +++ b/nameparser/_pipeline/_tokenize.py @@ -26,14 +26,15 @@ ) from nameparser._types import Role, Span -# Ported verbatim from v1 (nameparser/config/regexes.py, "emoji" and -# "bidi") -- layering forbids importing the config package here, so the -# patterns are duplicated by design with this provenance note. When -# editing, keep both copies in sync. -_EMOJI = re.compile('[' - '\U0001F300-\U0001F64F' # lgtm[py/overly-large-range] - '\U0001F680-\U0001F6FF' - '\u2600-\u26FF\u2700-\u27BF]+') +# Ported from v1 (nameparser/config/regexes.py, "emoji" and "bidi") -- +# layering forbids importing the config package here, so the tables are +# duplicated by design with this provenance note. When editing, keep +# both copies in sync (regexes.py builds its public re_emoji from the +# SAME codepoint pairs). Integer ranges, not a regex character class: +# the per-char test needs no regex, and CodeQL's py/overly-large-range +# false-positives on literal astral ranges (surrogate decomposition). +_EMOJI_RANGES = ((0x1F300, 0x1F64F), (0x1F680, 0x1F6FF), + (0x2600, 0x26FF), (0x2700, 0x27BF)) _BIDI = re.compile('[\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+') @@ -46,7 +47,10 @@ def _ignorable(ch: str, state: ParseState) -> bool: return False if state.policy.strip_bidi and _BIDI.match(ch): return True - return bool(state.policy.strip_emoji and _EMOJI.match(ch)) + if state.policy.strip_emoji: + cp = ord(ch) + return any(lo <= cp <= hi for lo, hi in _EMOJI_RANGES) + return False def _tokenize_region(state: ParseState, start: int, end: int, diff --git a/nameparser/config/regexes.py b/nameparser/config/regexes.py index a8116cf6..17716df3 100644 --- a/nameparser/config/regexes.py +++ b/nameparser/config/regexes.py @@ -1,10 +1,13 @@ import re -# emoji regex from https://stackoverflow.com/questions/26568722/remove-unicode-emoji-using-re-in-python -re_emoji = re.compile('[' - '\U0001F300-\U0001F64F' # lgtm[py/overly-large-range] - '\U0001F680-\U0001F6FF' - '\u2600-\u26FF\u2700-\u27BF]+') +# emoji ranges from https://stackoverflow.com/questions/26568722/remove-unicode-emoji-using-re-in-python +# Built from codepoint pairs rather than a literal character class: +# the compiled pattern is identical, but CodeQL's py/overly-large-range +# false-positives on literal astral ranges (surrogate decomposition). +_EMOJI_RANGES = ((0x1F300, 0x1F64F), (0x1F680, 0x1F6FF), + (0x2600, 0x26FF), (0x2700, 0x27BF)) +re_emoji = re.compile( + '[' + ''.join(f'{chr(lo)}-{chr(hi)}' for lo, hi in _EMOJI_RANGES) + ']+') # Invisible bidirectional formatting characters: ALM, LRM, RLM, the # embedding/override marks (LRE/RLE/PDF/LRO/RLO) and the isolates From ac43dc9fcd1eb30db146ab27afeaa141b57f8251 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 20:23:36 -0700 Subject: [PATCH 147/275] Add the locales package with lazy loading and the RU pack Co-Authored-By: Claude Fable 5 --- nameparser/locales/__init__.py | 57 ++++++++++++++++++++++++++++++++++ nameparser/locales/ru.py | 50 +++++++++++++++++++++++++++++ tests/v2/test_layering.py | 12 ++++++- tests/v2/test_locales.py | 43 +++++++++++++++++++++++++ 4 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 nameparser/locales/__init__.py create mode 100644 nameparser/locales/ru.py create mode 100644 tests/v2/test_locales.py diff --git a/nameparser/locales/__init__.py b/nameparser/locales/__init__.py new file mode 100644 index 00000000..3c39dd62 --- /dev/null +++ b/nameparser/locales/__init__.py @@ -0,0 +1,57 @@ +"""Locale packs: named (lexicon fragment, PolicyPatch) deltas folded in +by parser_for (locales spec §2). Packs are pure data with no +privileged capabilities; they dissolve at parser construction. + +Loaded lazily (PEP 562): importing nameparser.locales never imports a +pack module; ``locales.RU`` triggers ``nameparser.locales.ru`` on +first access and caches the Locale here. + +Layering: this package imports pack modules, which import _locale/ +_lexicon/_policy only (enforced by tests/v2/test_layering.py). +""" +from __future__ import annotations + +import importlib + +from nameparser._locale import Locale + +#: attribute name -> (module name, module attribute). Codes are the +#: lowercase module names; attribute constants are uppercase (spec §2). +_REGISTRY = { + "RU": ("nameparser.locales.ru", "RU"), + "TR_AZ": ("nameparser.locales.tr_az", "TR_AZ"), +} + + +def __getattr__(name: str) -> Locale: + entry = _REGISTRY.get(name) + if entry is None: + raise AttributeError( + f"module {__name__!r} has no locale {name!r}; available: " + f"{', '.join(sorted(_REGISTRY))}" + ) + module_name, attr = entry + locale: Locale = getattr(importlib.import_module(module_name), attr) + globals()[name] = locale # cache: next access skips __getattr__ + return locale + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(_REGISTRY)) + + +def available() -> tuple[str, ...]: + """The registered locale codes, lowercase, sorted.""" + return tuple(sorted(attr.lower() for attr in _REGISTRY)) + + +def get(code: str) -> Locale: + """Dynamic lookup by code ('ru'); raises KeyError listing the + available codes (spec §2).""" + attr = code.upper() if isinstance(code, str) else code + if not isinstance(code, str) or attr not in _REGISTRY: + raise KeyError( + f"unknown locale code {code!r}; available: " + f"{', '.join(available())}" + ) + return __getattr__(attr) diff --git a/nameparser/locales/ru.py b/nameparser/locales/ru.py new file mode 100644 index 00000000..7283ce7b --- /dev/null +++ b/nameparser/locales/ru.py @@ -0,0 +1,50 @@ +"""The Russian locale pack (locales spec §3): policy-only -- it turns +on the EAST_SLAVIC patronymic rule. The morphology data (-ovich/-ovna +endings, Cyrillic and transliterated) lives inside the rule +implementation in nameparser/_pipeline/_post_rules.py, not in the +Lexicon (mirrors v1's patronymic_name_order flag design, v1.3.0). +Cyrillic titles/conjunctions are default-lexicon vocabulary (#269), +not pack data (spec §2 sorting rule). + +Data sources: the v1.3.0 patronymic rule (PR #154 discussion and the +east-slavic test bank); no external lists -- the pack itself carries +no vocabulary. + +Declared deviations (spec §2 authoring requirement 3): applying this +pack changes only NO_COMMA names whose final token carries an East +Slavic patronymic ending while the middle token does not -- +DEVIATES(name) below is the machine-readable declaration the +non-interference gate checks. +""" +from __future__ import annotations + +import re + +from nameparser._lexicon import Lexicon +from nameparser._locale import Locale +from nameparser._policy import PatronymicRule, PolicyPatch + +RU = Locale( + code="ru", + lexicon=Lexicon.empty(), + policy=PolicyPatch( + patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})), +) + +# Ported by hand from nameparser/_pipeline/_post_rules.py's +# _EAST_SLAVIC/_EAST_SLAVIC_CYR (the rule is the source of truth; this +# predicate only DECLARES the deviation surface for the non-interference +# gate -- layering forbids importing the pipeline from a pack). Keep +# both alternations byte-identical to those two patterns. +_EAST_SLAVIC = re.compile( + r"(ovich|ovna|evich|evna|ichna|ilyich|kuzmich|lukich|fomich|fokich)$", + re.I) +_EAST_SLAVIC_CYR = re.compile( + r"(ович|овна|евич|евна|ична|ильич|кузьмич|лукич|фомич|фокич)$", re.I) + + +def DEVIATES(name: str) -> bool: + """True when this pack may parse `name` differently from the + default parser (the declared-deviation predicate the + non-interference gate consumes).""" + return bool(_EAST_SLAVIC.search(name) or _EAST_SLAVIC_CYR.search(name)) diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 5896fbc3..ed92c7f8 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -18,7 +18,7 @@ "_pipeline/_classify.py", "_pipeline/_group.py", "_pipeline/_assign.py", "_pipeline/_post_rules.py", "_pipeline/_assemble.py", "_parser.py", "_facade.py", - "_config_shim.py"} + "_config_shim.py", "locales/__init__.py", "locales/ru.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", @@ -75,6 +75,16 @@ "_config_shim.py": ("nameparser._lexicon", "nameparser._parser", "nameparser._policy", "nameparser.util", "nameparser.config"), + # PEP 562 lazy loader: the only static import is _locale (for the + # Locale return type); pack modules load via importlib.import_module + # (a runtime string, invisible to the AST walker) -- the + # "nameparser.locales." prefix documents that reach for humans even + # though the walker never exercises it. + "locales/__init__.py": ("nameparser._locale", "nameparser.locales."), + # a locale pack: pure data over the three base types, no pipeline + # or config access (locales spec §2) + "locales/ru.py": ("nameparser._lexicon", "nameparser._locale", + "nameparser._policy"), # v1 import-path preservation: thin re-exports of the facade/shim "parser.py": ("nameparser._facade",), "config/__init__.py": ("nameparser._config_shim",), diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py new file mode 100644 index 00000000..06cfa749 --- /dev/null +++ b/tests/v2/test_locales.py @@ -0,0 +1,43 @@ +"""The locale pack layer (locales spec §2-3): lazy access, the two +2.0.0 packs, composition, and the non-interference gate.""" +import pytest + +from nameparser import Locale + + +def test_locales_module_attribute_access() -> None: + from nameparser import locales + assert isinstance(locales.RU, Locale) + assert locales.RU.code == "ru" + assert locales.RU is locales.RU # cached, not rebuilt + + +def test_locales_get_and_available() -> None: + from nameparser import locales + assert locales.get("ru") is locales.RU + assert set(locales.available()) == {"ru", "tr_az"} + with pytest.raises(KeyError, match="ru, tr_az"): + locales.get("xx") + + +def test_locales_unknown_attribute() -> None: + from nameparser import locales + with pytest.raises(AttributeError, match="XX"): + locales.XX + + +def test_locales_import_is_lazy() -> None: + # importing the package must not import any pack module; PEP 562 + # loads them on first attribute access (spec §2: "importing + # nameparser never pays for pack data") + import importlib + import sys + + for mod in list(sys.modules): + if mod.startswith("nameparser.locales"): + del sys.modules[mod] + import nameparser.locales + assert "nameparser.locales.ru" not in sys.modules + nameparser.locales.RU + assert "nameparser.locales.ru" in sys.modules + importlib.reload(nameparser.locales) From e68db5824bfc6bde1e146ac323e1bd72d491d156 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 20:34:48 -0700 Subject: [PATCH 148/275] Add the TR_AZ pack Co-Authored-By: Claude Fable 5 --- nameparser/locales/__init__.py | 7 ++++- nameparser/locales/tr_az.py | 48 ++++++++++++++++++++++++++++++++++ tests/v2/test_layering.py | 7 ++++- tests/v2/test_locales.py | 30 +++++++++++++++++++++ 4 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 nameparser/locales/tr_az.py diff --git a/nameparser/locales/__init__.py b/nameparser/locales/__init__.py index 3c39dd62..7b9a99d5 100644 --- a/nameparser/locales/__init__.py +++ b/nameparser/locales/__init__.py @@ -32,7 +32,12 @@ def __getattr__(name: str) -> Locale: ) module_name, attr = entry locale: Locale = getattr(importlib.import_module(module_name), attr) - globals()[name] = locale # cache: next access skips __getattr__ + # cache: next access skips __getattr__. Benign race under free + # threading: two threads racing here both import the same module + # object (import machinery serializes/dedupes that) and assign the + # same singleton Locale back to the same name, so the last write + # wins with no observable difference. + globals()[name] = locale return locale diff --git a/nameparser/locales/tr_az.py b/nameparser/locales/tr_az.py new file mode 100644 index 00000000..cfed583f --- /dev/null +++ b/nameparser/locales/tr_az.py @@ -0,0 +1,48 @@ +"""The Turkish/Azerbaijani locale pack (locales spec §3): policy-only +-- it turns on the TURKIC patronymic rule. The marker data (oglu/qizi/ +uulu/kyzy and Cyrillic forms) lives inside the rule implementation in +nameparser/_pipeline/_post_rules.py (mirrors v1's flag design). + +Data sources: the v1 Turkic patronymic rule (its plan and test bank); +the pack carries no vocabulary. + +Declared deviations (spec §2 authoring requirement 3): applying this +pack changes only NO_COMMA names where some token is a standalone +Turkic patronymic marker -- DEVIATES(name) below is the +machine-readable declaration the non-interference gate checks. +""" +from __future__ import annotations + +import re + +from nameparser._lexicon import Lexicon +from nameparser._locale import Locale +from nameparser._policy import PatronymicRule, PolicyPatch + +TR_AZ = Locale( + code="tr_az", + lexicon=Lexicon.empty(), + policy=PolicyPatch( + patronymic_rules=frozenset({PatronymicRule.TURKIC})), +) + +# Ported by hand from nameparser/_pipeline/_post_rules.py's +# _TURKIC/_TURKIC_CYR (the rule is the source of truth; this predicate +# only DECLARES the deviation surface for the non-interference gate -- +# layering forbids importing the pipeline from a pack). Keep both +# alternations byte-identical to those two patterns. Note both are +# fullmatch-shaped (^...$): the rule tests a single WHOLE token, so +# DEVIATES below scans per-token rather than searching the whole name. +_TURKIC = re.compile( + r"^(oglu|oğlu|ogly|ogli|o['’ʻ]g['’ʻ]li" + r"|qizi|qızı|kizi|kyzy|gyzy|uly|uulu)$", re.I) +_TURKIC_CYR = re.compile( + r"^(оглу|оглы|оғлу|ўғли|угли|кызы|гызы|қызы|қизи|улы|ұлы|уулу)$", re.I) + + +def DEVIATES(name: str) -> bool: + """True when this pack may parse `name` differently from the + default parser (the declared-deviation predicate the + non-interference gate consumes).""" + return any(_TURKIC.match(tok) or _TURKIC_CYR.match(tok) + for tok in name.split()) diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index ed92c7f8..6a5d15fc 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -18,7 +18,8 @@ "_pipeline/_classify.py", "_pipeline/_group.py", "_pipeline/_assign.py", "_pipeline/_post_rules.py", "_pipeline/_assemble.py", "_parser.py", "_facade.py", - "_config_shim.py", "locales/__init__.py", "locales/ru.py"} + "_config_shim.py", "locales/__init__.py", "locales/ru.py", + "locales/tr_az.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", @@ -85,6 +86,10 @@ # or config access (locales spec §2) "locales/ru.py": ("nameparser._lexicon", "nameparser._locale", "nameparser._policy"), + # a locale pack: pure data over the three base types, no pipeline + # or config access (locales spec §2) + "locales/tr_az.py": ("nameparser._lexicon", "nameparser._locale", + "nameparser._policy"), # v1 import-path preservation: thin re-exports of the facade/shim "parser.py": ("nameparser._facade",), "config/__init__.py": ("nameparser._config_shim",), diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 06cfa749..4c076439 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -3,6 +3,8 @@ import pytest from nameparser import Locale +from nameparser._lexicon import Lexicon +from nameparser._policy import PatronymicRule def test_locales_module_attribute_access() -> None: @@ -20,6 +22,34 @@ def test_locales_get_and_available() -> None: locales.get("xx") +def test_tr_az_pack_contents() -> None: + from nameparser import locales + + assert locales.TR_AZ.code == "tr_az" + assert locales.TR_AZ.policy.patronymic_rules == frozenset( + {PatronymicRule.TURKIC}) + assert locales.TR_AZ.lexicon == Lexicon.empty() + + +def test_pack_marker_regexes_stay_in_sync_with_post_rules() -> None: + # ru.py/tr_az.py hand-copy their DEVIATES marker regexes from + # _pipeline._post_rules (layering forbids a pack importing the + # pipeline -- see each pack's module docstring). Assert pattern + # equality mechanically so drift fails here, not at the L6 + # non-interference gate. + from nameparser._pipeline import _post_rules + from nameparser.locales import ru, tr_az + + for pack_regex, rule_regex in ( + (ru._EAST_SLAVIC, _post_rules._EAST_SLAVIC), + (ru._EAST_SLAVIC_CYR, _post_rules._EAST_SLAVIC_CYR), + (tr_az._TURKIC, _post_rules._TURKIC), + (tr_az._TURKIC_CYR, _post_rules._TURKIC_CYR), + ): + assert pack_regex.pattern == rule_regex.pattern + assert pack_regex.flags == rule_regex.flags + + def test_locales_unknown_attribute() -> None: from nameparser import locales with pytest.raises(AttributeError, match="XX"): From be3f25e84c48032edee9def2bc9b198d782b9c6a Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 20:40:14 -0700 Subject: [PATCH 149/275] Scan ru.DEVIATES per token (whole-string search under-declared) Co-Authored-By: Claude Fable 5 --- nameparser/locales/ru.py | 11 ++++++++++- tests/v2/test_locales.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/nameparser/locales/ru.py b/nameparser/locales/ru.py index 7283ce7b..cea49b99 100644 --- a/nameparser/locales/ru.py +++ b/nameparser/locales/ru.py @@ -47,4 +47,13 @@ def DEVIATES(name: str) -> bool: """True when this pack may parse `name` differently from the default parser (the declared-deviation predicate the non-interference gate consumes).""" - return bool(_EAST_SLAVIC.search(name) or _EAST_SLAVIC_CYR.search(name)) + # Scan per token: the rule fires on the family-POSITION token, not + # the name's final characters, so a whole-string search would miss + # 'Ivan Petr Sidorovich Jr.' (a suffix after the ending -> the $ + # anchor never lands: under-declaration, the unsafe direction for + # the gate). Per-token scanning instead OVER-declares (e.g. + # 'Sidorovich Anna' matches though the rule's 1+1+1 shape never + # fires) -- the safe direction: DEVIATES may claim more than the + # rule changes, never less. + return any(_EAST_SLAVIC.search(tok) or _EAST_SLAVIC_CYR.search(tok) + for tok in name.split()) diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 4c076439..494e118a 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -50,6 +50,22 @@ def test_pack_marker_regexes_stay_in_sync_with_post_rules() -> None: assert pack_regex.flags == rule_regex.flags +def test_ru_deviates_scans_per_token() -> None: + # Regression: the rule rotates on the family-POSITION token, so a + # whole-string search missed a patronymic ending followed by a + # suffix -- under-declaration, the unsafe direction for the + # non-interference gate. The pack rotates this name; DEVIATES must + # say so. + from nameparser.locales import ru + + assert ru.DEVIATES("Ivan Petr Sidorovich Jr.") + # Over-declaration is the accepted safe direction: the ending + # matches a token although the rule's 1 given + 1 middle + + # 1 family shape never fires on a two-token name. + assert ru.DEVIATES("Sidorovich Anna") + assert not ru.DEVIATES("John Smith") + + def test_locales_unknown_attribute() -> None: from nameparser import locales with pytest.raises(AttributeError, match="XX"): From 6fc7031683331d39389fea398b0bc3457a373909 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 20:43:39 -0700 Subject: [PATCH 150/275] Add pack case-table segments through parser_for Co-Authored-By: Claude Fable 5 --- tests/v2/cases.py | 27 +++++++++++++++++++++++++++ tests/v2/pipeline/test_state.py | 8 ++++++++ tests/v2/test_cases.py | 10 ++++++++-- tests/v2/test_facade_cases.py | 6 ++++++ 4 files changed, 49 insertions(+), 2 deletions(-) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index d12b2aa2..2aca7829 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -26,10 +26,18 @@ class Case: text: str expect: dict[str, str] # field -> value; absent fields == "" policy: Policy | None = None + locale: str | None = None # locale CODE (keeps this table + # import-light); mutually exclusive + # with policy classification: str = "parity" ambiguities: tuple[str, ...] = () # expected AmbiguityKind values notes: str = "" + def __post_init__(self) -> None: + if self.policy is not None and self.locale is not None: + raise ValueError( + f"{self.id}: policy and locale are mutually exclusive") + _ES = Policy(patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})) _TK = Policy(patronymic_rules=frozenset({PatronymicRule.TURKIC})) @@ -287,6 +295,25 @@ class Case: Case("turkic", "Mammadova Aygun Ali kizi", {"given": "Aygun", "middle": "Ali kizi", "family": "Mammadova"}, policy=_TK), + Case("ru_pack_formal_rotation", "Сидоров Иван Петрович", + {"given": "Иван", "middle": "Петрович", "family": "Сидоров"}, + locale="ru", + notes="the RU pack end-to-end: same expectation as the " + "east_slavic synthetic row, through parser_for"), + Case("ru_pack_transliterated", "Petrov Ivan Sergeyevich", + {"given": "Ivan", "middle": "Sergeyevich", "family": "Petrov"}, + locale="ru"), + Case("ru_pack_comma_untouched", "Петров, Иван", + {"given": "Иван", "family": "Петров"}, + locale="ru", + notes="a comma is an explicit signal that suppresses the " + "rotation (spec §1)"), + Case("tr_az_pack_marker", "Mammadova Aygun Ali kizi", + {"given": "Aygun", "middle": "Ali kizi", "family": "Mammadova"}, + locale="tr_az", + notes="the TR_AZ pack end-to-end: same expectation as the " + "turkic synthetic row (pinned live during Plan 3), " + "through parser_for"), Case("empty", "", {}), Case("whitespace", " ", {}), Case("bare_ambiguous_acronym", "John Ed", diff --git a/tests/v2/pipeline/test_state.py b/tests/v2/pipeline/test_state.py index eed8d005..4b0c0423 100644 --- a/tests/v2/pipeline/test_state.py +++ b/tests/v2/pipeline/test_state.py @@ -68,6 +68,14 @@ def test_stage_field_ownership() -> None: "post_rules": {"role", "tags"}, } for case in CASES: + if case.locale is not None: + # locale rows dissolve into a Policy only through parser_for + # (nameparser._parser); this test exercises the raw stage + # pipeline directly, and the same inputs already run here + # under the equivalent synthetic Policy row (_ES/_TK) -- + # covering them again through a resolved locale policy would + # be redundant, not additive. + continue state = ParseState(original=case.text, lexicon=_Lexicon.default(), policy=case.policy or Policy()) for stage in STAGES: diff --git a/tests/v2/test_cases.py b/tests/v2/test_cases.py index bb14e671..3495a26f 100644 --- a/tests/v2/test_cases.py +++ b/tests/v2/test_cases.py @@ -2,16 +2,22 @@ runner (migration plan) consumes the same CASES.""" import pytest -from nameparser import Parser, Role +from nameparser import Parser, Role, locales, parser_for from .cases import CASES, Case _FIELDS = tuple(r.value for r in Role) # declaration order is canonical +def _parser_for_case(case: Case) -> Parser: + if case.locale is not None: + return parser_for(locales.get(case.locale)) + return Parser(policy=case.policy) if case.policy else Parser() + + @pytest.mark.parametrize("case", CASES, ids=lambda c: c.id) def test_case(case: Case) -> None: - parser = Parser(policy=case.policy) if case.policy else Parser() + parser = _parser_for_case(case) pn = parser.parse(case.text) actual = {f: getattr(pn, f) for f in _FIELDS if getattr(pn, f)} assert actual == case.expect, f"{case.text!r} ({case.classification})" diff --git a/tests/v2/test_facade_cases.py b/tests/v2/test_facade_cases.py index 58457755..90932553 100644 --- a/tests/v2/test_facade_cases.py +++ b/tests/v2/test_facade_cases.py @@ -56,6 +56,12 @@ def _constants_for(case: Case) -> Constants | None: @pytest.mark.parametrize("case", CASES, ids=lambda c: c.id) def test_facade_case(case: Case) -> None: + if case.locale is not None: + # v1 Constants' patronymic bool enables BOTH the East Slavic and + # Turkic rules at once, so it cannot express a single-rule pack + # faithfully -- these rows are core-only (proven via parser_for + # in test_cases.py instead). + pytest.skip("locale rows are core-only") constants = _constants_for(case) if constants is None: pytest.skip("policy not expressible through v1 Constants") From 822170adbe217dc0fb9e6b7a15467b6d0f513806 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 20:49:12 -0700 Subject: [PATCH 151/275] Prove the RU+TR_AZ composition example with the shipped packs Adds two composition tests over the real RU and TR_AZ packs (the synthetic composition tests in test_parser.py stay as-is): the union of both packs' patronymic rules parses one name from each pack's case segment correctly, and a custom-lexicon base survives the parser_for fold. Co-Authored-By: Claude Fable 5 --- tests/v2/test_locales.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 494e118a..54d8801b 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -72,6 +72,30 @@ def test_locales_unknown_attribute() -> None: locales.XX +def test_ru_plus_tr_az_unions_patronymic_rules() -> None: + from nameparser import locales, parser_for + from nameparser._policy import PatronymicRule + + p = parser_for(locales.RU, locales.TR_AZ) + assert p.policy.patronymic_rules == frozenset( + {PatronymicRule.EAST_SLAVIC, PatronymicRule.TURKIC}) + # both rules live: one name from each pack's case segment + ru = p.parse("Сидоров Иван Петрович") + assert ru.given == "Иван" + tr = p.parse("Mammadova Aygun Ali kizi") + assert (tr.given, tr.middle, tr.family) == ( + "Aygun", "Ali kizi", "Mammadova") + + +def test_pack_over_custom_base() -> None: + from nameparser import Lexicon, Parser, locales, parser_for + + base = Parser(lexicon=Lexicon.default().add(titles={"zqxcustom"})) + p = parser_for(locales.RU, base=base) + n = p.parse("Zqxcustom Иван Петрович") + assert n.title == "Zqxcustom" # base lexicon survives the fold + + def test_locales_import_is_lazy() -> None: # importing the package must not import any pack module; PEP 562 # loads them on first attribute access (spec §2: "importing From be28a9b3123f2e4f6f722ef2ef485451855ed4a5 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 20:51:27 -0700 Subject: [PATCH 152/275] Add the non-interference gate over the default corpus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spec §5.2: each pack (and both together) must parse the full default corpus -- the shared case table's untargeted rows plus the 486-name differential corpus at tools/differential/corpus.jsonl -- identically to the default parser, or the deviation must be declared via the pack's DEVIATES predicate. Ran clean: no undeclared deviations across RU (1 declared), TR_AZ (1 declared), or the combined pack (2 declared) -- the be3f25e predicate hardening already covers this corpus. Co-Authored-By: Claude Fable 5 --- tests/v2/test_locales.py | 61 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 54d8801b..61906d45 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -1,10 +1,23 @@ """The locale pack layer (locales spec §2-3): lazy access, the two 2.0.0 packs, composition, and the non-interference gate.""" +import json +from collections.abc import Callable, Iterable +from pathlib import Path + import pytest -from nameparser import Locale +from nameparser import Locale, Parser, locales, parser_for from nameparser._lexicon import Lexicon from nameparser._policy import PatronymicRule +from nameparser.locales import ru as _ru +from nameparser.locales import tr_az as _tr_az + +_CORPUS = [ + json.loads(line) + for line in (Path(__file__).parents[2] / "tools" / "differential" + / "corpus.jsonl").read_text().splitlines() + if line.strip() +] def test_locales_module_attribute_access() -> None: @@ -111,3 +124,49 @@ def test_locales_import_is_lazy() -> None: nameparser.locales.RU assert "nameparser.locales.ru" in sys.modules importlib.reload(nameparser.locales) + + +def _assert_non_interference( + packed: Parser, deviates: Callable[[str], bool], corpus: Iterable[str], +) -> int: + """Return the number of DECLARED deviations seen; fail on any + undeclared one (spec §5.2 = the pack-acceptance rejection rule).""" + default = Parser() + declared = 0 + for name in corpus: + base = default.parse(name).as_dict() + got = packed.parse(name).as_dict() + if got != base: + assert deviates(name), ( + f"UNDECLARED deviation: {name!r}\n" + f" default: {base}\n packed: {got}") + declared += 1 + return declared + + +def _default_corpus() -> list[str]: + from .cases import CASES + + return list(_CORPUS) + [c.text for c in CASES + if c.locale is None and c.policy is None] + + +def test_non_interference_ru() -> None: + corpus = _default_corpus() + declared = _assert_non_interference( + parser_for(locales.RU), _ru.DEVIATES, corpus) + # the positive side: the gate must be exercising something -- the + # corpus contains east-slavic bank names that DO rotate + assert declared > 0 + + +def test_non_interference_tr_az() -> None: + _assert_non_interference( + parser_for(locales.TR_AZ), _tr_az.DEVIATES, _default_corpus()) + + +def test_non_interference_combined() -> None: + _assert_non_interference( + parser_for(locales.RU, locales.TR_AZ), + lambda n: _ru.DEVIATES(n) or _tr_az.DEVIATES(n), + _default_corpus()) From 6028ab4fedf3c4e9c686453d0a06c0722755dea7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 21:01:40 -0700 Subject: [PATCH 153/275] Add non-Latin default vocabulary (#269): Cyrillic, Greek, Arabic, Hebrew MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prefix reasoning per new NON_FIRST_NAME_PREFIXES/PREFIXES entry (nameparser/config/prefixes.py): - بن ("bin"/"ibn", son of): added to NON_FIRST_NAME_PREFIXES. Latin 'bin' stays ambiguous only to protect the unrelated Chinese given name "Bin"; that collision risk doesn't exist in Arabic script, and بن is never a bare given name on its own. - بنت ("bint", daughter of): NON_FIRST_NAME_PREFIXES, mirroring the existing Latin 'bint' entry. - ابن ("ibn", son of, alternate spelling): NON_FIRST_NAME_PREFIXES, mirroring the existing Latin 'ibn' entry. - أبو / ابو ("abu", father of): left in plain PREFIXES (ambiguous), mirroring the existing Latin 'abu' entry -- "Abu Bakr" reads "Abu" as a given name. - آل ("aal", family/clan of, e.g. "Al Saud"): NON_FIRST_NAME_PREFIXES. Distinct from the excluded definite article "ال" (#269 explicitly excludes standalone "ال"); a clan prefix, never a bare given name. - בן ("ben", son of) / בת ("bat", daughter of): NON_FIRST_NAME_PREFIXES, same reasoning as بن -- no Latin collision risk, never a bare given name in Hebrew usage. Verification notes: - Every parametrized test row was verified live against a runtime-augmented Lexicon.default() before the data landed. Two rows came out differently than the task's first guess: a single-alphabetic-character conjunction (Cyrillic "и"/"і") hits v1's issue #11 initial carve-out and does not join a short 3-piece title chain, so the "и" conjunction row uses a longer, 5-piece chain instead; "الشيخ" was added to FIRST_NAME_TITLES (like its transliterated cousin "sheikh") so a lone following name reads as given, not family. - Geresh/gershayim gate (#269 step 2): probed 'ד"ר'/"גב'" (ASCII quotes) against extract_delimited live. Both survive untouched -- the quote sits mid-word and never satisfies _open_ok/_close_ok's boundary test -- so both the ASCII-quote and the typographic Unicode spellings ('ד״ר' U+05F4, 'גב׳' U+05F3) ship. - Cyrillic мл/ст (junior/senior) deliberately NOT shipped, per the issue's own collision caveat; deferral comment left in nameparser/config/suffixes.py. - Differential harness: adding native-script بن as a prefix changes one existing non-Latin corpus row ('‏محمد بن سلمان‏', RTL-marked) from v1's plain-middle-name reading to 2.0's prefix-chained family -- exactly the new-recognition #269 exists to add, not a Latin-corpus regression. Classified in tools/differential/expected_changes.toml rather than reverting the entry; harness exits 0 with 2 classified diffs (the pre-existing comma-family diff plus this one), 0 unexplained. Co-Authored-By: Claude Fable 5 --- nameparser/config/conjunctions.py | 6 +++ nameparser/config/prefixes.py | 29 ++++++++++ nameparser/config/suffixes.py | 5 ++ nameparser/config/titles.py | 40 ++++++++++++++ tests/v2/test_locales.py | 67 +++++++++++++++++++++++- tools/differential/expected_changes.toml | 13 +++++ 6 files changed, 159 insertions(+), 1 deletion(-) diff --git a/nameparser/config/conjunctions.py b/nameparser/config/conjunctions.py index 003f8c89..cf738084 100644 --- a/nameparser/config/conjunctions.py +++ b/nameparser/config/conjunctions.py @@ -7,6 +7,12 @@ 'the', 'und', 'y', + # #269: Cyrillic (ru/uk/bg) "and": и, і, та. + 'и', + 'і', + 'та', + # #269: Greek "and": και. + 'και', } """ Pieces that should join to their neighboring pieces, e.g. "and", "y" and "&". diff --git a/nameparser/config/prefixes.py b/nameparser/config/prefixes.py index 43ee5414..8f9f867e 100644 --- a/nameparser/config/prefixes.py +++ b/nameparser/config/prefixes.py @@ -32,6 +32,28 @@ 'vd', 'vom', 'zu', + + # #269: Arabic native-script patronymic/clan particles. Unlike their + # Latin transliterations, these live in a script namespace with no + # collision against an unrelated Latin given name (the reason 'bin' + # itself stays ambiguous above), so each is judged on its own + # semantics rather than mirrored blindly: + 'بن', # "bin"/"ibn" (son of) -- never a bare given name; Latin + # 'bin' stays ambiguous only to protect the Chinese given + # name "Bin", which cannot collide with this script. + 'بنت', # "bint" (daughter of) -- mirrors Latin 'bint' above. + 'ابن', # "ibn" (son of, alternate spelling) -- mirrors Latin + # 'ibn' above. + 'آل', # "aal" (family/clan of, e.g. "Al Saud") -- distinct from + # the excluded definite article "ال" (#269 explicitly + # excludes standalone "ال"); a clan prefix, never a bare + # given name. + + # #269: Hebrew native-script patronymic particles -- same + # reasoning as the Arabic ones above: no Latin-script collision, + # and neither functions as a standalone given name in Hebrew usage. + 'בן', # "ben" (son of) + 'בת', # "bat" (daughter of) } #: Name pieces that appear before a last name. Prefixes join to the piece @@ -86,6 +108,13 @@ 'vander', 'vel', 'von', + + # #269: Arabic "abu" (father of), left ambiguous like its Latin + # transliteration 'abu' above (both spellings): "Abu Bakr" reads + # "Abu" as a given name, so this stays a PREFIXES-only member, not + # NON_FIRST_NAME_PREFIXES. + 'أبو', + 'ابو', } # Guard the two invariants the docstring above promises, so a future edit that diff --git a/nameparser/config/suffixes.py b/nameparser/config/suffixes.py index 88962836..114ba5a5 100644 --- a/nameparser/config/suffixes.py +++ b/nameparser/config/suffixes.py @@ -1,4 +1,9 @@ SUFFIX_NOT_ACRONYMS = { + # #269: Cyrillic мл/ст (junior/senior, the jr/sr analogs) deferred + # pending collision vetting -- the issue itself flags 'ст' as a + # two-letter sequence that collides with real Cyrillic words/ + # abbreviations, and both are short enough to risk false positives + # inside the Cyrillic vocabulary. Not shipped in this pass. 'dr', 'esq', 'esquire', diff --git a/nameparser/config/titles.py b/nameparser/config/titles.py index dfc88daa..39cbed69 100644 --- a/nameparser/config/titles.py +++ b/nameparser/config/titles.py @@ -21,6 +21,10 @@ 'shaikh', 'cheikh', 'shekh', + # #269: Arabic -- "الشيخ" ("the sheikh") is the native-script form of + # the transliterated sheikh/sheik/... cluster above; same + # single-first-name-follows convention ("Sheikh Mohammed"). + 'الشيخ', } """ When these titles appear with a single other name, that name is a first name, e.g. @@ -683,4 +687,40 @@ 'woodman', 'writer', 'zoologist', + + # #269: Cyrillic (ru/uk) -- mr/mrs/dr/prof/academician/pan(i) + # honorifics, same title-then-family convention as 'mr'/'dr'/'prof' + # above (not FIRST_NAME_TITLES: "г-н Петров" families the surname + # just like "Mr. Smith" does). + 'г-н', + 'г-жа', + 'д-р', + 'проф', + 'акад', + 'пан', + 'пані', + + # #269: Greek -- kyrios/kyria/kyrios(abbr)/doctor/professor + # abbreviations; same plain-title convention as above. + 'κ', + 'κα', + 'κος', + 'δρ', + 'καθ', + + # #269: Hebrew -- "מר" ("Mr."), plain title like its Latin analog. + # Geresh/gershayim forms of "doctor"/"Mrs." -- both the ASCII-quote + # spelling ('ד"ר', "גב'") and the typographic Unicode spelling + # ('ד״ר' U+05F4 gershayim, 'גב׳' U+05F3 geresh) ship: probed live + # against extract_delimited's _open_ok/_close_ok boundary rules + # (2026-07-17) -- the quote chars sit mid-word (no preceding + # whitespace before the internal quote and, for the closing "'" one, + # no following boundary), so both fail the open/close boundary test + # and are left untouched as literal text. Extraction is provably + # inert on these two ASCII spellings; no delimiter-interaction risk. + 'מר', + 'ד"ר', + "גב'", + 'ד״ר', + 'גב׳', } diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 61906d45..5766aa46 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -6,7 +6,7 @@ import pytest -from nameparser import Locale, Parser, locales, parser_for +from nameparser import Locale, Parser, locales, parse, parser_for from nameparser._lexicon import Lexicon from nameparser._policy import PatronymicRule from nameparser.locales import ru as _ru @@ -170,3 +170,68 @@ def test_non_interference_combined() -> None: parser_for(locales.RU, locales.TR_AZ), lambda n: _ru.DEVIATES(n) or _tr_az.DEVIATES(n), _default_corpus()) + + +# -- #269: non-Latin default vocabulary (Cyrillic, Greek, Arabic, Hebrew) -- +# +# This is DEFAULT vocabulary (nameparser/config/titles.py, +# conjunctions.py, prefixes.py), not a locale pack -- it lives here +# because it's the non-Latin counterpart to the pack tests above. Every +# row was verified live against a runtime-augmented Lexicon.default() +# before the data landed (2026-07-17), so these pin actual observed +# behavior, not guesses -- see the per-script comments below for the +# rows that came out differently than a first guess would suggest. +@pytest.mark.parametrize("name, field, expected", [ + # Cyrillic (ru/uk) titles. + ("г-н Иван Петров", "title", "г-н"), + ("г-жа Мария Иванова", "title", "г-жа"), + ("д-р Мария Иванова", "title", "д-р"), + ("проф Петро Шевченко", "title", "проф"), + ("акад Іван Франко", "title", "акад"), + ("пан Тарас Шевченко", "title", "пан"), + ("пані Марія Іванова", "title", "пані"), + # Cyrillic conjunction "и": v1's issue #11 carve-out treats a bare + # single-alphabetic-character conjunction in a short name as more + # likely an initial (group._group_segment), so a 3-piece chain + # ("проф и акад Іван Франко") does NOT join -- "и" reads as a given + # name instead. The chain only joins once the segment has enough + # rootname pieces (total >= 4); pinned against that actual behavior + # with a 5-piece name instead of the shorter guess. + ("проф и акад Тарас Григорович Шевченко", "title", "проф и акад"), + ("проф та акад Іван Франко", "title", "проф та акад"), + # Greek titles + conjunction (και has 3 letters, so the single-char + # initial carve-out above never applies to it). + ("δρ Νίκος Παπαδόπουλος", "title", "δρ"), + ("κ Γιώργος Παπαδόπουλος", "title", "κ"), + ("καθ και δρ Νίκος Παπαδόπουλος", "title", "καθ και δρ"), + # Arabic patronymic/clan prefixes: non-leading "بن"/"بنت" chain onto + # the family, mirroring the Latin "von"/"bin" prefix-chain rule. + ("محمد بن سلمان", "family", "بن سلمان"), + ("فاطمة بنت محمد", "family", "بنت محمد"), + # "الشيخ" carries the FIRST_NAME_TITLES semantics of its + # transliterated cousin 'sheikh': a single following name reads as + # given, not family. + ("الشيخ محمد", "given", "محمد"), + # Hebrew patronymic prefixes: same non-leading chain-onto-family + # behavior. + ("דוד בן גוריון", "family", "בן גוריון"), + ("שרה בת אברהם", "family", "בת אברהם"), + # Hebrew "מר" title (plain title, not FIRST_NAME_TITLES -- like + # 'mr', the following name reads as family). + ("מר דוד לוי", "title", "מר"), + # Geresh/gershayim gate (#269 step 2): probed live against + # extract_delimited's _open_ok/_close_ok boundary rules. Both the + # ASCII-quote spelling and the typographic Unicode spelling of + # "doctor"/"Mrs." survive extraction untouched -- the quote sits + # mid-word (no preceding whitespace before '"', and for the + # trailing "'" no following word boundary), so it never satisfies + # the boundary-valid open/close test and is left as literal text. + # Both spellings are shipped; see the titles.py comment. + ('ד"ר דוד לוי', "title", 'ד"ר'), + ("גב' דוד לוי", "title", "גב'"), + ("ד״ר דוד לוי", "title", "ד״ר"), + ("גב׳ דוד לוי", "title", "גב׳"), +]) +def test_269_nonlatin_vocabulary_parses( + name: str, field: str, expected: str) -> None: + assert getattr(parse(name), field) == expected diff --git a/tools/differential/expected_changes.toml b/tools/differential/expected_changes.toml index 4e98c16a..6228feda 100644 --- a/tools/differential/expected_changes.toml +++ b/tools/differential/expected_changes.toml @@ -46,6 +46,19 @@ issue = "ambiguous-surname-acronym data change: parenthesized (MA)/(DO) now stay name_regex = "(?i)[(\"'](m\\.?a\\.?|d\\.?o\\.?)[)\"']" fields = ["suffix", "nickname"] +[[change]] +issue = "feat(#269) Arabic بن prefix chains onto family (non-Latin new-recognition)" +# '‏محمد بن سلمان‏': #269 adds the native-script Arabic +# patronymic particle بن ("bin"/"son of") to PREFIXES/ +# NON_FIRST_NAME_PREFIXES. v1 had no such entry, so it left بن a plain +# middle-name token ('سلمان' alone as last); 2.0 now chains it onto the +# family the same way 'von'/'bin' (Latin) do, giving family 'بن سلمان'. +# This is new-recognition on non-Latin input -- the exact behavior +# #269 exists to add -- not a Latin-corpus regression, so it is +# classified rather than reverted. +name_regex = "بن" +fields = ["middle", "last"] + # Deliberately NOT a [[change]] rule: 'Ph. D.' split-token healing # ('John Ph. D.', 'John Smith, Ph. D.') is PARITY, not a 2.0 behavior # change -- v1's fix_phd healing of the adjacent 'Ph.'/'D.' pair into From a053e8a5fef3da9555bb5c587073387a87b5f140 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 21:08:52 -0700 Subject: [PATCH 154/275] =?UTF-8?q?Drop=20bare=20Greek=20=CE=BA=20and=20ti?= =?UTF-8?q?ghten=20the=20#269=20allowlist=20rule?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up to 6028ab4: - Bare 'κ' removed from TITLES: _normalize strips the edge period, so the abbreviated initial 'Κ.' matched the entry and 'Κ. Παπαδόπουλος' parsed as title='Κ.' with an EMPTY given -- degrading the very common initial+surname shape. The single-letter #11 carve-out guards conjunctions only, and Latin TITLES deliberately has no bare single-letter entries for the same reason. Deferral note left next to the Greek entry group; regression test pins the correct initial+surname reading; the bare-κ test row is replaced with κος/κα rows (verified live before pinning). - expected_changes.toml: the #269 rule's name_regex tightened from "بن" to "\bبن\b" -- the bare form matches بن as a substring anywhere (including inside لبنان, 'Lebanon') and would silently absorb unrelated middle/last diffs. Verified live: re.search matches '‏محمد بن سلمان‏' and NOT 'لبنان'. Co-Authored-By: Claude Fable 5 --- nameparser/config/titles.py | 10 +++++++--- tests/v2/test_locales.py | 18 ++++++++++++++++-- tools/differential/expected_changes.toml | 4 +++- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/nameparser/config/titles.py b/nameparser/config/titles.py index 39cbed69..6961a626 100644 --- a/nameparser/config/titles.py +++ b/nameparser/config/titles.py @@ -700,9 +700,13 @@ 'пан', 'пані', - # #269: Greek -- kyrios/kyria/kyrios(abbr)/doctor/professor - # abbreviations; same plain-title convention as above. - 'κ', + # #269: Greek -- kyria/kyrios(abbr)/doctor/professor abbreviations; + # same plain-title convention as above. + # #269: bare κ deferred -- collides with the initial+surname shape + # ('Κ. Παπαδόπουλος' would parse as title 'Κ.' with an empty given; + # _normalize strips the edge period, so the entry matches the + # initial). Latin TITLES deliberately has no bare single-letter + # entries for the same reason. 'κα', 'κος', 'δρ', diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 5766aa46..0f1e3523 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -200,9 +200,12 @@ def test_non_interference_combined() -> None: ("проф и акад Тарас Григорович Шевченко", "title", "проф и акад"), ("проф та акад Іван Франко", "title", "проф та акад"), # Greek titles + conjunction (και has 3 letters, so the single-char - # initial carve-out above never applies to it). + # initial carve-out above never applies to it). Bare κ is NOT + # shipped -- it collides with the initial+surname shape (see the + # regression test below). ("δρ Νίκος Παπαδόπουλος", "title", "δρ"), - ("κ Γιώργος Παπαδόπουλος", "title", "κ"), + ("κος Γιώργος Παπαδόπουλος", "title", "κος"), + ("κα Μαρία Παπαδοπούλου", "title", "κα"), ("καθ και δρ Νίκος Παπαδόπουλος", "title", "καθ και δρ"), # Arabic patronymic/clan prefixes: non-leading "بن"/"بنت" chain onto # the family, mirroring the Latin "von"/"bin" prefix-chain rule. @@ -235,3 +238,14 @@ def test_non_interference_combined() -> None: def test_269_nonlatin_vocabulary_parses( name: str, field: str, expected: str) -> None: assert getattr(parse(name), field) == expected + + +def test_269_bare_greek_kappa_not_a_title() -> None: + # Regression for the deferred bare 'κ' entry: were it in TITLES, + # _normalize's edge-period strip would make the abbreviated-initial + # 'Κ.' match it, degrading the very common initial+surname shape to + # title='Κ.' with an EMPTY given. Pin the correct reading. + n = parse("Κ. Παπαδόπουλος") + assert n.title == "" + assert n.given == "Κ." + assert n.family == "Παπαδόπουλος" diff --git a/tools/differential/expected_changes.toml b/tools/differential/expected_changes.toml index 6228feda..6feddfe0 100644 --- a/tools/differential/expected_changes.toml +++ b/tools/differential/expected_changes.toml @@ -56,7 +56,9 @@ issue = "feat(#269) Arabic بن prefix chains onto family (non-Latin new-recogni # This is new-recognition on non-Latin input -- the exact behavior # #269 exists to add -- not a Latin-corpus regression, so it is # classified rather than reverted. -name_regex = "بن" +# Word-bounded: a bare "بن" would also match the substring inside e.g. +# لبنان ("Lebanon") and silently absorb unrelated middle/last diffs. +name_regex = "\\bبن\\b" fields = ["middle", "last"] # Deliberately NOT a [[change]] rule: 'Ph. D.' split-token healing From 759b217a5c8b438bd33299d0144b6ea7cd9bae5a Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 21:11:02 -0700 Subject: [PATCH 155/275] Add --locale to the CLI (closes the deferred migration deviation) Co-Authored-By: Claude Fable 5 --- nameparser/__main__.py | 13 ++++++++++++- tests/v2/test_cli.py | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/nameparser/__main__.py b/nameparser/__main__.py index eec2503c..083ef6bc 100644 --- a/nameparser/__main__.py +++ b/nameparser/__main__.py @@ -15,8 +15,19 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument("name", help="the name string to parse") ap.add_argument("--json", action="store_true", help="print the component dict as JSON") + ap.add_argument("--locale", metavar="CODE", + help="parse with a locale pack (e.g. 'ru'); see " + "nameparser.locales") args = ap.parse_args(argv) - n = parse(args.name) + if args.locale: + from nameparser import locales, parser_for + try: + parser = parser_for(locales.get(args.locale)) + except KeyError as exc: + ap.error(exc.args[0]) # exits 2, message to stderr + n = parser.parse(args.name) + else: + n = parse(args.name) if args.json: print(json.dumps(n.as_dict(), ensure_ascii=False)) return 0 diff --git a/tests/v2/test_cli.py b/tests/v2/test_cli.py index 36b85773..df8a360d 100644 --- a/tests/v2/test_cli.py +++ b/tests/v2/test_cli.py @@ -24,3 +24,19 @@ def test_cli_json() -> None: def test_cli_no_args_usage() -> None: proc = _run() assert proc.returncode != 0 + + +def test_cli_locale() -> None: + proc = _run("Сидоров Иван Петрович", "--locale", "ru") + assert proc.returncode == 0 + assert "Иван" in proc.stdout + + proc = _run("Сидоров Иван Петрович", "--locale", "ru", "--json") + data = json.loads(proc.stdout) + assert "Иван" in data["given"] + + +def test_cli_locale_unknown_code_lists_available() -> None: + proc = _run("John Smith", "--locale", "xx") + assert proc.returncode != 0 + assert "ru" in proc.stderr and "tr_az" in proc.stderr From b2ce60ae3834a3f6448c7df372b72b1c808990a3 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 21:12:13 -0700 Subject: [PATCH 156/275] Record the locales package in AGENTS.md and the release-log draft Co-Authored-By: Claude Fable 5 --- AGENTS.md | 4 ++-- docs/release_log.rst | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 83775acb..eca650d2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -113,9 +113,9 @@ Each named attribute (`title`, `first`, etc.) is a `@property` that joins its co The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These conventions apply to all new-API code and are stricter than the v1 sections above. The full design record (rationale, settled-decision logs, dated amendments) lives in untracked `docs/superpowers/specs/`; this section is the enforceable subset. **A commit that establishes or amends one of these conventions must update this section in the same commit** — grep-driven staleness sweeps miss paraphrased prose (see Workflow above), so write-time maintenance is the mechanism, audits are the backstop. -- **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`, `_pipeline/`, `_parser.py`, plus the facade layer: `_facade.py`, `_config_shim.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Since the M11 swap, the old paths are import-path-preserving re-exports: `nameparser.parser` re-exports the `_facade` `HumanName`, `nameparser.config` re-exports the `_config_shim` names (`Constants`, `CONSTANTS`, `SetManager`, `TupleManager`, `RegexTupleManager`); the `config/` DATA modules stay the vocabulary source through 2.x. The whole facade layer is deleted in 3.0. +- **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`, `_pipeline/`, `_parser.py`, plus the facade layer: `_facade.py`, `_config_shim.py`). The public import surface is exactly `nameparser` and `nameparser.locales`; `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Since the M11 swap, the old paths are import-path-preserving re-exports: `nameparser.parser` re-exports the `_facade` `HumanName`, `nameparser.config` re-exports the `_config_shim` names (`Constants`, `CONSTANTS`, `SetManager`, `TupleManager`, `RegexTupleManager`); the `config/` DATA modules stay the vocabulary source through 2.x. The whole facade layer is deleted in 3.0. - **Facade layer** (`_facade.py`, `_config_shim.py`): the v1-compat `HumanName`/`Constants` over the core. Key mechanisms: `Constants._generation` dirty-tracking (every mutation bumps; facades resolve their `Parser` lazily via `_cached_parser(lexicon, policy)`); `Constants._snapshot()` mirrors `_lexicon._default_lexicon()` (equality-pinned); the facade pickles v1-SHAPED state (component lists, one `__setstate__` path for 1.4 and 2.x blobs; components rebuild via `replace()`, never a re-parse); `_V1_HOOKS` overrides warn once per subclass (#280). The compat contract is the migration spec's promise: warning-free 1.4 code behaves identically except release-log-classified fixes — `tools/differential/` (dev-only, not shipped) verifies this against 1.4-on-PyPI over a 486-name corpus. `parser.py:NNNN` citations throughout the 2.0 code refer to the PRE-swap v1 file, deleted at the M11 swap; resolve them with `git show 2d5d8c2:nameparser/parser.py`. -- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers; `_parser` sits on everything except `_render`; the facade layer (`_facade`, `_config_shim`, `parser`, `config/__init__`, `__main__`) may import anything public plus `_render`. Extend the test's `ALLOWED` table when adding a module. +- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers; `_parser` sits on everything except `_render`; the facade layer (`_facade`, `_config_shim`, `parser`, `config/__init__`, `__main__`) may import anything public plus `_render`; locale pack modules (`locales/*.py`) import `_locale`/`_lexicon`/`_policy` only, and the `locales/__init__` additionally lazy-imports its packs (PEP 562). Extend the test's `ALLOWED` table when adding a module. - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. Sanctioned deviation, facade layer only: `HumanName` and the shim `Constants` organize by v1 concern groups (`# -- render defaults --`, `# -- config / parsing --`, `# -- fields --`, ..., dunders and pickle last) — the classes mirror v1's own surface and die in 3.0; the canonical order still binds every core type. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). diff --git a/docs/release_log.rst b/docs/release_log.rst index 63396aad..35a683d4 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -2,6 +2,12 @@ Release Log =========== * 2.0.0 - unreleased + **Locale packs (opt-in)** + + - Add ``nameparser.locales`` with the first two packs: ``locales.RU`` (East Slavic patronymic order) and ``locales.TR_AZ`` (Turkic patronymic markers). Packs are pure data folded in at ``parser_for(locales.RU)``; they compose (``parser_for(locales.RU, locales.TR_AZ)`` unions the rules) and are never auto-detected -- there is no reliable way to detect a name's language (closes the pack half of #270; #271/#272/#146 stay staged for 2.x) + - The CLI gains ``--locale CODE`` (``python -m nameparser --locale ru "..."``) + - Add non-Latin vocabulary to the default lexicon (#269): Cyrillic, Greek, Arabic and Hebrew titles, conjunctions, and name particles -- native-script entries cannot collide with Latin-script names. Deferred pending vetting: Cyrillic ``мл``/``ст`` suffixes and the bare Greek ``κ`` title (it collides with the initial+surname shape). Behavior note: ``محمد بن سلمان`` now chains ``بن`` onto the family name where 1.x read it as a middle name + **Behavior Changes (draft)** .. note:: From b49239decfe65fbbbca58f35a63b7471280fd160 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 21:13:16 -0700 Subject: [PATCH 157/275] Make the SetManager doctest reprs addition-proof Four customize.rst doctests pinned the titles set's LAST element ('zoologist') through the ELLIPSIS form -- #269's non-Latin entries sort after it and broke all four. The ellipsis now swallows to the closing brace, so vocabulary additions cannot break the docs again (the constant-content rule, applied to doctests). Co-Authored-By: Claude Fable 5 --- docs/customize.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index 399042af..22f954c0 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -371,7 +371,7 @@ constant so that "Hon" can be parsed as a first name. maiden: '' ]> >>> constants.titles.remove('hon') - SetManager({'10th', ..., 'zoologist'}) + SetManager({'10th', ...}) >>> hn = HumanName("Hon Solo", constants=constants) >>> hn >> from nameparser.config import Constants >>> constants = Constants() >>> constants.titles.add('dean', 'Chemistry') - SetManager({'10th', ..., 'zoologist'}) + SetManager({'10th', ...}) >>> hn = HumanName("Assoc Dean of Chemistry Robert Johns", constants=constants) >>> hn >> from nameparser import HumanName >>> instance = HumanName("") >>> instance.C.titles.add('dean') - SetManager({'10th', ..., 'zoologist'}) + SetManager({'10th', ...}) >>> other_instance = HumanName("Dean Robert Johns") >>> other_instance # Dean parses as title >> instance.has_own_config False >>> instance.C.titles.add('dean') - SetManager({'10th', ..., 'zoologist'}) + SetManager({'10th', ...}) >>> other_instance = HumanName("Dean Robert Johns", Constants()) # <-- fresh, private config >>> other_instance Date: Fri, 17 Jul 2026 21:57:53 -0700 Subject: [PATCH 158/275] fix: normalize vocabulary with lower(), not casefold() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit casefold()'s caseless-matching folds mutated stored vocabulary: the new Greek title 'κος' arrived in the lexicon as the misspelling 'κοσ' (final sigma flattened) and v1's 'großfürst' as 'grossfürst' -- while the v1 CONSTANTS path (lc() = lower()) kept both as authored, so the two surfaces disagreed about the vocabulary's spelling. lower() follows Unicode SpecialCasing contextually ('ΚΟΣ' -> 'κος'), matches v1's lc() exactly, and _normalize is the single fold for storage and match-time lookups, so matching stays symmetric. Pin the preserved spellings and the default-lexicon fidelity of both entries. (Review finding, 2026-07-17.) Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 18 +++++++++++++----- nameparser/config/maiden_markers.py | 2 +- tests/v2/test_lexicon.py | 25 ++++++++++++++++++++----- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 0cb4e34e..8c1d9960 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -25,13 +25,21 @@ def _normalize(word: str) -> str: - """Casefold, strip whitespace and EDGE periods -- v1's lc() + """Lowercase, strip whitespace and EDGE periods -- v1's lc() semantics. Interior periods survive on purpose: 'J.R.' must not collapse to 'jr' and hit the periodless vocabulary (v1 parity, pinned live 2026-07-17). Suffix-ACRONYM membership alone uses the period-free form (see _vocab.suffix_as_written), mirroring v1's - is_suffix, which removed periods only for the acronym test.""" - return word.casefold().strip().strip(".") + is_suffix, which removed periods only for the acronym test. + + lower(), NOT casefold(): casefold's caseless-matching folds mutate + the stored vocabulary itself -- 'κος' becomes the misspelling 'κοσ' + (final sigma flattened) and 'großfürst' becomes 'grossfürst' -- + while lower() applies Unicode SpecialCasing contextually and keeps + both as authored. This function is the single fold for storage AND + match-time lookups, so matching stays symmetric either way; lower() + is what v1's lc() used, preserving which cross-spellings match.""" + return word.lower().strip().strip(".") def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: @@ -65,7 +73,7 @@ def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: if not n: raise ValueError( f"Lexicon.{field_name} entry {w!r} normalizes to empty " - f"(casefold + strip periods/whitespace leaves nothing)" + f"(lowercase + strip periods/whitespace leaves nothing)" ) normalized.add(n) return frozenset(normalized) @@ -111,7 +119,7 @@ def _normpairs( if not normalized_key: raise ValueError( f"capitalization_exceptions key {k!r} normalizes to " - f"empty (casefold + strip periods/whitespace leaves " + f"empty (lowercase + strip periods/whitespace leaves " f"nothing)" ) deduped[normalized_key] = v diff --git a/nameparser/config/maiden_markers.py b/nameparser/config/maiden_markers.py index 22399e37..637028b1 100644 --- a/nameparser/config/maiden_markers.py +++ b/nameparser/config/maiden_markers.py @@ -21,7 +21,7 @@ (#274). French née/né/nee, German geb./geborene, Dutch geboren, Czech/Slovak roz./rozená, Danish/Norwegian født (Nynorsk fødd), Swedish född, Russian урожд./урождённая/урождённый (both ё and е spellings — -``str.casefold()`` does not fold them, and running text routinely +case normalization does not fold them, and running text routinely writes е). Both grammatical genders are listed where #274 or review attested them (née/né, урождённая/урождённый); Czech masculine rozený awaits the same vetting. Entries are stored normalized: lowercase, no diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index e0365e38..ba82fff7 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -19,15 +19,20 @@ def test_default_sources_v1_vocabulary() -> None: assert "dos" in lex.particles and "dos" not in lex.particles_ambiguous assert "van" in lex.particles_ambiguous # v1's CAPITALIZATION_EXCEPTIONS maps 'phd' -> 'Ph.D.' (verbatim, not - # normalized -- only keys are casefolded/period-stripped at + # normalized -- only keys are lowercased/period-stripped at # construction, values pass through unchanged). assert lex.capitalization_exceptions_map["phd"] == "Ph.D." # maiden markers source from the same data-module pattern (#274); # non-colliding Cyrillic entries live in the default per the locales # design's sorting rule, and both ё/е spellings are listed because - # casefold() does not fold them. + # case normalization does not fold them. assert "geborene" in lex.maiden_markers assert "урожденная" in lex.maiden_markers and "урождённая" in lex.maiden_markers + # #269 fidelity: entries whose spelling casefold() would mutate must + # arrive in the lexicon exactly as authored in the data modules + # (final sigma intact, ß intact) -- the review that caught 'κοσ'. + assert "κος" in lex.titles and "κοσ" not in lex.titles + assert "großfürst" in lex.titles and "grossfürst" not in lex.titles def test_default_is_cached_single_instance() -> None: @@ -181,14 +186,24 @@ def test_setstate_rejects_mismatched_field_layout() -> None: Lexicon.__new__(Lexicon).__setstate__(extra) -def test_normalization_casefolds_and_keeps_interior_periods() -> None: - # v1's lc() semantics plus casefold: EDGE periods trimmed, interior +def test_normalization_lowercases_and_keeps_interior_periods() -> None: + # v1's lc() semantics: lowercase, EDGE periods trimmed, interior # periods KEPT ('J.R.' must not collapse to 'jr' and hit the # periodless vocabulary -- v1 parity, pinned live 2026-07-17). # Suffix-ACRONYM membership alone strips periods (see # _vocab.suffix_as_written). lex = Lexicon(titles=frozenset({"STRAßE", "Ph.D", "Dr."})) - assert lex.titles == frozenset({"strasse", "ph.d", "dr"}) + assert lex.titles == frozenset({"straße", "ph.d", "dr"}) + + +def test_normalization_preserves_spellings_casefold_would_mutate() -> None: + # lower(), not casefold(): casefold's caseless-matching folds would + # rewrite stored vocabulary -- Greek final sigma flattens ('κος' -> + # the misspelling 'κοσ') and German ß expands ('großfürst' -> + # 'grossfürst'). lower() follows Unicode SpecialCasing contextually + # ('ΚΟΣ' -> 'κος') and keeps entries as authored, matching v1's lc(). + lex = Lexicon(titles=frozenset({"ΚΟΣ", "Großfürst"})) + assert lex.titles == frozenset({"κος", "großfürst"}) def test_suffix_ambiguous_must_be_subset_of_acronyms() -> None: From 81f8fa3f3fcea894803e533afd05c4aa7a89d3fe Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 21:58:06 -0700 Subject: [PATCH 159/275] test: give the non-interference gate teeth; close locale coverage gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shared corpus exercised TR_AZ with a single name, leaving that gate nearly vacuous. Add synthetic rotator names -- one per alternation branch of each pack's marker regexes, both scripts -- prove each deviates AND declares individually, and floor every gate's declared count at its rotator count. Also: TR_AZ per-token DEVIATES regression (mirror of the RU one), whole-token negative ('Ogluev'), rows for the untested #269 entries (أبو/ابو kunya ambiguity split, آل leading and chained, ابن, Ukrainian і), pack-policy-survives-custom-base and parser_for-chaining composition tests, a facade-path #269 smoke test, and monkeypatch-based module-cache handling in the laziness test. (Review findings, 2026-07-17.) Co-Authored-By: Claude Fable 5 --- tests/v2/test_locales.py | 184 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 171 insertions(+), 13 deletions(-) diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 0f1e3523..bf692ea6 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -79,6 +79,23 @@ def test_ru_deviates_scans_per_token() -> None: assert not ru.DEVIATES("John Smith") +def test_tr_az_deviates_scans_per_token() -> None: + # Mirror of the RU regression above: the Turkic marker regexes are + # fullmatch-shaped (^...$), so anything but a per-token scan would + # miss a marker followed by a suffix (under-declaration) -- and a + # whole-string match would never fire at all on multi-token names. + from nameparser.locales import tr_az + + assert tr_az.DEVIATES("Mammadova Aygun Ali kizi Jr.") + # over-declaration stays the safe direction: a bare marker-shaped + # token declares even where the rule won't rewrite anything + assert tr_az.DEVIATES("Kizi Anna") + assert not tr_az.DEVIATES("John Smith") + # markers are whole-token: a name merely CONTAINING one must not + # declare ("Ogluev" is not a marker) + assert not tr_az.DEVIATES("Ogluev Ivan") + + def test_locales_unknown_attribute() -> None: from nameparser import locales with pytest.raises(AttributeError, match="XX"): @@ -109,21 +126,52 @@ def test_pack_over_custom_base() -> None: assert n.title == "Zqxcustom" # base lexicon survives the fold -def test_locales_import_is_lazy() -> None: +def test_pack_preserves_custom_base_policy() -> None: + # the lexicon-survival twin above has a policy counterpart: a + # policy-only pack must ADD its patronymic rule without resetting + # unrelated policy fields the base customized + from nameparser import Parser, locales, parser_for + from nameparser._policy import Policy + + base = Parser(policy=Policy(strip_emoji=False)) + p = parser_for(locales.RU, base=base) + assert p.policy.strip_emoji is False + assert p.policy.patronymic_rules == frozenset( + {PatronymicRule.EAST_SLAVIC}) + + +def test_parser_for_results_chain_as_bases() -> None: + # parser_for's result is itself a valid base= -- packs applied in + # two steps accumulate exactly like one call with both packs + from nameparser import locales, parser_for + + chained = parser_for(locales.TR_AZ, base=parser_for(locales.RU)) + assert chained.policy.patronymic_rules == frozenset( + {PatronymicRule.EAST_SLAVIC, PatronymicRule.TURKIC}) + assert chained.parse("Сидоров Иван Петрович").given == "Иван" + assert chained.parse("Mammadova Aygun Ali kizi").family == "Mammadova" + + +def test_locales_import_is_lazy(monkeypatch: pytest.MonkeyPatch) -> None: # importing the package must not import any pack module; PEP 562 # loads them on first attribute access (spec §2: "importing - # nameparser never pays for pack data") - import importlib + # nameparser never pays for pack data"). monkeypatch snapshots + # sys.modules AND the parent package attribute (the fresh import + # below rebinds nameparser.locales), so everything rolls back even + # if an assert fails mid-test (manual del + reload left other tests + # running against a half-restored module cache on failure). import sys + import nameparser as _np + + monkeypatch.setattr(_np, "locales", _np.locales) for mod in list(sys.modules): if mod.startswith("nameparser.locales"): - del sys.modules[mod] + monkeypatch.delitem(sys.modules, mod) import nameparser.locales assert "nameparser.locales.ru" not in sys.modules nameparser.locales.RU assert "nameparser.locales.ru" in sys.modules - importlib.reload(nameparser.locales) def _assert_non_interference( @@ -151,25 +199,104 @@ def _default_corpus() -> list[str]: if c.locale is None and c.policy is None] +# Synthetic rotator names: one per alternation branch of each pack's +# marker regexes (both scripts), in the shape that makes the rule fire +# (RU: family-first, patronymic last, plain middle; TR_AZ: a standalone +# marker token). They serve the gate's POSITIVE side -- without them the +# shared corpus exercises TR_AZ with a single name, leaving the gate +# nearly vacuous (review finding, 2026-07-17). Branch named per row so a +# regex edit that orphans a branch is visible here. +_RU_ROTATORS = [ + "Sidorov Ivan Petrovich", # ovich + "Sidorova Anna Petrovna", # ovna + "Karpov Oleg Sergeevich", # evich + "Karpova Olga Sergeevna", # evna + "Petrova Maria Nikitichna", # ichna + "Ulyanov Vladimir Ilyich", # ilyich + "Bulgakov Ivan Kuzmich", # kuzmich + "Titov Pyotr Lukich", # lukich + "Orlov Semyon Fomich", # fomich + "Zaitsev Andrei Fokich", # fokich + "Сидоров Иван Петрович", # ович + "Сидорова Анна Петровна", # овна + "Карпов Олег Сергеевич", # евич + "Карпова Ольга Сергеевна", # евна + "Петрова Мария Никитична", # ична + "Ульянов Владимир Ильич", # ильич + "Булгаков Иван Кузьмич", # кузьмич + "Титов Пётр Лукич", # лукич + "Орлов Семён Фомич", # фомич + "Зайцев Андрей Фокич", # фокич +] +_TR_AZ_ROTATORS = [ + "Aliyev Ali Vali oglu", # oglu + "Aliyev Rashad Vali oğlu", # oğlu + "Mammadov Elchin Hasan ogly", # ogly + "Karimov Aziz Vali ogli", # ogli + "Karimov Alisher Vali o'g'li", # o['’ʻ]g['’ʻ]li, ASCII ' + "Yusupov Botir Vali o’g’li", # o['’ʻ]g['’ʻ]li, typographic ’ + "Mammadova Aygun Hasan qizi", # qizi + "Aliyeva Leyla Vali qızı", # qızı + "Mammadova Sevinj Ali kizi", # kizi + "Asanova Aigul Bolot kyzy", # kyzy + "Guliyeva Nigar Vali gyzy", # gyzy + "Nazarbayev Nursultan Abish uly", # uly + "Zheenbekov Sooronbay Sharip uulu", # uulu + "Алиев Али Вали оглу", # оглу + "Мамедов Эльчин Гасан оглы", # оглы + "Алиев Рашад Вали оғлу", # оғлу + "Каримов Алишер Вали ўғли", # ўғли + "Юсупов Ботир Вали угли", # угли + "Мамедова Айгюн Гасан кызы", # кызы + "Алиева Лейла Вали гызы", # гызы + "Назарбаева Дарига Нурсултан қызы", # қызы + "Каримова Лола Вали қизи", # қизи + "Назарбаев Нурсултан Абиш улы", # улы + "Токаев Касым Кемел ұлы", # ұлы + "Жээнбеков Сооронбай Шарип уулу", # уулу +] + + +@pytest.mark.parametrize("name", _RU_ROTATORS) +def test_ru_rotator_deviates_and_declares(name: str) -> None: + # every branch of the pack's marker regexes both FIRES (packed parse + # differs from default) and is DECLARED (DEVIATES says so) -- the + # per-name proof behind the gate's declared-count floor below + packed = parser_for(locales.RU) + assert packed.parse(name).as_dict() != Parser().parse(name).as_dict() + assert _ru.DEVIATES(name) + + +@pytest.mark.parametrize("name", _TR_AZ_ROTATORS) +def test_tr_az_rotator_deviates_and_declares(name: str) -> None: + packed = parser_for(locales.TR_AZ) + assert packed.parse(name).as_dict() != Parser().parse(name).as_dict() + assert _tr_az.DEVIATES(name) + + def test_non_interference_ru() -> None: - corpus = _default_corpus() + corpus = _default_corpus() + _RU_ROTATORS declared = _assert_non_interference( parser_for(locales.RU), _ru.DEVIATES, corpus) - # the positive side: the gate must be exercising something -- the - # corpus contains east-slavic bank names that DO rotate - assert declared > 0 + # the positive side: every synthetic rotator (plus the corpus's own + # east-slavic names) must actually flow through the declared branch + assert declared >= len(_RU_ROTATORS) def test_non_interference_tr_az() -> None: - _assert_non_interference( - parser_for(locales.TR_AZ), _tr_az.DEVIATES, _default_corpus()) + corpus = _default_corpus() + _TR_AZ_ROTATORS + declared = _assert_non_interference( + parser_for(locales.TR_AZ), _tr_az.DEVIATES, corpus) + assert declared >= len(_TR_AZ_ROTATORS) def test_non_interference_combined() -> None: - _assert_non_interference( + corpus = _default_corpus() + _RU_ROTATORS + _TR_AZ_ROTATORS + declared = _assert_non_interference( parser_for(locales.RU, locales.TR_AZ), lambda n: _ru.DEVIATES(n) or _tr_az.DEVIATES(n), - _default_corpus()) + corpus) + assert declared >= len(_RU_ROTATORS) + len(_TR_AZ_ROTATORS) # -- #269: non-Latin default vocabulary (Cyrillic, Greek, Arabic, Hebrew) -- @@ -199,6 +326,9 @@ def test_non_interference_combined() -> None: # with a 5-piece name instead of the shorter guess. ("проф и акад Тарас Григорович Шевченко", "title", "проф и акад"), ("проф та акад Іван Франко", "title", "проф та акад"), + # Ukrainian "і" is single-character like "и", so the same #11 + # initial carve-out applies: pinned with a 5-piece name. + ("проф і акад Тарас Григорович Шевченко", "title", "проф і акад"), # Greek titles + conjunction (και has 3 letters, so the single-char # initial carve-out above never applies to it). Bare κ is NOT # shipped -- it collides with the initial+surname shape (see the @@ -211,6 +341,21 @@ def test_non_interference_combined() -> None: # the family, mirroring the Latin "von"/"bin" prefix-chain rule. ("محمد بن سلمان", "family", "بن سلمان"), ("فاطمة بنت محمد", "family", "بنت محمد"), + # "ابن" (alternate spelling) and the clan prefix "آل" are + # never-given: non-leading they chain onto the family, and a + # LEADING "آل" consumes the whole name as family (no given), like + # "de Mesnil". + ("محمد ابن رشد", "family", "ابن رشد"), + ("محمد آل سعود", "family", "آل سعود"), + ("آل سعود", "family", "آل سعود"), + ("آل سعود", "given", ""), + # The kunya "أبو"/"ابو" is AMBIGUOUS (it can begin a standalone + # byname): leading it reads as the given name -- the "Van Johnson" + # rule -- while non-leading it chains onto the family like any + # particle. + ("أبو مازن", "given", "أبو"), + ("أحمد أبو خليل", "family", "أبو خليل"), + ("علي ابو خالد", "family", "ابو خالد"), # "الشيخ" carries the FIRST_NAME_TITLES semantics of its # transliterated cousin 'sheikh': a single following name reads as # given, not family. @@ -240,6 +385,19 @@ def test_269_nonlatin_vocabulary_parses( assert getattr(parse(name), field) == expected +def test_269_vocabulary_reaches_the_v1_facade() -> None: + # one row per script through HumanName: the facade resolves its + # lexicon from the CONSTANTS shim snapshot, a different path from + # parse()'s Lexicon.default() -- proves the #269 data modules feed + # both surfaces + from nameparser import HumanName + + assert HumanName("г-н Иван Петров").title == "г-н" + assert HumanName("κος Γιώργος Παπαδόπουλος").title == "κος" + assert HumanName("محمد بن سلمان").last == "بن سلمان" + assert HumanName('ד"ר דוד לוי').title == 'ד"ר' + + def test_269_bare_greek_kappa_not_a_title() -> None: # Regression for the deferred bare 'κ' entry: were it in TITLES, # _normalize's edge-period strip would make the abbreviated-initial From affdd5be5f2dd77583a0d1c5de0f4a30634ffa77 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 21:58:07 -0700 Subject: [PATCH 160/275] fix: --locale '' errors instead of silently using the default parser Truthiness swallowed an empty code (empty shell variable, typo) into the default-parser branch; 'is not None' routes it to locales.get(), which exits 2 listing the available codes like any unknown code. (Review finding, 2026-07-17.) Co-Authored-By: Claude Fable 5 --- nameparser/__main__.py | 4 +++- tests/v2/test_cli.py | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/nameparser/__main__.py b/nameparser/__main__.py index 083ef6bc..fd1dde53 100644 --- a/nameparser/__main__.py +++ b/nameparser/__main__.py @@ -19,7 +19,9 @@ def main(argv: list[str] | None = None) -> int: help="parse with a locale pack (e.g. 'ru'); see " "nameparser.locales") args = ap.parse_args(argv) - if args.locale: + # `is not None`, not truthiness: --locale "" must reach get() and + # exit 2 listing the codes, not silently fall back to the default + if args.locale is not None: from nameparser import locales, parser_for try: parser = parser_for(locales.get(args.locale)) diff --git a/tests/v2/test_cli.py b/tests/v2/test_cli.py index df8a360d..861e25d6 100644 --- a/tests/v2/test_cli.py +++ b/tests/v2/test_cli.py @@ -40,3 +40,12 @@ def test_cli_locale_unknown_code_lists_available() -> None: proc = _run("John Smith", "--locale", "xx") assert proc.returncode != 0 assert "ru" in proc.stderr and "tr_az" in proc.stderr + + +def test_cli_locale_empty_string_errors_not_silent_default() -> None: + # --locale "" is a mistake (empty shell variable, typo), not a + # request for the default parser; a truthiness check would swallow + # it silently -- it must error like any other unknown code + proc = _run("John Smith", "--locale", "") + assert proc.returncode != 0 + assert "ru" in proc.stderr and "tr_az" in proc.stderr From 7d6071c5f96df59418e6cde7c931858fbc1f5430 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 17 Jul 2026 21:58:07 -0700 Subject: [PATCH 161/275] docs: comment accuracy pass from the locale review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the invented 'protect the Chinese given name Bin' rationale from prefixes.py -- no source in the repo history attests it; state only the observable fact that Latin 'bin' stays outside NON_FIRST_NAME_PREFIXES. Soften the мл/ст deferral comment to what #269 actually says (both need within-script collision vetting; the 'ст' specificity was ours, not the issue's). Cite issue #185 and its test bank in tr_az.py for parity with ru.py's data-sources citation. Co-Authored-By: Claude Fable 5 --- nameparser/config/prefixes.py | 11 +++++------ nameparser/config/suffixes.py | 8 ++++---- nameparser/locales/tr_az.py | 5 +++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/nameparser/config/prefixes.py b/nameparser/config/prefixes.py index 8f9f867e..4f3a1cb9 100644 --- a/nameparser/config/prefixes.py +++ b/nameparser/config/prefixes.py @@ -35,12 +35,11 @@ # #269: Arabic native-script patronymic/clan particles. Unlike their # Latin transliterations, these live in a script namespace with no - # collision against an unrelated Latin given name (the reason 'bin' - # itself stays ambiguous above), so each is judged on its own - # semantics rather than mirrored blindly: - 'بن', # "bin"/"ibn" (son of) -- never a bare given name; Latin - # 'bin' stays ambiguous only to protect the Chinese given - # name "Bin", which cannot collide with this script. + # collision against an unrelated Latin given name, so each is judged + # on its own semantics rather than mirrored blindly: + 'بن', # "bin"/"ibn" (son of) -- never a bare given name. Latin + # 'bin' is in PREFIXES but not in this set; that judgment + # is unchanged by adding the Arabic-script form. 'بنت', # "bint" (daughter of) -- mirrors Latin 'bint' above. 'ابن', # "ibn" (son of, alternate spelling) -- mirrors Latin # 'ibn' above. diff --git a/nameparser/config/suffixes.py b/nameparser/config/suffixes.py index 114ba5a5..027a6f9d 100644 --- a/nameparser/config/suffixes.py +++ b/nameparser/config/suffixes.py @@ -1,9 +1,9 @@ SUFFIX_NOT_ACRONYMS = { # #269: Cyrillic мл/ст (junior/senior, the jr/sr analogs) deferred - # pending collision vetting -- the issue itself flags 'ст' as a - # two-letter sequence that collides with real Cyrillic words/ - # abbreviations, and both are short enough to risk false positives - # inside the Cyrillic vocabulary. Not shipped in this pass. + # pending the within-script collision vetting the issue asks for; + # 'ст' especially is a plausible false-positive risk (many two- + # letter Cyrillic abbreviations exist), and both are short enough + # to worry about. Not shipped in this pass. 'dr', 'esq', 'esquire', diff --git a/nameparser/locales/tr_az.py b/nameparser/locales/tr_az.py index cfed583f..7e2133a6 100644 --- a/nameparser/locales/tr_az.py +++ b/nameparser/locales/tr_az.py @@ -3,8 +3,9 @@ uulu/kyzy and Cyrillic forms) lives inside the rule implementation in nameparser/_pipeline/_post_rules.py (mirrors v1's flag design). -Data sources: the v1 Turkic patronymic rule (its plan and test bank); -the pack carries no vocabulary. +Data sources: the v1 Turkic patronymic rule (issue #185 and its test +bank, tests/test_turkic_patronymic_order.py); the pack carries no +vocabulary. Declared deviations (spec §2 authoring requirement 3): applying this pack changes only NO_COMMA names where some token is a standalone From cf46e3cfd3f26b9dd59dd53f1543872bd5ae11af Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 00:38:48 -0700 Subject: [PATCH 162/275] test: simplify locale test scaffolding (/simplify review) Drop ~13 function-scope re-imports that restated module-scope names (hoisting Policy to the top); share one default-parser baseline (functools.cache) and module-level packed parsers across the gate and rotator tests, so the suite scales corpus + packs instead of corpus x packs; factor the per-pack layering ALLOWED tuples into a shared _LOCALE_PACK_ALLOWED constant mirroring _PIPELINE_STAGE_ALLOWED; and replace the rotator lists' per-row branch comments as the only coverage guarantee with a mechanical test that every alternation branch of every marker regex is exercised by some rotator token. Document why the corpus loader stays a local copy of compare.py's convention. Co-Authored-By: Claude Fable 5 --- tests/v2/test_layering.py | 17 +++--- tests/v2/test_locales.py | 117 ++++++++++++++++++++++++-------------- 2 files changed, 82 insertions(+), 52 deletions(-) diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 6a5d15fc..fe6120f2 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -28,6 +28,13 @@ "nameparser._pipeline.", ) +# a locale pack: pure data over the three base types, no pipeline or +# config access (locales spec §2) -- one contract shared by every pack, +# like _PIPELINE_STAGE_ALLOWED above, so tightening it is a one-line edit +_LOCALE_PACK_ALLOWED = ( + "nameparser._lexicon", "nameparser._locale", "nameparser._policy", +) + # module -> prefixes it may import from within nameparser ALLOWED = { # call-time imports only (inside the rendering-delegate method @@ -82,14 +89,8 @@ # "nameparser.locales." prefix documents that reach for humans even # though the walker never exercises it. "locales/__init__.py": ("nameparser._locale", "nameparser.locales."), - # a locale pack: pure data over the three base types, no pipeline - # or config access (locales spec §2) - "locales/ru.py": ("nameparser._lexicon", "nameparser._locale", - "nameparser._policy"), - # a locale pack: pure data over the three base types, no pipeline - # or config access (locales spec §2) - "locales/tr_az.py": ("nameparser._lexicon", "nameparser._locale", - "nameparser._policy"), + "locales/ru.py": _LOCALE_PACK_ALLOWED, + "locales/tr_az.py": _LOCALE_PACK_ALLOWED, # v1 import-path preservation: thin re-exports of the facade/shim "parser.py": ("nameparser._facade",), "config/__init__.py": ("nameparser._config_shim",), diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index bf692ea6..936b1594 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -1,6 +1,8 @@ """The locale pack layer (locales spec §2-3): lazy access, the two 2.0.0 packs, composition, and the non-interference gate.""" +import functools import json +import re from collections.abc import Callable, Iterable from pathlib import Path @@ -8,10 +10,14 @@ from nameparser import Locale, Parser, locales, parse, parser_for from nameparser._lexicon import Lexicon -from nameparser._policy import PatronymicRule +from nameparser._policy import PatronymicRule, Policy from nameparser.locales import ru as _ru from nameparser.locales import tr_az as _tr_az +# one JSON-encoded name string per line, blank lines skipped -- the same +# convention tools/differential/compare.py::main() reads (kept as two +# small copies on purpose: tools/ is a standalone uv-run script tree, +# not an importable package, and making it one costs more than 3 lines) _CORPUS = [ json.loads(line) for line in (Path(__file__).parents[2] / "tools" / "differential" @@ -19,16 +25,29 @@ if line.strip() ] +# shared across the gate and rotator tests: the default-parser baseline +# is identical everywhere (only the packed side varies per test), so +# parse each name once for the whole module instead of once per test +_DEFAULT_PARSER = Parser() +_RU_PACKED = parser_for(locales.RU) +_TR_AZ_PACKED = parser_for(locales.TR_AZ) +_COMBINED_PACKED = parser_for(locales.RU, locales.TR_AZ) + + +@functools.cache +def _default_parse(name: str) -> dict: + """Default-parser components for `name` (treat as read-only -- the + dict is cached and shared across callers).""" + return _DEFAULT_PARSER.parse(name).as_dict() + def test_locales_module_attribute_access() -> None: - from nameparser import locales assert isinstance(locales.RU, Locale) assert locales.RU.code == "ru" assert locales.RU is locales.RU # cached, not rebuilt def test_locales_get_and_available() -> None: - from nameparser import locales assert locales.get("ru") is locales.RU assert set(locales.available()) == {"ru", "tr_az"} with pytest.raises(KeyError, match="ru, tr_az"): @@ -36,8 +55,6 @@ def test_locales_get_and_available() -> None: def test_tr_az_pack_contents() -> None: - from nameparser import locales - assert locales.TR_AZ.code == "tr_az" assert locales.TR_AZ.policy.patronymic_rules == frozenset( {PatronymicRule.TURKIC}) @@ -51,13 +68,12 @@ def test_pack_marker_regexes_stay_in_sync_with_post_rules() -> None: # equality mechanically so drift fails here, not at the L6 # non-interference gate. from nameparser._pipeline import _post_rules - from nameparser.locales import ru, tr_az for pack_regex, rule_regex in ( - (ru._EAST_SLAVIC, _post_rules._EAST_SLAVIC), - (ru._EAST_SLAVIC_CYR, _post_rules._EAST_SLAVIC_CYR), - (tr_az._TURKIC, _post_rules._TURKIC), - (tr_az._TURKIC_CYR, _post_rules._TURKIC_CYR), + (_ru._EAST_SLAVIC, _post_rules._EAST_SLAVIC), + (_ru._EAST_SLAVIC_CYR, _post_rules._EAST_SLAVIC_CYR), + (_tr_az._TURKIC, _post_rules._TURKIC), + (_tr_az._TURKIC_CYR, _post_rules._TURKIC_CYR), ): assert pack_regex.pattern == rule_regex.pattern assert pack_regex.flags == rule_regex.flags @@ -69,14 +85,12 @@ def test_ru_deviates_scans_per_token() -> None: # suffix -- under-declaration, the unsafe direction for the # non-interference gate. The pack rotates this name; DEVIATES must # say so. - from nameparser.locales import ru - - assert ru.DEVIATES("Ivan Petr Sidorovich Jr.") + assert _ru.DEVIATES("Ivan Petr Sidorovich Jr.") # Over-declaration is the accepted safe direction: the ending # matches a token although the rule's 1 given + 1 middle + # 1 family shape never fires on a two-token name. - assert ru.DEVIATES("Sidorovich Anna") - assert not ru.DEVIATES("John Smith") + assert _ru.DEVIATES("Sidorovich Anna") + assert not _ru.DEVIATES("John Smith") def test_tr_az_deviates_scans_per_token() -> None: @@ -84,29 +98,23 @@ def test_tr_az_deviates_scans_per_token() -> None: # fullmatch-shaped (^...$), so anything but a per-token scan would # miss a marker followed by a suffix (under-declaration) -- and a # whole-string match would never fire at all on multi-token names. - from nameparser.locales import tr_az - - assert tr_az.DEVIATES("Mammadova Aygun Ali kizi Jr.") + assert _tr_az.DEVIATES("Mammadova Aygun Ali kizi Jr.") # over-declaration stays the safe direction: a bare marker-shaped # token declares even where the rule won't rewrite anything - assert tr_az.DEVIATES("Kizi Anna") - assert not tr_az.DEVIATES("John Smith") + assert _tr_az.DEVIATES("Kizi Anna") + assert not _tr_az.DEVIATES("John Smith") # markers are whole-token: a name merely CONTAINING one must not # declare ("Ogluev" is not a marker) - assert not tr_az.DEVIATES("Ogluev Ivan") + assert not _tr_az.DEVIATES("Ogluev Ivan") def test_locales_unknown_attribute() -> None: - from nameparser import locales with pytest.raises(AttributeError, match="XX"): locales.XX def test_ru_plus_tr_az_unions_patronymic_rules() -> None: - from nameparser import locales, parser_for - from nameparser._policy import PatronymicRule - - p = parser_for(locales.RU, locales.TR_AZ) + p = _COMBINED_PACKED assert p.policy.patronymic_rules == frozenset( {PatronymicRule.EAST_SLAVIC, PatronymicRule.TURKIC}) # both rules live: one name from each pack's case segment @@ -118,8 +126,6 @@ def test_ru_plus_tr_az_unions_patronymic_rules() -> None: def test_pack_over_custom_base() -> None: - from nameparser import Lexicon, Parser, locales, parser_for - base = Parser(lexicon=Lexicon.default().add(titles={"zqxcustom"})) p = parser_for(locales.RU, base=base) n = p.parse("Zqxcustom Иван Петрович") @@ -130,9 +136,6 @@ def test_pack_preserves_custom_base_policy() -> None: # the lexicon-survival twin above has a policy counterpart: a # policy-only pack must ADD its patronymic rule without resetting # unrelated policy fields the base customized - from nameparser import Parser, locales, parser_for - from nameparser._policy import Policy - base = Parser(policy=Policy(strip_emoji=False)) p = parser_for(locales.RU, base=base) assert p.policy.strip_emoji is False @@ -143,9 +146,7 @@ def test_pack_preserves_custom_base_policy() -> None: def test_parser_for_results_chain_as_bases() -> None: # parser_for's result is itself a valid base= -- packs applied in # two steps accumulate exactly like one call with both packs - from nameparser import locales, parser_for - - chained = parser_for(locales.TR_AZ, base=parser_for(locales.RU)) + chained = parser_for(locales.TR_AZ, base=_RU_PACKED) assert chained.policy.patronymic_rules == frozenset( {PatronymicRule.EAST_SLAVIC, PatronymicRule.TURKIC}) assert chained.parse("Сидоров Иван Петрович").given == "Иван" @@ -179,10 +180,9 @@ def _assert_non_interference( ) -> int: """Return the number of DECLARED deviations seen; fail on any undeclared one (spec §5.2 = the pack-acceptance rejection rule).""" - default = Parser() declared = 0 for name in corpus: - base = default.parse(name).as_dict() + base = _default_parse(name) got = packed.parse(name).as_dict() if got != base: assert deviates(name), ( @@ -257,27 +257,56 @@ def _default_corpus() -> list[str]: ] +def _alternation_branches(pattern: str) -> list[str]: + """Split a marker regex of the shape '(a|b|...)$' / '^(a|b|...)$' + into its alternatives. The packs' character classes contain no '|', + so a flat split is safe (and the shape assert guards the day one + stops being true).""" + inner = pattern.removeprefix("^").removesuffix("$") + assert inner.startswith("(") and inner.endswith(")") and "|" not in ( + pattern.replace(inner, "")) + return inner[1:-1].split("|") + + +@pytest.mark.parametrize("regex, rotators, anchored", [ + (_ru._EAST_SLAVIC, _RU_ROTATORS, False), + (_ru._EAST_SLAVIC_CYR, _RU_ROTATORS, False), + (_tr_az._TURKIC, _TR_AZ_ROTATORS, True), + (_tr_az._TURKIC_CYR, _TR_AZ_ROTATORS, True), +], ids=["east-slavic", "east-slavic-cyr", "turkic", "turkic-cyr"]) +def test_rotators_cover_every_marker_branch( + regex: re.Pattern, rotators: list[str], anchored: bool) -> None: + # The rotator lists' per-row branch comments are a human convention; + # this enforces them mechanically: every alternation branch of every + # marker regex must be hit by some rotator token, so a regex edit + # that adds a branch (kept in sync with _post_rules by the pattern- + # equality test above) fails HERE until a rotator name covers it. + tokens = [tok for name in rotators for tok in name.split()] + for branch in _alternation_branches(regex.pattern): + shape = f"^({branch})$" if anchored else f"({branch})$" + branch_re = re.compile(shape, re.I) + assert any(branch_re.search(tok) for tok in tokens), ( + f"no rotator name exercises marker branch {branch!r}") + + @pytest.mark.parametrize("name", _RU_ROTATORS) def test_ru_rotator_deviates_and_declares(name: str) -> None: # every branch of the pack's marker regexes both FIRES (packed parse # differs from default) and is DECLARED (DEVIATES says so) -- the # per-name proof behind the gate's declared-count floor below - packed = parser_for(locales.RU) - assert packed.parse(name).as_dict() != Parser().parse(name).as_dict() + assert _RU_PACKED.parse(name).as_dict() != _default_parse(name) assert _ru.DEVIATES(name) @pytest.mark.parametrize("name", _TR_AZ_ROTATORS) def test_tr_az_rotator_deviates_and_declares(name: str) -> None: - packed = parser_for(locales.TR_AZ) - assert packed.parse(name).as_dict() != Parser().parse(name).as_dict() + assert _TR_AZ_PACKED.parse(name).as_dict() != _default_parse(name) assert _tr_az.DEVIATES(name) def test_non_interference_ru() -> None: corpus = _default_corpus() + _RU_ROTATORS - declared = _assert_non_interference( - parser_for(locales.RU), _ru.DEVIATES, corpus) + declared = _assert_non_interference(_RU_PACKED, _ru.DEVIATES, corpus) # the positive side: every synthetic rotator (plus the corpus's own # east-slavic names) must actually flow through the declared branch assert declared >= len(_RU_ROTATORS) @@ -286,14 +315,14 @@ def test_non_interference_ru() -> None: def test_non_interference_tr_az() -> None: corpus = _default_corpus() + _TR_AZ_ROTATORS declared = _assert_non_interference( - parser_for(locales.TR_AZ), _tr_az.DEVIATES, corpus) + _TR_AZ_PACKED, _tr_az.DEVIATES, corpus) assert declared >= len(_TR_AZ_ROTATORS) def test_non_interference_combined() -> None: corpus = _default_corpus() + _RU_ROTATORS + _TR_AZ_ROTATORS declared = _assert_non_interference( - parser_for(locales.RU, locales.TR_AZ), + _COMBINED_PACKED, lambda n: _ru.DEVIATES(n) or _tr_az.DEVIATES(n), corpus) assert declared >= len(_RU_ROTATORS) + len(_TR_AZ_ROTATORS) From d4aaafa24971d754ba2d0e4f43989e3d11134c22 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 00:53:13 -0700 Subject: [PATCH 163/275] test: make the pack registry the gate contract (design note, option C) The non-interference gate, rotator lists, and branch-coverage test now iterate the registered packs (_REGISTRY -> _PACKS/_PACKED/_ROTATORS dicts) instead of hand-wiring each pack by name: the per-pack gate tests collapse into one parametrized test, the combined test derives its predicate by OR-ing every registered DEVIATES, and marker regexes are discovered as the pack module's re.Pattern attributes (anchoring read from the pattern's own shape). A contract meta-test requires every registered pack to declare DEVIATES and ship a rotator list, so pack #3 fails structurally until it does -- accepted option C of the 2026-07-18 DEVIATES design note, keeping Locale pure data. Co-Authored-By: Claude Fable 5 --- tests/v2/test_locales.py | 146 +++++++++++++++++++++++---------------- 1 file changed, 86 insertions(+), 60 deletions(-) diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 936b1594..295df832 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -1,10 +1,12 @@ """The locale pack layer (locales spec §2-3): lazy access, the two 2.0.0 packs, composition, and the non-interference gate.""" import functools +import importlib import json import re from collections.abc import Callable, Iterable from pathlib import Path +from types import ModuleType import pytest @@ -25,13 +27,26 @@ if line.strip() ] +# The registry is the pack contract (design note 2026-07-18, option C): +# the DEVIATES requirement, the rotator lists, and the non-interference +# gate below all iterate the registered packs, so adding a pack is one +# registry entry plus one rotator list -- the contract meta-test fails +# until both exist, instead of a human remembering to extend N tests. +def _pack_modules() -> dict[str, ModuleType]: + out: dict[str, ModuleType] = {} + for modname, attr in locales._REGISTRY.values(): + module = importlib.import_module(modname) + out[getattr(module, attr).code] = module + return out + + +_PACKS = _pack_modules() + # shared across the gate and rotator tests: the default-parser baseline # is identical everywhere (only the packed side varies per test), so # parse each name once for the whole module instead of once per test _DEFAULT_PARSER = Parser() -_RU_PACKED = parser_for(locales.RU) -_TR_AZ_PACKED = parser_for(locales.TR_AZ) -_COMBINED_PACKED = parser_for(locales.RU, locales.TR_AZ) +_PACKED = {code: parser_for(locales.get(code)) for code in _PACKS} @functools.cache @@ -114,7 +129,7 @@ def test_locales_unknown_attribute() -> None: def test_ru_plus_tr_az_unions_patronymic_rules() -> None: - p = _COMBINED_PACKED + p = parser_for(locales.RU, locales.TR_AZ) assert p.policy.patronymic_rules == frozenset( {PatronymicRule.EAST_SLAVIC, PatronymicRule.TURKIC}) # both rules live: one name from each pack's case segment @@ -146,7 +161,7 @@ def test_pack_preserves_custom_base_policy() -> None: def test_parser_for_results_chain_as_bases() -> None: # parser_for's result is itself a valid base= -- packs applied in # two steps accumulate exactly like one call with both packs - chained = parser_for(locales.TR_AZ, base=_RU_PACKED) + chained = parser_for(locales.TR_AZ, base=_PACKED["ru"]) assert chained.policy.patronymic_rules == frozenset( {PatronymicRule.EAST_SLAVIC, PatronymicRule.TURKIC}) assert chained.parse("Сидоров Иван Петрович").given == "Иван" @@ -199,14 +214,15 @@ def _default_corpus() -> list[str]: if c.locale is None and c.policy is None] -# Synthetic rotator names: one per alternation branch of each pack's -# marker regexes (both scripts), in the shape that makes the rule fire -# (RU: family-first, patronymic last, plain middle; TR_AZ: a standalone -# marker token). They serve the gate's POSITIVE side -- without them the -# shared corpus exercises TR_AZ with a single name, leaving the gate -# nearly vacuous (review finding, 2026-07-17). Branch named per row so a -# regex edit that orphans a branch is visible here. -_RU_ROTATORS = [ +# Synthetic rotator names, keyed by pack code: one per alternation +# branch of each pack's marker regexes (both scripts), in the shape that +# makes the rule fire (RU: family-first, patronymic last, plain middle; +# TR_AZ: a standalone marker token). They serve the gate's POSITIVE side +# -- without them the shared corpus exercises TR_AZ with a single name, +# leaving the gate nearly vacuous (review finding, 2026-07-17). Branch +# named per row, enforced by the branch-coverage test below. +_ROTATORS: dict[str, list[str]] = {} +_ROTATORS["ru"] = [ "Sidorov Ivan Petrovich", # ovich "Sidorova Anna Petrovna", # ovna "Karpov Oleg Sergeevich", # evich @@ -228,7 +244,7 @@ def _default_corpus() -> list[str]: "Орлов Семён Фомич", # фомич "Зайцев Андрей Фокич", # фокич ] -_TR_AZ_ROTATORS = [ +_ROTATORS["tr_az"] = [ "Aliyev Ali Vali oglu", # oglu "Aliyev Rashad Vali oğlu", # oğlu "Mammadov Elchin Hasan ogly", # ogly @@ -257,6 +273,20 @@ def _default_corpus() -> list[str]: ] +def test_registry_is_the_pack_contract() -> None: + # spec §2 authoring requirement 3, enforced structurally (design + # note 2026-07-18, option C): every registered pack module must + # declare its deviation surface, and every pack must feed the gate's + # positive side -- a new pack fails HERE until it ships both, rather + # than a reviewer remembering to extend the gate by hand. + assert set(_PACKS) == set(locales.available()) + for code, module in _PACKS.items(): + assert callable(getattr(module, "DEVIATES", None)), ( + f"pack {code!r} does not declare DEVIATES") + assert set(_ROTATORS) == set(_PACKS), ( + "every pack needs a rotator list (and only registered packs)") + + def _alternation_branches(pattern: str) -> list[str]: """Split a marker regex of the shape '(a|b|...)$' / '^(a|b|...)$' into its alternatives. The packs' character classes contain no '|', @@ -268,64 +298,60 @@ def _alternation_branches(pattern: str) -> list[str]: return inner[1:-1].split("|") -@pytest.mark.parametrize("regex, rotators, anchored", [ - (_ru._EAST_SLAVIC, _RU_ROTATORS, False), - (_ru._EAST_SLAVIC_CYR, _RU_ROTATORS, False), - (_tr_az._TURKIC, _TR_AZ_ROTATORS, True), - (_tr_az._TURKIC_CYR, _TR_AZ_ROTATORS, True), -], ids=["east-slavic", "east-slavic-cyr", "turkic", "turkic-cyr"]) -def test_rotators_cover_every_marker_branch( - regex: re.Pattern, rotators: list[str], anchored: bool) -> None: +@pytest.mark.parametrize("code", sorted(_PACKS)) +def test_rotators_cover_every_marker_branch(code: str) -> None: # The rotator lists' per-row branch comments are a human convention; # this enforces them mechanically: every alternation branch of every - # marker regex must be hit by some rotator token, so a regex edit - # that adds a branch (kept in sync with _post_rules by the pattern- - # equality test above) fails HERE until a rotator name covers it. - tokens = [tok for name in rotators for tok in name.split()] - for branch in _alternation_branches(regex.pattern): - shape = f"^({branch})$" if anchored else f"({branch})$" - branch_re = re.compile(shape, re.I) - assert any(branch_re.search(tok) for tok in tokens), ( - f"no rotator name exercises marker branch {branch!r}") - - -@pytest.mark.parametrize("name", _RU_ROTATORS) -def test_ru_rotator_deviates_and_declares(name: str) -> None: + # marker regex a pack defines (any module-level re.Pattern) must be + # hit by some rotator token, so a regex edit that adds a branch + # (kept in sync with _post_rules by the pattern-equality test above) + # fails HERE until a rotator name covers it. Anchoring follows the + # pattern's own shape: '^(...)$' = whole-token marker, '(...)$' = + # token ending. + tokens = [tok for name in _ROTATORS[code] for tok in name.split()] + patterns = [v for v in vars(_PACKS[code]).values() + if isinstance(v, re.Pattern)] + assert patterns, f"pack {code!r} defines no marker regexes" + for regex in patterns: + for branch in _alternation_branches(regex.pattern): + anchored = regex.pattern.startswith("^") + branch_re = re.compile( + f"^({branch})$" if anchored else f"({branch})$", re.I) + assert any(branch_re.search(tok) for tok in tokens), ( + f"no {code!r} rotator exercises marker branch {branch!r}") + + +@pytest.mark.parametrize("code, name", [ + (code, name) for code in sorted(_ROTATORS) + for name in _ROTATORS[code]]) +def test_rotator_deviates_and_declares(code: str, name: str) -> None: # every branch of the pack's marker regexes both FIRES (packed parse # differs from default) and is DECLARED (DEVIATES says so) -- the # per-name proof behind the gate's declared-count floor below - assert _RU_PACKED.parse(name).as_dict() != _default_parse(name) - assert _ru.DEVIATES(name) - + assert _PACKED[code].parse(name).as_dict() != _default_parse(name) + assert _PACKS[code].DEVIATES(name) -@pytest.mark.parametrize("name", _TR_AZ_ROTATORS) -def test_tr_az_rotator_deviates_and_declares(name: str) -> None: - assert _TR_AZ_PACKED.parse(name).as_dict() != _default_parse(name) - assert _tr_az.DEVIATES(name) - -def test_non_interference_ru() -> None: - corpus = _default_corpus() + _RU_ROTATORS - declared = _assert_non_interference(_RU_PACKED, _ru.DEVIATES, corpus) - # the positive side: every synthetic rotator (plus the corpus's own - # east-slavic names) must actually flow through the declared branch - assert declared >= len(_RU_ROTATORS) - - -def test_non_interference_tr_az() -> None: - corpus = _default_corpus() + _TR_AZ_ROTATORS +@pytest.mark.parametrize("code", sorted(_PACKS)) +def test_non_interference_each_pack(code: str) -> None: + corpus = _default_corpus() + _ROTATORS[code] declared = _assert_non_interference( - _TR_AZ_PACKED, _tr_az.DEVIATES, corpus) - assert declared >= len(_TR_AZ_ROTATORS) + _PACKED[code], _PACKS[code].DEVIATES, corpus) + # the positive side: every synthetic rotator (plus the corpus's own + # in-scope names) must actually flow through the declared branch + assert declared >= len(_ROTATORS[code]) -def test_non_interference_combined() -> None: - corpus = _default_corpus() + _RU_ROTATORS + _TR_AZ_ROTATORS +def test_non_interference_all_packs_combined() -> None: + all_rotators = [n for code in sorted(_ROTATORS) + for n in _ROTATORS[code]] + corpus = _default_corpus() + all_rotators + packed = parser_for(*(locales.get(code) for code in sorted(_PACKS))) declared = _assert_non_interference( - _COMBINED_PACKED, - lambda n: _ru.DEVIATES(n) or _tr_az.DEVIATES(n), + packed, + lambda n: any(m.DEVIATES(n) for m in _PACKS.values()), corpus) - assert declared >= len(_RU_ROTATORS) + len(_TR_AZ_ROTATORS) + assert declared >= len(all_rotators) # -- #269: non-Latin default vocabulary (Cyrillic, Greek, Arabic, Hebrew) -- From 0fd386c3890bef17e7a4ffb508aded221e865ff0 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 00:59:57 -0700 Subject: [PATCH 164/275] fix: point patronymic_rules=True at the v1 migration path v1's patronymic_name_order was a bool flag, so True is the likeliest wrong value a migrator passes to the 2.0 field; it failed loudly but with the raw "'bool' object is not iterable". Probe with iter() -- NOT a try/except around tuple()/frozenset(), which would also rewrite exceptions raised inside a caller's generator (pinned property) -- and raise a message naming the v1 flag and both working replacements. Same guard on PolicyPatch's union fields, with the v1 hint only where it applies. Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 39 +++++++++++++++++++++++++++++++++------ tests/v2/test_policy.py | 14 ++++++++++++++ 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index b6ad6a48..8493c871 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -85,12 +85,24 @@ def __post_init__(self) -> None: f"patronymic_rules must be an iterable of rule names, " f"not a bare string: {self.patronymic_rules!r}" ) - # Materialize before converting (the _normset pattern): a - # non-iterable raises its natural TypeError here, and an exception - # raised inside a caller's generator propagates untouched instead - # of being rewritten as an unknown-rule error. Only the enum - # lookup itself gets the enriched message, naming the offender. - items = tuple(self.patronymic_rules) + # Probe with iter() rather than wrapping tuple(): non-iterables + # (True especially -- v1's patronymic_name_order was a bool flag, + # so it's the likeliest wrong value here) get the migration- + # pointing message, while an exception raised inside a caller's + # generator still propagates untouched from the tuple() below + # instead of being rewritten. Only the enum lookup itself gets + # the unknown-rule message, naming the offender. + try: + rule_iter = iter(self.patronymic_rules) + except TypeError: + raise TypeError( + f"patronymic_rules must be an iterable of PatronymicRule " + f"names, got {self.patronymic_rules!r}; v1's " + f"patronymic_name_order=True becomes " + f"patronymic_rules={{PatronymicRule.EAST_SLAVIC}} " + f"(or parser_for(locales.RU))" + ) from None + items = tuple(rule_iter) rules = set() for r in items: try: @@ -232,6 +244,21 @@ def __post_init__(self) -> None: f"{f.name} must be an iterable, " f"not a bare string: {value!r}" ) + # same iter() probe as Policy: curated message for + # non-iterables (with the v1-flag hint where it applies), + # caller-generator exceptions propagate from frozenset() + try: + iter(value) + except TypeError: + hint = ( + "; v1's patronymic_name_order=True becomes " + "patronymic_rules={PatronymicRule.EAST_SLAVIC} " + "(or parser_for(locales.RU))" + if f.name == "patronymic_rules" else "" + ) + raise TypeError( + f"{f.name} must be an iterable, got {value!r}{hint}" + ) from None object.__setattr__(self, f.name, frozenset(value)) diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index eedce628..de929fa9 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -76,6 +76,20 @@ def test_patronymic_rules_rejects_bare_string_and_non_iterable() -> None: Policy(patronymic_rules=5) # type: ignore[arg-type] +def test_patronymic_rules_true_points_at_the_v1_migration() -> None: + # v1's patronymic_name_order was a BOOL flag, so True is the single + # likeliest wrong value a migrator passes here -- the message must + # name the v1 flag and the working replacements, not just say + # "not iterable". Same guard on the PolicyPatch path (packs). + with pytest.raises(TypeError, match="patronymic_name_order"): + Policy(patronymic_rules=True) # type: ignore[arg-type] + with pytest.raises(TypeError, match="patronymic_name_order"): + PolicyPatch(patronymic_rules=True) # type: ignore[arg-type] + # non-patronymic union fields get the generic curated message + with pytest.raises(TypeError, match="extra_suffix_delimiters"): + PolicyPatch(extra_suffix_delimiters=True) # type: ignore[arg-type] + + def test_policy_delimiters_coerce_to_frozensets() -> None: p = Policy(nickname_delimiters=[("(", ")")]) # type: ignore[arg-type] assert isinstance(p.nickname_delimiters, frozenset) From 17efbbe6aacd8a2b96ca91c2ccc590c063a5b939 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 01:06:49 -0700 Subject: [PATCH 165/275] docs+test: record the comparison_key casefold deviation from 1.4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1.4's comparison_key()/matches() lowercased; 2.0 casefolds (both the new API and the facade). The deviation is strictly more permissive -- ß/SS and final-sigma case pairs now compare equal -- and comparison is the one surface where casefold's aggressive folds are wanted: the key is opaque, so the vocabulary spelling-mutation casefold caused in storage (fixed to lower() in 3e6e08d) cannot happen here. Release-log behavior-changes entry + a pin test on both API paths, so the next casefold audit finds the intent recorded. Co-Authored-By: Claude Fable 5 --- docs/release_log.rst | 1 + tests/v2/test_types.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/docs/release_log.rst b/docs/release_log.rst index 35a683d4..583ef393 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -23,6 +23,7 @@ Release Log - Fix maiden-name markers (``née``/``nee``/``born``/``geb.``/``roz.``) being folded into ``middle``/``last`` in 1.x (e.g. ``"Jane Smith née Jones"`` → ``middle="Smith née"``); 2.0 recognizes the marker and routes the following name to the new ``maiden`` field (closes #274; ``tests/v2/cases.py`` row ``maiden_marker``; not present in the differential corpus) - Data change: ``ma``/``do`` added to ``suffix_acronyms_ambiguous`` -- ambiguous acronyms count as suffixes only when written with periods, so a bare common surname (``"Jack Ma"``, ``"Anh Do"``) keeps its family name under 2.0's keep-recognized-suffixes routing, matching 1.4's output (``tests/v2/cases.py`` row ``ambiguous_surname_acronyms``). Side effect: parenthesized/quoted ``"(MA)"``/``"(DO)"`` (no periods) no longer escape to ``suffix`` the way 1.x did -- they now fall through to nickname parsing like any other ambiguous-acronym delimited content. Not present in the differential corpus - Change suffix-delimiter rendering: with a custom ``Policy``/``Constants`` suffix delimiter configured (e.g. ``suffix_delimiter="/"``, ``"John Smith, RN/CRNA"``), 1.x split the token and rendered ``suffix="RN, CRNA"``; 2.0 keeps the no-space delimiter-core token whole (``suffix="RN/CRNA"``) -- role assignment is unchanged, only rendering differs (anti-#100, migration plan deviation 5; ``tests/v2/cases.py`` row ``suffix_delimiter_no_space_core``). Only fires with a non-default policy, so it does not appear in the (default-policy) differential corpus + - Change ``comparison_key()``/``matches()`` (both APIs) to fold components with ``str.casefold()`` where 1.4 used ``str.lower()``: Unicode case-pair forms now compare equal -- ``"STRASSE"`` matches ``"Straße"``, and a Greek final-sigma variant matches its regular-sigma form. Strictly more permissive (anything 1.4 matched still matches); parse output is unaffected -- vocabulary normalization itself uses ``lower()``, v1-parity (``tests/v2/test_types.py`` row ``test_matches_casefolds_unicode_case_pairs``) Everything else in the 486-name differential corpus (built from the v1 test banks as of commit ``2d5d8c2``, pre-dating the M12 test diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 84952eb3..7df3c4d9 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -303,6 +303,24 @@ def test_comparison_key_is_casefolded_canonical_seven_tuple() -> None: assert upper.comparison_key() == lower.comparison_key() +def test_matches_casefolds_unicode_case_pairs() -> None: + # Deliberate 1.4 deviation (release log): comparison folds with + # casefold() where 1.4 used lower(). Comparison is the one surface + # where casefold's aggressive folds are WANTED -- ß/SS and final- + # sigma forms are the same name under different case conventions, + # and the key is opaque, so the spelling-mutation bug casefold + # caused in vocabulary storage (which stays lower(), v1 lc parity + # -- see _lexicon._normalize) cannot happen here. + from nameparser import HumanName, parse + + assert parse("Hans Straße").matches("HANS STRASSE") # ß <-> SS + assert parse("Νίκος Παπαδόπουλος").matches("Νίκοσ Παπαδόπουλοσ") # ς <-> σ + # same fold on the facade path -- both APIs share the deviation + assert HumanName("Hans Straße").matches("HANS STRASSE") + key = parse("Hans Straße").comparison_key() + assert key == parse("HANS STRASSE").comparison_key() + + def test_token_rejects_bare_string_and_mapping_tags() -> None: # frozenset("particle") is the set(str) footgun: eight single chars. with pytest.raises(TypeError, match="bare string"): From 38b8dcb3f271072aa2104eb4271bf7c38c9ae52b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 01:09:40 -0700 Subject: [PATCH 166/275] style: ParsedName repr indents with 4 spaces, matching HumanName's The facade (and v1.4 before it) already indent the multi-line repr body with 4 spaces; ParsedName was the lone tab hold-out. Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 5 +++-- tests/v2/test_reprs.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/nameparser/_types.py b/nameparser/_types.py index 6dd26cc2..76fce80f 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -297,14 +297,15 @@ def __str__(self) -> str: return self.render() def __repr__(self) -> str: + # 4-space indent, matching HumanName's repr (v1 style) lines = [] for role in Role: text = self._text_for(role) if text: - lines.append(f"\t{role.value}: {text!r}") + lines.append(f" {role.value}: {text!r}") if self.ambiguities: kinds = [a.kind.value for a in self.ambiguities] - lines.append(f"\tambiguities: {kinds!r}") + lines.append(f" ambiguities: {kinds!r}") body = "\n".join(lines) return f"" if lines else "" diff --git a/tests/v2/test_reprs.py b/tests/v2/test_reprs.py index 61c31e17..60b41ea0 100644 --- a/tests/v2/test_reprs.py +++ b/tests/v2/test_reprs.py @@ -23,7 +23,7 @@ def test_parsedname_repr_lists_nonempty_fields_in_canonical_order() -> None: Token("Smith", Span(5, 10), Role.FAMILY), )) assert repr(pn) == ( - "" + "" ) From bcd58b7591aceda06000e5173fac82d9c399c2fc Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 01:53:06 -0700 Subject: [PATCH 167/275] =?UTF-8?q?docs:=20add=20concepts=20page=20?= =?UTF-8?q?=E2=80=94=20the=202.0=20model=20in=20four=20essays?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- docs/concepts.rst | 123 ++++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 124 insertions(+) create mode 100644 docs/concepts.rst diff --git a/docs/concepts.rst b/docs/concepts.rst new file mode 100644 index 00000000..f25e8c0c --- /dev/null +++ b/docs/concepts.rst @@ -0,0 +1,123 @@ +How the parser thinks +====================== + +Four short essays on the model behind the 2.0 API: how a string +becomes a :class:`~nameparser.ParsedName`, why configuration lives in +exactly three places sorted by what varies, why parsers are plain +values, and what the parser does when a name is genuinely ambiguous. + +From string to name +-------------------- + +The pipeline, in one breath: the original string becomes a sequence of +tokens with spans (character positions into the original string), +each span gets assigned a role — ``title``, ``given``, ``middle``, +``family``, ``suffix``, ``nickname``, ``maiden`` — and every string you +read off a :class:`~nameparser.ParsedName` (``.given``, ``.family``, +``str(name)``, and so on) is a view computed over those roles at read +time, not stored text. + +Spans matter because of what they replace. v1's ``HumanName`` worked +on plain lists of strings and re-found pieces by searching the list +for a matching value — so a name with a repeated word could make the +parser rewrite the wrong occurrence. That whole bug family (starting +with `issue #100 `_) +traces back to value-based lookup. A token's span is a position, not a +value, so nothing in the 2.0 pipeline is ever re-found by scanning for +a string that looks like it. + +:class:`~nameparser.ParsedName` is frozen: there is no attribute +assignment, ever. If a parse is almost right and you want to fix one +field, you call ``.replace()``, which returns a new +:class:`~nameparser.ParsedName` with that field changed and everything +else — tokens, spans, the rest of the roles — carried over unchanged. +``str()`` renders the default view; nothing about calling it mutates +the value you called it on. + +The three containers +--------------------- + +Every piece of nameparser configuration falls into exactly one of +three places, and which one is decided by a single question: what does +this setting vary with? + +* varies by **language** → :class:`~nameparser.Lexicon` (vocabulary: + titles, particles, suffixes, conjunctions, and the rest of the + word lists the parser matches against) +* varies by **data source or application** → :class:`~nameparser.Policy` + (behavior switches: name order, patronymic rules, delimiters, strip + flags — anything that changes how the pipeline runs, not what words + it recognizes) +* varies by **output destination** → a rendering argument (the + ``spec`` you pass to ``render(spec)``, or a keyword to + ``initials()``/``capitalized()``) + +"Chancellor" being a title is a fact about German-language input, not +about any one dataset or report — that's a :class:`~nameparser.Lexicon` +entry. A CRM that always exports "Family, Given" strings is a fact +about that one data source, not about the language of the names in +it — that's a :class:`~nameparser.Policy`. One particular report +wanting names formatted as "Family, Given" while every other consumer +of the same parsed data wants "Given Family" is a fact about where the +string is going next, decided at the moment you render it — that's a +rendering argument, not something baked into how the name was parsed. + +This replaces v1's single ``Constants`` object, which mixed all three +concerns — vocabulary, behavior, and output formatting — into one +mutable bag plus a ``string_format`` template string. Sorting a +setting into the right container is largely mechanical once you ask +the "varies by what?" question above; see :doc:`migrate` for the +attribute-by-attribute mapping from the old ``Constants`` to the new +containers. + +Parsers are values +-------------------- + +``parse()`` is a convenience function over a module-level default +:class:`~nameparser.Parser`. You only need to build your own +:class:`~nameparser.Parser` (directly, or via +:func:`~nameparser.parser_for`) when you want non-default vocabulary +or behavior — and when you do, build it once and reuse it. Constructing +a :class:`~nameparser.Parser` validates its configuration up front, so +it's cheap but not free; parsing individual names is the hot path, and +a :class:`~nameparser.Parser` is designed to be called many times +without reconstruction. + +:class:`~nameparser.Lexicon`, :class:`~nameparser.Policy`, +:class:`~nameparser.Parser`, and :class:`~nameparser.ParsedName` are +all frozen and hashable. That means they're safe to share across +threads without locking, safe to use as dict keys or cache keys, and +equality means exactly what it says — two :class:`~nameparser.Parser` +instances built from equal configuration are equal values, not merely +two objects that happen to behave the same. Every piece of +configuration in the 2.0 API is a frozen value — including the +module-level default parser itself. + +Honest ambiguity +------------------ + +Parsing never raises. Pass in a string that doesn't look like a name +at all, and you get back a :class:`~nameparser.ParsedName` with empty +fields, not an exception. The parser's job is to make a reasonable +call on real-world text, not to reject it. + +Some calls are irreducibly ambiguous — both readings are legitimate, +and no amount of rule-tuning resolves them without breaking some other +name. Those surface as entries on ``ParsedName.ambiguities`` instead +of being silently guessed away. The canonical example: a leading "Van" +in "Van Johnson" reads as a given name (that's the common case for +that shape), but "Van" is also a family-name particle in plenty of +other names, so the parse records a ``particle-or-given`` ambiguity +alongside its answer. You can inspect ``ambiguities`` to decide, case +by case, whether your data needs a second look. + +Tokens also carry tags — a second, independent label alongside their +role, recording how a token was classified rather than what part of +the name it belongs to — but only a handful of them are part of the +stable API: ``particle``, ``conjunction``, ``initial``, and ``joined``. +Any tag written with a namespace prefix, like ``vocab:...``, is +provenance information for debugging how a token got classified — it +can change shape between releases and isn't something to match against +in your own code. If you need to branch on how a token was +classified, branch on role or on one of the four stable tags above, +not on a namespaced one. diff --git a/docs/index.rst b/docs/index.rst index f7ce0760..59fc5d0f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -57,6 +57,7 @@ Parsing Names :maxdepth: 2 usage + concepts customize **Developer Documentation** From 684f854110abee7c508e4aefd1422d605130d25f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 02:10:03 -0700 Subject: [PATCH 168/275] docs: rewrite usage.rst as the 2.0 getting-started tour Co-Authored-By: Claude Fable 5 --- docs/usage.rst | 377 +++++++++++-------------------------------------- 1 file changed, 84 insertions(+), 293 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 8c9ae36e..d295c06e 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1,328 +1,119 @@ -Using the HumanName Parser -========================== +Getting started +=============== -Example Usage -------------- +Requires Python 3.11+. ``pip install nameparser`` -Requires Python 3.10+. +Parse a name +------------ .. doctest:: - :options: +NORMALIZE_WHITESPACE - >>> from nameparser import HumanName - >>> name = HumanName("Dr. Juan Q. Xavier de la Vega III") - >>> name.title - 'Dr.' - >>> name["title"] - 'Dr.' - >>> name.first - 'Juan' - >>> name.middle - 'Q. Xavier' - >>> name.last - 'de la Vega' - >>> name.last_base - 'Vega' - >>> name.last_prefixes - 'de la' - >>> name.suffix - 'III' - >>> name.surnames - 'Q. Xavier de la Vega' - >>> name.given_names - 'Juan Q. Xavier' - >>> name.full_name = "Juan Q. Xavier Velasquez y Garcia, Jr." - >>> name - - >>> name.middle = "Jason Alexander" - >>> name.middle - 'Jason Alexander' - >>> name - - >>> name.middle = ["custom","values"] - >>> name.middle - 'custom values' - >>> name.full_name = 'Doe-Ray, Jonathan "John" A. Harris' - >>> name.as_dict() - {'title': '', 'first': 'Jonathan', 'middle': 'A. Harris', 'last': 'Doe-Ray', 'suffix': '', 'nickname': 'John', 'maiden': ''} - >>> name.as_dict(False) # add False to hide keys with empty values - {'first': 'Jonathan', 'middle': 'A. Harris', 'last': 'Doe-Ray', 'nickname': 'John'} - >>> name = HumanName("Dr. Juan Q. Xavier de la Vega III") - >>> name2 = HumanName("de la vega, dr. juan Q. xavier III") - >>> name.matches(name2) - True - >>> name.matches("de la vega, dr. juan Q. xavier III") - True - -``name == other`` and ``hash(name)`` are deprecated and will be removed in -2.0; use ``matches()`` for comparison and ``comparison_key()`` for sets, -dicts, and dedup (see `issue #223 -`_). Slicing a name -by position (``name[1:-3]``) and item assignment (``name['first'] = ...``) -are likewise deprecated and will be removed in 2.0; use the named attributes -instead (see `issue #258 -`_). - -Empty or unparsable input does not raise an error; it produces a name whose -attributes are all empty. Check ``len(name) == 0`` (or ``str(name) == ''``) -to detect that nothing was parsed. - - -Capitalization Support ----------------------- - -The HumanName class can try to guess the correct capitalization of name -entered in all upper or lower case. By default, it will not adjust -the case of names entered in mixed case. To run capitalization on a -`HumanName` instance, pass the parameter `force=True`. - - Capitalize the name. + >>> from nameparser import parse + >>> name = parse("Dr. Juan Q. Xavier de la Vega III") + >>> name.given, name.family + ('Juan', 'de la Vega') + >>> name.title, name.middle, name.suffix + ('Dr.', 'Q. Xavier', 'III') - * bob v. de la macdole-eisenhower phd -> Bob V. de la MacDole-Eisenhower Ph.D. +A parsed name has seven fields: ``title``, ``given``, ``middle``, +``family``, ``suffix``, ``nickname``, and ``maiden``. Parsing never +raises; unparseable input yields a :class:`~nameparser.ParsedName` with +empty fields plus any ``ambiguities`` the parser noticed along the way +(see :doc:`concepts`). -.. doctest:: capitalize - - >>> name = HumanName("bob v. de la macdole-eisenhower phd") - >>> name.capitalize() - >>> str(name) - 'Bob V. de la MacDole-Eisenhower Ph.D.' - >>> name = HumanName('Shirley Maclaine') # Don't change mixed case names - >>> name.capitalize() - >>> str(name) - 'Shirley Maclaine' - >>> name.capitalize(force=True) - >>> str(name) - 'Shirley MacLaine' - -To apply capitalization to all `HumanName` instances, set -:py:attr:`~nameparser.config.Constants.capitalize_name` to `True`. - -.. doctest:: capitalize_name - :options: +NORMALIZE_WHITESPACE - - >>> from nameparser.config import CONSTANTS - >>> CONSTANTS.capitalize_name = True - >>> name = HumanName("bob v. de la macdole-eisenhower phd") - >>> str(name) - 'Bob V. de la MacDole-Eisenhower Ph.D.' - >>> CONSTANTS.capitalize_name = False - -To force the capitalization of mixed case strings on all `HumanName` instances, -set :py:attr:`~nameparser.config.Constants.force_mixed_case_capitalization` to `True`. +Aggregate views +---------------- -.. doctest:: force_mixed_case_capitalization - :options: +NORMALIZE_WHITESPACE +.. doctest:: - >>> from nameparser.config import CONSTANTS - >>> CONSTANTS.force_mixed_case_capitalization = True - >>> name = HumanName('Shirley Maclaine') - >>> name.capitalize() - >>> str(name) - 'Shirley MacLaine' - >>> CONSTANTS.force_mixed_case_capitalization = False + >>> name.given_names # given + middle + 'Juan Q. Xavier' + >>> name.family_base, name.family_particles # family, split apart + ('Vega', 'de la') +``surnames`` (``middle`` + ``family``) is the mirror-image aggregate. +The plural is the tell: ``given`` and ``family`` are single fields, +while ``given_names`` and ``surnames`` roll several fields together — +the same sense in which a passport form asks for your "given names" as +one blank that can hold more than one word. -Nickname Handling +Dicts and strings ------------------ -The content of parenthesis or quotes in the name will be -available from the nickname attribute. - -.. doctest:: nicknames - :options: +NORMALIZE_WHITESPACE - - >>> name = HumanName('Jonathan "John" A. Smith') - >>> name - - -Exception: content that looks like a suffix (a member of -:py:data:`~nameparser.config.SUFFIX_ACRONYMS` or -:py:data:`~nameparser.config.SUFFIX_NOT_ACRONYMS`, or anything ending in a -period) is treated as a suffix instead of a nickname, since that's usually -what's meant, e.g. a retired military title or a professional designation -written in parenthesis. - -.. doctest:: nicknames - :options: +NORMALIZE_WHITESPACE - - >>> name = HumanName('Andrew Perkins (MBA)') - >>> name - - -A few suffix acronyms, listed in -:py:data:`~nameparser.config.SUFFIX_ACRONYMS_AMBIGUOUS`, also work as common -given-name nicknames on their own (e.g. "JD", "Ed"). These stay nicknames -when found alone in parenthesis or quotes, since that's the more common -reading in that ambiguous context: - -.. doctest:: nicknames - :options: +NORMALIZE_WHITESPACE - - >>> name = HumanName('JEFFREY (JD) BRICKEN') - >>> name - - -Leading Period-Abbreviation Titles ------------------------------------ - -An unrecognized, multi-letter word ending in a period, found anywhere in the -leading title run (i.e. before the first name is set), is treated as a title --- this covers military ranks and other abbreviations that aren't in the -built-in titles list, including chained abbreviations like -``"Foo. Xyz. John Smith"``. Single-letter initials (``"J."``) and -internal-period abbreviations (``"E.T."``) are not affected, and the same -word appearing after the first name is left as a middle name. - -.. doctest:: leading_period_titles - :options: +NORMALIZE_WHITESPACE - - >>> name = HumanName("Major. Dona Smith") - >>> name - - >>> name = HumanName("J. Smith") - >>> name.first - 'J.' - >>> name.title - '' - -Change the output string with string formatting ------------------------------------------------ - -The string representation of a `HumanName` instance is controlled by its `string_format` attribute. -The default value, `"{title} {first} {middle} {last} {suffix} ({nickname})"`, includes parenthesis -around nicknames. Trailing commas and empty quotes and parenthesis are automatically removed if the -name has no nickname pieces. +.. doctest:: -You can change the default formatting for all `HumanName` instances by setting a new -:py:attr:`~nameparser.config.Constants.string_format` value on the shared -:py:class:`~nameparser.config.CONSTANTS` configuration instance. + >>> name.as_dict(include_empty=False) + {'title': 'Dr.', 'given': 'Juan', 'middle': 'Q. Xavier', 'family': 'de la Vega', 'suffix': 'III'} + >>> str(name) + 'Dr. Juan Q. Xavier de la Vega III' + >>> name.render("{family}, {given}") + 'de la Vega, Juan' + >>> name.initials() + 'J. Q. X. V.' -.. doctest:: string format +Fixing case +----------- - >>> from nameparser.config import CONSTANTS - >>> CONSTANTS.string_format = "{title} {first} ({nickname}) {middle} {last} {suffix}" - >>> name = HumanName('Robert Johnson') - >>> str(name) - 'Robert Johnson' - >>> name = HumanName('Robert "Rob" Johnson') - >>> str(name) - 'Robert (Rob) Johnson' +.. doctest:: -You can control the order and presence of any name fields by changing the -:py:attr:`~nameparser.config.Constants.string_format` attribute of the shared CONSTANTS instance. -Don't want to include nicknames in your output? No problem. Just omit that keyword from the -`string_format` attribute. + >>> str(parse("juan de la vega").capitalized()) + 'Juan de la Vega' -.. doctest:: string format +Nicknames and maiden names +---------------------------- - >>> from nameparser.config import CONSTANTS - >>> CONSTANTS.string_format = "{title} {first} {last}" - >>> name = HumanName("Dr. Juan Ruiz de la Vega III (Doc Vega)") - >>> str(name) - 'Dr. Juan de la Vega' +.. doctest:: + >>> parse("Jonathan 'Jack' Kennedy").nickname + 'Jack' + >>> parse("Jane Smith née Jones").maiden + 'Jones' -Initials Support +Comparing names ---------------- -The HumanName class can try to get the correct representation of initials. -Initials can be tricky as different format usages exist. -To exclude any of the name parts from the initials, change the initials format string: -:py:attr:`~nameparser.config.Constants.initials_format` -Three attributes exist for the format, `first`, `middle` and `last`. +``==`` is strict value equality — two :class:`~nameparser.ParsedName` +instances are equal only if every field matches exactly. For "is this +the same name, allowing for order and case?" use ``matches()`` or +``comparison_key()`` instead. -.. doctest:: initials format - - >>> from nameparser.config import CONSTANTS - >>> CONSTANTS.initials_format = "{first} {middle}" - >>> HumanName("Doe, John A. Kenneth, Jr.").initials() - 'J. A. K.' - >>> HumanName("Doe, John A. Kenneth, Jr.", initials_format="{last}, {first}").initials() - 'D., J.' - >>> CONSTANTS.initials_format = "{first} {middle} {last}" - - -Furthermore, the delimiter for the string output can be set through: -:py:attr:`~nameparser.config.Constants.initials_delimiter` +.. doctest:: -.. doctest:: initials delimiter + >>> parse("de la Vega, Juan").matches("Juan de la Vega") + True + >>> parse("JUAN DE LA VEGA").comparison_key() == parse("Juan de la Vega").comparison_key() + True - >>> HumanName("Doe, John A. Kenneth, Jr.", initials_delimiter=";").initials() - 'J; A; K; D;' +Correcting a parse +-------------------- -The separator between consecutive initials *within* a name group (e.g. two middle -names) is controlled by :py:attr:`~nameparser.config.Constants.initials_separator`, -which defaults to ``" "``. Setting it to ``""`` removes that space within a group; -spacing *between* groups is still governed by ``initials_format``. +:class:`~nameparser.ParsedName` is immutable, so a correction is a new +value: ``replace()`` returns a copy with the given fields changed and +everything else carried over. -``initials_delimiter``, ``initials_separator``, and ``initials_format`` work together: +.. doctest:: -- ``initials_delimiter`` — appended *after* each individual initial (default ``"."``) -- ``initials_separator`` — placed *after* the delimiter between consecutive initials in the same group (default ``" "``), so with ``delimiter="."`` and ``separator=" "`` you get ``A. K.`` -- ``initials_format`` — controls how the first, middle, and last groups are arranged + >>> name = parse("Juan de la Vega") + >>> corrected = name.replace(title="Dr.") + >>> str(corrected) + 'Dr. Juan de la Vega' + >>> name.title + '' -For example, to produce compact period-separated initials with no spaces: +Command line +------------ -.. doctest:: initials separator +:: - >>> HumanName("Doe, John A. Kenneth, Jr.", initials_separator="", initials_format="{first}{middle}{last}").initials() - 'J.A.K.D.' - >>> HumanName("Doe, John A. Kenneth, Jr.", initials_delimiter="", initials_separator="", initials_format="{first}{middle}{last}").initials() - 'JAKD' + $ python -m nameparser --json "Doe, John" -To get a list representation of the initials, use :py:meth:`~nameparser.HumanName.initials_list`. -This function is unaffected by :py:attr:`~nameparser.config.Constants.initials_format` +Add ``--locale`` to parse with a locale pack (for example ``--locale +ru``); see :doc:`locales`. -.. doctest:: list format +Where next +---------- - >>> HumanName("Doe, John A. Kenneth, Jr.", initials_delimiter=";").initials_list() - ['J', 'A', 'K', 'D'] - +* :doc:`concepts` — how the parser thinks +* :doc:`customize` — your own vocabulary and behavior +* :doc:`locales` — locale packs +* :doc:`migrate` — coming from 1.x ``HumanName`` From e9548252b42a35cdfb3b3e0ec7092c270981de28 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 02:19:47 -0700 Subject: [PATCH 169/275] docs: rewrite customize.rst principle-first for the 2.0 API Co-Authored-By: Claude Fable 5 --- docs/customize.rst | 714 +++++++++------------------------------------ 1 file changed, 134 insertions(+), 580 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index 22f954c0..f39e510b 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -1,603 +1,157 @@ -Customizing the Parser with Your Own Configuration -================================================== +Customizing the parser +======================= -:py:class:`~nameparser.config.Constants` is for application-level -configuration, set once at startup: the shared module-level ``CONSTANTS`` -instance is the only channel that reaches parses happening in code you don't -own -- helpers, pipelines, a third-party library using nameparser internally --- the same role ``logging`` and ``locale`` play elsewhere. For anything -scoped to one dataset, one library, or one test, pass your own ``Constants`` -instance instead -- see the three explicit forms under "Module-level Shared -Configuration Instance" below. +Every piece of nameparser configuration sorts into one of three places +by asking what it varies with: vocabulary varies by **language** +(:class:`~nameparser.Lexicon`), behavior varies by **data source or +application** (:class:`~nameparser.Policy`), and presentation varies by +**output destination** (a rendering argument). See :doc:`concepts` for +why the split is drawn there. -Recognition of titles, prefixes, suffixes and conjunctions is handled by -matching the lower case characters of a name piece with pre-defined sets -of strings located in :py:mod:`nameparser.config`. You can adjust -these predefined sets to help fine tune the parser for your dataset. - -Changing the Parser Constants ------------------------------ - -There are a few ways to adjust the parser configuration depending on your -needs. The config is available in two places. - -The first is via ``from nameparser.config import CONSTANTS``. - -.. doctest:: - - >>> from nameparser.config import CONSTANTS - >>> CONSTANTS # doctest: +ELLIPSIS - - -The other is the ``C`` attribute of a ``HumanName`` instance, e.g. -``hn.C``. +Vocabulary: Lexicon +-------------------- .. doctest:: - >>> from nameparser import HumanName - >>> hn = HumanName("Dean Robert Johns") - >>> hn.C # doctest: +ELLIPSIS - - -Both places are usually a reference to the same shared module-level -:py:class:`~nameparser.config.CONSTANTS` instance, depending on how you -instantiate the :py:class:`~nameparser.parser.HumanName` class (see below). - - - -Editable attributes of nameparser.config.CONSTANTS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -* :py:data:`~nameparser.config.TITLES` - Pieces that come before the name. Includes all `first_name_titles`. Cannot include things that may be first names. -* :py:data:`~nameparser.config.FIRST_NAME_TITLES` - Titles that, when followed by a single name, that name is a first name, e.g. "King David". -* :py:data:`~nameparser.config.SUFFIX_ACRONYMS` - Pieces that come at the end of the name that may or may not have periods separating the letters, e.g. "m.d.". -* :py:data:`~nameparser.config.SUFFIX_NOT_ACRONYMS` - Pieces that come at the end of the name that never have periods separating the letters, e.g. "Jr.". -* :py:data:`~nameparser.config.SUFFIX_ACRONYMS_AMBIGUOUS` - Acronym suffixes from ``SUFFIX_ACRONYMS`` that also plausibly work as a name on their own -- a given-name nickname (e.g. "JD", "Ed") or a common surname (e.g. "Ma", "Do"). An ambiguous acronym counts as a suffix only when written with periods: ``"M.A."`` is a credential, ``"Jack Ma"`` keeps its family name. When one appears alone in parenthesis or quotes (e.g. ``'JEFFREY (JD) BRICKEN'``), it's kept as a nickname rather than reclassified as a suffix, since that's the more common reading in ambiguous, delimiter-only context (see the "Nickname Handling" section in the usage guide). -* :py:data:`~nameparser.config.CONJUNCTIONS` - Connectors like "and" that join the preceding piece to the following piece. -* :py:data:`~nameparser.config.PREFIXES` - Connectors like "del" and "bin" that join to the following piece but not the preceding, similar to titles but can appear anywhere in the name. -* :py:data:`~nameparser.config.CAPITALIZATION_EXCEPTIONS` - Dictionary of pieces that do not capitalize the first letter, e.g. "Ph.D". -* :py:data:`~nameparser.config.REGEXES` - Regular expressions used to find words, initials, nicknames, etc. - -Each set-valued constant comes with :py:func:`~nameparser.config.SetManager.add`, :py:func:`~nameparser.config.SetManager.remove`, and :py:func:`~nameparser.config.SetManager.discard` methods for tuning -the constants for your project. These methods automatically lower case and -remove punctuation to normalize them for comparison. The two dict-valued -constants (``CAPITALIZATION_EXCEPTIONS`` and ``REGEXES``) are edited with -normal dict operations. - -Adding Custom Nickname Delimiters -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:py:meth:`~nameparser.parser.HumanName.parse_nicknames` recognizes delimiters -through two per-bucket collections: -:py:obj:`~nameparser.config.Constants.nickname_delimiters` (default: the -three built-ins -- ``quoted_word``, ``double_quotes`` and ``parenthesis``, -each resolved live from :py:attr:`~nameparser.config.Constants.regexes`, so -overriding e.g. ``CONSTANTS.regexes.parenthesis`` still works exactly as -before) and :py:obj:`~nameparser.config.Constants.maiden_delimiters` (empty -by default -- see "Routing to Maiden Name" below). - -In 2.0, custom delimiter patterns on ``Constants`` were removed; adding a -non-built-in key raises ``TypeError``. Custom delimiter *pairs* are a -first-class feature of the new API instead -- pass them to -:py:class:`~nameparser.Policy`: + >>> from nameparser import Lexicon, Parser + >>> lex = Lexicon.default().add(titles={"chancellor"}) + >>> Parser(lexicon=lex).parse("Chancellor Angela Merkel").title + 'Chancellor' + +:meth:`~nameparser.Lexicon.add` and :meth:`~nameparser.Lexicon.remove` +both return a new :class:`~nameparser.Lexicon` — the one you started +from (here, ``Lexicon.default()``) is never mutated. Every field +accepts a plain set of lowercase words, keyword by field name (``titles`` +above; ``particles``, ``suffix_words``, and the rest work the same +way) — see :doc:`modules` for the full field list. + +``capitalization_exceptions`` is the one pair-valued field — each entry +maps a lowercase key to its exact-cased replacement (``"phd"`` → +``"PhD"``), so it isn't a fit for ``add()``/``remove()``. Change it with +``dataclasses.replace()`` instead: ``dataclasses.replace(lex, +capitalization_exceptions=(("phd", "PhD"),))``. + +Two fields — ``suffix_acronyms_ambiguous`` and ``particles_ambiguous`` +— mark entries from ``suffix_acronyms`` and ``particles`` that are also +plausible as ordinary name words on their own (an acronym suffix that +doubles as a nickname, a particle that doubles as a given name). They +don't add new vocabulary by themselves; they narrow how an existing +entry is read when it appears alone. If you're not sure whether a word +you're adding is one of these ambiguous cases, leave it out — an +unrecognized word usually still parses reasonably, while a wrongly +disambiguated one silently picks the less likely reading. + +Behavior: Policy +----------------- + +When your data source or application needs different parsing behavior +— a different name order, stricter suffix rules, extra delimiters — +set it on :class:`~nameparser.Policy`, a small, closed set of fields, +listed below. + +.. list-table:: + :header-rows: 1 + :widths: 22 28 50 + + * - Field + - Type + - Effect + * - ``name_order`` + - ``tuple[Role, Role, Role]`` + - Assigns positional (no-comma) input to given/middle/family in + this order. Use the exported ``GIVEN_FIRST`` (default), + ``FAMILY_FIRST``, or ``FAMILY_FIRST_GIVEN_LAST`` constants. + * - ``patronymic_rules`` + - ``frozenset[PatronymicRule]`` + - Reorders patronymic-shaped names via opt-in detectors — East + Slavic formal order (``EAST_SLAVIC``) or Turkic reversed order + (``TURKIC``). Defaults to empty. + * - ``middle_as_family`` + - ``bool`` + - Folds ``middle`` into ``family`` instead of splitting them — + for naming systems with no middle-name concept. Defaults to + ``False``. + * - ``nickname_delimiters`` + - ``frozenset[tuple[str, str]]`` + - Routes content enclosed by these delimiter pairs to + ``nickname``. Defaults to quotes and parentheses. + * - ``maiden_delimiters`` + - ``frozenset[tuple[str, str]]`` + - Routes content enclosed by these delimiter pairs to ``maiden`` + instead. Defaults to empty — see the routing example below. + * - ``extra_suffix_delimiters`` + - ``frozenset[str]`` + - Adds separators (beyond a comma) that split suffix groups, e.g. + ``" - "`` for ``"Jane Smith, RN - CRNA"``. + * - ``lenient_comma_suffixes`` + - ``bool`` + - Allows a segment after a comma to be read as a suffix using + the lenient, initial-shaped test. Set ``False`` to require the + strict acronym form. + * - ``strip_emoji`` + - ``bool`` + - Removes emoji from the input before parsing. Defaults to + ``True``. + * - ``strip_bidi`` + - ``bool`` + - Removes bidirectional control characters from the input before + parsing. Defaults to ``True``. .. doctest:: >>> from nameparser import Parser, Policy >>> policy = Policy( - ... nickname_delimiters=Policy().nickname_delimiters | {("{", "}")}) - >>> Parser(policy=policy).parse("Benjamin {Ben} Franklin").nickname - 'Ben' - -Routing to Maiden Name -~~~~~~~~~~~~~~~~~~~~~~~ - -Parenthesized (or otherwise delimited) alternate/maiden surnames -- -``"Baker (Johnson), Jenny"`` -- go to ``nickname`` by default, same as any -other delimited content. To route a delimiter to the first-class ``maiden`` -field instead, move its key from ``nickname_delimiters`` to -``maiden_delimiters`` on a ``Constants`` instance (a plain ``dict.pop()`` + -assign -- this preserves the live link back to ``regexes`` for the three -built-ins) *before* parsing a name with it, the same way you'd configure -``patronymic_name_order`` or ``middle_name_as_last``: - -.. doctest:: - - >>> from nameparser import HumanName - >>> from nameparser.config import Constants - >>> C = Constants() - >>> C.maiden_delimiters['parenthesis'] = C.nickname_delimiters.pop('parenthesis') - >>> hn = HumanName("Baker (Johnson), Jenny", constants=C) - >>> hn.first, hn.last, hn.maiden - ('Jenny', 'Baker', 'Johnson') - -This also strips the parenthesized maiden name from the no-comma written -form, since routing happens before positional parsing: - -.. doctest:: - - >>> hn = HumanName("Jenny Baker (Johnson)", constants=C) - >>> hn.first, hn.last, hn.maiden - ('Jenny', 'Baker', 'Johnson') - -Routing an already-active built-in delimiter on an *existing* ``HumanName`` -instance and calling ``parse_full_name()`` again will not work: only the -``full_name`` setter resets the working copy of the name string back to the -original input, so re-parsing in place has nothing left for the moved -delimiter to match if it already matched during the first parse. Configure -the ``Constants`` first, as above. - -``maiden`` is not included in the default :py:obj:`~nameparser.config.Constants.string_format`, -so ``str(hn)`` is unaffected unless you add ``{maiden}`` to your own format. - -Other editable attributes -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -* :py:obj:`~nameparser.config.Constants.string_format` - controls output from `str()` -* :py:obj:`~nameparser.config.Constants.empty_attribute_default` - value returned by empty attributes, defaults to empty string -* :py:obj:`~nameparser.config.Constants.capitalize_name` - If set, applies :py:meth:`~nameparser.parser.HumanName.capitalize` to :py:class:`~nameparser.parser.HumanName` instance. -* :py:obj:`~nameparser.config.Constants.force_mixed_case_capitalization` - If set, forces the capitalization of mixed case strings when :py:meth:`~nameparser.parser.HumanName.capitalize` is called. -* :py:obj:`~nameparser.config.Constants.suffix_delimiter` - additional delimiter used to split suffix groups after comma-splitting, e.g. ``" - "`` for names like ``"Jane Smith, RN - CRNA"``. Defaults to ``None`` (disabled). -* :py:obj:`~nameparser.config.Constants.initials_separator` - string placed between consecutive initials within the same name group (after the delimiter). Defaults to ``" "``, so ``"A. K."``; set to ``""`` for compact ``"A.K."``. -* :py:obj:`~nameparser.config.Constants.patronymic_name_order` - If set, detects Russian formal-order names (``Surname GivenName Patronymic``) via a trailing East-Slavic patronymic suffix and rotates the parts to Western order (``first=GivenName``, ``middle=Patronymic``, ``last=Surname``). Also detects reversed-order Azerbaijani/Central-Asian Turkic patronymics (``Surname GivenName PatronymicRoot Marker``, e.g. ``oglu``/``qizi``). Opt-in; see subsections below. -* :py:obj:`~nameparser.config.Constants.middle_name_as_last` - If set, folds middle names into the last name (``.last`` becomes what ``.surnames`` already was, ``.middle`` becomes empty). Opt-in; see subsection below. - - -Russian Formal Name Order -~~~~~~~~~~~~~~~~~~~~~~~~~ - -By default the parser treats all three-part names as ``First Middle Last``. For -Russian data in formal order (``Surname GivenName Patronymic``), enable -``patronymic_name_order``:: - - >>> from nameparser import HumanName - >>> from nameparser.config import Constants - >>> C = Constants(patronymic_name_order=True) - >>> hn = HumanName("Ivanov Ivan Ivanovich", constants=C) - >>> hn.first, hn.middle, hn.last - ('Ivan', 'Ivanovich', 'Ivanov') - -Detection is anchored on a recognised East-Slavic patronymic suffix -(``-ovich``, ``-ovna``, ``-evich``, ``-evna``, ``-ichna``, and the irregular -forms ``-ilyich``, ``-kuzmich``, ``-lukich``, ``-fomich``, ``-fokich``; same -patterns in Cyrillic). A comma activates the parser's standard -Last, First Middle path, which already handles Russian formal order — -reordering is suppressed to avoid a double-transformation. - -**Opt-in tradeoff:** when the flag is on, any name whose last token happens to -end in a patronymic suffix is reordered — including Western names with -patronymic-form surnames such as ``"David Michael Abramovich"``. Enable this -flag only when your data is predominantly Russian formal-order names. - - -Turkic Patronymics -~~~~~~~~~~~~~~~~~~ - -Azerbaijani and Central-Asian formal names follow a different shape: a -4-word ``[Given] [Father's given name] [Marker] [Family]``, where the -marker is a standalone word (``oglu``/``oğlu`` "son of", -``qizi``/``qızı`` "daughter of", and further variants — see below), not a -bound suffix. The same ``patronymic_name_order`` flag also detects and -rotates the reversed, no-comma form of this shape:: - - >>> from nameparser import HumanName - >>> from nameparser.config import Constants - >>> C = Constants(patronymic_name_order=True) - >>> hn = HumanName("Aliyev Vusal Said oglu", constants=C) - >>> hn.first, hn.middle, hn.last - ('Vusal', 'Said oglu', 'Aliyev') - -Natural order (``"Vusal Said oglu Aliyev"``) and comma order -(``"Aliyev, Vusal Said oglu"``) already parse correctly without this flag -and are left unchanged. - -Detection is scoped strictly to the 4-token shape (single-token first/last, -exactly two middle tokens, last token a recognised marker) — matching the -East-Slavic guard's token-count strictness above. Unlike that guard, there's -no additional check on the given-name token, since Turkic markers are a -small, closed set unlikely to coincide with an ordinary given name (whereas -East-Slavic patronymic suffixes can coincide with real Western surnames). -Recognised markers cover common transliterations and native orthographies: -Latin ``oglu``, ``oğlu``, ``ogly``, ``ogli``, ``o'g'li`` (and its Uzbek -modifier-apostrophe and right-single-quote variants), ``qizi``, ``qızı``, -``kizi``, ``kyzy``, ``gyzy``, ``uly``, ``uulu``; and Cyrillic ``оглу``, -``оглы``, ``оғлу``, ``ўғли``, ``угли``, ``кызы``, ``гызы``, ``қызы``, -``қизи``, ``улы``, ``ұлы``, ``уулу``. Matching is case-insensitive. - - -Suppressing Middle Names -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Some naming systems have no middle-name concept — everything after the given -name is lineage or family (e.g. Arabic patronymic chaining: given + father + -grandfather + family). Enable ``middle_name_as_last`` to fold the middle name -into the last name instead of splitting them:: - - >>> from nameparser import HumanName - >>> from nameparser.config import Constants - >>> C = Constants(middle_name_as_last=True) - >>> hn = HumanName("Mohamad Ahmad Ali Hassan", constants=C) - >>> hn.first, hn.middle, hn.last - ('Mohamad', '', 'Ahmad Ali Hassan') - -The fold applies uniformly to comma input too, so both written forms of a name -converge on the same result:: - - >>> hn2 = HumanName("Hassan, Mohamad Ahmad Ali", constants=C) - >>> hn2.first, hn2.last - ('Mohamad', 'Ahmad Ali Hassan') - - -Splitting last-name prefix particles -------------------------------------- - -The :py:attr:`~nameparser.parser.HumanName.last_base` and -:py:attr:`~nameparser.parser.HumanName.last_prefixes` properties split the last -name at the boundary between leading prefix particles and the core surname. They -use the same ``PREFIXES`` set, so adding a particle makes the split pick it up -automatically:: - - >>> from nameparser import HumanName - >>> from nameparser.config import CONSTANTS - >>> CONSTANTS.prefixes.add('op') # doctest: +ELLIPSIS - SetManager({...}) - >>> HumanName("Jan op den Berg").last_base - 'Berg' - >>> HumanName("Jan op den Berg").last_prefixes - 'op den' - >>> CONSTANTS.prefixes.remove('op') # doctest: +ELLIPSIS - SetManager({...}) - -Note the ``remove`` call at the end — ``customize.rst`` examples share global -``CONSTANTS``, so mutations must be reversed to avoid affecting later examples. - -Because ``last_base`` is a plain string property, sorting a list of names by -core surname (ignoring prefix particles like *van*, *de la*) is just a key -function:: - - names = [ - HumanName("Vincent van Gogh"), - HumanName("Juan de la Vega"), - HumanName("John Smith"), - ] - sorted_names = sorted(names, key=lambda n: n.last_base.lower()) - # sort keys: 'gogh', 'smith', 'vega' → van Gogh, Smith, de la Vega - -To sort by first name when two people share the same ``last_base``, add it as -a secondary key:: - - sorted_names = sorted(names, key=lambda n: (n.last_base.lower(), n.first.lower())) - -Bound First Names ------------------- - -``CONSTANTS.bound_first_names`` controls bound given-name prefixes that attach -to the following word to form one first name. By default it contains -``{'abdul', 'abdel', 'abdal', 'abu', 'abou', 'umm'}``. - -Example:: - - >>> from nameparser import HumanName - >>> hn = HumanName("abdul salam ahmed salem") - >>> hn.first, hn.middle, hn.last - ('abdul salam', 'ahmed', 'salem') - -To **disable** the feature entirely:: - - >>> from nameparser.config import CONSTANTS - >>> CONSTANTS.bound_first_names.clear() - -To **add** a word (e.g. if your data uses ``mohamad`` as a bound prefix):: - - >>> CONSTANTS.bound_first_names.add('mohamad') - -To **remove** a single entry:: - - >>> CONSTANTS.bound_first_names.remove('umm') - -You can also pass a custom set per ``Constants`` instance:: - - >>> from nameparser.config import Constants - >>> c = Constants(bound_first_names={'abu', 'umm'}) - >>> hn2 = HumanName("abu bakr al saud", constants=c) - >>> hn2.first, hn2.last - ('abu bakr', 'al saud') - -Non-First-Name Prefixes ------------------------ - -``CONSTANTS.non_first_name_prefixes`` is the subset of prefixes that are *never* -a standalone first name (``de``, ``dos``, ``ibn``, ...). When a name **starts** -with one of these, there is no first name -- the whole thing is a surname. - -Example:: - - >>> from nameparser import HumanName - >>> hn = HumanName("de Mesnil") - >>> hn.first, hn.last - ('', 'de Mesnil') - -A member must be a prefix that is never a given name in any culture, and the set -must stay **disjoint** from ``bound_first_names`` (a word cannot both join to -the first name and never be a first name). Ambiguous particles that *can* be -given names (``van``, ``von``, ``della``, ``di``, ``del``, ...) are intentionally -excluded; add them yourself if your data warrants it:: - - >>> from nameparser.config import CONSTANTS - >>> CONSTANTS.non_first_name_prefixes.add('von') # doctest: +SKIP - -To **disable** the feature entirely:: - - >>> CONSTANTS.non_first_name_prefixes.clear() # doctest: +SKIP - -Parser Customization Examples ------------------------------ - -Removing a Title -~~~~~~~~~~~~~~~~ - -Take a look at the :py:mod:`nameparser.config` documentation to see what's -in the constants. Here's a quick walk through of some examples where you -might want to adjust them. - -"Hon" is a common abbreviation for "Honorable", a title used when -addressing judges, and is included in the default tiles constants. This -means it will never be considered a first name, because titles are the -pieces before first names. - -But "Hon" is also sometimes a first name. If your dataset contains more -"Hon"s than "Honorable"s, you may wish to remove it from the titles -constant so that "Hon" can be parsed as a first name. - -.. doctest:: - :options: +ELLIPSIS, +NORMALIZE_WHITESPACE - - >>> from nameparser import HumanName - >>> from nameparser.config import Constants - >>> constants = Constants() - >>> hn = HumanName("Hon Solo", constants=constants) - >>> hn - - >>> constants.titles.remove('hon') - SetManager({'10th', ...}) - >>> hn = HumanName("Hon Solo", constants=constants) - >>> hn - - - -If you don't want to detect any titles at all, you can remove all of them: - -.. doctest:: - :options: +ELLIPSIS, +NORMALIZE_WHITESPACE - - >>> constants.titles.clear() - SetManager(set()) - - -Adding a Title -~~~~~~~~~~~~~~~~ - -You can also pass a ``Constants`` instance to ``HumanName`` on instantiation. - -"Dean" is a common first name so it is not included in the default titles -constant. But in some contexts it is more common as a title. If you would -like "Dean" to be parsed as a title, simply add it to the titles constant. - -You can pass multiple strings to both the :py:func:`~nameparser.config.SetManager.add` -and :py:func:`~nameparser.config.SetManager.remove` -methods and each string will be added or removed. Both functions -automatically normalize the strings for the parser's comparison method by -making them lower case and removing periods. - -.. doctest:: - :options: +ELLIPSIS, +NORMALIZE_WHITESPACE - - >>> from nameparser import HumanName - >>> from nameparser.config import Constants - >>> constants = Constants() - >>> constants.titles.add('dean', 'Chemistry') - SetManager({'10th', ...}) - >>> hn = HumanName("Assoc Dean of Chemistry Robert Johns", constants=constants) - >>> hn - - - -Module-level Shared Configuration Instance ------------------------------------------- - -As established above, ``CONSTANTS`` is shared by every ``HumanName`` created -without its own config -- that's what makes it the right place for -application-level setup, and also the source of the one gotcha it carries: -changing the config on one instance changes the behavior of every other -instance that shares it, which can be surprising if you only meant to -configure the one you're holding. Parsing itself never modifies the -configuration — only your own ``add`` and ``remove`` calls do — so the shared -instance is safe to read concurrently, e.g. parsing names on multiple threads. - -.. doctest:: module config - :options: +ELLIPSIS, +NORMALIZE_WHITESPACE - - >>> from nameparser import HumanName - >>> instance = HumanName("") - >>> instance.C.titles.add('dean') - SetManager({'10th', ...}) - >>> other_instance = HumanName("Dean Robert Johns") - >>> other_instance # Dean parses as title - - - -If you'd prefer new instances to have their own config values, pass your own -:py:class:`~nameparser.config.Constants` instance as the ``constants`` -argument when instantiating ``HumanName``. There are three spellings, -depending on which config you want the new instance to start from: - -.. code-block:: - - HumanName(name) # shared CONSTANTS (unchanged) - HumanName(name, constants=Constants()) # private, fresh library defaults - HumanName(name, constants=CONSTANTS.copy()) # private, snapshot of the current shared config - -The middle and last forms both give the instance an independent config that -further changes to ``CONSTANTS`` won't reach, but they answer different -questions: ``Constants()`` ignores any customization already made to -``CONSTANTS`` and starts clean, while ``CONSTANTS.copy()`` carries those -customizations over into the private copy. Each instance always has a ``C`` -attribute, but if you didn't pass one of the private forms to the -``constants`` argument then it's a reference to the module-level config -values with the behavior described above. - -.. doctest:: module config - :options: +ELLIPSIS, +NORMALIZE_WHITESPACE - - >>> from nameparser import HumanName - >>> from nameparser.config import Constants - >>> instance = HumanName("Dean Robert Johns") - >>> instance.has_own_config - False - >>> instance.C.titles.add('dean') - SetManager({'10th', ...}) - >>> other_instance = HumanName("Dean Robert Johns", Constants()) # <-- fresh, private config - >>> other_instance - - >>> other_instance.has_own_config - True - -.. deprecated:: 1.4.0 - Passing ``None`` as the ``constants`` argument also builds a fresh - ``Constants()``, but is deprecated: ``None`` conventionally means "use - the default," which here is the *shared* ``CONSTANTS`` -- the opposite of - what passing ``None`` actually does. It emits a ``DeprecationWarning`` - and will raise ``TypeError`` in 2.0 (issue #260); use one of the two - explicit forms above instead. - -Don't Remove Emojis -~~~~~~~~~~~~~~~~~~~ - -By default, all emojis are removed from the input string before the name is -parsed. In 2.0 the ``regexes.emoji`` override was removed (assignment raises -``TypeError``); the switch is the :py:class:`~nameparser.Policy` flag -``strip_emoji``: - -.. doctest:: - - >>> from nameparser import Parser, Policy - >>> parser = Parser(policy=Policy(strip_emoji=False)) - >>> str(parser.parse("Sam 😊 Smith")) - 'Sam 😊 Smith' - -Don't Remove Bidi Control Characters -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -By default, invisible bidirectional control characters (the left-to-right and -right-to-left marks and friends, common in copy-pasted right-to-left names) are -removed from the input string before the name is parsed. In 2.0 the -``regexes.bidi`` override was removed (assignment raises ``TypeError``); the -switch is the :py:class:`~nameparser.Policy` flag ``strip_bidi``: - -.. doctest:: - - >>> from nameparser import Parser, Policy - >>> parser = Parser(policy=Policy(strip_bidi=False)) - >>> parser.parse("\u200fJohn\u200f Smith").given == "\u200fJohn\u200f" - True - -Config Changes May Need Parse Refresh -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The full name is parsed upon assignment to the ``full_name`` attribute or -instantiation. Sometimes after making changes to configuration or other inner -data after assigning the full name, the name will need to be re-parsed with the -:py:func:`~nameparser.parser.HumanName.parse_full_name()` method before you see -those changes with ``repr()``. + ... nickname_delimiters=frozenset({("'", "'"), ('"', '"')}), + ... maiden_delimiters=frozenset({("(", ")")}), + ... ) + >>> Parser(policy=policy).parse("Jane (Jones) Smith").maiden + 'Jones' +Routing parentheses to ``maiden`` means moving that pair out of +``nickname_delimiters`` as well as into ``maiden_delimiters``, as +above — a delimiter pair present in *both* buckets routes to +``nickname``. -Adjusting names after parsing them -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Presentation: rendering arguments +---------------------------------- -Each attribute has a corresponding ordered list of name pieces. If you're doing -pre- or post-processing you may wish to manipulate these lists directly. -The strings returned by the attribute names just join these lists with spaces. +Once a name is parsed, how it's displayed is a separate decision made +at the point of output, not baked into the parse: -* o.title_list -* o.first_list -* o.middle_list -* o.last_list -* o.suffix_list -* o.nickname_list -* o.maiden_list +.. code-block:: python -:: + def render(self, spec: str = "{title} {given} {middle} {family} {suffix}") -> str: ... + def initials(self, spec: str = "{given} {middle} {family}", + delimiter: str = ".", separator: str = " ") -> str: ... + def capitalized(self, lexicon: Lexicon | None = None, *, force: bool = False) -> ParsedName: ... - >>> hn = HumanName("Juan Q. Xavier Velasquez y Garcia, Jr.") - >>> hn.middle_list - ['Q.', 'Xavier'] - >>> hn.middle_list += ["Ricardo"] - >>> hn.middle_list - ['Q.', 'Xavier', 'Ricardo'] +Looking for v1's ``string_format``? It's the ``render(spec)`` argument +now — pass your own format string per call instead of setting it once +on a shared config object. See :doc:`usage` for worked examples of all +three. +Sharing a configured parser +---------------------------- -You can also replace any name bucket's contents by assigning a string or a list -directly to the attribute. +A :class:`~nameparser.Parser` is a frozen value, so the way to share +one configuration across a codebase is the same way you'd share any +other constant: build it once at module level and import it wherever +you parse. -:: +.. code-block:: python - >>> hn = HumanName("Dr. John A. Kenneth Doe") - >>> hn.title = ["Associate","Professor"] - >>> hn.suffix = "Md." - >>> hn - + # myapp/names.py + from nameparser import Lexicon, Parser, Policy + lex = Lexicon.default().add(titles={"chancellor"}) + policy = Policy(strip_emoji=False) + parser = Parser(lexicon=lex, policy=policy) + # elsewhere + from myapp.names import parser + name = parser.parse(raw_name) +Because :class:`~nameparser.Parser` and its ``lexicon``/``policy`` are +immutable and hashable, ``parser`` is safe to import and call from +multiple threads with no locking — there is no shared mutable state to +protect, unlike v1's module-level ``CONSTANTS``. From f817d221a9fb458e660b2c499b21765da359b629 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 02:33:04 -0700 Subject: [PATCH 170/275] =?UTF-8?q?docs:=20add=20locales=20page=20?= =?UTF-8?q?=E2=80=94=20using,=20creating,=20contributing=20packs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- docs/index.rst | 1 + docs/locales.rst | 144 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 docs/locales.rst diff --git a/docs/index.rst b/docs/index.rst index 59fc5d0f..0c26e98b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -59,6 +59,7 @@ Parsing Names usage concepts customize + locales **Developer Documentation** diff --git a/docs/locales.rst b/docs/locales.rst new file mode 100644 index 00000000..97f2fdc2 --- /dev/null +++ b/docs/locales.rst @@ -0,0 +1,144 @@ +Locale packs +============ + +A locale pack is an opt-in bundle of policy — and, when a naming +tradition needs it, vocabulary — for one specific pattern: East Slavic +patronymics, Turkic patronymic markers, and so on. Packs apply only +when you ask for one by name, and every pack makes the same promise: +a pack never changes a name it doesn't declare. + +Using a pack +------------ + +:func:`~nameparser.parser_for` folds one or more packs onto a base +:class:`~nameparser.Parser` (the module default, unless you pass +``base=``). Here the Russian pack reads "Сидоров Иван Петрович" +(Sidorov Ivan Petrovich — surname/given/patronymic order) the way a +formal Russian document intends: + +.. doctest:: + + >>> from nameparser import locales, parser_for + >>> ru = parser_for(locales.RU) + >>> ru.parse("Сидоров Иван Петрович").given + 'Иван' + +Packs stack: pass more than one pack and their policies fold together +in order. + +.. doctest:: + + >>> both = parser_for(locales.RU, locales.TR_AZ) + >>> sorted(rule.name for rule in both.policy.patronymic_rules) + ['EAST_SLAVIC', 'TURKIC'] + +Find what's shipped with :func:`~nameparser.locales.available`, and +look one up dynamically by its lowercase code with +:func:`~nameparser.locales.get` — the same code the ``--locale`` flag +takes: + +.. doctest:: + + >>> locales.available() + ('ru', 'tr_az') + >>> locales.get("ru") is locales.RU + True + +The command line accepts the same codes: ``python -m nameparser +--locale ru --json "Сидоров Иван Петрович"`` applies the pack before +parsing, equivalent to ``parser_for(locales.get("ru"))``. + +.. list-table:: Shipped packs + :header-rows: 1 + :widths: 15 85 + + * - Code + - Turns on + * - ``ru`` + - East Slavic patronymic order — detects a formal + given/patronymic/family shape (Cyrillic and transliterated + ``-ovich``/``-ovna``-style endings) and assigns it accordingly. + * - ``tr_az`` + - Turkic patronymic markers — detects a standalone marker token + (``oglu``, ``qizi``, ``uulu``, and their Latin- and + Cyrillic-script variants) and reads the name around it as + given/middle/family. + +Both shipped packs are policy-only — they carry no vocabulary of their +own; see :doc:`concepts` for why that split (language vocabulary vs. +behavior) is drawn where it is. + +Creating your own Locale +------------------------- + +You don't need to touch nameparser's registry to use your own pack — +:class:`~nameparser.Locale` is a plain, constructible value: +``Locale(code=..., lexicon=..., policy=PolicyPatch(...))``. A +:class:`~nameparser.PolicyPatch` is a :class:`~nameparser.Policy`-shaped +patch: every field defaults to :data:`~nameparser.UNSET` (leave it +alone) instead of to a concrete value, so a pack only ever states what +it changes. + +.. doctest:: + + >>> from nameparser import Lexicon, Locale, PolicyPatch, parser_for + >>> lex = Lexicon.empty().add(titles={"kapitan"}) + >>> mine = Locale(code="mycorp", lexicon=lex, + ... policy=PolicyPatch(middle_as_family=True)) + >>> name = parser_for(mine).parse("Kapitan Anna Maria Schmidt") + >>> name.title, name.given, name.family + ('Kapitan', 'Anna', 'Maria Schmidt') + +That pack does two things at once: the :class:`~nameparser.Lexicon` +fragment teaches the parser that ``kapitan`` is a title, and the +:class:`~nameparser.PolicyPatch` turns on ``middle_as_family`` so any +remaining given-position words after the first fold into ``family`` +instead of ``middle`` — compare this to the default parser's reading +of the same string, which has no title and splits ``given='Kapitan'``, +``middle='Anna Maria'``, ``family='Schmidt'``. + +When ``parser_for`` folds one or more packs onto a base, lexicons +union (a pack's words are added to the base's, never removed); policy +fields declared as set-valued in :class:`~nameparser.PolicyPatch` +(``patronymic_rules`` and the delimiter fields) union the same way; +and every other, scalar field is later-wins — if two packs (or a pack +and an explicit conflicting value) set the same scalar field, the last +one applied wins and a ``UserWarning`` is raised so the conflict +isn't silent. + +Contributing a pack to nameparser +---------------------------------- + +Shipping a pack in nameparser itself (rather than keeping it local to +your own code) means meeting the in-repo contract, checked mechanically +by ``tests/v2/test_locales.py``: + +#. Add a registry entry in ``nameparser/locales/__init__.py`` — a + ``"CODE": ("module.path", "ATTR")`` row in ``_REGISTRY``, so the + pack loads lazily on first access (importing ``nameparser.locales`` + never imports pack modules). +#. Declare a module-level ``DEVIATES(name)`` predicate: given a name + string, return whether *this pack alone* might parse it differently + from the default parser. Over-declaring is safe; under-declaring is + not — when in doubt, ``DEVIATES`` should say yes. +#. Add a rotator list to ``tests/v2/test_locales.py`` with at least one + name exercising every alternation branch of every marker regex the + pack defines — ``test_rotators_cover_every_marker_branch`` fails + until each branch is hit. +#. Keep the non-interference gate green over the shared corpus plus + your rotators: every name the packed parser parses differently from + the default must be one your ``DEVIATES`` predicate flags — no + silent, undeclared side effects on names outside the pack's stated + scope. +#. Keep the pack policy-only in 2.0 — ``ru`` and ``tr_az`` both ship + an empty :class:`~nameparser.Lexicon`; a pack that wants to carry + its own vocabulary is a later conversation. +#. Curate vocabulary conservatively, the same rule as + :doc:`customize`: when you're unsure whether a word or a marker + belongs, leave it out. + +``nameparser/locales/ru.py`` is the reference implementation to copy +from. Staged packs in progress are tracked in issues `#271 +`_, `#272 +`_, and `#146 +`_. From 26e0dead854a7c1b56d9c85ad86df57d640dfce5 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 02:46:51 -0700 Subject: [PATCH 171/275] docs: add Migrating from HumanName page (#262 successor) Co-Authored-By: Claude Fable 5 --- docs/index.rst | 1 + docs/migrate.rst | 240 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 241 insertions(+) create mode 100644 docs/migrate.rst diff --git a/docs/index.rst b/docs/index.rst index 0c26e98b..b03e5cbd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -60,6 +60,7 @@ Parsing Names concepts customize locales + migrate **Developer Documentation** diff --git a/docs/migrate.rst b/docs/migrate.rst new file mode 100644 index 00000000..8a97758d --- /dev/null +++ b/docs/migrate.rst @@ -0,0 +1,240 @@ +Migrating from HumanName +========================= + +Nothing breaks. 2.0 keeps ``HumanName`` and ``CONSTANTS`` working +exactly as before — same imports, same attributes, same mutation API +(``name.C.titles.add(...)``, ``name.first = "..."``, and so on). +Upgrading to 2.0 and migrating your code to the new +:class:`~nameparser.Parser`/:class:`~nameparser.Lexicon`/ +:class:`~nameparser.Policy` API are two separate decisions — you can do +the former today and the latter whenever it's convenient, or never. The +compatibility layer (``HumanName`` and ``nameparser.config``) is +removed in 3.0; that release is not scheduled. + +The full 1.x documentation remains the canonical reference for +``HumanName`` and stays online at the readthedocs ``stable`` build, +currently the 1.4.0 release: https://nameparser.readthedocs.io/en/stable/ + +This page exists for the other direction: translating a v1 +customization or a v1-shaped comparison into the 2.0 API, one row per +old name. + +Attribute map +------------- + +``HumanName``'s seven fields and their aggregates map onto +:class:`~nameparser.ParsedName` like this: + +.. list-table:: + :header-rows: 1 + :widths: 30 40 30 + + * - ``HumanName`` + - ``ParsedName`` + - Note + * - ``title`` + - ``title`` + - + * - ``first`` + - ``given`` + - + * - ``middle`` + - ``middle`` + - + * - ``last`` + - ``family`` + - + * - ``suffix`` + - ``suffix`` + - + * - ``nickname`` + - ``nickname`` + - + * - ``maiden`` + - ``maiden`` + - New field (added 1.4) + * - ``title_list``, ``first_list``, ``middle_list``, ``last_list``, + ``suffix_list``, ``nickname_list``, ``maiden_list`` + - ``tokens_for(Role.TITLE)``, ``tokens_for(Role.GIVEN)``, ... + - Returns the raw :class:`~nameparser.Token` tuple for that role; + read ``.text`` off each token for the string a ``_list`` + attribute gave you + * - ``given_names`` / ``given_names_list`` + - ``given_names`` + - Unchanged name; ``middle`` folded into ``first`` + * - ``surnames`` / ``surnames_list`` + - ``surnames`` + - Unchanged name; ``middle`` folded into ``last`` + * - ``last_base`` / ``last_base_list`` + - ``family_base`` + - The surname with leading particles split off + * - ``last_prefixes`` / ``last_prefixes_list`` + - ``family_particles`` + - The particles ``family_base`` was split from (e.g. ``"de la"``) + * - ``string_format`` + - ``render(spec)`` + - A per-call argument now, not stored config — see :doc:`customize` + * - ``initials_format``, ``initials_delimiter``, ``initials_separator`` + - ``initials(spec, delimiter, separator)`` + - Same three knobs, now call-site arguments to + :meth:`~nameparser.ParsedName.initials` + * - ``suffix_delimiter`` + - ``Policy(extra_suffix_delimiters={...})`` + - Moves from a ``HumanName``/``Constants`` scalar to a ``Policy`` + set field, so more than one custom delimiter can be active at + once + * - ``capitalize()``, ``force_mixed_case_capitalization`` + - ``capitalized(lexicon, force=...)`` + - :meth:`~nameparser.ParsedName.capitalized` returns a new value + rather than mutating in place + +Verified live, for example: + +.. doctest:: + + >>> from nameparser import HumanName, parse, Role + >>> hn = HumanName("Dr. Juan Q. Xavier de la Vega III") + >>> n = parse("Dr. Juan Q. Xavier de la Vega III") + >>> hn.title == n.title, hn.first == n.given, hn.last == n.family + (True, True, True) + >>> hn.first_list == [t.text for t in n.tokens_for(Role.GIVEN)] + True + >>> hn.given_names == n.given_names, hn.surnames == n.surnames + (True, True) + >>> hn.last_base == n.family_base, hn.last_prefixes == n.family_particles + (True, True) + +Config map +---------- + +``CONSTANTS``' vocabulary sets map onto :class:`~nameparser.Lexicon` +fields: + +.. list-table:: + :header-rows: 1 + :widths: 30 40 30 + + * - ``CONSTANTS`` + - ``Lexicon`` + - Note + * - ``titles`` + - ``titles`` + - + * - ``first_name_titles`` + - ``given_name_titles`` + - + * - ``suffix_acronyms`` + - ``suffix_acronyms`` + - + * - ``suffix_not_acronyms`` + - ``suffix_words`` + - + * - ``suffix_acronyms_ambiguous`` + - ``suffix_acronyms_ambiguous`` + - + * - ``prefixes`` + - ``particles`` + - + * - ``non_first_name_prefixes`` + - ``particles_ambiguous`` + - **Flipped** — see the warning below + * - ``conjunctions`` + - ``conjunctions`` + - + * - ``bound_first_names`` + - ``bound_given_names`` + - + * - ``capitalization_exceptions`` + - ``capitalization_exceptions`` + - Pair-valued; set it via ``dataclasses.replace(lexicon, + capitalization_exceptions={...})``, not ``add()``/``remove()`` + +And behavior/render scalars map onto :class:`~nameparser.Policy` (or a +rendering argument, where the 2.0 equivalent isn't config at all): + +.. list-table:: + :header-rows: 1 + :widths: 30 40 30 + + * - ``CONSTANTS`` + - 2.0 equivalent + - Note + * - ``patronymic_name_order`` + - ``Policy(patronymic_rules={PatronymicRule.EAST_SLAVIC})`` + - v1's bare ``True`` becomes the East Slavic rule explicitly (or + use ``parser_for(locales.RU)``); see :doc:`locales` + * - ``middle_name_as_last`` + - ``Policy.middle_as_family`` + - + * - ``nickname_delimiters`` + - ``Policy.nickname_delimiters`` + - Was a three-sentinel dict; now a plain ``frozenset`` of + ``(open, close)`` pairs + * - ``maiden_delimiters`` + - ``Policy.maiden_delimiters`` + - Same shape change as ``nickname_delimiters`` + * - ``regexes.bidi`` + - ``Policy.strip_bidi`` + - ``regexes.bidi = False`` becomes ``Policy(strip_bidi=False)`` + * - ``regexes.emoji`` + - ``Policy.strip_emoji`` + - ``regexes.emoji = False`` becomes ``Policy(strip_emoji=False)`` + * - ``force_mixed_case_capitalization`` + - ``capitalized(force=True)`` + - Moves from stored config to a per-call argument + * - ``capitalize_name`` + - *(no equivalent)* + - 2.0 never capitalizes automatically during ``parse()``; call + ``.capitalized()`` explicitly on the result instead + +Every other ``regexes.*`` entry (``word``, ``spaces``, and the rest of +the compiled-pattern proxy) has no 2.0 replacement — parsing behavior +is configured entirely through named ``Policy`` fields now, not by +handing the parser a regex. + +.. warning:: + + ``non_first_name_prefixes`` and ``particles_ambiguous`` mark + **complementary** sets, not the same set under a new name. + ``non_first_name_prefixes`` lists particles that are *never* read as + a given name; ``particles_ambiguous`` lists the particles that + *may* be read as one. Translating a customization means flipping + the set: ``lexicon.particles_ambiguous == lexicon.particles - + constants.non_first_name_prefixes``. Copying + ``non_first_name_prefixes`` straight into ``particles_ambiguous`` + silently inverts which particles are allowed to double as a given + name. + +Comparison +---------- + +``HumanName.__eq__``/``__hash__`` were deprecated in 1.3.0 and are gone +in 2.0's core API; use ``matches()`` for "is this the same name?" and +``comparison_key()`` for dedup, dict keys, and sorting — both exist on +``HumanName`` and on :class:`~nameparser.ParsedName` with the same +behavior: + +.. doctest:: + + >>> from nameparser import HumanName, parse + >>> hn = HumanName("de la Vega, Juan") + >>> n = parse("de la Vega, Juan") + >>> hn.matches("Juan de la Vega"), n.matches("Juan de la Vega") + (True, True) + >>> hn.comparison_key() == n.comparison_key() + True + +One behavior changed underneath both methods: components now fold with +``str.casefold()`` instead of ``str.lower()``, so more Unicode +case-pairs compare equal than did under 1.4 (see the 2.0.0 section of +:doc:`release_log` for the exact rule and examples). + +Behavior changes +----------------- + +Beyond the API surface mapped above, a handful of parse *outputs* +differ between 1.4 and 2.0 for specific input shapes — comma-suffix +routing, maiden-marker detection, an ambiguous-acronym data change, and +one rendering difference under a custom suffix delimiter. These are +listed with their reasoning and test coverage in the 2.0.0 section of +:doc:`release_log`; they aren't repeated here. From 6b68fda3d070f61c916f1a270597f8d3f780846c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 02:52:11 -0700 Subject: [PATCH 172/275] fix: migration hint names both rules v1's patronymic flag enabled The shim translates patronymic_name_order=True to {EAST_SLAVIC, TURKIC} (v1's single bool governed both detectors); the True-guard's hint said EAST_SLAVIC only, which would migrate users to narrower behavior than the compat layer gives them. Caught by the migrate-page table audit. Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 8493c871..bf9d30c4 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -98,9 +98,10 @@ def __post_init__(self) -> None: raise TypeError( f"patronymic_rules must be an iterable of PatronymicRule " f"names, got {self.patronymic_rules!r}; v1's " - f"patronymic_name_order=True becomes " - f"patronymic_rules={{PatronymicRule.EAST_SLAVIC}} " - f"(or parser_for(locales.RU))" + f"patronymic_name_order=True enabled both rules -- " + f"patronymic_rules={{PatronymicRule.EAST_SLAVIC, " + f"PatronymicRule.TURKIC}} (or pick one via " + f"parser_for(locales.RU) / locales.TR_AZ)" ) from None items = tuple(rule_iter) rules = set() @@ -251,9 +252,10 @@ def __post_init__(self) -> None: iter(value) except TypeError: hint = ( - "; v1's patronymic_name_order=True becomes " - "patronymic_rules={PatronymicRule.EAST_SLAVIC} " - "(or parser_for(locales.RU))" + "; v1's patronymic_name_order=True enabled both rules" + " -- patronymic_rules={PatronymicRule.EAST_SLAVIC, " + "PatronymicRule.TURKIC} (or pick one via " + "parser_for(locales.RU) / locales.TR_AZ)" if f.name == "patronymic_rules" else "" ) raise TypeError( From 7306724cc85fae1f4cdf518793ebc1a2e3d32977 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 02:53:00 -0700 Subject: [PATCH 173/275] docs: correct two migrate-table facts from the table audit patronymic_name_order=True maps to BOTH patronymic rules (what the shim actually does); the maiden field landed in 1.3, not 1.4. Co-Authored-By: Claude Fable 5 --- docs/migrate.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/migrate.rst b/docs/migrate.rst index 8a97758d..f7b71d0e 100644 --- a/docs/migrate.rst +++ b/docs/migrate.rst @@ -52,7 +52,7 @@ Attribute map - * - ``maiden`` - ``maiden`` - - New field (added 1.4) + - New field (added 1.3) * - ``title_list``, ``first_list``, ``middle_list``, ``last_list``, ``suffix_list``, ``nickname_list``, ``maiden_list`` - ``tokens_for(Role.TITLE)``, ``tokens_for(Role.GIVEN)``, ... @@ -88,7 +88,7 @@ Attribute map - :meth:`~nameparser.ParsedName.capitalized` returns a new value rather than mutating in place -Verified live, for example: +Side by side: .. doctest:: @@ -160,9 +160,11 @@ rendering argument, where the 2.0 equivalent isn't config at all): - 2.0 equivalent - Note * - ``patronymic_name_order`` - - ``Policy(patronymic_rules={PatronymicRule.EAST_SLAVIC})`` - - v1's bare ``True`` becomes the East Slavic rule explicitly (or - use ``parser_for(locales.RU)``); see :doc:`locales` + - ``Policy(patronymic_rules={PatronymicRule.EAST_SLAVIC, + PatronymicRule.TURKIC})`` + - v1's single flag enabled both detectors at once; pick one rule + (or a locale pack, see :doc:`locales`) if you only want one + tradition * - ``middle_name_as_last`` - ``Policy.middle_as_family`` - @@ -199,7 +201,7 @@ handing the parser a regex. ``non_first_name_prefixes`` lists particles that are *never* read as a given name; ``particles_ambiguous`` lists the particles that *may* be read as one. Translating a customization means flipping - the set: ``lexicon.particles_ambiguous == lexicon.particles - + the set: ``particles_ambiguous = lexicon.particles - constants.non_first_name_prefixes``. Copying ``non_first_name_prefixes`` straight into ``particles_ambiguous`` silently inverts which particles are allowed to double as a given From b29861c0646ecf8c041900a817f3af6ad43b9c61 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 02:59:52 -0700 Subject: [PATCH 174/275] docs: regroup reference into 2.0 API + compat layer; new front page Co-Authored-By: Claude Fable 5 --- docs/index.rst | 69 +++++++---------------------- docs/modules.rst | 110 +++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 108 insertions(+), 71 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index b03e5cbd..e85eb681 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,74 +6,38 @@ Python Human Name Parser ======================== -Version |release| +Version |release| -A simple Python module for parsing human names into their individual -components. +nameparser parses human names into seven fields — title, given, +middle, family, suffix, nickname, maiden. Results are immutable, +configuration is composable (:class:`~nameparser.Lexicon` for +vocabulary, :class:`~nameparser.Policy` for behavior), and locale +packs are opt-in. Requires Python 3.11+. -* hn.title -* hn.first -* hn.middle -* hn.last -* hn.suffix -* hn.nickname -* hn.maiden +.. doctest:: -Supports 3 different comma placement variations in the input string. + >>> from nameparser import parse + >>> name = parse("Dr. Juan Q. Xavier de la Vega III") + >>> name.given, name.family + ('Juan', 'de la Vega') + >>> name.render("{family}, {given}") + 'de la Vega, Juan' -1. Title Firstname "Nickname" Middle Middle Lastname Suffix -2. Lastname [Suffix], Title Firstname (Nickname) Middle Middle[,] Suffix [, Suffix] -3. Title Firstname M Lastname [Suffix], Suffix [Suffix] [, Suffix] - - -It attempts the best guess that can be made with a simple, rule-based -approach. It's not perfect, but it gets you pretty far. - -Its main use case is English, but it may be useful for other latin-based languages, especially -if you are willing to `customize it`_, but it is not likely to be useful for languages -that do not share the same structure as English names. - -.. _customize it: customize.html - -Instantiating the `HumanName` class with a string splits on commas and then spaces, -classifying name parts based on placement in the string and matches against known name -pieces like titles. It joins name pieces on conjunctions and special prefixes to last names like -"del". Titles can be chained together and include conjunctions to handle -titles like "Asst Secretary of State". It can also try to correct -capitalization. - -It does not attempt to correct input mistakes. When there is ambiguity that cannot be resolved by a rule-based approach, -HumanName prefers to handle the most common cases correctly. For example, -"Dean" is not parsed as title because it is more common as a first name -(You can customize this behavior though, see `Parser Customization Examples`_). - -.. _Parser Customization Examples: customize.html#parser-customization-examples - - -Parsing Names -------------- +Coming from 1.x? :doc:`migrate`. .. toctree:: :maxdepth: 2 - + usage concepts customize locales migrate - -**Developer Documentation** - -.. toctree:: - :maxdepth: 2 - modules - resources release_log + resources contributing - - Indices and tables ================== @@ -83,4 +47,3 @@ Indices and tables **GitHub Project**: https://github.com/derek73/python-nameparser - diff --git a/docs/modules.rst b/docs/modules.rst index 8f3b1990..85d808ab 100644 --- a/docs/modules.rst +++ b/docs/modules.rst @@ -1,41 +1,115 @@ -HumanName Class Documentation -============================== +API reference +============= + +The 2.0 API +----------- + +Parsing +~~~~~~~ + +.. autofunction:: nameparser.parse + +.. autoclass:: nameparser.Parser + :members: + +.. autofunction:: nameparser.parser_for + +Results +~~~~~~~ + +.. autoclass:: nameparser.ParsedName + :members: + +.. autoclass:: nameparser.Token + :members: + +.. autoclass:: nameparser.Span + :members: + +.. autoclass:: nameparser.Role + :members: + +.. autoclass:: nameparser.Ambiguity + :members: + +.. autoclass:: nameparser.AmbiguityKind + :members: + +Configuration +~~~~~~~~~~~~~ + +.. autoclass:: nameparser.Lexicon + :members: + +.. autoclass:: nameparser.Policy + :members: + +.. autoclass:: nameparser.PolicyPatch + :members: + +.. autodata:: nameparser.UNSET + +.. autoclass:: nameparser.PatronymicRule + :members: + +.. autodata:: nameparser.GIVEN_FIRST + +.. autodata:: nameparser.FAMILY_FIRST + +.. autodata:: nameparser.FAMILY_FIRST_GIVEN_LAST + +Locales +~~~~~~~ + +.. autoclass:: nameparser.Locale + :members: + +.. automodule:: nameparser.locales + :members: get, available + +1.x compatibility layer +------------------------ + +.. note:: + + ``HumanName`` and ``nameparser.config`` are the 1.x API, kept + working through 2.x and removed in 3.0. New code should use the + 2.0 API above; see :doc:`migrate`. HumanName.parser ----------------- +~~~~~~~~~~~~~~~~ .. py:module:: nameparser.parser .. py:class:: HumanName - :noindex: + :noindex: .. autoclass:: HumanName - :members: - :special-members: __eq__, __init__ + :members: + :special-members: __eq__, __init__ HumanName.config ----------------- +~~~~~~~~~~~~~~~~ .. automodule:: nameparser.config - :members: + :members: HumanName.config Defaults -------------------------- - +~~~~~~~~~~~~~~~~~~~~~~~~~~ .. automodule:: nameparser.config.titles - :members: + :members: .. automodule:: nameparser.config.suffixes - :members: + :members: .. automodule:: nameparser.config.prefixes - :members: + :members: .. automodule:: nameparser.config.bound_first_names - :members: + :members: .. automodule:: nameparser.config.conjunctions - :members: + :members: .. automodule:: nameparser.config.maiden_markers - :members: + :members: .. automodule:: nameparser.config.capitalization - :members: + :members: .. automodule:: nameparser.config.regexes - :members: + :members: From 432e39e5ed22ebdba53b491be1e186d635525045 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 04:36:09 -0700 Subject: [PATCH 175/275] docs: rewrite README new-API-first (doubles as the PyPI page) Co-Authored-By: Claude Fable 5 --- README.rst | 173 +++++++++++------------------------------------------ 1 file changed, 35 insertions(+), 138 deletions(-) diff --git a/README.rst b/README.rst index 7a0f8b4d..f2379c9a 100644 --- a/README.rst +++ b/README.rst @@ -3,67 +3,9 @@ Name Parser |Build Status| |PyPI| |PyPI version| |Documentation| |License| |Downloads| |Codecov| -📣 **nameparser 2.0 is in design** — a new immutable core API, with full -compatibility for existing code through 2.x. Read the `design RFC -`__ and share -feedback on `the discussion issue -`__. - -A simple Python (3.10+) module for parsing human names into their -individual components. - -* hn.title -* hn.first -* hn.middle -* hn.last -* hn.suffix -* hn.nickname -* hn.maiden -* hn.surnames *(middle + last)* -* hn.given_names *(first + middle)* -* hn.initials *(first initial of each name part)* -* hn.last_base *(last, minus any prefixes)* -* hn.last_prefixes *(leading last-name particles, e.g. "van der")* - -Supported Name Structures -~~~~~~~~~~~~~~~~~~~~~~~~~ - -The supported name structure is generally "Title First Middle Last Suffix", where all pieces -are optional. Comma-separated format like "Last, First" is also supported. - -1. Title Firstname "Nickname" Middle Middle Lastname Suffix -2. Lastname [Suffix], Title Firstname (Nickname) Middle Middle[,] Suffix [, Suffix] -3. Title Firstname M Lastname [Suffix], Suffix [Suffix] [, Suffix] - -How It Works -~~~~~~~~~~~~ - -The parser works in two layers. - -A **vocabulary layer** recognizes name pieces by what they are, using -configurable sets of known words: titles ("Dr."), suffixes ("III", "PhD"), -last-name prefixes ("de la"), conjunctions ("y", "&"), and delimited -nicknames ("Doc"). Titles and conjunctions chain together to handle complex -titles like "Asst Secretary of State"; prefixes join forward so "de la Vega" -stays one last name. This layer doesn't care where in the string a word -appears — and it's the layer you customize, by adding or removing entries -in the sets to fit your dataset. - -A **positional layer** then assigns everything the vocabulary layer didn't -claim, based purely on where it sits: the first unclaimed word is the first -name, the last one is the last name, and anything between them is a middle -name. There is no semantic understanding — "Dr" is a title when it comes -before a name and a suffix when it comes after ("pre-nominal" and -"post-nominal" would probably be better names) — and no attempt to correct -mistakes in the input. - -It attempts the best guess that can be made with a simple, deterministic, -rule-based approach — no statistical models or machine learning; the same -input always parses the same way. The positional layer assumes Western name -order (given name first), so the main use case is English and other -languages that share that structure. It can also try to correct the -capitalization of names that are all upper- or lowercase. It's not perfect, -but it gets you pretty far. +nameparser parses human names into seven fields — title, given, middle, +family, suffix, nickname, maiden. Results are immutable, configuration is +composable, and locale packs are opt-in. Installation ------------ @@ -72,95 +14,50 @@ Installation pip install nameparser -If you want to try out the latest code from GitHub you can -install with pip using the command below. - -``pip install -e git+https://github.com/derek73/python-nameparser.git`` - -If you need to handle lists of names, check out -`namesparser `_, a -compliment to this module that handles multiple names in a string. - +Requires Python 3.11+. Quick Start Example -------------------- - -:: - - >>> from nameparser import HumanName - >>> name = HumanName("Dr. Juan Q. Xavier de la Vega III (Doc Vega)") - >>> name - - >>> name.last - 'de la Vega' - >>> name.as_dict() - {'title': 'Dr.', 'first': 'Juan', 'middle': 'Q. Xavier', 'last': 'de la Vega', 'suffix': 'III', 'nickname': 'Doc Vega', 'maiden': ''} - >>> str(name) - 'Dr. Juan Q. Xavier de la Vega III (Doc Vega)' - >>> name.string_format = "{first} {last}" - >>> str(name) - 'Juan de la Vega' - - -Because the positional layer has no semantic understanding, position is -everything: +-------------------- -:: - - >>> name = HumanName("1 & 2, 3 4 5, Mr.") - >>> name - - -Customization -------------- +.. code-block:: python -Your project may need some adjustment for your dataset. Most customization -is vocabulary — `customizing the configured pre-defined sets`_ of titles, -prefixes, etc. that the vocabulary layer matches against. You can also do -your own pre- or post-processing, or subclass the `HumanName` class for -deeper changes. See the `full documentation`_ for more information. + >>> from nameparser import parse + >>> name = parse("Dr. Juan Q. Xavier de la Vega III") + >>> name.given, name.family + ('Juan', 'de la Vega') + >>> name.render("{family}, {given}") + 'de la Vega, Juan' +Those seven fields are ``title``, ``given``, ``middle``, ``family``, +``suffix``, ``nickname``, and ``maiden`` — plus aggregate views like +``given_names``, ``surnames``, ``family_base``, and ``family_particles`` +for combining or splitting them further. -`Full documentation`_ -~~~~~~~~~~~~~~~~~~~~~ +Learn more +---------- -.. _customizing the configured pre-defined sets: http://nameparser.readthedocs.org/en/latest/customize.html -.. _Full documentation: http://nameparser.readthedocs.org/en/latest/ - - -Contributing ------------- +* `Getting started `__ — the full tour: parsing, aggregates, dicts, rendering, dedup +* `Customizing the parser `__ — vocabulary, behavior, and presentation +* `Locale packs `__ — opt-in bundles for East Slavic patronymics, Turkic markers, and more +* There's also a CLI: ``python -m nameparser --json "Doe, John"`` -If you come across name piece that you think should be in the default config, you're -probably right. `Start a New Issue`_ and we can get them added. +Coming from 1.x +---------------- -Please let me know if there are ways this library could be structured to make -it easier for you to use in your projects. Read CONTRIBUTING.md_ for more info -on running the tests and contributing to the project. +Nothing breaks. 2.0 keeps ``HumanName`` and ``CONSTANTS`` working exactly +as before — same imports, same attributes, same mutation API. See +`Migrating from HumanName `__ +for translating a v1 customization into the new API, whenever that's +convenient for you. -**GitHub Project** +See the `release log `__ +for the full list of changes in the 2.0 series. -https://github.com/derek73/python-nameparser +License +------- -.. _CONTRIBUTING.md: https://github.com/derek73/python-nameparser/tree/master/CONTRIBUTING.md -.. _Start a New Issue: https://github.com/derek73/python-nameparser/issues -.. _click here to propose changes to the titles: https://github.com/derek73/python-nameparser/edit/master/nameparser/config/titles.py +LGPL licensed. See `LICENSE `__ +for details. .. |Build Status| image:: https://github.com/derek73/python-nameparser/actions/workflows/python-package.yml/badge.svg :target: https://github.com/derek73/python-nameparser/actions/workflows/python-package.yml From 5063056a9e915d17fcae0a5952fcdd00236f49a3 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 04:43:34 -0700 Subject: [PATCH 176/275] docs: drop the nonexistent _static path so -W builds run clean Co-Authored-By: Claude Fable 5 --- docs/conf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 6449d56d..28f3301c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -139,7 +139,9 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# (empty: docs/_static does not exist; a non-existent entry warns on +# every build and blocks -W) +html_static_path = [] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied From b809ccd290d11d82f897cdfb45824dc7d5446cab Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 10:50:07 -0700 Subject: [PATCH 177/275] docs: show the CLI example's JSON output in usage.rst Co-Authored-By: Claude Fable 5 --- docs/usage.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/usage.rst b/docs/usage.rst index d295c06e..85829ac3 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -106,6 +106,7 @@ Command line :: $ python -m nameparser --json "Doe, John" + {"title": "", "given": "John", "middle": "", "family": "Doe", "suffix": "", "nickname": "", "maiden": ""} Add ``--locale`` to parse with a locale pack (for example ``--locale ru``); see :doc:`locales`. From cc7063f6fcae110b9dbd0fbdcb0c59962f5ffc04 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 11:22:57 -0700 Subject: [PATCH 178/275] feat: maiden_delimiters wins on overlap; export DEFAULT_NICKNAME_DELIMITERS Routing parens to maiden used to require editing both delimiter buckets in tandem, with the nickname default's contents discovered by digging: a pair in both buckets silently parsed as nickname. Now Policy canonicalizes the effective nickname set to nickname_delimiters - maiden_delimiters (listing a pair in the specific bucket IS the intent), so Policy(maiden_delimiters={('(', ')')}) is the whole recipe -- same canonicalization precedent as the name_order coercion, equal values converge. The nickname default is now the public, documented DEFAULT_NICKNAME_DELIMITERS constant for explicit set math. The 1.x facade keeps v1's nickname-wins precedence (pinned by the v1 suite) via a shim-side pre-subtraction on the maiden bucket, pinned by test_snapshot_overlap_keeps_v1_nickname_precedence. Case row nickname_bucket_wins_when_shared becomes maiden_delimiters_win_when_shared. Docs: customize one-liner example, migrate precedence note, release-log entry, reference autodata. Co-Authored-By: Claude Fable 5 --- docs/customize.rst | 22 ++++++++++++---------- docs/migrate.rst | 5 ++++- docs/modules.rst | 2 ++ docs/release_log.rst | 1 + nameparser/__init__.py | 4 +++- nameparser/_config_shim.py | 6 +++++- nameparser/_policy.py | 27 ++++++++++++++++++++++----- tests/v2/cases.py | 12 +++++++----- tests/v2/test_config_shim.py | 17 +++++++++++++++++ tests/v2/test_layering.py | 3 ++- tests/v2/test_policy.py | 36 ++++++++++++++++++++++++++++++++++++ 11 files changed, 111 insertions(+), 24 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index f39e510b..6177a76f 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -74,11 +74,14 @@ listed below. * - ``nickname_delimiters`` - ``frozenset[tuple[str, str]]`` - Routes content enclosed by these delimiter pairs to - ``nickname``. Defaults to quotes and parentheses. + ``nickname``. Defaults to + :data:`~nameparser.DEFAULT_NICKNAME_DELIMITERS` — quotes and + parentheses. * - ``maiden_delimiters`` - ``frozenset[tuple[str, str]]`` - Routes content enclosed by these delimiter pairs to ``maiden`` - instead. Defaults to empty — see the routing example below. + instead, and drops them from the effective nickname set. + Defaults to empty — see the routing example below. * - ``extra_suffix_delimiters`` - ``frozenset[str]`` - Adds separators (beyond a comma) that split suffix groups, e.g. @@ -100,17 +103,16 @@ listed below. .. doctest:: >>> from nameparser import Parser, Policy - >>> policy = Policy( - ... nickname_delimiters=frozenset({("'", "'"), ('"', '"')}), - ... maiden_delimiters=frozenset({("(", ")")}), - ... ) + >>> policy = Policy(maiden_delimiters=frozenset({("(", ")")})) >>> Parser(policy=policy).parse("Jane (Jones) Smith").maiden 'Jones' -Routing parentheses to ``maiden`` means moving that pair out of -``nickname_delimiters`` as well as into ``maiden_delimiters``, as -above — a delimiter pair present in *both* buckets routes to -``nickname``. +A pair routes to exactly one field, and ``maiden_delimiters`` states +the specific intent — so listing a pair there automatically drops it +from the effective ``nickname_delimiters`` set, and the one-liner +above is the whole recipe. To *add* delimiters instead of rerouting +them, build on the named default: +``nickname_delimiters=DEFAULT_NICKNAME_DELIMITERS | {("«", "»")}``. Presentation: rendering arguments ---------------------------------- diff --git a/docs/migrate.rst b/docs/migrate.rst index f7b71d0e..fa961e49 100644 --- a/docs/migrate.rst +++ b/docs/migrate.rst @@ -174,7 +174,10 @@ rendering argument, where the 2.0 equivalent isn't config at all): ``(open, close)`` pairs * - ``maiden_delimiters`` - ``Policy.maiden_delimiters`` - - Same shape change as ``nickname_delimiters`` + - Same shape change as ``nickname_delimiters``. Precedence + differs: in the 2.0 API a pair listed here wins over + ``nickname_delimiters``; through the 1.x facade a pair in both + buckets keeps parsing as ``nickname`` (v1 behavior) * - ``regexes.bidi`` - ``Policy.strip_bidi`` - ``regexes.bidi = False`` becomes ``Policy(strip_bidi=False)`` diff --git a/docs/modules.rst b/docs/modules.rst index 85d808ab..a4593a6e 100644 --- a/docs/modules.rst +++ b/docs/modules.rst @@ -58,6 +58,8 @@ Configuration .. autodata:: nameparser.FAMILY_FIRST_GIVEN_LAST +.. autodata:: nameparser.DEFAULT_NICKNAME_DELIMITERS + Locales ~~~~~~~ diff --git a/docs/release_log.rst b/docs/release_log.rst index 583ef393..b94b7b77 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -24,6 +24,7 @@ Release Log - Data change: ``ma``/``do`` added to ``suffix_acronyms_ambiguous`` -- ambiguous acronyms count as suffixes only when written with periods, so a bare common surname (``"Jack Ma"``, ``"Anh Do"``) keeps its family name under 2.0's keep-recognized-suffixes routing, matching 1.4's output (``tests/v2/cases.py`` row ``ambiguous_surname_acronyms``). Side effect: parenthesized/quoted ``"(MA)"``/``"(DO)"`` (no periods) no longer escape to ``suffix`` the way 1.x did -- they now fall through to nickname parsing like any other ambiguous-acronym delimited content. Not present in the differential corpus - Change suffix-delimiter rendering: with a custom ``Policy``/``Constants`` suffix delimiter configured (e.g. ``suffix_delimiter="/"``, ``"John Smith, RN/CRNA"``), 1.x split the token and rendered ``suffix="RN, CRNA"``; 2.0 keeps the no-space delimiter-core token whole (``suffix="RN/CRNA"``) -- role assignment is unchanged, only rendering differs (anti-#100, migration plan deviation 5; ``tests/v2/cases.py`` row ``suffix_delimiter_no_space_core``). Only fires with a non-default policy, so it does not appear in the (default-policy) differential corpus - Change ``comparison_key()``/``matches()`` (both APIs) to fold components with ``str.casefold()`` where 1.4 used ``str.lower()``: Unicode case-pair forms now compare equal -- ``"STRASSE"`` matches ``"Straße"``, and a Greek final-sigma variant matches its regular-sigma form. Strictly more permissive (anything 1.4 matched still matches); parse output is unaffected -- vocabulary normalization itself uses ``lower()``, v1-parity (``tests/v2/test_types.py`` row ``test_matches_casefolds_unicode_case_pairs``) + - Change delimiter-overlap precedence in the 2.0 API: a pair listed in ``Policy.maiden_delimiters`` is dropped from the effective ``nickname_delimiters`` set (maiden wins), so ``Policy(maiden_delimiters={("(", ")")})`` alone routes parenthesized content to ``maiden`` -- no need to rebuild the nickname set. The default nickname set is now the public ``DEFAULT_NICKNAME_DELIMITERS`` constant. The 1.x facade keeps v1's nickname-wins precedence on overlap via a shim-side pre-subtraction, so no ``HumanName`` behavior changes (``tests/v2/cases.py`` row ``maiden_delimiters_win_when_shared``; ``tests/v2/test_config_shim.py`` row ``test_snapshot_overlap_keeps_v1_nickname_precedence``) Everything else in the 486-name differential corpus (built from the v1 test banks as of commit ``2d5d8c2``, pre-dating the M12 test diff --git a/nameparser/__init__.py b/nameparser/__init__.py index fb9575fc..9e6e7011 100644 --- a/nameparser/__init__.py +++ b/nameparser/__init__.py @@ -10,6 +10,7 @@ from nameparser._locale import Locale from nameparser._parser import Parser, parse, parser_for from nameparser._policy import ( + DEFAULT_NICKNAME_DELIMITERS, FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, GIVEN_FIRST, @@ -33,6 +34,7 @@ # v2 core "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET", - "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "Locale", + "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", + "DEFAULT_NICKNAME_DELIMITERS", "Locale", "Parser", "parse", "parser_for", ] diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index 3513b027..d71dbc68 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -950,8 +950,12 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: middle_as_family=self.middle_name_as_last, nickname_delimiters=frozenset( _SENTINEL_PAIRS[k] for k in self.nickname_delimiters), + # v1 precedence: a pair in BOTH v1 buckets parses as nickname. + # Policy itself resolves overlap the other way (maiden wins), + # so pre-subtract here to keep the facade at v1 behavior. maiden_delimiters=frozenset( - _SENTINEL_PAIRS[k] for k in self.maiden_delimiters), + _SENTINEL_PAIRS[k] for k in self.maiden_delimiters + if k not in self.nickname_delimiters), # suffix_delimiter is a _RenderDefaults-only field here; the # facade layers it onto extra_suffix_delimiters per instance # (a later task) -- _snapshot() itself stays pure translation diff --git a/nameparser/_policy.py b/nameparser/_policy.py index bf9d30c4..d9ae6790 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -27,6 +27,13 @@ class PatronymicRule(StrEnum): _NAME_ROLES = frozenset({Role.GIVEN, Role.MIDDLE, Role.FAMILY}) +#: Policy.nickname_delimiters' default (v1 parity: quotes + parentheses). +#: Public and named so customizations read as set math against a +#: documented value -- e.g. ``DEFAULT_NICKNAME_DELIMITERS | {("«", "»")}`` +#: -- instead of a rebuilt literal the user had to go discover. +DEFAULT_NICKNAME_DELIMITERS = frozenset( + {("'", "'"), ('"', '"'), ("(", ")")}) + def _reject_bare_string_order(value: object) -> None: # tuple("gmf") would be ("g", "m", "f") -- catch the bare string @@ -45,11 +52,11 @@ class Policy: patronymic_rules: frozenset[PatronymicRule] = frozenset() middle_as_family: bool = False # v1's middle_name_as_last # v1 default delimiter set (#273) - nickname_delimiters: frozenset[tuple[str, str]] = frozenset( - {("'", "'"), ('"', '"'), ("(", ")")} - ) - # empty by default (v1 parity); route ("(", ")") here to send - # parenthesized content to maiden instead of nickname (#274) + nickname_delimiters: frozenset[tuple[str, str]] = DEFAULT_NICKNAME_DELIMITERS + # empty by default (v1 parity); a pair listed here routes to maiden + # AND is dropped from the effective nickname set (maiden wins, see + # __post_init__), so maiden_delimiters={("(", ")")} is the whole + # parenthesized-maiden recipe (#274) maiden_delimiters: frozenset[tuple[str, str]] = frozenset() extra_suffix_delimiters: frozenset[str] = frozenset() lenient_comma_suffixes: bool = True @@ -129,6 +136,16 @@ def __post_init__(self) -> None: f"strings, got {pair!r}" ) object.__setattr__(self, pairs_name, frozenset(pairs)) + # Maiden wins: a pair can route to exactly one field, and listing + # it in maiden_delimiters is the specific intent, so the effective + # nickname set drops it. Canonicalization, not validation (the + # name_order coercion precedent): differently-written but + # equivalent Policies converge to equal values. The v1 facade + # keeps v1's nickname-wins precedence via a pre-subtraction in + # _config_shim's snapshot instead. + object.__setattr__( + self, "nickname_delimiters", + self.nickname_delimiters - self.maiden_delimiters) if isinstance(self.extra_suffix_delimiters, str): raise TypeError( f"extra_suffix_delimiters must be an iterable of strings, " diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 2aca7829..ca104fba 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -117,13 +117,15 @@ def __post_init__(self) -> None: notes="v1 renders each tail comma segment as ONE suffix " "entry; words within an entry space-join via the " "'joined' tag"), - Case("nickname_bucket_wins_when_shared", + Case("maiden_delimiters_win_when_shared", 'Baker (Johnson), Jenny', - {"given": "Jenny", "family": "Baker", "nickname": "Johnson"}, + {"given": "Jenny", "family": "Baker", "maiden": "Johnson"}, policy=Policy(maiden_delimiters=frozenset({("(", ")")})), - notes="when the same delimiter pair sits in both buckets the " - "nickname reading wins (v1 parse_nicknames order); the " - "bucket-move idiom removes it from nickname first"), + notes="listing a pair in maiden_delimiters drops it from the " + "effective nickname set (maiden wins, 2026-07-19) -- the " + "one-liner replaces the bucket-move idiom; the v1 facade " + "keeps v1's nickname-wins precedence via the shim's " + "pre-subtraction (pinned in test_config_shim)"), Case("family_segment_trailing_suffix", "Smith Jr., John", {"given": "John", "family": "Smith", "suffix": "Jr."}, notes="v1: the family part may have suffixes in it " diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index 7cee0d66..dd5933f2 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -361,6 +361,23 @@ def test_snapshot_delimiter_bucket_move() -> None: assert ("(", ")") not in policy.nickname_delimiters +def test_snapshot_overlap_keeps_v1_nickname_precedence() -> None: + # v1 semantics: a pair present in BOTH v1 buckets parses as nickname. + # 2.0's Policy resolves overlap the other way (maiden wins), so the + # snapshot pre-subtracts on the maiden side -- a v1 user who added + # parens to maiden_delimiters WITHOUT removing them from + # nickname_delimiters keeps their 1.x parse through the facade. + c = Constants() + c.maiden_delimiters["parenthesis"] = ("(", ")") # nickname still has it + _, policy, _ = c._snapshot() + assert ("(", ")") in policy.nickname_delimiters + assert ("(", ")") not in policy.maiden_delimiters + from nameparser import HumanName + + n = HumanName("Jane (Jones) Smith", constants=c) + assert n.nickname == "Jones" and n.maiden == "" + + def test_snapshot_ambiguous_removed_acronym_intersects() -> None: c = Constants() c.suffix_acronyms.remove("jd") # 'jd' stays in ambiguous diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index fe6120f2..7ddcf910 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -166,7 +166,8 @@ def test_public_exports() -> None: expected = { "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET", - "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "Locale", + "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", + "DEFAULT_NICKNAME_DELIMITERS", "Locale", "Parser", "parse", "parser_for", } assert expected <= set(nameparser.__all__) diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index de929fa9..170876b4 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -90,6 +90,42 @@ def test_patronymic_rules_true_points_at_the_v1_migration() -> None: PolicyPatch(extra_suffix_delimiters=True) # type: ignore[arg-type] +def test_maiden_delimiters_win_over_nickname_defaults() -> None: + # A pair routes to exactly one field, and listing it in + # maiden_delimiters is the specific intent -- so the one-liner works + # without knowing (or rebuilding) the nickname default set. The + # review that prompted this: routing parens to maiden used to + # require both buckets edited in tandem. + p = Policy(maiden_delimiters=frozenset({("(", ")")})) + assert ("(", ")") in p.maiden_delimiters + assert ("(", ")") not in p.nickname_delimiters + # canonicalization: the explicit-removal spelling converges to the + # same value (equal AND same hash -- cache keys agree) + explicit = Policy( + nickname_delimiters=frozenset({("'", "'"), ('"', '"')}), + maiden_delimiters=frozenset({("(", ")")}), + ) + assert p == explicit and hash(p) == hash(explicit) + + +def test_maiden_precedence_applies_through_policy_patch() -> None: + # apply_patch re-runs Policy's constructor, so a patch (e.g. from a + # locale pack) adding a maiden pair gets the same subtraction + patched = apply_patch( + Policy(), PolicyPatch(maiden_delimiters=frozenset({("(", ")")}))) + assert ("(", ")") in patched.maiden_delimiters + assert ("(", ")") not in patched.nickname_delimiters + + +def test_default_nickname_delimiters_constant_is_the_default() -> None: + from nameparser import DEFAULT_NICKNAME_DELIMITERS + + assert Policy().nickname_delimiters == DEFAULT_NICKNAME_DELIMITERS + # the documented set-math idiom stays valid input + p = Policy(nickname_delimiters=DEFAULT_NICKNAME_DELIMITERS | {("«", "»")}) + assert ("«", "»") in p.nickname_delimiters + + def test_policy_delimiters_coerce_to_frozensets() -> None: p = Policy(nickname_delimiters=[("(", ")")]) # type: ignore[arg-type] assert isinstance(p.nickname_delimiters, frozenset) From 0ab19fdbbd9d7969f6d303fe4fa28b3c04e500b5 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 11:33:01 -0700 Subject: [PATCH 179/275] docs: use dean, not chancellor, as the add-a-title example chancellor is already in the default TITLES set, so the doctest's add() was a no-op demonstrating nothing. dean is genuinely absent (the old v1 docs' example word) -- and absent for a teachable reason, which the ambiguous-entries paragraph now uses: 'Dean' doubles as a given name, so a default that swallowed it would misparse 'Dean Martin'. Co-Authored-By: Claude Fable 5 --- docs/concepts.rst | 6 +++--- docs/customize.rst | 14 +++++++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index f25e8c0c..f8395585 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -52,9 +52,9 @@ this setting vary with? ``spec`` you pass to ``render(spec)``, or a keyword to ``initials()``/``capitalized()``) -"Chancellor" being a title is a fact about German-language input, not -about any one dataset or report — that's a :class:`~nameparser.Lexicon` -entry. A CRM that always exports "Family, Given" strings is a fact +"Dean" being a title in your data is a fact about the language and +domain the names come from (academic rosters, say), not about any one +dataset or report — that's a :class:`~nameparser.Lexicon` entry. A CRM that always exports "Family, Given" strings is a fact about that one data source, not about the language of the names in it — that's a :class:`~nameparser.Policy`. One particular report wanting names formatted as "Family, Given" while every other consumer diff --git a/docs/customize.rst b/docs/customize.rst index 6177a76f..c391b7ae 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -14,9 +14,9 @@ Vocabulary: Lexicon .. doctest:: >>> from nameparser import Lexicon, Parser - >>> lex = Lexicon.default().add(titles={"chancellor"}) - >>> Parser(lexicon=lex).parse("Chancellor Angela Merkel").title - 'Chancellor' + >>> lex = Lexicon.default().add(titles={"dean"}) + >>> Parser(lexicon=lex).parse("Dean Robert Johns").title + 'Dean' :meth:`~nameparser.Lexicon.add` and :meth:`~nameparser.Lexicon.remove` both return a new :class:`~nameparser.Lexicon` — the one you started @@ -39,7 +39,11 @@ don't add new vocabulary by themselves; they narrow how an existing entry is read when it appears alone. If you're not sure whether a word you're adding is one of these ambiguous cases, leave it out — an unrecognized word usually still parses reasonably, while a wrongly -disambiguated one silently picks the less likely reading. +disambiguated one silently picks the less likely reading. (That +conservatism is why ``dean`` above isn't in the default vocabulary in +the first place: "Dean" is also a common given name, and a default +that swallowed it as a title would misparse "Dean Martin" for +everyone.) Behavior: Policy ----------------- @@ -145,7 +149,7 @@ you parse. # myapp/names.py from nameparser import Lexicon, Parser, Policy - lex = Lexicon.default().add(titles={"chancellor"}) + lex = Lexicon.default().add(titles={"dean"}) policy = Policy(strip_emoji=False) parser = Parser(lexicon=lex, policy=policy) From f609d4037de958611734a4ab3589803ad7a9de77 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 11:43:18 -0700 Subject: [PATCH 180/275] docs: user-facing docstrings across the public API surface The task pages delegate detail to the reference ('see modules for the full field list'), but Token/Ambiguity/Lexicon/Policy/Locale had no class docstrings (autodoc rendered bare constructor signatures), Role inherited Enum's 'Create a collection of name/value pairs', and no dataclass field carried a #: doc. Every public class now opens with what it is / how you get one / how it relates to its neighbors, every field has a one-line #: doc that renders in the reference, and parse()/Parser.parse()'s first sentences speak to users instead of citing spec sections. Maintainer notes stay, demoted below the user-facing text. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 31 ++++++++++++++++++++ nameparser/_locale.py | 12 ++++++++ nameparser/_parser.py | 30 +++++++++++++------- nameparser/_policy.py | 29 +++++++++++++++---- nameparser/_types.py | 64 ++++++++++++++++++++++++++++++++++++------ 5 files changed, 143 insertions(+), 23 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 8c1d9960..bf124a7c 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -128,16 +128,47 @@ def _normpairs( @dataclass(frozen=True, slots=True) class Lexicon: + """The vocabulary a parser matches against: which words are + titles, particles, suffixes, and so on. Immutable and hashable. + Start from :meth:`default` (the shipped vocabulary) or + :meth:`empty`, derive variants with :meth:`add` / :meth:`remove` / + ``|`` (union), and pass the result to ``Parser(lexicon=...)``. + Entries are normalized at construction -- lowercased, edge periods + stripped -- so matching is case-insensitive.""" + + #: Pre-nominal titles ("dr", "sir", "capt"). titles: frozenset[str] = frozenset() + #: Titles whose single following name reads as a GIVEN name + #: ("sheikh", "sister") rather than a family name. given_name_titles: frozenset[str] = frozenset() + #: Post-nominal acronym suffixes, matched with or without periods + #: ("phd" matches "PhD" and "Ph.D."). suffix_acronyms: frozenset[str] = frozenset() + #: Post-nominal word suffixes ("jr", "esquire", "iii"). suffix_words: frozenset[str] = frozenset() + #: Subset of suffix_acronyms counted as suffixes only when written + #: WITH periods -- their bare forms are common surnames ("ma", + #: "do": "Jack Ma" keeps his family name). suffix_acronyms_ambiguous: frozenset[str] = frozenset() + #: Family-name particles that chain onto the following piece + #: ("van", "de", "bin"). particles: frozenset[str] = frozenset() + #: Subset of particles that can also BE a given name: a leading + #: one reads as given and records a particle-or-given ambiguity + #: ("Van Johnson"). particles_ambiguous: frozenset[str] = frozenset() + #: Words that join surrounding pieces into one ("y", "and", "и"). conjunctions: frozenset[str] = frozenset() + #: Given-name prefixes that bind to the following word to form one + #: given name ("abdul" -> "Abdul Salam"); never standalone names. bound_given_names: frozenset[str] = frozenset() + #: Marker words introducing a birth surname, routed to the maiden + #: field ("née", "geb.", "roz."). maiden_markers: frozenset[str] = frozenset() + #: Lowercase word -> exact-cased replacement used by capitalized() + #: ("phd" -> "Ph.D."). Pair-valued: change it with + #: dataclasses.replace(), not add()/remove(); read it as a mapping + #: via capitalization_exceptions_map. # Canonical storage: sorted tuple of (key, value) pairs. The # constructor tolerates any Mapping (or pair iterable) at runtime and # canonicalizes here; this closes the caller-aliasing hole and keeps diff --git a/nameparser/_locale.py b/nameparser/_locale.py index d50cd6ed..e1cd0fa5 100644 --- a/nameparser/_locale.py +++ b/nameparser/_locale.py @@ -20,8 +20,20 @@ @dataclass(frozen=True, slots=True) class Locale: + """A named, shareable bundle of vocabulary and behavior for a + naming tradition: a lexicon fragment plus a policy patch, applied + together by :func:`nameparser.parser_for`. The packs shipped with + nameparser live in :mod:`nameparser.locales`; building your own + needs no registration -- construct one and pass it to + ``parser_for``.""" + + #: Identifier, lowercase ``[a-z0-9_]+`` (e.g. "ru", "tr_az"). code: str + #: Vocabulary ADDED to the base parser's lexicon (unioned; a pack + #: never removes base vocabulary). lexicon: Lexicon + #: Behavior changes folded onto the base policy (set-valued fields + #: union; scalars override, later pack wins). policy: PolicyPatch = PolicyPatch() # in the class body so @dataclass(slots=True) keeps them diff --git a/nameparser/_parser.py b/nameparser/_parser.py index 21e6073c..93b6545a 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -24,12 +24,18 @@ @dataclass(frozen=True, slots=True) class Parser: - """Immutable, thread-safe, picklable by construction (spec §4): all - validity checking happens at construction; a Parser that constructs - successfully cannot fail at parse time on any str content. The None - field defaults resolve in __post_init__; after construction both - fields are always non-None (the annotations state the steady-state - truth, hence the assignment ignores on the defaults).""" + """A configured name parser: a :class:`Lexicon` (vocabulary) plus + a :class:`Policy` (behavior), both defaulted when omitted. Build + one when you need non-default configuration, build it once, and + call :meth:`parse` many times -- it is immutable, thread-safe, and + picklable by construction: all validity checking happens at + construction, so a Parser that constructs successfully cannot fail + at parse time on any str content. + + (The None field defaults resolve in __post_init__; after + construction both fields are always non-None -- the annotations + state the steady-state truth, hence the assignment ignores on the + defaults.)""" lexicon: Lexicon = None # type: ignore[assignment] # None -> default() policy: Policy = None # type: ignore[assignment] # None -> Policy() @@ -55,9 +61,10 @@ def __repr__(self) -> str: return f"Parser({self.lexicon!r}, {self.policy!r})" def parse(self, text: str) -> ParsedName: - """Total over str (spec §5a): content never raises; non-str - raises TypeError eagerly, with a decode hint for bytes (#245: - bytes support ended with 1.x).""" + """Parse one name string into a :class:`ParsedName`. Never + raises on string content (unparseable input yields empty + fields plus ambiguities); non-str raises TypeError eagerly, + with a decode hint for bytes (bytes support ended with 1.x).""" if isinstance(text, bytes): raise TypeError( "parse() takes str, not bytes -- decode first, e.g. " @@ -75,7 +82,10 @@ def _default_parser() -> Parser: def parse(text: str) -> ParsedName: - """Module-level convenience over a lazily created default Parser.""" + """Parse a name with the default configuration and return a + :class:`ParsedName`. Equivalent to ``Parser().parse(text)``; build + your own :class:`Parser` (or use :func:`parser_for`) for custom + vocabulary or behavior. Never raises on string content.""" return _default_parser().parse(text) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index d9ae6790..b11d77e2 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -48,19 +48,38 @@ def _reject_bare_string_order(value: object) -> None: @dataclass(frozen=True, slots=True) class Policy: + """The behavior switches a parser runs with: name order, + patronymic rules, delimiter routing, input scrubbing. Immutable + and hashable; every field has a safe default, so construct with + only what you change -- ``Policy(maiden_delimiters={("(", ")")})`` + -- and pass the result to ``Parser(policy=...)``.""" + + #: How positional (no-comma) input maps onto given/middle/family; + #: one of the exported GIVEN_FIRST (default), FAMILY_FIRST, + #: FAMILY_FIRST_GIVEN_LAST orders. name_order: tuple[Role, Role, Role] = GIVEN_FIRST + #: Opt-in detectors that reorder patronymic-shaped names + #: (EAST_SLAVIC, TURKIC); usually set via a locale pack. patronymic_rules: frozenset[PatronymicRule] = frozenset() + #: Folds middle into family instead of splitting them (v1's + #: middle_name_as_last). middle_as_family: bool = False # v1's middle_name_as_last - # v1 default delimiter set (#273) + #: (open, close) pairs whose enclosed content becomes the nickname + #: field. Defaults to DEFAULT_NICKNAME_DELIMITERS (#273). nickname_delimiters: frozenset[tuple[str, str]] = DEFAULT_NICKNAME_DELIMITERS - # empty by default (v1 parity); a pair listed here routes to maiden - # AND is dropped from the effective nickname set (maiden wins, see - # __post_init__), so maiden_delimiters={("(", ")")} is the whole - # parenthesized-maiden recipe (#274) + #: (open, close) pairs whose enclosed content becomes the maiden + #: field instead; a pair listed here is dropped from the effective + #: nickname set (maiden wins, see __post_init__), so + #: maiden_delimiters={("(", ")")} is the whole recipe (#274). maiden_delimiters: frozenset[tuple[str, str]] = frozenset() + #: Separators beyond a comma that split suffix groups (e.g. " - "). extra_suffix_delimiters: frozenset[str] = frozenset() + #: Allows a post-comma segment to read as a suffix via the lenient + #: initial-shaped test; False requires the strict acronym form. lenient_comma_suffixes: bool = True + #: Removes emoji from the input before parsing. strip_emoji: bool = True + #: Removes bidirectional control characters before parsing. strip_bidi: bool = True # =False replaces v1's opt-out CONSTANTS.regexes.bidi = False # in the class body so @dataclass(slots=True) keeps them diff --git a/nameparser/_types.py b/nameparser/_types.py index 76fce80f..e3ba741f 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -25,6 +25,12 @@ class Role(Enum): + """The seven fields of a parsed name, one per :class:`Token`. + Declaration order is the canonical field order everywhere + (``as_dict()``, ``comparison_key()``, rendering). Pass a member to + :meth:`ParsedName.tokens_for`; a member's ``.value`` is the field's + string name (``Role.GIVEN.value == "given"``).""" + # Declaration order IS the canonical field order (conventions §3): # every listing of the seven fields anywhere derives from this. TITLE = "title" @@ -37,9 +43,15 @@ class Role(Enum): class Span(NamedTuple): - """Provenance range into ParsedName.original. end is exclusive.""" + """Where a :class:`Token` came from: a character range into + :attr:`ParsedName.original` such that ``original[start:end]`` is + the token's source text (``end`` exclusive). A plain two-int + NamedTuple; ``None`` in :attr:`Token.span` marks a synthetic token + with no source position.""" + #: First character index (0-based). start: int + #: One past the last character index. end: int def __add__(self, other: object) -> NoReturn: # type: ignore[override] @@ -120,9 +132,23 @@ def _guarded_setstate(self: object, state: dict[str, object]) -> None: @dataclass(frozen=True, slots=True) class Token: + """One classified word of a parsed name: its text, where it came + from, which field it belongs to, and how it was classified. Read + tokens off :attr:`ParsedName.tokens` or + :meth:`ParsedName.tokens_for`; you only construct one directly + when hand-building a :class:`ParsedName`.""" + + #: The word exactly as written in the input (never empty). text: str - span: Span | None # None = synthetic (from replace()) + #: Position in ParsedName.original; None marks a synthetic token + #: (e.g. introduced by replace()) with no source position. + span: Span | None + #: The field this token belongs to. role: Role + #: Classification labels. Exactly the four in STABLE_TAGS + #: ("particle", "conjunction", "initial", "joined") are API; + #: namespaced tags like "vocab:..." are unstable debugging + #: provenance -- never match against them. tags: frozenset[str] = frozenset() # in the class body so @dataclass(slots=True) keeps them @@ -187,7 +213,10 @@ def __repr__(self) -> str: class AmbiguityKind(StrEnum): - """Stable identifiers (API); members ARE their string values.""" + """The stable vocabulary of :class:`Ambiguity` kinds. A StrEnum: + members ARE their string values, so ``kind == "particle-or-given"`` + compares directly. New kinds may be added in minor releases; + existing values never change meaning.""" ORDER = "order" SUFFIX_OR_NICKNAME = "suffix-or-nickname" @@ -198,8 +227,17 @@ class AmbiguityKind(StrEnum): @dataclass(frozen=True, slots=True) class Ambiguity: + """A call the parser made that could legitimately have gone the + other way, surfaced on :attr:`ParsedName.ambiguities` instead of + silently guessed away. The parse still commits to one reading -- + an Ambiguity is a flag for review, not an error.""" + + #: Which known ambiguity shape this is (stable API values). kind: AmbiguityKind + #: Human-readable specifics of this occurrence (wording unstable). detail: str + #: The tokens involved -- always a subset of the owning + #: ParsedName's tokens; may be empty (e.g. unbalanced-delimiter). tokens: tuple[Token, ...] # in the class body so @dataclass(slots=True) keeps them @@ -232,15 +270,25 @@ def __repr__(self) -> str: @dataclass(frozen=True, slots=True) class ParsedName: - """Immutable result of a parse. Constructor-enforced invariants: - spans ascending, non-overlapping, in bounds of `original`; every - Ambiguity's tokens are a subset of `tokens`. Provenance semantics - (text == original[span] for parser-produced names) are documented, - not enforced -- transforms like replace() legitimately break them. + """The immutable result of parsing one name string. Read the seven + fields as strings (``.given``, ``.family``, ...); inspect structure + through :attr:`tokens` / :meth:`tokens_for`; correct a parse with + :meth:`replace` (returns a new value); produce output with + :meth:`render`, :meth:`initials`, :meth:`capitalized`, or ``str()``. + + Constructor-enforced invariants: spans ascending, non-overlapping, + in bounds of `original`; every Ambiguity's tokens are a subset of + `tokens`. Provenance semantics (text == original[span] for + parser-produced names) are documented, not enforced -- transforms + like replace() legitimately break them. """ + #: The input string exactly as passed to parse(). original: str + #: Every classified token, in document order. tokens: tuple[Token, ...] + #: Judgment calls that could have gone the other way; empty for + #: most names (see Ambiguity). ambiguities: tuple[Ambiguity, ...] = () # in the class body so @dataclass(slots=True) keeps them From 76e85dca935f72e6150fbe600e765c661dc0f4eb Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 11:59:36 -0700 Subject: [PATCH 181/275] docs: introduce and properly describe the module-level constants autodata on a re-export can't see the defining module's #: doc comments, so every module-level constant rendered its TYPE's docstring -- GIVEN_FIRST et al. showed tuple's 'Built-in immutable sequence', DEFAULT_NICKNAME_DELIMITERS showed frozenset's constructor help, UNSET rendered blank. Replace them with manual py:data directives carrying real descriptions, under two new reference sub-headings ('Name-order constants' -- stating that name_order accepts ONLY these three, arbitrary Role tuples raise -- and 'Delimiter defaults'). The customize table's name_order type cell now says 'one of the three exported order constants' instead of the tuple[Role, Role, Role] annotation that invited hand-rolled tuples. The #: comments stay in _policy.py for source readers. Co-Authored-By: Claude Fable 5 --- docs/customize.rst | 2 +- docs/modules.rst | 46 ++++++++++++++++++++++++++++++++++++++----- nameparser/_policy.py | 10 ++++++++++ 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index c391b7ae..a2fb1c14 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -61,7 +61,7 @@ listed below. - Type - Effect * - ``name_order`` - - ``tuple[Role, Role, Role]`` + - one of the three exported order constants - Assigns positional (no-comma) input to given/middle/family in this order. Use the exported ``GIVEN_FIRST`` (default), ``FAMILY_FIRST``, or ``FAMILY_FIRST_GIVEN_LAST`` constants. diff --git a/docs/modules.rst b/docs/modules.rst index a4593a6e..4aa63e57 100644 --- a/docs/modules.rst +++ b/docs/modules.rst @@ -47,18 +47,54 @@ Configuration .. autoclass:: nameparser.PolicyPatch :members: -.. autodata:: nameparser.UNSET +.. py:data:: nameparser.UNSET + + Sentinel meaning "this patch does not set this field" — the default + of every :class:`~nameparser.PolicyPatch` field, distinguishable + from every real value including ``None`` and ``False``. .. autoclass:: nameparser.PatronymicRule :members: -.. autodata:: nameparser.GIVEN_FIRST +Name-order constants +^^^^^^^^^^^^^^^^^^^^ + +The three valid values for ``Policy(name_order=...)``. ``name_order`` +is deliberately restricted to these exported constants — an arbitrary +tuple of :class:`~nameparser.Role` members raises ``ValueError``, +because only these three orders have defined assignment semantics. + +.. py:data:: nameparser.GIVEN_FIRST + :value: (Role.GIVEN, Role.MIDDLE, Role.FAMILY) + + Western order (the default): the first word of positional input is + the given name, the last is the family name, everything between is + middle. + +.. py:data:: nameparser.FAMILY_FIRST + :value: (Role.FAMILY, Role.GIVEN, Role.MIDDLE) + + Family name first, given name second, remaining words middle — + e.g. Hungarian, or East Asian order. + +.. py:data:: nameparser.FAMILY_FIRST_GIVEN_LAST + :value: (Role.FAMILY, Role.MIDDLE, Role.GIVEN) + + Family name first, given name *last*, the words between middle — + e.g. Vietnamese full-name order. -.. autodata:: nameparser.FAMILY_FIRST +Delimiter defaults +^^^^^^^^^^^^^^^^^^ -.. autodata:: nameparser.FAMILY_FIRST_GIVEN_LAST +.. py:data:: nameparser.DEFAULT_NICKNAME_DELIMITERS + :value: frozenset({("'", "'"), ('"', '"'), ("(", ")")}) -.. autodata:: nameparser.DEFAULT_NICKNAME_DELIMITERS + The default :attr:`~nameparser.Policy.nickname_delimiters` set — + quotes and parentheses. Build on it for additive customizations, + e.g. ``nickname_delimiters=DEFAULT_NICKNAME_DELIMITERS | + {("«", "»")}``; to *reroute* a pair to ``maiden``, just list it in + :attr:`~nameparser.Policy.maiden_delimiters` (it is dropped from + the effective nickname set automatically). Locales ~~~~~~~ diff --git a/nameparser/_policy.py b/nameparser/_policy.py index b11d77e2..4a81568a 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -21,8 +21,18 @@ class PatronymicRule(StrEnum): # Order-spec constants (#270). Each reads as its contents because roles # are named given/family, not first/last. + +#: Western order (the default): the first word of positional input is +#: the given name, the last is the family name, everything between is +#: middle. One of the three valid ``Policy(name_order=...)`` values. GIVEN_FIRST = (Role.GIVEN, Role.MIDDLE, Role.FAMILY) +#: Family name first, given name second, remaining words middle +#: (e.g. Hungarian, or East Asian order). One of the three valid +#: ``Policy(name_order=...)`` values. FAMILY_FIRST = (Role.FAMILY, Role.GIVEN, Role.MIDDLE) +#: Family name first, given name LAST, words between middle +#: (e.g. Vietnamese full-name order). One of the three valid +#: ``Policy(name_order=...)`` values. FAMILY_FIRST_GIVEN_LAST = (Role.FAMILY, Role.MIDDLE, Role.GIVEN) _NAME_ROLES = frozenset({Role.GIVEN, Role.MIDDLE, Role.FAMILY}) From 45315389d7e4fe5a0a54d5fe106e4855466532d3 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 12:05:26 -0700 Subject: [PATCH 182/275] docs: say when UNSET is actually needed Co-Authored-By: Claude Fable 5 --- docs/modules.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/modules.rst b/docs/modules.rst index 4aa63e57..1e3d1746 100644 --- a/docs/modules.rst +++ b/docs/modules.rst @@ -51,7 +51,11 @@ Configuration Sentinel meaning "this patch does not set this field" — the default of every :class:`~nameparser.PolicyPatch` field, distinguishable - from every real value including ``None`` and ``False``. + from every real value including ``None`` and ``False``. You rarely + need it: omit a field instead of passing it. Import it to test + whether a patch sets a field (``patch.name_order is UNSET``) or to + leave a field conditionally unset when building patches + programmatically. .. autoclass:: nameparser.PatronymicRule :members: From d7df3ce7e0eae2c6b1e6ef7155c429cb50d1acf7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 12:08:58 -0700 Subject: [PATCH 183/275] docs: strip flags exclude characters from tokenization, not the input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 'Removes ... from the input' contradicted the spans-into-original invariant: parse('John 😀 Smith').original keeps the emoji; the tokenizer just never includes those characters in any token, so they appear in no field or rendered view. Say that. Co-Authored-By: Claude Fable 5 --- docs/customize.rst | 7 ++++--- nameparser/_policy.py | 7 +++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index a2fb1c14..fbd67b9b 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -97,12 +97,13 @@ listed below. strict acronym form. * - ``strip_emoji`` - ``bool`` - - Removes emoji from the input before parsing. Defaults to + - Excludes emoji from tokenization — they appear in no field or + rendered view, though ``original`` keeps them. Defaults to ``True``. * - ``strip_bidi`` - ``bool`` - - Removes bidirectional control characters from the input before - parsing. Defaults to ``True``. + - Excludes bidirectional control characters the same way. + Defaults to ``True``. .. doctest:: diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 4a81568a..ae97e1a9 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -87,9 +87,12 @@ class Policy: #: Allows a post-comma segment to read as a suffix via the lenient #: initial-shaped test; False requires the strict acronym form. lenient_comma_suffixes: bool = True - #: Removes emoji from the input before parsing. + #: Excludes emoji from tokenization: they appear in no token, + #: field, or rendered view. The original string keeps them (input + #: is never modified -- spans stay true). strip_emoji: bool = True - #: Removes bidirectional control characters before parsing. + #: Excludes bidirectional control characters from tokenization the + #: same way. strip_bidi: bool = True # =False replaces v1's opt-out CONSTANTS.regexes.bidi = False # in the class body so @dataclass(slots=True) keeps them From 4e657a8103643c7c635efc389fe7349dff6d3e4b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 12:14:22 -0700 Subject: [PATCH 184/275] docs: name_order states its valid values and comma-input scope Verified: assign consults name_order only for the NO_COMMA structure; 'Yamamoto, Haruki' takes family from the comma format with name_order never read. Co-Authored-By: Claude Fable 5 --- docs/customize.rst | 2 ++ nameparser/_policy.py | 10 +++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index fbd67b9b..607e48dc 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -65,6 +65,8 @@ listed below. - Assigns positional (no-comma) input to given/middle/family in this order. Use the exported ``GIVEN_FIRST`` (default), ``FAMILY_FIRST``, or ``FAMILY_FIRST_GIVEN_LAST`` constants. + Ignored for comma-format input ("Yamamoto, Haruki"), which + states its own order. * - ``patronymic_rules`` - ``frozenset[PatronymicRule]`` - Reorders patronymic-shaped names via opt-in detectors — East diff --git a/nameparser/_policy.py b/nameparser/_policy.py index ae97e1a9..a5bc0d0a 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -64,9 +64,13 @@ class Policy: only what you change -- ``Policy(maiden_delimiters={("(", ")")})`` -- and pass the result to ``Parser(policy=...)``.""" - #: How positional (no-comma) input maps onto given/middle/family; - #: one of the exported GIVEN_FIRST (default), FAMILY_FIRST, - #: FAMILY_FIRST_GIVEN_LAST orders. + #: How positional (no-comma) input maps onto given/middle/family. + #: Valid values are exactly the three exported constants -- + #: GIVEN_FIRST (the default), FAMILY_FIRST, and + #: FAMILY_FIRST_GIVEN_LAST; any other tuple of Roles raises + #: ValueError. Ignored when the input contains a comma: + #: comma-format input ("Yamamoto, Haruki") states its own order + #: explicitly. name_order: tuple[Role, Role, Role] = GIVEN_FIRST #: Opt-in detectors that reorder patronymic-shaped names #: (EAST_SLAVIC, TURKIC); usually set via a locale pack. From 79ff5f58f1ebf00c6e01dab6a1572f99abc3b6ba Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 12:20:11 -0700 Subject: [PATCH 185/275] docs: reference lists members in source order; strip_bidi self-contained autodoc's default alphabetical member sort scrambled the deliberate field ordering (strip_bidi rendered before strip_emoji, orphaning its 'the same way' reference; Lexicon's vocabulary grouping interleaved). autodoc_member_order = 'bysource' matches the codebase's declaration-order-is-canonical rule and the customize.rst table; the strip_bidi doc is also now self-contained regardless of ordering. Co-Authored-By: Claude Fable 5 --- docs/conf.py | 6 ++++++ nameparser/_policy.py | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 28f3301c..e69d2547 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -36,6 +36,12 @@ 'sphinx.ext.viewcode', ] +# Declaration order is canonical in this codebase (Role order drives the +# seven-field order everywhere), so the reference lists members in +# source order, not alphabetically -- Lexicon's vocabulary grouping and +# Policy's field ordering match the customize.rst table. +autodoc_member_order = 'bysource' + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/nameparser/_policy.py b/nameparser/_policy.py index a5bc0d0a..5aaa2ed3 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -95,8 +95,9 @@ class Policy: #: field, or rendered view. The original string keeps them (input #: is never modified -- spans stay true). strip_emoji: bool = True - #: Excludes bidirectional control characters from tokenization the - #: same way. + #: Excludes bidirectional control characters from tokenization: + #: they appear in no token, field, or rendered view; the original + #: string keeps them. strip_bidi: bool = True # =False replaces v1's opt-out CONSTANTS.regexes.bidi = False # in the class body so @dataclass(slots=True) keeps them From b746792732377565a13304de468529acb6f4a0e9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 12:24:14 -0700 Subject: [PATCH 186/275] docs: extra_suffix_delimiters is additive; the comma is not replaceable Verified live: with ' - ' configured, comma-split suffix groups parse unchanged. The comma is structural (the same reading that parses 'Family, Given' input), not a configurable suffix delimiter. Co-Authored-By: Claude Fable 5 --- docs/customize.rst | 5 +++-- nameparser/_policy.py | 6 +++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index 607e48dc..40cce522 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -90,8 +90,9 @@ listed below. Defaults to empty — see the routing example below. * - ``extra_suffix_delimiters`` - ``frozenset[str]`` - - Adds separators (beyond a comma) that split suffix groups, e.g. - ``" - "`` for ``"Jane Smith, RN - CRNA"``. + - Adds separators that split suffix groups, e.g. ``" - "`` for + ``"Jane Smith, RN - CRNA"``. Additions only — the comma always + splits suffix groups and cannot be replaced. * - ``lenient_comma_suffixes`` - ``bool`` - Allows a segment after a comma to be read as a suffix using diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 5aaa2ed3..dcc4164d 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -86,7 +86,11 @@ class Policy: #: nickname set (maiden wins, see __post_init__), so #: maiden_delimiters={("(", ")")} is the whole recipe (#274). maiden_delimiters: frozenset[tuple[str, str]] = frozenset() - #: Separators beyond a comma that split suffix groups (e.g. " - "). + #: Additional separators that split suffix groups (e.g. " - " for + #: "Jane Smith, RN - CRNA"). Additive only: the comma always + #: splits suffix groups and cannot be replaced -- comma handling + #: is structural (the same comma reading that parses + #: "Family, Given" input), not a configurable delimiter. extra_suffix_delimiters: frozenset[str] = frozenset() #: Allows a post-comma segment to read as a suffix via the lenient #: initial-shaped test; False requires the strict acronym form. From 28078c7358e63b8bcbc704c352c94941eda719af Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 12:26:22 -0700 Subject: [PATCH 187/275] docs: name_order links to the Name-order constants reference section Co-Authored-By: Claude Fable 5 --- docs/modules.rst | 2 ++ nameparser/_policy.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/modules.rst b/docs/modules.rst index 1e3d1746..7b23d399 100644 --- a/docs/modules.rst +++ b/docs/modules.rst @@ -60,6 +60,8 @@ Configuration .. autoclass:: nameparser.PatronymicRule :members: +.. _name-order-constants: + Name-order constants ^^^^^^^^^^^^^^^^^^^^ diff --git a/nameparser/_policy.py b/nameparser/_policy.py index dcc4164d..8d66930e 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -65,7 +65,8 @@ class Policy: -- and pass the result to ``Parser(policy=...)``.""" #: How positional (no-comma) input maps onto given/middle/family. - #: Valid values are exactly the three exported constants -- + #: Valid values are exactly the three exported + #: :ref:`name-order constants ` -- #: GIVEN_FIRST (the default), FAMILY_FIRST, and #: FAMILY_FIRST_GIVEN_LAST; any other tuple of Roles raises #: ValueError. Ignored when the input contains a comma: From f37cc4886384457dad8561776ba2659bf156430c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 12:29:31 -0700 Subject: [PATCH 188/275] docs: list the AmbiguityKind and PatronymicRule members in the reference Enum members without doc comments are skipped by autodoc, so the 'stable vocabulary of kinds' was invisible on the very page that promised it. Each member now carries a #: doc -- honestly marking ORDER and SUFFIX_OR_NICKNAME as reserved (no emitter yet; planned 2.x) and describing the emitter behavior of the three live kinds. PatronymicRule's two members get the same treatment. Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 10 +++++++++- nameparser/_types.py | 13 +++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 8d66930e..74eb07ec 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -13,9 +13,17 @@ class PatronymicRule(StrEnum): - """Stable rule names (API); implementations live in the pipeline.""" + """Stable rule names (API); implementations live in the pipeline. + Enable via ``Policy(patronymic_rules={...})`` or, more commonly, a + locale pack (:mod:`nameparser.locales`).""" + #: East Slavic formal order: "Sidorov Ivan Petrovich" + #: (family, given, patronymic) is detected by the patronymic + #: ending and reordered. Enabled by locales.RU. EAST_SLAVIC = "east-slavic" + #: Turkic patronymic markers: a standalone "oglu"/"qizi"/"kyzy" + #: (etc.) binds to the preceding name as a patronymic. Enabled by + #: locales.TR_AZ. TURKIC = "turkic" diff --git a/nameparser/_types.py b/nameparser/_types.py index e3ba741f..41bc82e3 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -218,10 +218,23 @@ class AmbiguityKind(StrEnum): compares directly. New kinds may be added in minor releases; existing values never change meaning.""" + #: Reserved: the name's field order itself is uncertain (e.g. a + #: two-word name under a non-default name_order). Not yet emitted; + #: planned for 2.x. ORDER = "order" + #: Reserved: a trailing word reads plausibly as either a suffix or + #: a nickname. Not yet emitted; planned for 2.x. SUFFIX_OR_NICKNAME = "suffix-or-nickname" + #: A leading ambiguous particle was read as a given name -- "Van + #: Johnson" parses given="Van", but "Van" is also a family-name + #: particle in other names. PARTICLE_OR_GIVEN = "particle-or-given" + #: A nickname/maiden delimiter opened without closing (or closed + #: without opening); the text was kept as literal name content. + #: May carry no tokens. UNBALANCED_DELIMITER = "unbalanced-delimiter" + #: More comma-separated segments than any recognized name shape; + #: the parse is best-effort over the extra segments. COMMA_STRUCTURE = "comma-structure" From 8c87959b5ed04ef5b2e3ac6b2165f66179166656 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 12:34:34 -0700 Subject: [PATCH 189/275] docs: explain lenient_comma_suffixes by its observable behavior 'Lenient initial-shaped test' was internal jargon. The flag decides 'John Smith, V': suffix (the fifth) by default, or given-initial V when False; multi-letter suffixes are unaffected. Verified live both ways before wording. Co-Authored-By: Claude Fable 5 --- docs/customize.rst | 8 +++++--- nameparser/_policy.py | 7 +++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index 40cce522..d4227257 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -95,9 +95,11 @@ listed below. splits suffix groups and cannot be replaced. * - ``lenient_comma_suffixes`` - ``bool`` - - Allows a segment after a comma to be read as a suffix using - the lenient, initial-shaped test. Set ``False`` to require the - strict acronym form. + - Reads an initial-shaped suffix word after a comma as a suffix: + ``"John Smith, V"`` is John Smith the fifth when ``True`` + (default); ``False`` reads ``V`` as a given-name initial + instead. Multi-letter suffixes (``III``, ``MD``) are + unaffected. * - ``strip_emoji`` - ``bool`` - Excludes emoji from tokenization — they appear in no field or diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 74eb07ec..e86e696f 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -101,8 +101,11 @@ class Policy: #: is structural (the same comma reading that parses #: "Family, Given" input), not a configurable delimiter. extra_suffix_delimiters: frozenset[str] = frozenset() - #: Allows a post-comma segment to read as a suffix via the lenient - #: initial-shaped test; False requires the strict acronym form. + #: Governs "Family, Suffix"-shaped input where the suffix word is + #: also initial-shaped: "John Smith, V" reads as John Smith the + #: fifth when True (the default, v1 behavior); False reads "V" as + #: a given-name initial instead (family "John Smith", given "V"). + #: Multi-letter suffixes ("III", "MD") parse the same either way. lenient_comma_suffixes: bool = True #: Excludes emoji from tokenization: they appear in no token, #: field, or rendered view. The original string keeps them (input From c8313a0268176c1d7ce2c741846ecf39cdf08e6d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 12:36:03 -0700 Subject: [PATCH 190/275] docs: define initial-shaped in the lenient_comma_suffixes doc Single letter, bare or period-written; of the default suffix words that means exactly the roman numerals I and V. Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index e86e696f..43878848 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -102,10 +102,12 @@ class Policy: #: "Family, Given" input), not a configurable delimiter. extra_suffix_delimiters: frozenset[str] = frozenset() #: Governs "Family, Suffix"-shaped input where the suffix word is - #: also initial-shaped: "John Smith, V" reads as John Smith the - #: fifth when True (the default, v1 behavior); False reads "V" as - #: a given-name initial instead (family "John Smith", given "V"). - #: Multi-letter suffixes ("III", "MD") parse the same either way. + #: also initial-shaped (a single letter, bare or period-written -- + #: of the default vocabulary that means the roman numerals "I" and + #: "V"): "John Smith, V" reads as John Smith the fifth when True + #: (the default, v1 behavior); False reads "V" as a given-name + #: initial instead (family "John Smith", given "V"). Multi-letter + #: suffixes ("III", "MD") parse the same either way. lenient_comma_suffixes: bool = True #: Excludes emoji from tokenization: they appear in no token, #: field, or rendered view. The original string keeps them (input From 0a5e93b6fb04d30ef95a147de266ad5446f7c0a1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 12:39:50 -0700 Subject: [PATCH 191/275] docs: comma-order example a reader can verify without cultural context 'Yamamoto, Haruki' only demonstrates the point if you already know which word is the surname. 'Thomas, John' is ambiguous to an English-reading audience in exactly the way the comma resolves. Co-Authored-By: Claude Fable 5 --- docs/customize.rst | 4 ++-- nameparser/_policy.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index d4227257..e7cb4157 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -65,8 +65,8 @@ listed below. - Assigns positional (no-comma) input to given/middle/family in this order. Use the exported ``GIVEN_FIRST`` (default), ``FAMILY_FIRST``, or ``FAMILY_FIRST_GIVEN_LAST`` constants. - Ignored for comma-format input ("Yamamoto, Haruki"), which - states its own order. + Ignored for comma-format input — the comma itself states the + order ("Thomas, John" puts the family name first). * - ``patronymic_rules`` - ``frozenset[PatronymicRule]`` - Reorders patronymic-shaped names via opt-in detectors — East diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 43878848..1ee2d7ac 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -77,9 +77,10 @@ class Policy: #: :ref:`name-order constants ` -- #: GIVEN_FIRST (the default), FAMILY_FIRST, and #: FAMILY_FIRST_GIVEN_LAST; any other tuple of Roles raises - #: ValueError. Ignored when the input contains a comma: - #: comma-format input ("Yamamoto, Haruki") states its own order - #: explicitly. + #: ValueError. Ignored when the input contains a comma: the comma + #: itself states the order -- "Thomas, John" puts the family name + #: first no matter which words could otherwise be either + #: ("Thomas" and "John" both work as given or family names). name_order: tuple[Role, Role, Role] = GIVEN_FIRST #: Opt-in detectors that reorder patronymic-shaped names #: (EAST_SLAVIC, TURKIC); usually set via a locale pack. From d1f2749850b36c37b31ddd0eeb66b3a2b4fccac4 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 12:41:56 -0700 Subject: [PATCH 192/275] docs: conjunctions are words or characters; show and/& examples Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index bf124a7c..d8257110 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -157,7 +157,8 @@ class Lexicon: #: one reads as given and records a particle-or-given ambiguity #: ("Van Johnson"). particles_ambiguous: frozenset[str] = frozenset() - #: Words that join surrounding pieces into one ("y", "and", "и"). + #: Words or characters that join surrounding pieces into one + #: ("and", "&", "y", "и"). conjunctions: frozenset[str] = frozenset() #: Given-name prefixes that bind to the following word to form one #: given name ("abdul" -> "Abdul Salam"); never standalone names. From 7716e9b896f49c0fbd013e99d592532f76f157eb Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 12:43:13 -0700 Subject: [PATCH 193/275] docs: Lexicon example lists read as examples, not full contents Append ellipses to the field docs' example parentheticals and teach the inspection idiom once in the class docstring (Lexicon.default(). is the full list). Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index d8257110..6e4cf739 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -134,37 +134,39 @@ class Lexicon: :meth:`empty`, derive variants with :meth:`add` / :meth:`remove` / ``|`` (union), and pass the result to ``Parser(lexicon=...)``. Entries are normalized at construction -- lowercased, edge periods - stripped -- so matching is case-insensitive.""" + stripped -- so matching is case-insensitive. Field docs below show + examples, not full contents; inspect any field's shipped vocabulary + directly, e.g. ``Lexicon.default().conjunctions``.""" - #: Pre-nominal titles ("dr", "sir", "capt"). + #: Pre-nominal titles ("dr", "sir", "capt", ...). titles: frozenset[str] = frozenset() #: Titles whose single following name reads as a GIVEN name - #: ("sheikh", "sister") rather than a family name. + #: ("sheikh", "sister", ...) rather than a family name. given_name_titles: frozenset[str] = frozenset() #: Post-nominal acronym suffixes, matched with or without periods #: ("phd" matches "PhD" and "Ph.D."). suffix_acronyms: frozenset[str] = frozenset() - #: Post-nominal word suffixes ("jr", "esquire", "iii"). + #: Post-nominal word suffixes ("jr", "esquire", "iii", ...). suffix_words: frozenset[str] = frozenset() #: Subset of suffix_acronyms counted as suffixes only when written #: WITH periods -- their bare forms are common surnames ("ma", #: "do": "Jack Ma" keeps his family name). suffix_acronyms_ambiguous: frozenset[str] = frozenset() #: Family-name particles that chain onto the following piece - #: ("van", "de", "bin"). + #: ("van", "de", "bin", ...). particles: frozenset[str] = frozenset() #: Subset of particles that can also BE a given name: a leading #: one reads as given and records a particle-or-given ambiguity #: ("Van Johnson"). particles_ambiguous: frozenset[str] = frozenset() #: Words or characters that join surrounding pieces into one - #: ("and", "&", "y", "и"). + #: ("and", "&", "y", "и", ...). conjunctions: frozenset[str] = frozenset() #: Given-name prefixes that bind to the following word to form one #: given name ("abdul" -> "Abdul Salam"); never standalone names. bound_given_names: frozenset[str] = frozenset() #: Marker words introducing a birth surname, routed to the maiden - #: field ("née", "geb.", "roz."). + #: field ("née", "geb.", "roz.", ...). maiden_markers: frozenset[str] = frozenset() #: Lowercase word -> exact-cased replacement used by capitalized() #: ("phd" -> "Ph.D."). Pair-valued: change it with From d2cc767692978dbf4fc936fd9b1d7d780cfc3d4f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 12:46:58 -0700 Subject: [PATCH 194/275] docs: every Lexicon field links to its full default list Each field doc now ends with a same-page link to the config data constant it sources from (the constants render their complete contents in the reference's compat section). The renamed pairs (suffix_words -> SUFFIX_NOT_ACRONYMS, bound_given_names -> BOUND_FIRST_NAMES) double as visible v1<->v2 mapping documentation, and particles_ambiguous -- which has no constant -- documents its derivation from NON_FIRST_NAME_PREFIXES instead. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 6e4cf739..f83fc7b2 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -138,40 +138,55 @@ class Lexicon: examples, not full contents; inspect any field's shipped vocabulary directly, e.g. ``Lexicon.default().conjunctions``.""" - #: Pre-nominal titles ("dr", "sir", "capt", ...). + #: Pre-nominal titles ("dr", "sir", "capt", ...). Full default + #: list: :data:`~nameparser.config.titles.TITLES`. titles: frozenset[str] = frozenset() #: Titles whose single following name reads as a GIVEN name - #: ("sheikh", "sister", ...) rather than a family name. + #: ("sheikh", "sister", ...) rather than a family name. Full + #: default list: :data:`~nameparser.config.titles.FIRST_NAME_TITLES`. given_name_titles: frozenset[str] = frozenset() #: Post-nominal acronym suffixes, matched with or without periods - #: ("phd" matches "PhD" and "Ph.D."). + #: ("phd" matches "PhD" and "Ph.D."). Full default list: + #: :data:`~nameparser.config.suffixes.SUFFIX_ACRONYMS`. suffix_acronyms: frozenset[str] = frozenset() - #: Post-nominal word suffixes ("jr", "esquire", "iii", ...). + #: Post-nominal word suffixes ("jr", "esquire", "iii", ...). Full + #: default list: + #: :data:`~nameparser.config.suffixes.SUFFIX_NOT_ACRONYMS`. suffix_words: frozenset[str] = frozenset() #: Subset of suffix_acronyms counted as suffixes only when written #: WITH periods -- their bare forms are common surnames ("ma", - #: "do": "Jack Ma" keeps his family name). + #: "do": "Jack Ma" keeps his family name). Full default list: + #: :data:`~nameparser.config.suffixes.SUFFIX_ACRONYMS_AMBIGUOUS`. suffix_acronyms_ambiguous: frozenset[str] = frozenset() #: Family-name particles that chain onto the following piece - #: ("van", "de", "bin", ...). + #: ("van", "de", "bin", ...). Full default list: + #: :data:`~nameparser.config.prefixes.PREFIXES`. particles: frozenset[str] = frozenset() #: Subset of particles that can also BE a given name: a leading #: one reads as given and records a particle-or-given ambiguity - #: ("Van Johnson"). + #: ("Van Johnson"). No constant of its own -- the default derives + #: as particles minus + #: :data:`~nameparser.config.prefixes.NON_FIRST_NAME_PREFIXES` + #: (which marks the opposite, never-given subset). particles_ambiguous: frozenset[str] = frozenset() #: Words or characters that join surrounding pieces into one - #: ("and", "&", "y", "и", ...). + #: ("and", "&", "y", "и", ...). Full default list: + #: :data:`~nameparser.config.conjunctions.CONJUNCTIONS`. conjunctions: frozenset[str] = frozenset() #: Given-name prefixes that bind to the following word to form one #: given name ("abdul" -> "Abdul Salam"); never standalone names. + #: Full default list: + #: :data:`~nameparser.config.bound_first_names.BOUND_FIRST_NAMES`. bound_given_names: frozenset[str] = frozenset() #: Marker words introducing a birth surname, routed to the maiden - #: field ("née", "geb.", "roz.", ...). + #: field ("née", "geb.", "roz.", ...). Full default list: + #: :data:`~nameparser.config.maiden_markers.MAIDEN_MARKERS`. maiden_markers: frozenset[str] = frozenset() #: Lowercase word -> exact-cased replacement used by capitalized() #: ("phd" -> "Ph.D."). Pair-valued: change it with #: dataclasses.replace(), not add()/remove(); read it as a mapping - #: via capitalization_exceptions_map. + #: via capitalization_exceptions_map. Full default mapping: + #: :data:`~nameparser.config.capitalization.CAPITALIZATION_EXCEPTIONS`. # Canonical storage: sorted tuple of (key, value) pairs. The # constructor tolerates any Mapping (or pair iterable) at runtime and # canonicalizes here; this closes the caller-aliasing hole and keeps From f0bd837d32aa0f59cc26e744dda0822e6db14bfa Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 12:54:26 -0700 Subject: [PATCH 195/275] docs: Role members document the seven fields in the reference Same enum-member invisibility as AmbiguityKind: without doc comments autodoc skipped them, so the class docstring promised seven fields the page never listed. Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nameparser/_types.py b/nameparser/_types.py index 41bc82e3..8be0953c 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -33,12 +33,21 @@ class Role(Enum): # Declaration order IS the canonical field order (conventions §3): # every listing of the seven fields anywhere derives from this. + + #: Pre-nominal titles and honorifics ("Dr.", "Sir", "Capt."). TITLE = "title" + #: The given (first) name, or its initial. GIVEN = "given" + #: Names between given and family -- middle names or initials. MIDDLE = "middle" + #: The family (last) name, including any particles ("de la Vega"). FAMILY = "family" + #: Post-nominal pieces ("III", "Jr.", "PhD"). SUFFIX = "suffix" + #: Delimited nickname content ("Jonathan 'Jack' Kennedy" -> "Jack"). NICKNAME = "nickname" + #: A birth surname, from a marker word ("Jane Smith née Jones" -> + #: "Jones") or a delimiter pair routed via Policy.maiden_delimiters. MAIDEN = "maiden" From bdc4cfdf508376328abccded95cc58ea5eb75735 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 12:59:08 -0700 Subject: [PATCH 196/275] docs: middle_as_family use case; nickname default links to its constant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fold example (verified live): 'Gabriel García Márquez' -> family 'García Márquez' for multi-part-surname data. Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 1ee2d7ac..82a967cd 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -86,10 +86,14 @@ class Policy: #: (EAST_SLAVIC, TURKIC); usually set via a locale pack. patronymic_rules: frozenset[PatronymicRule] = frozenset() #: Folds middle into family instead of splitting them (v1's - #: middle_name_as_last). + #: middle_name_as_last) -- for data where unrecognized interior + #: words are surname parts, not middle names: multi-part surnames + #: like Spanish/Portuguese dual surnames ("Gabriel García Márquez" + #: -> family "García Márquez" instead of middle "García"). middle_as_family: bool = False # v1's middle_name_as_last #: (open, close) pairs whose enclosed content becomes the nickname - #: field. Defaults to DEFAULT_NICKNAME_DELIMITERS (#273). + #: field. Defaults to + #: :data:`~nameparser.DEFAULT_NICKNAME_DELIMITERS` (#273). nickname_delimiters: frozenset[tuple[str, str]] = DEFAULT_NICKNAME_DELIMITERS #: (open, close) pairs whose enclosed content becomes the maiden #: field instead; a pair listed here is dropped from the effective From 895ecb3f9ad82ef423c3f3ec465ee8eecf02aad2 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 13:02:06 -0700 Subject: [PATCH 197/275] docs: Locale docstring links to the locale packs guide Co-Authored-By: Claude Fable 5 --- nameparser/_locale.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nameparser/_locale.py b/nameparser/_locale.py index e1cd0fa5..f22c3eb8 100644 --- a/nameparser/_locale.py +++ b/nameparser/_locale.py @@ -25,7 +25,8 @@ class Locale: together by :func:`nameparser.parser_for`. The packs shipped with nameparser live in :mod:`nameparser.locales`; building your own needs no registration -- construct one and pass it to - ``parser_for``.""" + ``parser_for``. See the :doc:`locale packs guide ` for + using, creating, and contributing packs.""" #: Identifier, lowercase ``[a-z0-9_]+`` (e.g. "ru", "tr_az"). code: str From 717101e184a4cad1b0a6bb3d6518d2b7dc589d52 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 13:15:47 -0700 Subject: [PATCH 198/275] =?UTF-8?q?docs:=20concepts=20page=20reads=20plain?= =?UTF-8?q?ly=20=E2=80=94=20accurate=20title,=20concrete=20walkthrough?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback (Derek): 'How the parser thinks' implied nondeterminism -> 'How the parser works'; the 'four short essays' opener and 'pipeline, in one breath' phrasing replaced with direct statements; 'From string to name' now walks the canonical example concretely (8 tokens, Dr. span (0,3), family as a view joining de/la/Vega) and defines span before explaining what it prevents; the three-container bullets lead with the container names; the Dean paragraph returns to the v1 docs' problem-first framing. Co-Authored-By: Claude Fable 5 --- docs/concepts.rst | 82 ++++++++++++++++++++++++++--------------------- docs/usage.rst | 2 +- 2 files changed, 47 insertions(+), 37 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index f8395585..8f9a0407 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -1,30 +1,37 @@ -How the parser thinks -====================== +How the parser works +==================== -Four short essays on the model behind the 2.0 API: how a string -becomes a :class:`~nameparser.ParsedName`, why configuration lives in -exactly three places sorted by what varies, why parsers are plain -values, and what the parser does when a name is genuinely ambiguous. +``parse()`` turns a name string into a +:class:`~nameparser.ParsedName`. This page explains the model behind +that call: how a string becomes tokens and tokens become fields, +where configuration lives and why it is split the way it is, why +parsers are plain values, and what happens when a name is genuinely +ambiguous. The task pages all build on these four ideas. From string to name -------------------- -The pipeline, in one breath: the original string becomes a sequence of -tokens with spans (character positions into the original string), -each span gets assigned a role — ``title``, ``given``, ``middle``, -``family``, ``suffix``, ``nickname``, ``maiden`` — and every string you -read off a :class:`~nameparser.ParsedName` (``.given``, ``.family``, -``str(name)``, and so on) is a view computed over those roles at read -time, not stored text. - -Spans matter because of what they replace. v1's ``HumanName`` worked -on plain lists of strings and re-found pieces by searching the list -for a matching value — so a name with a repeated word could make the -parser rewrite the wrong occurrence. That whole bug family (starting -with `issue #100 `_) -traces back to value-based lookup. A token's span is a position, not a -value, so nothing in the 2.0 pipeline is ever re-found by scanning for -a string that looks like it. +Every parse follows the same path: the input string is split into +tokens, each token is assigned one of the seven roles — ``title``, +``given``, ``middle``, ``family``, ``suffix``, ``nickname``, +``maiden`` — and every string you read off the result is computed +from those tokens at read time. + +Parsing ``"Dr. Juan Q. Xavier de la Vega III"`` produces eight +tokens. The first is ``Dr.`` with the ``title`` role; ``de``, ``la``, +and ``Vega`` each carry the ``family`` role, which is why +``name.family`` returns ``"de la Vega"`` — the field is a view that +joins the family-role tokens in order, not a stored string. + +Each token also records where it came from. A span is a pair of +character positions bounding the token in the original string: +``Dr.`` has span ``(0, 3)``, and ``name.original[0:3]`` is exactly +``"Dr."``. Internally, spans let the pipeline refer to a token by +position instead of by text. v1 re-found name pieces by searching for +matching text, so a name with a repeated word could make the parser +rewrite the wrong occurrence (`issue #100 +`_ and its +relatives); a position cannot be confused with a look-alike. :class:`~nameparser.ParsedName` is frozen: there is no attribute assignment, ever. If a parse is almost right and you want to fix one @@ -41,20 +48,23 @@ Every piece of nameparser configuration falls into exactly one of three places, and which one is decided by a single question: what does this setting vary with? -* varies by **language** → :class:`~nameparser.Lexicon` (vocabulary: - titles, particles, suffixes, conjunctions, and the rest of the - word lists the parser matches against) -* varies by **data source or application** → :class:`~nameparser.Policy` - (behavior switches: name order, patronymic rules, delimiters, strip - flags — anything that changes how the pipeline runs, not what words - it recognizes) -* varies by **output destination** → a rendering argument (the - ``spec`` you pass to ``render(spec)``, or a keyword to - ``initials()``/``capitalized()``) - -"Dean" being a title in your data is a fact about the language and -domain the names come from (academic rosters, say), not about any one -dataset or report — that's a :class:`~nameparser.Lexicon` entry. A CRM that always exports "Family, Given" strings is a fact +* :class:`~nameparser.Lexicon` holds everything that varies by + **language**: the vocabulary — titles, particles, suffixes, + conjunctions, and the rest of the word lists the parser matches + against. +* :class:`~nameparser.Policy` holds everything that varies by + **data source or application**: the behavior switches — name order, + patronymic rules, delimiters, strip flags — anything that changes + how the pipeline runs, not what words it recognizes. +* Rendering arguments cover everything that varies by **output + destination**: the ``spec`` you pass to ``render(spec)``, or a + keyword to ``initials()``/``capitalized()``. + +"Dean" is a common given name, so it is not in the default titles +vocabulary — but in some data it is more common as a title. Which +reading is right is a fact about the language and domain the names +come from, not about any one dataset or report: that makes it a +:class:`~nameparser.Lexicon` entry. A CRM that always exports "Family, Given" strings is a fact about that one data source, not about the language of the names in it — that's a :class:`~nameparser.Policy`. One particular report wanting names formatted as "Family, Given" while every other consumer diff --git a/docs/usage.rst b/docs/usage.rst index 85829ac3..e39fb4f7 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -114,7 +114,7 @@ ru``); see :doc:`locales`. Where next ---------- -* :doc:`concepts` — how the parser thinks +* :doc:`concepts` — how the parser works * :doc:`customize` — your own vocabulary and behavior * :doc:`locales` — locale packs * :doc:`migrate` — coming from 1.x ``HumanName`` From 5b21b975130c71d4a83e5753426399e0e60ccbea Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 13:17:45 -0700 Subject: [PATCH 199/275] docs: rendering-arguments bullet links to the customize section Co-Authored-By: Claude Fable 5 --- docs/concepts.rst | 6 +++--- docs/customize.rst | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 8f9a0407..5533413d 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -56,9 +56,9 @@ this setting vary with? **data source or application**: the behavior switches — name order, patronymic rules, delimiters, strip flags — anything that changes how the pipeline runs, not what words it recognizes. -* Rendering arguments cover everything that varies by **output - destination**: the ``spec`` you pass to ``render(spec)``, or a - keyword to ``initials()``/``capitalized()``. +* :ref:`Rendering arguments ` cover everything + that varies by **output destination**: the ``spec`` you pass to + ``render(spec)``, or a keyword to ``initials()``/``capitalized()``. "Dean" is a common given name, so it is not in the default titles vocabulary — but in some data it is more common as a title. Which diff --git a/docs/customize.rst b/docs/customize.rst index e7cb4157..172d0cfe 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -124,6 +124,8 @@ above is the whole recipe. To *add* delimiters instead of rerouting them, build on the named default: ``nickname_delimiters=DEFAULT_NICKNAME_DELIMITERS | {("«", "»")}``. +.. _rendering-arguments: + Presentation: rendering arguments ---------------------------------- From 7ee6e3a747feb6269cef0692758c5b2a1bd881ef Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 13:43:44 -0700 Subject: [PATCH 200/275] feat: typographic nickname delimiters as defaults (closes #273) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DEFAULT_NICKNAME_DELIMITERS grows by the eight conventions the issue proposed: smart quotes, low-high (de/pl/cs/hu), right-right (sv/fi), guillemets both directions (« Petit » inner spacing tolerated), CJK corner brackets, fullwidth parentheses. Curly single quotes stay excluded (U+2019 is the apostrophe in O'Connor, pinned). Both APIs get them: the shim's sentinel map gains named keys (smart_double_quotes, guillemets, ...) so the v1 keyed idioms (pop/move/del) work on the new pairs like the original trio. The conventions share characters in opposite roles ('"' closes the German pair but opens the English one; '»' closes guillemets and opens the reversed pair), which made extraction emit spurious unbalanced-delimiter ambiguities: extract now tracks unmatched-open offsets and drops candidates inside a region another pair successfully masked. Genuine unbalanced opens still flag. TDD throughout (extract suppression -> defaults -> shim sentinels, each red first); 9 case rows through both runners; differential harness exit 0 unchanged (no typographic quotes in the corpus); release-log feat entry; reference/customize/migrate docs updated. Co-Authored-By: Claude Fable 5 --- docs/customize.rst | 5 ++-- docs/migrate.rst | 5 ++-- docs/modules.rst | 17 +++++++++----- docs/release_log.rst | 1 + nameparser/_config_shim.py | 16 +++++++++++-- nameparser/_pipeline/_extract.py | 22 ++++++++++++----- nameparser/_policy.py | 25 +++++++++++++++----- tests/test_constants.py | 14 +++++++---- tests/v2/cases.py | 39 +++++++++++++++++++++++++++++++ tests/v2/pipeline/test_extract.py | 26 +++++++++++++++++++++ tests/v2/test_config_shim.py | 23 ++++++++++++++++-- tests/v2/test_policy.py | 9 ++++--- 12 files changed, 168 insertions(+), 34 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index 172d0cfe..1398a6a3 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -81,8 +81,9 @@ listed below. - ``frozenset[tuple[str, str]]`` - Routes content enclosed by these delimiter pairs to ``nickname``. Defaults to - :data:`~nameparser.DEFAULT_NICKNAME_DELIMITERS` — quotes and - parentheses. + :data:`~nameparser.DEFAULT_NICKNAME_DELIMITERS` — straight + quotes and parentheses plus the typographic conventions (smart + quotes, guillemets, CJK brackets, ...). * - ``maiden_delimiters`` - ``frozenset[tuple[str, str]]`` - Routes content enclosed by these delimiter pairs to ``maiden`` diff --git a/docs/migrate.rst b/docs/migrate.rst index fa961e49..48fe1472 100644 --- a/docs/migrate.rst +++ b/docs/migrate.rst @@ -170,8 +170,9 @@ rendering argument, where the 2.0 equivalent isn't config at all): - * - ``nickname_delimiters`` - ``Policy.nickname_delimiters`` - - Was a three-sentinel dict; now a plain ``frozenset`` of - ``(open, close)`` pairs + - Was a dict of named sentinels; now a plain ``frozenset`` of + ``(open, close)`` pairs. Both APIs gained the #273 typographic + defaults (smart quotes, guillemets, CJK brackets, ...) in 2.0 * - ``maiden_delimiters`` - ``Policy.maiden_delimiters`` - Same shape change as ``nickname_delimiters``. Precedence diff --git a/docs/modules.rst b/docs/modules.rst index 7b23d399..8cec591a 100644 --- a/docs/modules.rst +++ b/docs/modules.rst @@ -93,12 +93,17 @@ Delimiter defaults ^^^^^^^^^^^^^^^^^^ .. py:data:: nameparser.DEFAULT_NICKNAME_DELIMITERS - :value: frozenset({("'", "'"), ('"', '"'), ("(", ")")}) - - The default :attr:`~nameparser.Policy.nickname_delimiters` set — - quotes and parentheses. Build on it for additive customizations, - e.g. ``nickname_delimiters=DEFAULT_NICKNAME_DELIMITERS | - {("«", "»")}``; to *reroute* a pair to ``maiden``, just list it in + :value: frozenset({("'", "'"), ('"', '"'), ("(", ")"), ("“", "”"), ("„", "“"), ("”", "”"), ("«", "»"), ("»", "«"), ("「", "」"), ("『", "』"), ("（", "）")}) + + The default :attr:`~nameparser.Policy.nickname_delimiters` set: + straight quotes and parentheses plus the typographic conventions — + smart quotes, German/Polish low-high quotes, Swedish right-right + quotes, guillemets in both directions, CJK corner brackets, and + fullwidth parentheses (#273). Curly *single* quotes are deliberately + absent: U+2019 is the typographic apostrophe ("O’Connor"). Build on + the constant for additive customizations, e.g. + ``nickname_delimiters=DEFAULT_NICKNAME_DELIMITERS | {("｟", "｠")}``; + to *reroute* a pair to ``maiden``, just list it in :attr:`~nameparser.Policy.maiden_delimiters` (it is dropped from the effective nickname set automatically). diff --git a/docs/release_log.rst b/docs/release_log.rst index b94b7b77..0701d76e 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -25,6 +25,7 @@ Release Log - Change suffix-delimiter rendering: with a custom ``Policy``/``Constants`` suffix delimiter configured (e.g. ``suffix_delimiter="/"``, ``"John Smith, RN/CRNA"``), 1.x split the token and rendered ``suffix="RN, CRNA"``; 2.0 keeps the no-space delimiter-core token whole (``suffix="RN/CRNA"``) -- role assignment is unchanged, only rendering differs (anti-#100, migration plan deviation 5; ``tests/v2/cases.py`` row ``suffix_delimiter_no_space_core``). Only fires with a non-default policy, so it does not appear in the (default-policy) differential corpus - Change ``comparison_key()``/``matches()`` (both APIs) to fold components with ``str.casefold()`` where 1.4 used ``str.lower()``: Unicode case-pair forms now compare equal -- ``"STRASSE"`` matches ``"Straße"``, and a Greek final-sigma variant matches its regular-sigma form. Strictly more permissive (anything 1.4 matched still matches); parse output is unaffected -- vocabulary normalization itself uses ``lower()``, v1-parity (``tests/v2/test_types.py`` row ``test_matches_casefolds_unicode_case_pairs``) - Change delimiter-overlap precedence in the 2.0 API: a pair listed in ``Policy.maiden_delimiters`` is dropped from the effective ``nickname_delimiters`` set (maiden wins), so ``Policy(maiden_delimiters={("(", ")")})`` alone routes parenthesized content to ``maiden`` -- no need to rebuild the nickname set. The default nickname set is now the public ``DEFAULT_NICKNAME_DELIMITERS`` constant. The 1.x facade keeps v1's nickname-wins precedence on overlap via a shim-side pre-subtraction, so no ``HumanName`` behavior changes (``tests/v2/cases.py`` row ``maiden_delimiters_win_when_shared``; ``tests/v2/test_config_shim.py`` row ``test_snapshot_overlap_keeps_v1_nickname_precedence``) + - Recognize typographic nickname delimiters by default in BOTH APIs (closes #273): smart quotes (``“Jack”``), German/Polish low-high quotes (``„Hansi“``), Swedish right-right quotes (``”Ann”``), guillemets in both directions (``«Petit»``, ``»Hansi«``, inner spacing tolerated), CJK corner brackets (``「タロ」``/``『ハナ』``), and fullwidth parentheses. In 1.x these leaked into ``middle`` as literal text. Curly *single* quotes stay excluded (U+2019 is the apostrophe in "O’Connor" -- pinned by ``curly_apostrophe_stays_literal``). The v1 keyed idioms work on the new named sentinels (``smart_double_quotes``, ``guillemets``, ...). A delimiter character consumed by another pair's extraction no longer emits a spurious ``unbalanced-delimiter`` ambiguity (``tests/v2/cases.py`` rows ``*_nickname``; ``tests/v2/pipeline/test_extract.py``). Not present in the differential corpus Everything else in the 486-name differential corpus (built from the v1 test banks as of commit ``2d5d8c2``, pre-dating the M12 test diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index d71dbc68..8fc1e7ae 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -407,12 +407,24 @@ def __setstate__(self, state: dict[str, object]) -> None: self._on_change = None # rewired by the owning Constants -#: v1's three named delimiter buckets, translated to the ``Policy`` -#: (open, close) pairs they stand for (spec §3). +#: The named delimiter buckets, translated to the ``Policy`` +#: (open, close) pairs they stand for (spec §3). The first three are +#: v1's; the rest are the #273 typographic conventions, named so the +#: v1 keyed idioms (pop/move/del) work on them like the originals. +#: Keep in sync with DEFAULT_NICKNAME_DELIMITERS in _policy.py (pinned +#: by the default-Constants equality test). _SENTINEL_PAIRS = { "quoted_word": ("'", "'"), "double_quotes": ('"', '"'), "parenthesis": ("(", ")"), + "smart_double_quotes": ("“", "”"), + "low_high_quotes": ("„", "“"), + "right_double_quotes": ("”", "”"), + "guillemets": ("«", "»"), + "reversed_guillemets": ("»", "«"), + "corner_brackets": ("「", "」"), + "white_corner_brackets": ("『", "』"), + "fullwidth_parenthesis": ("（", "）"), } #: derived, so the manager's accepted keys and _snapshot()'s diff --git a/nameparser/_pipeline/_extract.py b/nameparser/_pipeline/_extract.py index db31c4b9..0f6672fd 100644 --- a/nameparser/_pipeline/_extract.py +++ b/nameparser/_pipeline/_extract.py @@ -59,7 +59,9 @@ def extract_delimited(state: ParseState) -> ParseState: text = state.original extracted: list[tuple[Role, Span]] = [] masked: list[Span] = [] - ambiguities: list[PendingAmbiguity] = [] + # candidates, not final: each carries the offset of the unmatched + # open so cross-pair overlaps can be filtered once all pairs ran + unbalanced: list[tuple[int, PendingAmbiguity]] = [] # nickname first (v1 parse_nicknames order): when the same # delimiter pair sits in BOTH buckets, the nickname reading wins; # the documented bucket-move idiom removes it from nickname, so @@ -79,11 +81,11 @@ def extract_delimited(state: ParseState) -> ParseState: and not _close_ok(text, j, len(close))): j = text.find(close, j + 1) if j == -1: - ambiguities.append(PendingAmbiguity( + unbalanced.append((i, PendingAmbiguity( AmbiguityKind.UNBALANCED_DELIMITER, f"unmatched {open_!r} at offset {i}; treated as " f"literal text", - )) + ))) if open_ == close: # _close_ok is open-independent, so a failed # close-walk means NO boundary-valid close @@ -94,11 +96,11 @@ def extract_delimited(state: ParseState) -> ParseState: scan = i + len(open_) while (k := text.find(open_, scan)) != -1: if _open_ok(text, k): - ambiguities.append(PendingAmbiguity( + unbalanced.append((k, PendingAmbiguity( AmbiguityKind.UNBALANCED_DELIMITER, f"unmatched {open_!r} at offset {k}; " f"treated as literal text", - )) + ))) scan = k + 1 break pos = i + len(open_) @@ -125,6 +127,14 @@ def extract_delimited(state: ParseState) -> ParseState: pos = j + len(close) extracted.sort(key=lambda pair: pair[1]) masked.sort() + # A delimiter character consumed by another pair's successful + # extraction is literal for every other pair: drop unbalanced + # candidates whose offset lies inside a masked region (#273 -- the + # conventions share characters in opposite roles: '“' closes „…“ + # but opens “…”, '»' closes «…» but opens »…«). + ambiguities = tuple( + a for offset, a in unbalanced + if not any(m.start <= offset < m.end for m in masked)) return dataclasses.replace( state, extracted=tuple(extracted), masked=tuple(masked), - ambiguities=state.ambiguities + tuple(ambiguities)) + ambiguities=state.ambiguities + ambiguities) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 82a967cd..3fd6f71a 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -45,12 +45,25 @@ class PatronymicRule(StrEnum): _NAME_ROLES = frozenset({Role.GIVEN, Role.MIDDLE, Role.FAMILY}) -#: Policy.nickname_delimiters' default (v1 parity: quotes + parentheses). -#: Public and named so customizations read as set math against a -#: documented value -- e.g. ``DEFAULT_NICKNAME_DELIMITERS | {("«", "»")}`` -#: -- instead of a rebuilt literal the user had to go discover. -DEFAULT_NICKNAME_DELIMITERS = frozenset( - {("'", "'"), ('"', '"'), ("(", ")")}) +#: Policy.nickname_delimiters' default. Public and named so +#: customizations read as set math against a documented value -- e.g. +#: ``DEFAULT_NICKNAME_DELIMITERS | {("｟", "｠")}`` -- instead of a +#: rebuilt literal the user had to go discover. The v1 trio (straight +#: quotes + parentheses) plus the typographic conventions (#273): +#: smart quotes, low-high and right-right quotes, guillemets both +#: directions, CJK corner brackets, fullwidth parentheses. Curly +#: SINGLE quotes are deliberately absent: U+2019 is the typographic +#: apostrophe ("O’Connor"). +DEFAULT_NICKNAME_DELIMITERS = frozenset({ + ("'", "'"), ('"', '"'), ("(", ")"), # v1 trio + ("“", "”"), # smart quotes (en, zh) + ("„", "“"), # low-high (de, pl, cs, hu) + ("”", "”"), # right-right (sv, fi) + ("«", "»"), # guillemets (fr, ru, it, el) + ("»", "«"), # reversed guillemets (de alt) + ("「", "」"), ("『", "』"), # CJK corner brackets (ja) + ("（", "）"), # fullwidth parentheses (CJK) +}) def _reject_bare_string_order(value: object) -> None: diff --git a/tests/test_constants.py b/tests/test_constants.py index 233860f9..1a7ff53d 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -545,11 +545,15 @@ def test_nickname_delimiters_default_builtins_resolve_live(self) -> None: # see test_overriding_builtin_regex_still_affects_nickname_parsing in # test_nicknames.py. c = Constants() - self.assertEqual(dict(c.nickname_delimiters), { - 'quoted_word': 'quoted_word', - 'double_quotes': 'double_quotes', - 'parenthesis': 'parenthesis', - }) + entries = dict(c.nickname_delimiters) + # v1 trio still present, still stored name-as-value + for name in ('quoted_word', 'double_quotes', 'parenthesis'): + self.assertEqual(entries[name], name) + # 2.0 adds the #273 typographic sentinels alongside them, same + # name-as-value scheme (full list pinned in tests/v2/ + # test_config_shim.py against _SENTINEL_PAIRS) + assert all(value == name for name, value in entries.items()) + assert 'smart_double_quotes' in entries self.assertEqual(dict(c.maiden_delimiters), {}) def test_extra_nickname_delimiters_removed(self) -> None: diff --git a/tests/v2/cases.py b/tests/v2/cases.py index ca104fba..5e4b5bc4 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -126,6 +126,45 @@ def __post_init__(self) -> None: "one-liner replaces the bucket-move idiom; the v1 facade " "keeps v1's nickname-wins precedence via the shim's " "pre-subtraction (pinned in test_config_shim)"), + # #273: typographic nickname delimiters ship as defaults -- one row + # per new pair; expectations verified live 2026-07-19 before the + # pairs were added to DEFAULT_NICKNAME_DELIMITERS. + Case("smart_double_quotes_nickname", 'John “Jack” Kennedy', + {"given": "John", "family": "Kennedy", "nickname": "Jack"}, + classification="feat(#273)"), + Case("low_high_quotes_nickname", 'Hans „Hansi“ Müller', + {"given": "Hans", "family": "Müller", "nickname": "Hansi"}, + classification="feat(#273)", + notes="the closing '“' doubles as the English pair's opener; " + "no spurious unbalanced-delimiter ambiguity (pinned in " + "test_extract)"), + Case("guillemets_nickname_inner_spaces", 'Jean « Petit » Dupont', + {"given": "Jean", "family": "Dupont", "nickname": "Petit"}, + classification="feat(#273)", + notes="French spacing: inner padding is trimmed from the " + "extracted nickname"), + Case("reversed_guillemets_nickname", 'Hans »Hansi« Müller', + {"given": "Hans", "family": "Müller", "nickname": "Hansi"}, + classification="feat(#273)"), + Case("swedish_right_quotes_nickname", 'Anna ”Ann” Larsson', + {"given": "Anna", "family": "Larsson", "nickname": "Ann"}, + classification="feat(#273)"), + Case("cjk_corner_bracket_nickname", '山田「タロ」太郎', + {"given": "山田", "family": "太郎", "nickname": "タロ"}, + classification="feat(#273)", + notes="extraction also splits the unspaced remainder -- the " + "masked region acts as a token boundary"), + Case("cjk_white_corner_bracket_nickname", '田中『ハナ』花子', + {"given": "田中", "family": "花子", "nickname": "ハナ"}, + classification="feat(#273)"), + Case("fullwidth_paren_nickname", 'John （Jack） Kennedy', + {"given": "John", "family": "Kennedy", "nickname": "Jack"}, + classification="feat(#273)"), + Case("curly_apostrophe_stays_literal", 'Sean O’Connor', + {"given": "Sean", "family": "O’Connor"}, + notes="U+2019 is the typographic apostrophe; curly single " + "quotes are deliberately NOT delimiters (#273 excludes " + "them)"), Case("family_segment_trailing_suffix", "Smith Jr., John", {"given": "John", "family": "Smith", "suffix": "Jr."}, notes="v1: the family part may have suffixes in it " diff --git a/tests/v2/pipeline/test_extract.py b/tests/v2/pipeline/test_extract.py index 0ee365d7..b6d27393 100644 --- a/tests/v2/pipeline/test_extract.py +++ b/tests/v2/pipeline/test_extract.py @@ -43,6 +43,32 @@ def test_unbalanced_delimiter_left_literal_with_ambiguity() -> None: assert out.ambiguities[0].kind is AmbiguityKind.UNBALANCED_DELIMITER +def test_no_spurious_unbalanced_from_role_overlapping_pairs() -> None: + # #273: '“' closes the German „…“ pair but OPENS the English “…” + # pair, and '»' closes «…» but opens the reversed »…« pair. A + # delimiter character consumed by another pair's successful + # extraction is literal for every other pair -- it must not + # surface as an unbalanced-delimiter ambiguity. + policy = Policy(nickname_delimiters=frozenset( + {("“", "”"), ("„", "“"), ("«", "»"), ("»", "«")})) + for text, inner in (("Hans „Hansi“ Müller", "Hansi"), + ("Jean «Petit» Dupont", "Petit")): + out = extract_delimited(_state(text, policy)) + assert [text[s.start:s.end] for _, s in out.extracted] == [inner] + assert out.ambiguities == () + + +def test_genuine_unbalanced_still_flagged_alongside_overlapping_pairs() -> None: + # the suppression must not swallow REAL unbalanced opens: here the + # German open has no close anywhere, and no other pair extracts + policy = Policy(nickname_delimiters=frozenset( + {("“", "”"), ("„", "“")})) + out = extract_delimited(_state("Hans „Hansi Müller", policy)) + assert out.extracted == () + assert [a.kind for a in out.ambiguities] == [ + AmbiguityKind.UNBALANCED_DELIMITER] + + def test_maiden_delimiters_route_to_maiden() -> None: policy = dataclasses.replace( Policy(), diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index dd5933f2..d267b529 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -206,8 +206,14 @@ def test_constants_default_fields_present() -> None: assert c.force_mixed_case_capitalization is False assert c.string_format == "{title} {first} {middle} {last} {suffix} ({nickname})" assert c.suffix_delimiter is None - assert set(c.nickname_delimiters) == {"quoted_word", "double_quotes", - "parenthesis"} + # every named sentinel is a default nickname bucket (the v1 trio + # plus the #273 typographic pairs); derived from the map itself so + # a new sentinel can't leave this pin stale + from nameparser._config_shim import _SENTINEL_PAIRS + + assert set(c.nickname_delimiters) == set(_SENTINEL_PAIRS) + assert {"quoted_word", "double_quotes", "parenthesis", + "smart_double_quotes", "guillemets"} <= set(_SENTINEL_PAIRS) assert len(c.maiden_delimiters) == 0 @@ -353,6 +359,19 @@ def test_snapshot_patronymic_and_middle_flags() -> None: assert policy.middle_as_family is True +def test_typographic_delimiter_sentinels_present_and_movable() -> None: + # #273: the eight typographic pairs surface in the v1 API as new + # named sentinels, so the documented keyed idioms (pop/move/del) + # work on them exactly like the original trio + c = Constants() + assert "smart_double_quotes" in c.nickname_delimiters + assert "guillemets" in c.nickname_delimiters + c.maiden_delimiters["guillemets"] = c.nickname_delimiters.pop("guillemets") + _, policy, _ = c._snapshot() + assert ("«", "»") in policy.maiden_delimiters + assert ("«", "»") not in policy.nickname_delimiters + + def test_snapshot_delimiter_bucket_move() -> None: c = Constants() c.maiden_delimiters["parenthesis"] = c.nickname_delimiters.pop("parenthesis") diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 170876b4..60ddb648 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -99,10 +99,13 @@ def test_maiden_delimiters_win_over_nickname_defaults() -> None: p = Policy(maiden_delimiters=frozenset({("(", ")")})) assert ("(", ")") in p.maiden_delimiters assert ("(", ")") not in p.nickname_delimiters - # canonicalization: the explicit-removal spelling converges to the - # same value (equal AND same hash -- cache keys agree) + # canonicalization: the explicit-removal spelling (the documented + # set-math idiom) converges to the same value (equal AND same hash + # -- cache keys agree) + from nameparser import DEFAULT_NICKNAME_DELIMITERS + explicit = Policy( - nickname_delimiters=frozenset({("'", "'"), ('"', '"')}), + nickname_delimiters=DEFAULT_NICKNAME_DELIMITERS - {("(", ")")}, maiden_delimiters=frozenset({("(", ")")}), ) assert p == explicit and hash(p) == hash(explicit) From 9dc7e1394012960c6e9c6d3fde69bb9096a2507e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 13:48:35 -0700 Subject: [PATCH 201/275] feat: Arabic-script bound given names (#269 follow-up) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BOUND_FIRST_NAMES had only the transliterations; the script originals now join too: عبد (one entry covers abdul/abdel/abdal -- script attaches the article to the following word), the kunya pair أبو/ابو, and أم/ام. Behavior mirrors the Latin twins, probed live before pinning: bound join fires with 3+ tokens (عبد الرحمن محمد -> given 'عبد الرحمن'), the two-token kunya stays split (أبو مازن unchanged), and non-leading أبو still prefix-chains onto family (أحمد أبو خليل unchanged -- both prior pins hold). TDD: 4 case rows red in both runners first. Differential exit 0 unchanged. Co-Authored-By: Claude Fable 5 --- docs/release_log.rst | 1 + nameparser/config/bound_first_names.py | 11 +++++++++++ tests/v2/cases.py | 21 +++++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/docs/release_log.rst b/docs/release_log.rst index 0701d76e..ff71ea6f 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -7,6 +7,7 @@ Release Log - Add ``nameparser.locales`` with the first two packs: ``locales.RU`` (East Slavic patronymic order) and ``locales.TR_AZ`` (Turkic patronymic markers). Packs are pure data folded in at ``parser_for(locales.RU)``; they compose (``parser_for(locales.RU, locales.TR_AZ)`` unions the rules) and are never auto-detected -- there is no reliable way to detect a name's language (closes the pack half of #270; #271/#272/#146 stay staged for 2.x) - The CLI gains ``--locale CODE`` (``python -m nameparser --locale ru "..."``) - Add non-Latin vocabulary to the default lexicon (#269): Cyrillic, Greek, Arabic and Hebrew titles, conjunctions, and name particles -- native-script entries cannot collide with Latin-script names. Deferred pending vetting: Cyrillic ``мл``/``ст`` suffixes and the bare Greek ``κ`` title (it collides with the initial+surname shape). Behavior note: ``محمد بن سلمان`` now chains ``بن`` onto the family name where 1.x read it as a middle name + - Add the Arabic-script bound given names (#269 follow-up): ``عبد``, the kunya pair ``أبو``/``ابو``, and ``أم``/``ام`` join the following word into the given name like their transliterations (``abdul``, ``abu``, ``umm``) always did -- ``عبد الرحمن محمد`` now parses given ``عبد الرحمن``, family ``محمد``, where 1.x split it into given + middle (``tests/v2/cases.py`` rows ``arabic_bound_given_*``). Not present in the differential corpus **Behavior Changes (draft)** diff --git a/nameparser/config/bound_first_names.py b/nameparser/config/bound_first_names.py index 07bd2bce..d0602417 100644 --- a/nameparser/config/bound_first_names.py +++ b/nameparser/config/bound_first_names.py @@ -9,4 +9,15 @@ 'abu', 'abou', 'umm', + + # #269 follow-up: the Arabic-script originals of the entries above. + # Script writes "Abdul Rahman" as two words (عبد + الرحمن -- the + # article attaches to the following word), so عبد alone covers the + # abdul/abdel/abdal variants. Both kunya spellings ship, matching + # the أبو/ابو prefix pair. + 'عبد', # "abd" (servant of) -- عبد الرحمن -> given "عبد الرحمن" + 'أبو', # "abu" (father of), hamza spelling + 'ابو', # "abu", hamza-less spelling + 'أم', # "umm" (mother of), hamza spelling + 'ام', # "umm", hamza-less spelling } diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 5e4b5bc4..004638aa 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -126,6 +126,27 @@ def __post_init__(self) -> None: "one-liner replaces the bucket-move idiom; the v1 facade " "keeps v1's nickname-wins precedence via the shim's " "pre-subtraction (pinned in test_config_shim)"), + # #269 follow-up: Arabic-script bound given names, mirroring the + # Latin transliterations' behavior (probed live 2026-07-19: bound + # join fires only with 3+ tokens, eats the NEXT token into given; + # two-token kunya stays split -- 'أبو مازن' pinned in test_locales; + # non-leading أبو still prefix-chains onto family). + Case("arabic_bound_given_abd", 'عبد الرحمن محمد', + {"given": "عبد الرحمن", "family": "محمد"}, + classification="feat(#269)", + notes="script twin of 'Abdul Rahman Mohammed'; عبد was the " + "missing bound entry -- 1.x split it into given عبد + " + "middle الرحمن"), + Case("arabic_bound_given_kunya", 'أبو بكر أحمد', + {"given": "أبو بكر", "family": "أحمد"}, + classification="feat(#269)"), + Case("arabic_bound_given_kunya_hamzaless", 'ابو بكر احمد', + {"given": "ابو بكر", "family": "احمد"}, + classification="feat(#269)", + notes="both kunya spellings ship, like the أبو/ابو prefix pair"), + Case("arabic_bound_given_umm", 'أم كلثوم إبراهيم', + {"given": "أم كلثوم", "family": "إبراهيم"}, + classification="feat(#269)"), # #273: typographic nickname delimiters ship as defaults -- one row # per new pair; expectations verified live 2026-07-19 before the # pairs were added to DEFAULT_NICKNAME_DELIMITERS. From 00b73b872486c511b923adc78a3bd14c5972f6a0 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 13:58:33 -0700 Subject: [PATCH 202/275] =?UTF-8?q?feat:=20Arabic=20honorific=20titles=20a?= =?UTF-8?q?nd=20the=20conjunction=20=D9=88=20(#269=20follow-up)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Twelve given-name titles -- Arabic honorifics precede the given name, so all join الشيخ in FIRST_NAME_TITLES: the article forms (never given names) الدكتور/الدكتورة/الأستاذ/الأستاذة/الحاج/الحاجة/الشيخة and the bare doctor/professor/engineer forms دكتور/دكتورة/أستاذ/أستاذة/مهندس. Deferred under the collision rule (recorded in the data-module comment): bare سيد/شيخ/أمير/سلطان (all common given names), the د. abbreviation (bare د would swallow single-letter initials, the bare-κ trap), and the Ottoman post-nominals باشا/بك/أفندي (survive as family names). و ('and') joins conjunctions: formal script attaches it to the next word, but the standalone informal spacing is common in real data; single-character like y/и, so the #11 carve-out protects short names. TDD: 20 rows red first, incl. the الحاج + عبد الرحمن compound (title + bound join + family) and the 6-piece و title chain. Differential exit 0 unchanged. Co-Authored-By: Claude Fable 5 --- docs/release_log.rst | 1 + nameparser/config/conjunctions.py | 6 ++++++ nameparser/config/titles.py | 21 +++++++++++++++++++++ tests/v2/test_locales.py | 27 +++++++++++++++++++++++++++ 4 files changed, 55 insertions(+) diff --git a/docs/release_log.rst b/docs/release_log.rst index ff71ea6f..c9df8dfc 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -8,6 +8,7 @@ Release Log - The CLI gains ``--locale CODE`` (``python -m nameparser --locale ru "..."``) - Add non-Latin vocabulary to the default lexicon (#269): Cyrillic, Greek, Arabic and Hebrew titles, conjunctions, and name particles -- native-script entries cannot collide with Latin-script names. Deferred pending vetting: Cyrillic ``мл``/``ст`` suffixes and the bare Greek ``κ`` title (it collides with the initial+surname shape). Behavior note: ``محمد بن سلمان`` now chains ``بن`` onto the family name where 1.x read it as a middle name - Add the Arabic-script bound given names (#269 follow-up): ``عبد``, the kunya pair ``أبو``/``ابو``, and ``أم``/``ام`` join the following word into the given name like their transliterations (``abdul``, ``abu``, ``umm``) always did -- ``عبد الرحمن محمد`` now parses given ``عبد الرحمن``, family ``محمد``, where 1.x split it into given + middle (``tests/v2/cases.py`` rows ``arabic_bound_given_*``). Not present in the differential corpus + - Add Arabic honorific titles and the conjunction ``و`` (#269 follow-up): the doctor/professor/hajj/sheikha/engineer forms (``الدكتور``/``الدكتورة``/``دكتور``/``دكتورة``, ``الأستاذ``/``الأستاذة``/``أستاذ``/``أستاذة``, ``الحاج``/``الحاجة``, ``الشيخة``, ``مهندس``) as given-name titles -- Arabic honorifics precede the given name, like ``الشيخ``. Deferred under the collision rule: bare ``سيد``/``شيخ``/``أمير``/``سلطان`` (all common given names), the ``د.`` abbreviation (bare ``د`` would swallow initials), and the Ottoman post-nominals ``باشا``/``بك``/``أفندي`` (survive as family names). Not present in the differential corpus **Behavior Changes (draft)** diff --git a/nameparser/config/conjunctions.py b/nameparser/config/conjunctions.py index cf738084..a30cc3f9 100644 --- a/nameparser/config/conjunctions.py +++ b/nameparser/config/conjunctions.py @@ -13,6 +13,12 @@ 'та', # #269: Greek "and": και. 'και', + # #269 follow-up: Arabic "and". Formal script attaches و to the + # following word (وفاطمة), so a standalone و token appears only in + # informal spacing -- common in real data. Single-character like + # 'y'/'и': the #11 initial carve-out protects short names (joins + # only with enough rootname pieces). + 'و', } """ Pieces that should join to their neighboring pieces, e.g. "and", "y" and "&". diff --git a/nameparser/config/titles.py b/nameparser/config/titles.py index 6961a626..2e4666c3 100644 --- a/nameparser/config/titles.py +++ b/nameparser/config/titles.py @@ -25,6 +25,27 @@ # the transliterated sheikh/sheik/... cluster above; same # single-first-name-follows convention ("Sheikh Mohammed"). 'الشيخ', + # #269 follow-up: Arabic honorifics precede the GIVEN name, so all + # belong here rather than in plain TITLES. Article forms are never + # given names; the bare doctor/professor/engineer forms are not + # used as given names either. Deferred under the collision rule + # (like мл/ст in suffixes.py): bare سيد (Sayyid is a common given + # name), bare شيخ (Shaikha is a common female given name), أمير + # and سلطان (Amir/Sultan, common given names), and the 'د.' + # abbreviation -- edge-period normalization would leave bare 'د', + # which swallows single-letter initials (the bare-κ trap above). + 'الدكتور', # "the doctor" (m) + 'الدكتورة', # "the doctor" (f) + 'دكتور', # doctor (m), article-less + 'دكتورة', # doctor (f), article-less + 'الأستاذ', # "the professor"/Mr. honorific (m) + 'الأستاذة', # professor/Mrs. honorific (f) + 'أستاذ', # professor (m), article-less + 'أستاذة', # professor (f), article-less + 'الحاج', # hajj honorific (m) + 'الحاجة', # hajj honorific (f) + 'الشيخة', # female counterpart of الشيخ + 'مهندس', # engineer (a genuine title in Egyptian usage) } """ When these titles appear with a single other name, that name is a first name, e.g. diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 295df832..9a8ab0f9 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -415,6 +415,33 @@ def test_non_interference_all_packs_combined() -> None: # transliterated cousin 'sheikh': a single following name reads as # given, not family. ("الشيخ محمد", "given", "محمد"), + # Arabic honorifics precede the GIVEN name, so all ship in + # given_name_titles like الشيخ: article forms (never given names) + # and the bare doctor/professor/engineer forms (not used as given + # names). Deferred with the same collision rule as мл/ст: bare + # سيد/شيخ/أمير/سلطان (common given names) and the 'د.' + # abbreviation (bare 'د' would swallow initials, the bare-κ trap). + ("الدكتور أحمد", "given", "أحمد"), + ("الدكتور أحمد", "title", "الدكتور"), + ("الدكتورة منى السيد", "title", "الدكتورة"), + ("الدكتورة منى السيد", "family", "السيد"), + ("دكتور أحمد", "given", "أحمد"), + ("دكتورة منى", "given", "منى"), + ("الأستاذ محمود", "given", "محمود"), + ("أستاذ محمود", "given", "محمود"), + ("أستاذة سلمى", "given", "سلمى"), + ("الأستاذة سلمى", "given", "سلمى"), + ("مهندس حسن", "given", "حسن"), + ("الشيخة موزة", "given", "موزة"), + ("الحاجة فاطمة", "given", "فاطمة"), + # compound with the bound given name: title + bound join + family + ("الحاج عبد الرحمن السيد", "title", "الحاج"), + ("الحاج عبد الرحمن السيد", "given", "عبد الرحمن"), + ("الحاج عبد الرحمن السيد", "family", "السيد"), + # "و" ("and") joins title chains like Cyrillic "и"; single-char + # conjunctions need the same #11 carve-out headroom (enough + # rootname pieces), so pinned with the 6-piece shape the и row uses + ("الأستاذ و الدكتور طارق حسن السيد", "title", "الأستاذ و الدكتور"), # Hebrew patronymic prefixes: same non-leading chain-onto-family # behavior. ("דוד בן גוריון", "family", "בן גוריון"), From f0fbf658725e240dac968ed07878207d4ec4f39f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 14:11:09 -0700 Subject: [PATCH 203/275] feat: Hebrew honorifics + post-nominals, Devanagari titles (#269 follow-up) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hebrew titles גברת/פרופ'/פרופ׳/פרופסור/עו"ד/עו״ד/הרב join מר as plain titles (Israeli convention: family follows), each abbreviation in both geresh/gershayim spellings per the ד"ר precedent. Hebrew gains the suffix category Arabic deferred: ז"ל and שליט"א (both spellings) -- post-nominals ubiquitous in genealogical data, never names, mid-word quotes inert in extraction. Devanagari becomes a #269 script: श्री/श्रीमती/डॉ (डॉ. matches via edge-period normalization). Latin sri/shri deliberately NOT added -- the transliterations collide with real given names (Sri Mulyani); the native script cannot. Deferred under the collision rule, recorded in the data-module comments: bare רב (the ordinary word 'many') and בר as a particle (Bar is a common modern Israeli given name; the surname spelling is hyphenated anyway). TDD: 15 rows red first across title/suffix/script categories. Differential exit 0 unchanged. Co-Authored-By: Claude Fable 5 --- docs/release_log.rst | 1 + nameparser/config/suffixes.py | 8 ++++++++ nameparser/config/titles.py | 21 +++++++++++++++++++++ tests/v2/test_locales.py | 27 +++++++++++++++++++++++++++ 4 files changed, 57 insertions(+) diff --git a/docs/release_log.rst b/docs/release_log.rst index c9df8dfc..56b61400 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -8,6 +8,7 @@ Release Log - The CLI gains ``--locale CODE`` (``python -m nameparser --locale ru "..."``) - Add non-Latin vocabulary to the default lexicon (#269): Cyrillic, Greek, Arabic and Hebrew titles, conjunctions, and name particles -- native-script entries cannot collide with Latin-script names. Deferred pending vetting: Cyrillic ``мл``/``ст`` suffixes and the bare Greek ``κ`` title (it collides with the initial+surname shape). Behavior note: ``محمد بن سلمان`` now chains ``بن`` onto the family name where 1.x read it as a middle name - Add the Arabic-script bound given names (#269 follow-up): ``عبد``, the kunya pair ``أبو``/``ابو``, and ``أم``/``ام`` join the following word into the given name like their transliterations (``abdul``, ``abu``, ``umm``) always did -- ``عبد الرحمن محمد`` now parses given ``عبد الرحمن``, family ``محمد``, where 1.x split it into given + middle (``tests/v2/cases.py`` rows ``arabic_bound_given_*``). Not present in the differential corpus + - Add Hebrew honorifics, Hebrew post-nominals, and Devanagari titles (#269 follow-up): the Israeli honorifics ``גברת``, ``פרופ'``/``פרופ׳``, ``פרופסור``, ``עו"ד``/``עו״ד``, and ``הרב`` as plain titles; the post-nominals ``ז"ל``/``ז״ל`` and ``שליט"א``/``שליט״א`` as suffixes (both gershayim spellings each); and a new #269 script -- Devanagari ``श्री``, ``श्रीमती``, and ``डॉ`` (``डॉ.`` matches via edge-period normalization). Latin ``sri``/``shri`` deliberately NOT added (they collide with real given names; the native script cannot). Deferred: bare ``רב`` (ordinary word "many") and ``בר`` as a particle (Bar is a common modern given name). Not present in the differential corpus - Add Arabic honorific titles and the conjunction ``و`` (#269 follow-up): the doctor/professor/hajj/sheikha/engineer forms (``الدكتور``/``الدكتورة``/``دكتور``/``دكتورة``, ``الأستاذ``/``الأستاذة``/``أستاذ``/``أستاذة``, ``الحاج``/``الحاجة``, ``الشيخة``, ``مهندس``) as given-name titles -- Arabic honorifics precede the given name, like ``الشيخ``. Deferred under the collision rule: bare ``سيد``/``شيخ``/``أمير``/``سلطان`` (all common given names), the ``د.`` abbreviation (bare ``د`` would swallow initials), and the Ottoman post-nominals ``باشا``/``بك``/``أفندي`` (survive as family names). Not present in the differential corpus **Behavior Changes (draft)** diff --git a/nameparser/config/suffixes.py b/nameparser/config/suffixes.py index 027a6f9d..3582df0d 100644 --- a/nameparser/config/suffixes.py +++ b/nameparser/config/suffixes.py @@ -26,6 +26,14 @@ # literally instead of going through nickname/suffix disambiguation). 'ret', 'vet', + + # #269 follow-up: Hebrew post-nominals, both gershayim spellings + # (ASCII '"' and U+05F4); the mid-word quote is inert in + # extraction, like the ד"ר title. Neither is ever a name. + 'ז"ל', # "of blessed memory" (deceased), ASCII quote + 'ז״ל', # same, U+05F4 gershayim + 'שליט"א', # honorific for a living rabbi, ASCII quote + 'שליט״א', # same, U+05F4 gershayim } """ diff --git a/nameparser/config/titles.py b/nameparser/config/titles.py index 2e4666c3..30c60254 100644 --- a/nameparser/config/titles.py +++ b/nameparser/config/titles.py @@ -748,4 +748,25 @@ "גב'", 'ד״ר', 'גב׳', + # #269 follow-up: the rest of the common Israeli honorifics, same + # plain-title bucket (family follows) and the same dual geresh/ + # gershayim spelling rule as above. Deferred under the collision + # rule: bare 'רב' (also the ordinary word "many"); 'בר' as a + # particle (Bar is a common modern given name, and the surname + # spelling is hyphenated anyway). + 'גברת', # Mrs./Ms., full form + "פרופ'", # professor abbreviation, ASCII apostrophe + 'פרופ׳', # professor abbreviation, U+05F3 geresh + 'פרופסור', # professor, full form + 'עו"ד', # advocate/lawyer, ASCII quote + 'עו״ד', # advocate/lawyer, U+05F4 gershayim + 'הרב', # "the rabbi" (article form; bare רב deferred) + + # #269 follow-up: Devanagari (hi/mr). NO Latin twins on purpose: + # transliterated sri/shri collide with real given names (Sri + # Mulyani); the native-script forms cannot. "डॉ." matches via the + # edge-period normalization, like Latin "Dr.". + 'श्री', # Shri (Mr.) + 'श्रीमती', # Shrimati (Mrs.) + 'डॉ', # Dr. abbreviation } diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 9a8ab0f9..2724c6cd 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -449,6 +449,33 @@ def test_non_interference_all_packs_combined() -> None: # Hebrew "מר" title (plain title, not FIRST_NAME_TITLES -- like # 'mr', the following name reads as family). ("מר דוד לוי", "title", "מר"), + # Hebrew title/suffix sweep (#269 follow-up): plain titles (Israeli + # convention: family follows, the מר precedent); both geresh/ + # gershayim spellings ship where an abbreviation carries one. + ("גברת רות כהן", "title", "גברת"), + ("פרופ' דוד לוי", "title", "פרופ'"), + ("פרופ׳ דוד לוי", "title", "פרופ׳"), + ("פרופסור רות כהן", "title", "פרופסור"), + ('עו"ד דוד לוי', "title", 'עו"ד'), + ("עו״ד דוד לוי", "title", "עו״ד"), + ("הרב עובדיה יוסף", "title", "הרב"), + # post-nominal Hebrew suffixes: ז"ל ("of blessed memory") and + # שליט"א (honorific for living rabbis) -- suffix words, exact + # token incl. the mid-word gershayim (inert in extraction, like + # the ד"ר title) + ('משה כהן ז"ל', "suffix", 'ז"ל'), + ("משה כהן ז״ל", "suffix", "ז״ל"), + ('הרב משה פיינשטיין שליט"א', "suffix", 'שליט"א'), + ("הרב משה פיינשטיין שליט״א", "title", "הרב"), + # Devanagari (hi/mr) titles -- a new #269 script: श्री/श्रीमती are + # the Mr./Mrs. analogs, डॉ the Dr. abbreviation (edge-period + # normalization makes "डॉ." match too). NO Latin twins on purpose: + # transliterated sri/shri collide with real given names (Sri + # Mulyani); the native script cannot. + ("श्री राम शर्मा", "title", "श्री"), + ("श्रीमती सीता शर्मा", "title", "श्रीमती"), + ("डॉ राम शर्मा", "title", "डॉ"), + ("डॉ. राम शर्मा", "title", "डॉ."), # Geresh/gershayim gate (#269 step 2): probed live against # extract_delimited's _open_ok/_close_ok boundary rules. Both the # ASCII-quote spelling and the typographic Unicode spelling of From 05fe693ec41237d7df1aaccac4f8c422599cd9f7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 15:16:31 -0700 Subject: [PATCH 204/275] fix: extraction scans by position, not pair order (review critical) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three review agents independently found the same defect in the #273 defaults: each pair scanned the full text independently in sorted order, so when two conventions sharing a character in opposite roles BOTH occurred ('Hans „Erster" und "Zweiter" Müller'), the earlier-sorted pair claimed the other's close as its own open, extracted a bogus span across the real boundary, and the legitimate match was dropped by the overlap check with zero ambiguity signal. extract_delimited now runs one position-driven scan: at each position the leftmost boundary-valid opener among ALL pairs wins, with per-pair forward-only cursors keeping the scan linear. Overlapping matches become impossible by construction, so the overlap check on matches is gone; the unmatched-open filter remains (now via _overlaps) for bulk-recorded dangling opens consumed by later matches. Bucket precedence is documented where it actually lives: Policy's maiden-wins canonicalization (facade: shim pre-subtraction) -- the stale 'maiden scanned first' module-docstring claim and the contradicting 'nickname first' comment are both gone. Also the review's quantified perf fix: a cached delimiter-charset prescreen short-circuits the no-delimiter common case (isdisjoint), more than paying back the 3->11 default-pair growth. TDD: four mixed-convention/coexistence tests red first; 19 extract tests green; differential exit 0 unchanged. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_extract.py | 204 ++++++++++++++++++------------ tests/v2/pipeline/test_extract.py | 65 ++++++++++ 2 files changed, 188 insertions(+), 81 deletions(-) diff --git a/nameparser/_pipeline/_extract.py b/nameparser/_pipeline/_extract.py index 0f6672fd..634fb233 100644 --- a/nameparser/_pipeline/_extract.py +++ b/nameparser/_pipeline/_extract.py @@ -6,21 +6,31 @@ UNBALANCED_DELIMITER ambiguities for opens with no close. Reads: Policy.nickname_delimiters, Policy.maiden_delimiters. -Matching rules (the #273 mechanism): pairs scan the original text left -to right, no nesting. For pairs whose open == close (quotes), the open -must sit at a word boundary (start of text or after whitespace) and the -close before one (end, whitespace, or a comma char) -- this is what -keeps the apostrophe in O'Connor literal. maiden_delimiters are scanned -before nickname_delimiters, so routing a pair to maiden (#274) wins if -a pair appears in both sets. Empty enclosures are masked (removed from -the token stream) but extract nothing. Overlapping candidate regions -resolve by scan order (maiden pairs, then nickname pairs, sorted within -each): the first match masks the region, and any delimiter chars inside -an extracted span stay literal. +Matching rules (the #273 mechanism): one left-to-right scan over the +original text, no nesting. At each position the LEFTMOST boundary-valid +opener among ALL configured pairs wins -- position order, never pair +order, decides between conventions that share a character in opposite +roles ('“' closes „…“ but opens “…”; '»' closes «…» but opens »…«), so +"Hans „Erster“ und “Zweiter” Müller" extracts both names. For pairs +whose open == close (quotes), the open must sit at a word boundary +(start of text or after whitespace) and the close before one (end, +whitespace, or a comma char) -- this is what keeps the apostrophe in +O'Connor literal. Empty enclosures are masked (removed from the token +stream) but extract nothing; delimiter characters inside a matched +region are literal content for every other pair. + +Bucket precedence is NOT decided here: Policy canonicalizes overlap +away before parsing (a pair listed in maiden_delimiters is dropped +from the effective nickname set -- maiden wins; the v1 facade restores +v1's nickname-wins reading via a pre-subtraction in _config_shim), so +the two buckets are always disjoint by the time this stage runs. The +nickname-before-maiden candidate order below is only a same-position +tie-break for exotic configs where two pairs share an OPEN character. """ from __future__ import annotations import dataclasses +import functools from nameparser._lexicon import Lexicon, _normalize from nameparser._pipeline._state import ( @@ -55,86 +65,118 @@ def _overlaps(span: Span, taken: list[Span]) -> bool: return any(span.start < t.end and t.start < span.end for t in taken) +@functools.lru_cache(maxsize=128) +def _delimiter_chars( + nickname_pairs: frozenset[tuple[str, str]], + maiden_pairs: frozenset[tuple[str, str]], +) -> frozenset[str]: + """Every character appearing in any configured delimiter, cached on + the (hashable) policy frozensets: the common no-delimiter name pays + one isdisjoint() instead of a per-pair scan.""" + return frozenset( + ch + for pairs in (nickname_pairs, maiden_pairs) + for pair in pairs + for part in pair + for ch in part + ) + + +def _unmatched(open_: str, offset: int) -> tuple[int, PendingAmbiguity]: + return (offset, PendingAmbiguity( + AmbiguityKind.UNBALANCED_DELIMITER, + f"unmatched {open_!r} at offset {offset}; treated as literal text", + )) + + def extract_delimited(state: ParseState) -> ParseState: text = state.original + policy = state.policy + if _delimiter_chars(policy.nickname_delimiters, + policy.maiden_delimiters).isdisjoint(text): + return state + # Candidate order matters only as a same-position tie-break (see + # module docstring); the scan itself is position-driven. + order = tuple( + (role, open_, close) + for role, pairs in ((Role.NICKNAME, policy.nickname_delimiters), + (Role.MAIDEN, policy.maiden_delimiters)) + for open_, close in sorted(pairs) + ) extracted: list[tuple[Role, Span]] = [] masked: list[Span] = [] # candidates, not final: each carries the offset of the unmatched - # open so cross-pair overlaps can be filtered once all pairs ran + # open so ones consumed by a later match can be filtered at the end unbalanced: list[tuple[int, PendingAmbiguity]] = [] - # nickname first (v1 parse_nicknames order): when the same - # delimiter pair sits in BOTH buckets, the nickname reading wins; - # the documented bucket-move idiom removes it from nickname, so - # maiden still gets it after a move - for role, pairs in ( - (Role.NICKNAME, state.policy.nickname_delimiters), - (Role.MAIDEN, state.policy.maiden_delimiters), - ): - for open_, close in sorted(pairs): - pos = 0 - while (i := text.find(open_, pos)) != -1: - if open_ == close and not _open_ok(text, i): - pos = i + 1 - continue - j = text.find(close, i + len(open_)) - while (open_ == close and j != -1 - and not _close_ok(text, j, len(close))): - j = text.find(close, j + 1) - if j == -1: - unbalanced.append((i, PendingAmbiguity( - AmbiguityKind.UNBALANCED_DELIMITER, - f"unmatched {open_!r} at offset {i}; treated as " - f"literal text", - ))) - if open_ == close: - # _close_ok is open-independent, so a failed - # close-walk means NO boundary-valid close - # exists anywhere to the right: every remaining - # boundary-valid open is unmatched too. Record - # each in one forward pass without re-walking - # closes (keeps adversarial input linear). - scan = i + len(open_) - while (k := text.find(open_, scan)) != -1: - if _open_ok(text, k): - unbalanced.append((k, PendingAmbiguity( - AmbiguityKind.UNBALANCED_DELIMITER, - f"unmatched {open_!r} at offset {k}; " - f"treated as literal text", - ))) - scan = k + 1 - break - pos = i + len(open_) - continue - full = Span(i, j + len(close)) - if not _overlaps(full, masked): - inner = Span(i + len(open_), j) - if inner.start < inner.end and _suffix_shaped( - text[inner.start:inner.end], state.lexicon): - # v1 parse_nicknames: suffix-shaped delimited - # content is left IN PLACE (undelimited) for - # normal downstream parsing -- 'Andrew Perkins - # (MBA)' keeps MBA a suffix, not a nickname. - # Spans index the original (anti-#100), so the - # v2 spelling masks only the two delimiter - # spans and lets the inner content join the - # main token stream. - masked.append(Span(i, i + len(open_))) - masked.append(Span(j, j + len(close))) - else: - if inner.start < inner.end: - extracted.append((role, inner)) - masked.append(full) - pos = j + len(close) + # per-candidate cursor cache: next boundary-valid open at or after + # the position it was computed for. find() calls only ever move + # forward, keeping the whole scan linear in len(text) per pair. + cursors: dict[tuple[Role, str, str], int] = {} + exhausted: set[tuple[Role, str, str]] = set() + pos = 0 + while pos < len(text): + best: tuple[int, Role, str, str] | None = None + for key in order: + if key in exhausted: + continue + _, open_, close = key + i = cursors.get(key, -2) + if i != -1 and i < pos: + i = text.find(open_, pos) + while (i != -1 and open_ == close + and not _open_ok(text, i)): + i = text.find(open_, i + 1) + cursors[key] = i + if i != -1 and (best is None or i < best[0]): + best = (i, *key) + if best is None: + break + i, role, open_, close = best + j = text.find(close, i + len(open_)) + while (open_ == close and j != -1 + and not _close_ok(text, j, len(close))): + j = text.find(close, j + 1) + if j == -1: + # No (boundary-valid) close exists anywhere to the right -- + # the walk above ran to end of text -- so every remaining + # open of this pair is unmatched too. Record them all in + # one forward pass and retire the pair; other pairs keep + # scanning from the same position. + unbalanced.append(_unmatched(open_, i)) + scan = i + len(open_) + while (k := text.find(open_, scan)) != -1: + if open_ != close or _open_ok(text, k): + unbalanced.append(_unmatched(open_, k)) + scan = k + 1 + exhausted.add((role, open_, close)) + continue + inner = Span(i + len(open_), j) + if inner.start < inner.end and _suffix_shaped( + text[inner.start:inner.end], state.lexicon): + # v1 parse_nicknames: suffix-shaped delimited content is + # left IN PLACE (undelimited) for normal downstream parsing + # -- 'Andrew Perkins (MBA)' keeps MBA a suffix, not a + # nickname. Spans index the original (anti-#100), so the v2 + # spelling masks only the two delimiter spans and lets the + # inner content join the main token stream. + masked.append(Span(i, i + len(open_))) + masked.append(Span(j, j + len(close))) + else: + if inner.start < inner.end: + extracted.append((role, inner)) + masked.append(Span(i, j + len(close))) + # position-driven scanning makes overlapping matches + # impossible by construction: every later open is found at or + # after this match's end + pos = j + len(close) extracted.sort(key=lambda pair: pair[1]) masked.sort() - # A delimiter character consumed by another pair's successful - # extraction is literal for every other pair: drop unbalanced - # candidates whose offset lies inside a masked region (#273 -- the - # conventions share characters in opposite roles: '“' closes „…“ - # but opens “…”, '»' closes «…» but opens »…«). + # An unmatched-open candidate whose character was consumed by a + # later successful match (the bulk pass above runs ahead of the + # main scan) is literal content there, not a dangling delimiter. ambiguities = tuple( a for offset, a in unbalanced - if not any(m.start <= offset < m.end for m in masked)) + if not _overlaps(Span(offset, offset + 1), masked)) return dataclasses.replace( state, extracted=tuple(extracted), masked=tuple(masked), ambiguities=state.ambiguities + ambiguities) diff --git a/tests/v2/pipeline/test_extract.py b/tests/v2/pipeline/test_extract.py index b6d27393..6f07c810 100644 --- a/tests/v2/pipeline/test_extract.py +++ b/tests/v2/pipeline/test_extract.py @@ -58,6 +58,71 @@ def test_no_spurious_unbalanced_from_role_overlapping_pairs() -> None: assert out.ambiguities == () +def test_mixed_conventions_both_extract_leftmost_wins() -> None: + # Review finding (2026-07-19, three agents independently): with the + # default #273 pairs, a string containing TWO delimited asides in + # different conventions mis-extracted -- the earlier-sorted pair + # claimed the other's close character ('“' closes „…“ but opens + # “…”) as its own open and swallowed a bogus span across the real + # boundary, silently dropping the legitimate match. The scan must + # be position-driven: the leftmost genuine opener wins. + text = "Hans „Erster“ und “Zweiter” Müller" + out = extract_delimited(_state(text)) + assert [text[s.start:s.end] for _, s in out.extracted] == [ + "Erster", "Zweiter"] + assert out.ambiguities == () + + +def test_mixed_guillemet_directions_both_extract() -> None: + text = "Jean «Petit» Dupont »Rex« Martin" + out = extract_delimited(_state(text)) + assert [text[s.start:s.end] for _, s in out.extracted] == [ + "Petit", "Rex"] + assert out.ambiguities == () + + +def test_mixed_conventions_across_maiden_and_nickname_buckets() -> None: + # same shape across ROLES: guillemets routed to maiden must not let + # the reversed nickname pair steal the maiden close + policy = Policy(maiden_delimiters=frozenset({("«", "»")})) + text = "Jean «Dupont» Martin »Rex« Smith" + out = extract_delimited(_state(text, policy)) + got = {(role, text[s.start:s.end]) for role, s in out.extracted} + assert got == {(Role.MAIDEN, "Dupont"), (Role.NICKNAME, "Rex")} + assert out.ambiguities == () + + +def test_extraction_and_genuine_unbalanced_coexist() -> None: + # the kept path: a successful extraction must not suppress a REAL + # dangling open elsewhere in the same string + text = 'John (Jack) Smith "Extra' + out = extract_delimited(_state(text)) + assert [text[s.start:s.end] for _, s in out.extracted] == ["Jack"] + assert [a.kind for a in out.ambiguities] == [ + AmbiguityKind.UNBALANCED_DELIMITER] + + +def test_same_char_bulk_unbalanced_filtered_by_other_pairs_mask() -> None: + # the same-char forward-scan records EVERY dangling quote; one that + # sits inside a region another pair successfully extracts is + # literal content, not a dangling delimiter -- only the truly + # dangling ones surface. All three quotes here fail the close-walk + # (no quote is followed by a boundary), so the bulk pass records + # all three; the middle one lands inside the paren match. + # 01234567890123456789 + text = 'Jon "A (x "y) Bob "C' + out = extract_delimited(_state(text)) + assert [text[s.start:s.end] for _, s in out.extracted] == ['x "y'] + # quotes at 4 and 18 are genuinely dangling; the one at 10 was + # consumed by the parenthesized extraction + assert sorted(a.detail for a in out.ambiguities) == [ + "unmatched '\"' at offset 18; treated as literal text", + "unmatched '\"' at offset 4; treated as literal text", + ] + assert all(a.kind is AmbiguityKind.UNBALANCED_DELIMITER + for a in out.ambiguities) + + def test_genuine_unbalanced_still_flagged_alongside_overlapping_pairs() -> None: # the suppression must not swallow REAL unbalanced opens: here the # German open has no close anywhere, and no other pair extracts From ed9938a851198f32bcb8f751f607de848ab6a8d4 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 15:16:31 -0700 Subject: [PATCH 205/275] docs: review accuracy corrections; hoist the migration hint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - name_order's comma claim narrowed: only a family-separating comma ignores it; 'John Smith, Jr.' still obeys name_order (live-proven, SUFFIX_COMMA reads it). - The 'issue #11' carve-out citations were pointing at GitHub #11 (a 2014 'meant to create a milestone' accident); the real source is GOOGLE CODE issue 11, the 'john e smith' bug -- v1's own comment carries the full URL. All six sites now say so. - _DelimiterManager no longer claims 'only the three named sentinels'. - The בר particle deferral moved to prefixes.py with the other particle decisions. - The patronymic migration hint is a single module constant shared by Policy and PolicyPatch. - apply_patch documents maiden-wins through patches as intended (decided 2026-07-19): maiden membership IS the routing decision, whoever contributes it. Co-Authored-By: Claude Fable 5 --- docs/customize.rst | 5 ++-- nameparser/_config_shim.py | 4 +-- nameparser/_pipeline/_group.py | 3 ++- nameparser/_policy.py | 42 ++++++++++++++++++------------- nameparser/config/conjunctions.py | 3 ++- nameparser/config/prefixes.py | 3 +++ nameparser/config/titles.py | 6 ++--- tests/v2/pipeline/test_group.py | 3 ++- tests/v2/test_locales.py | 21 +++++++++++++--- 9 files changed, 59 insertions(+), 31 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index 1398a6a3..17369b43 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -65,8 +65,9 @@ listed below. - Assigns positional (no-comma) input to given/middle/family in this order. Use the exported ``GIVEN_FIRST`` (default), ``FAMILY_FIRST``, or ``FAMILY_FIRST_GIVEN_LAST`` constants. - Ignored for comma-format input — the comma itself states the - order ("Thomas, John" puts the family name first). + Ignored when a comma separates family from given ("Thomas, + John" puts the family name first); a comma that only sets off + suffixes ("John Smith, Jr.") leaves it governing the name part. * - ``patronymic_rules`` - ``frozenset[PatronymicRule]`` - Reorders patronymic-shaped names via opt-in detectors — East diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index 8fc1e7ae..096df927 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -449,8 +449,8 @@ class -- via ``TupleManager.__reduce__``'s ``(type(self), (), state)`` class _DelimiterManager(TupleManager): """v1 ``nickname_delimiters``/``maiden_delimiters`` bucket. In 2.0 - only the three named sentinels exist (spec §3) -- each maps to the - name of a ``_RegexesProxy`` entry it stays linked to; assigning any + only the named sentinels in ``_DELIMITER_SENTINELS`` exist (spec + §3; the v1 trio plus the #273 typographic pairs) -- assigning any other key raises so a caller reaches for a custom-delimiter Policy kwarg instead of a dict entry that silently does nothing. ``pop()``/ ``__setitem__``/``__delitem__`` stay open (inherited) for the diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index b7643b6a..675a88ac 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -132,7 +132,8 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), merge(k, k + 2, add={"conjunction"}) else: k += 1 - # each conjunction joins its neighbors (v1 issue #11 carve-out: + # each conjunction joins its neighbors (v1's Google Code issue 11 + # carve-out, the "john e smith" bug: # a single-letter alphabetic conjunction in a short name is more # likely an initial) k = 0 diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 3fd6f71a..9aa6d859 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -45,6 +45,16 @@ class PatronymicRule(StrEnum): _NAME_ROLES = frozenset({Role.GIVEN, Role.MIDDLE, Role.FAMILY}) +# Single source for the migration hint raised by both Policy and +# PolicyPatch when patronymic_rules gets a non-iterable (True is the +# likeliest wrong value -- v1's flag was a bool that enabled BOTH rules). +_PATRONYMIC_MIGRATION_HINT = ( + "v1's patronymic_name_order=True enabled both rules -- " + "patronymic_rules={PatronymicRule.EAST_SLAVIC, " + "PatronymicRule.TURKIC} (or pick one via " + "parser_for(locales.RU) / locales.TR_AZ)" +) + #: Policy.nickname_delimiters' default. Public and named so #: customizations read as set math against a documented value -- e.g. #: ``DEFAULT_NICKNAME_DELIMITERS | {("｟", "｠")}`` -- instead of a @@ -90,10 +100,11 @@ class Policy: #: :ref:`name-order constants ` -- #: GIVEN_FIRST (the default), FAMILY_FIRST, and #: FAMILY_FIRST_GIVEN_LAST; any other tuple of Roles raises - #: ValueError. Ignored when the input contains a comma: the comma - #: itself states the order -- "Thomas, John" puts the family name - #: first no matter which words could otherwise be either - #: ("Thomas" and "John" both work as given or family names). + #: ValueError. Ignored when a comma separates family from given: + #: "Thomas, John" puts the family name first no matter which words + #: could otherwise be either ("Thomas" and "John" both work as + #: given or family names). A comma that only sets off suffixes + #: ("John Smith, Jr.") leaves name_order governing the name part. name_order: tuple[Role, Role, Role] = GIVEN_FIRST #: Opt-in detectors that reorder patronymic-shaped names #: (EAST_SLAVIC, TURKIC); usually set via a locale pack. @@ -177,11 +188,8 @@ def __post_init__(self) -> None: except TypeError: raise TypeError( f"patronymic_rules must be an iterable of PatronymicRule " - f"names, got {self.patronymic_rules!r}; v1's " - f"patronymic_name_order=True enabled both rules -- " - f"patronymic_rules={{PatronymicRule.EAST_SLAVIC, " - f"PatronymicRule.TURKIC}} (or pick one via " - f"parser_for(locales.RU) / locales.TR_AZ)" + f"names, got {self.patronymic_rules!r}; " + f"{_PATRONYMIC_MIGRATION_HINT}" ) from None items = tuple(rule_iter) rules = set() @@ -341,13 +349,8 @@ def __post_init__(self) -> None: try: iter(value) except TypeError: - hint = ( - "; v1's patronymic_name_order=True enabled both rules" - " -- patronymic_rules={PatronymicRule.EAST_SLAVIC, " - "PatronymicRule.TURKIC} (or pick one via " - "parser_for(locales.RU) / locales.TR_AZ)" - if f.name == "patronymic_rules" else "" - ) + hint = ("; " + _PATRONYMIC_MIGRATION_HINT + if f.name == "patronymic_rules" else "") raise TypeError( f"{f.name} must be an iterable, got {value!r}{hint}" ) from None @@ -356,7 +359,12 @@ def __post_init__(self) -> None: def apply_patch(policy: Policy, patch: PolicyPatch) -> Policy: """Fold a PolicyPatch onto a Policy. Policy.__post_init__ re-runs via - dataclasses.replace, so patched values are revalidated for free.""" + dataclasses.replace, so patched values are revalidated for free -- + including the maiden-wins canonicalization: a patch that adds a + maiden pair removes that pair from the base's effective nickname + set, exactly as if the combined Policy had been constructed + directly. Intended (decided 2026-07-19): maiden_delimiters + membership IS the routing decision, whoever contributes it.""" updates: dict[str, object] = {} for f in dataclasses.fields(PolicyPatch): value = getattr(patch, f.name) diff --git a/nameparser/config/conjunctions.py b/nameparser/config/conjunctions.py index a30cc3f9..52f01b34 100644 --- a/nameparser/config/conjunctions.py +++ b/nameparser/config/conjunctions.py @@ -16,7 +16,8 @@ # #269 follow-up: Arabic "and". Formal script attaches و to the # following word (وفاطمة), so a standalone و token appears only in # informal spacing -- common in real data. Single-character like - # 'y'/'и': the #11 initial carve-out protects short names (joins + # 'y'/'и': the single-letter carve-out (Google Code issue 11, + # the "john e smith" bug) protects short names (joins # only with enough rootname pieces). 'و', } diff --git a/nameparser/config/prefixes.py b/nameparser/config/prefixes.py index 4f3a1cb9..efa92bac 100644 --- a/nameparser/config/prefixes.py +++ b/nameparser/config/prefixes.py @@ -51,6 +51,9 @@ # #269: Hebrew native-script patronymic particles -- same # reasoning as the Arabic ones above: no Latin-script collision, # and neither functions as a standalone given name in Hebrew usage. + # Deferred under the collision rule: 'בר' (Aramaic son-of, as in + # Bar-Lev) -- Bar is a common modern Israeli given name, and the + # surname spelling is hyphenated anyway. 'בן', # "ben" (son of) 'בת', # "bat" (daughter of) } diff --git a/nameparser/config/titles.py b/nameparser/config/titles.py index 30c60254..0f1d9c86 100644 --- a/nameparser/config/titles.py +++ b/nameparser/config/titles.py @@ -751,9 +751,9 @@ # #269 follow-up: the rest of the common Israeli honorifics, same # plain-title bucket (family follows) and the same dual geresh/ # gershayim spelling rule as above. Deferred under the collision - # rule: bare 'רב' (also the ordinary word "many"); 'בר' as a - # particle (Bar is a common modern given name, and the surname - # spelling is hyphenated anyway). + # rule: bare 'רב' (also the ordinary word "many"). The 'בר' + # particle deferral is recorded in prefixes.py with the other + # particle decisions. 'גברת', # Mrs./Ms., full form "פרופ'", # professor abbreviation, ASCII apostrophe 'פרופ׳', # professor abbreviation, U+05F3 geresh diff --git a/tests/v2/pipeline/test_group.py b/tests/v2/pipeline/test_group.py index 0d3150d2..182a603d 100644 --- a/tests/v2/pipeline/test_group.py +++ b/tests/v2/pipeline/test_group.py @@ -51,7 +51,8 @@ def test_contiguous_conjunctions_join_first() -> None: def test_single_letter_conjunction_prefers_initial_when_short() -> None: - # v1 issue #11: 3 rootname parts, single-letter conjunction "y" + # v1 Google Code issue 11 ("john e smith"): 3 rootname parts, + # single-letter conjunction "y" out = _grouped("John y Smith") assert _piece_texts(out) == [["John", "y", "Smith"]] diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 2724c6cd..2ea1753e 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -372,7 +372,8 @@ def test_non_interference_all_packs_combined() -> None: ("акад Іван Франко", "title", "акад"), ("пан Тарас Шевченко", "title", "пан"), ("пані Марія Іванова", "title", "пані"), - # Cyrillic conjunction "и": v1's issue #11 carve-out treats a bare + # Cyrillic conjunction "и": v1's single-letter carve-out (Google + # Code issue 11, the "john e smith" bug) treats a bare # single-alphabetic-character conjunction in a short name as more # likely an initial (group._group_segment), so a 3-piece chain # ("проф и акад Іван Франко") does NOT join -- "и" reads as a given @@ -381,13 +382,20 @@ def test_non_interference_all_packs_combined() -> None: # with a 5-piece name instead of the shorter guess. ("проф и акад Тарас Григорович Шевченко", "title", "проф и акад"), ("проф та акад Іван Франко", "title", "проф та акад"), - # Ukrainian "і" is single-character like "и", so the same #11 - # initial carve-out applies: pinned with a 5-piece name. + # Ukrainian "і" is single-character like "и", so the same + # single-letter carve-out applies: pinned with a 5-piece name. ("проф і акад Тарас Григорович Шевченко", "title", "проф і акад"), # Greek titles + conjunction (και has 3 letters, so the single-char # initial carve-out above never applies to it). Bare κ is NOT # shipped -- it collides with the initial+surname shape (see the # regression test below). + # Match-time contract of the lower()-not-casefold() normalization + # (review 2026-07-19): case variants that lower() folds still + # match; the ASCII-SS spelling of ß does NOT (v1-parity -- casefold + # would have matched it, at the cost of mutating stored spellings). + ("ΚΟΣ Γιώργος Παπαδόπουλος", "title", "ΚΟΣ"), + ("GROẞFÜRST Otto Schmidt", "title", "GROẞFÜRST"), + ("GROSSFÜRST Otto Schmidt", "title", ""), ("δρ Νίκος Παπαδόπουλος", "title", "δρ"), ("κος Γιώργος Παπαδόπουλος", "title", "κος"), ("κα Μαρία Παπαδοπούλου", "title", "κα"), @@ -439,7 +447,7 @@ def test_non_interference_all_packs_combined() -> None: ("الحاج عبد الرحمن السيد", "given", "عبد الرحمن"), ("الحاج عبد الرحمن السيد", "family", "السيد"), # "و" ("and") joins title chains like Cyrillic "и"; single-char - # conjunctions need the same #11 carve-out headroom (enough + # conjunctions need the same single-letter carve-out headroom (enough # rootname pieces), so pinned with the 6-piece shape the и row uses ("الأستاذ و الدكتور طارق حسن السيد", "title", "الأستاذ و الدكتور"), # Hebrew patronymic prefixes: same non-leading chain-onto-family @@ -505,6 +513,11 @@ def test_269_vocabulary_reaches_the_v1_facade() -> None: assert HumanName("κος Γιώργος Παπαδόπουλος").title == "κος" assert HumanName("محمد بن سلمان").last == "بن سلمان" assert HumanName('ד"ר דוד לוי').title == 'ד"ר' + # one row per NEW vocabulary category (2026-07-19 batches): bound + # given name, Hebrew post-nominal suffix, Devanagari title + assert HumanName("عبد الرحمن محمد").first == "عبد الرحمن" + assert HumanName('משה כהן ז"ל').suffix == 'ז"ל' + assert HumanName("डॉ. राम शर्मा").title == "डॉ." def test_269_bare_greek_kappa_not_a_title() -> None: From 59cb4cbe44d245238fb923918700bd78a6686816 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 15:16:31 -0700 Subject: [PATCH 206/275] test: regression guards from the review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Match-time contract of lower()-not-casefold() pinned at parse level (ΚΟΣ and GROẞFÜRST match; ASCII-SS GROSSFÜRST deliberately does not, v1 parity); facade rows for each new vocabulary category (bound given, Hebrew suffix, Devanagari title); maiden-wins pinned on a typographic pair; a pre-#273 three-key pickle restores as exactly its own keys (setstate replaces, never merges defaults). Co-Authored-By: Claude Fable 5 --- tests/test_constants.py | 15 +++++++++++++++ tests/v2/test_policy.py | 10 ++++++++++ 2 files changed, 25 insertions(+) diff --git a/tests/test_constants.py b/tests/test_constants.py index 1a7ff53d..a8ebe82d 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -549,6 +549,21 @@ def test_nickname_delimiters_default_builtins_resolve_live(self) -> None: # v1 trio still present, still stored name-as-value for name in ('quoted_word', 'double_quotes', 'parenthesis'): self.assertEqual(entries[name], name) + # A pre-#273 pickle restores as exactly its own three keys -- + # __setstate__ REPLACES the bucket, never merges defaults in -- + # so old configs don't silently gain the typographic pairs. + legacy = Constants() + state = legacy.__getstate__() + state['nickname_delimiters'] = { + 'quoted_word': 'quoted_word', 'double_quotes': 'double_quotes', + 'parenthesis': 'parenthesis'} + restored = Constants.__new__(Constants) + restored.__setstate__(state) + self.assertEqual(set(restored.nickname_delimiters), + {'quoted_word', 'double_quotes', 'parenthesis'}) + from nameparser import HumanName + assert HumanName('John «Jack» Kennedy', constants=restored).nickname == '' + assert HumanName('John (Jack) Kennedy', constants=restored).nickname == 'Jack' # 2.0 adds the #273 typographic sentinels alongside them, same # name-as-value scheme (full list pinned in tests/v2/ # test_config_shim.py against _SENTINEL_PAIRS) diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 60ddb648..47047d55 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -111,6 +111,16 @@ def test_maiden_delimiters_win_over_nickname_defaults() -> None: assert p == explicit and hash(p) == hash(explicit) +def test_maiden_wins_applies_to_typographic_pairs_too() -> None: + # the subtraction is pair-agnostic; pin one #273 pair end-to-end + from nameparser import Parser + + p = Policy(maiden_delimiters=frozenset({("«", "»")})) + assert ("«", "»") not in p.nickname_delimiters + n = Parser(policy=p).parse("Jean «Dupont» Martin") + assert n.maiden == "Dupont" and n.nickname == "" + + def test_maiden_precedence_applies_through_policy_patch() -> None: # apply_patch re-runs Policy's constructor, so a patch (e.g. from a # locale pack) adding a maiden pair gets the same subtraction From cb3e5b5470f072ad8f0a091070d162d960d80d51 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 18:37:38 -0700 Subject: [PATCH 207/275] docs: inline unresolvable docs/superpowers spec citations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "core spec §5a/§5b/§6" pointed at docs/superpowers/, which is gitignored and not committed on any branch -- a future reader can never resolve the citation. Replace each with the actual rule content inline instead (review finding on #288). --- nameparser/_pipeline/__init__.py | 12 +++++++----- nameparser/_pipeline/_assign.py | 4 ++-- nameparser/_pipeline/_segment.py | 3 ++- nameparser/_pipeline/_tokenize.py | 8 ++++---- nameparser/_render.py | 11 +++++------ nameparser/_types.py | 15 ++++++++------- 6 files changed, 28 insertions(+), 25 deletions(-) diff --git a/nameparser/_pipeline/__init__.py b/nameparser/_pipeline/__init__.py index d9a3bfeb..f69c2188 100644 --- a/nameparser/_pipeline/__init__.py +++ b/nameparser/_pipeline/__init__.py @@ -1,8 +1,8 @@ """The 2.0 parse pipeline: pure stages folded over ParseState. -Stage names and ParseState are NOT public API (core spec §6). The stage -list is data; runners other than the plain fold (explain(), parse_all()) -arrive in 2.x minors without changing any stage signature. +Stage names and ParseState are NOT public API. The stage list is data; +runners other than the plain fold (explain(), parse_all()) arrive in +2.x minors without changing any stage signature. Layering: imports only nameparser._pipeline.* (enforced by tests/v2/test_layering.py). @@ -20,7 +20,7 @@ from nameparser._pipeline._state import ParseState from nameparser._pipeline._tokenize import tokenize -#: The full seven-stage fold (spec §6). +#: The full seven-stage fold. STAGES: tuple[Callable[[ParseState], ParseState], ...] = ( extract_delimited, tokenize, segment, classify, group, assign, post_rules, @@ -29,7 +29,9 @@ def run(state: ParseState) -> ParseState: """Fold the stages over the initial state. Pure: each stage returns - a new ParseState; content never raises (totality, spec §5a).""" + a new ParseState; parse is total over str input, so no stage may + raise on the *content* of a name (malformed policy/config is a + separate, allowed failure mode).""" for stage in STAGES: state = stage(state) return state diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 1a3d95c7..a742bb66 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -79,8 +79,8 @@ def _name_positions(order: tuple[Role, Role, Role], """Roles for `count` name pieces (titles/suffixes already peeled), per name_order. GIVEN_FIRST: given, middles..., family. FAMILY_FIRST: family, given, middles... FAMILY_FIRST_GIVEN_LAST: - family, middles..., given. One piece takes order[0]'s role - (spec §5a); two pieces take order[0] and the other primary.""" + family, middles..., given. One piece takes order[0]'s role; two + pieces take order[0] and the other primary.""" first, second = order[0], order[1] if count == 1: return [first] diff --git a/nameparser/_pipeline/_segment.py b/nameparser/_pipeline/_segment.py index c9762d16..6a734f8b 100644 --- a/nameparser/_pipeline/_segment.py +++ b/nameparser/_pipeline/_segment.py @@ -15,7 +15,8 @@ lenient-suffix AND >1 word before the first comma -> SUFFIX_COMMA; otherwise FAMILY_COMMA ("Family, Given ..."), with segments beyond the second that are not lenient-suffix flagged COMMA_STRUCTURE (they are -still best-effort consumed as suffixes by assign, spec §5a). +still best-effort consumed as suffixes by assign, since parse must +stay total over str input and never raise on content). """ from __future__ import annotations diff --git a/nameparser/_pipeline/_tokenize.py b/nameparser/_pipeline/_tokenize.py index 85fcbcfa..8351e8a7 100644 --- a/nameparser/_pipeline/_tokenize.py +++ b/nameparser/_pipeline/_tokenize.py @@ -6,10 +6,10 @@ slice), comma_offsets (segmentation points; never tokens). Reads: Policy.strip_emoji, Policy.strip_bidi. -There is NO text-rewriting normalize stage (core spec §6): whitespace -collapsing and emoji/bidi stripping are character-classification rules -here -- ignorable characters act as separators and never enter a token, -so spans always index the original exactly as given. +There is NO text-rewriting normalize stage: whitespace collapsing and +emoji/bidi stripping are character-classification rules here -- +ignorable characters act as separators and never enter a token, so +spans always index the original exactly as given. v1's squash_emoji/squash_bidi REMOVED the char and joined neighbors ('A\U0001f600B' -> 'AB'); here an ignorable char is a SEPARATOR diff --git a/nameparser/_render.py b/nameparser/_render.py index 18399b71..ad236f6e 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -42,11 +42,10 @@ def _collapse(rendered: str) -> str: - """The #254 collapse, normative (core spec §5b): empty fields - substitute '' and every artifact of that is removed -- dangling - empty-nickname wrappers, space runs, space-before-comma, one - trailing comma character (any script), leading/trailing ', ' - debris.""" + """The #254 collapse: empty fields substitute '' and every artifact + of that is removed -- dangling empty-nickname wrappers, space runs, + space-before-comma, one trailing comma character (any script), + leading/trailing ', ' debris.""" rendered = (rendered.replace(" ()", "") .replace(" ''", "") .replace(' ""', "")) @@ -136,7 +135,7 @@ def _cap_text(text: str, role: Role, lex: Lexicon) -> str: def capitalized(name: ParsedName, lexicon: Lexicon | None, *, force: bool) -> ParsedName: """Case-fixing transform -> new ParsedName, same spans, new token - texts (core spec §5b). Gate (v1 parity): only single-case input is + texts. Gate (v1 parity): only single-case input is touched unless force=True; the gate reads the joined token texts (not render() output -- the case gate stays decoupled from spec formatting and the #254 collapse). diff --git a/nameparser/_types.py b/nameparser/_types.py index 8be0953c..e0f6ddb7 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -66,8 +66,8 @@ class Span(NamedTuple): def __add__(self, other: object) -> NoReturn: # type: ignore[override] # Inherited tuple + would concatenate two spans into a 4-tuple. # There is deliberately NO covering-span operation: grouping is - # index-run based (spec §6, the anti-#100 invariant) and never - # merges spans. + # index-run based (the anti-#100 invariant: never resolve by + # joining text back together) and never merges spans. raise TypeError( "Span does not support +; tuple concatenation is not a " "covering span" @@ -530,9 +530,10 @@ def matches(self, other: str | ParsedName, *, return self.comparison_key() == other.comparison_key() # -- rendering delegates ---------------------------------------------- - # One-line delegation to nameparser._render (core spec §5b): parsing - # code physically cannot import formatting logic, so these import at - # call time -- module level stays internal-import-free. + # One-line delegation to nameparser._render: parsing code physically + # cannot import formatting logic (layering rule, enforced by + # tests/v2/test_layering.py), so these import at call time -- module + # level stays internal-import-free. def render(self, spec: str = "{title} {given} {middle} {family} {suffix}") -> str: """Fill the str.format spec from the seven role fields and the @@ -544,8 +545,8 @@ def render(self, spec: str = "{title} {given} {middle} {family} {suffix}") -> st def initials(self, spec: str = "{given} {middle} {family}", delimiter: str = ".", separator: str = " ") -> str: """Initials per group; v1's initials_format/_delimiter/_separator - become call-site arguments (core spec §5b). Valid spec keys: - given, middle, family.""" + become call-site arguments instead of Config-wide settings. + Valid spec keys: given, middle, family.""" import nameparser._render as _render return _render.initials(self, spec, delimiter, separator) From e5e83e7d3663d47d5b61a3653da9013be28ee4eb Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 18:37:50 -0700 Subject: [PATCH 208/275] fix: name the field in non-iterable TypeErrors for Policy/Lexicon patronymic_rules already probed with iter() to give non-iterable values (an int, a bool) a curated, field-naming TypeError. The other iterable-typed fields (name_order, nickname_delimiters, maiden_delimiters, extra_suffix_delimiters, every Lexicon vocab field, capitalization_exceptions) fell through to whatever bare "'int' object is not iterable" the first tuple()/iter() call happened to raise. Apply the same _require_iterable probe everywhere, with regression tests (review finding on #288). --- nameparser/_lexicon.py | 15 ++++++++++++++- nameparser/_policy.py | 23 ++++++++++++++++++++--- tests/v2/test_lexicon.py | 15 +++++++++++++++ tests/v2/test_policy.py | 16 ++++++++++++++++ 4 files changed, 65 insertions(+), 4 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index f83fc7b2..efff1bbd 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -59,7 +59,13 @@ def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: f"Lexicon.{field_name} must be an iterable of strings, not a " f"mapping (only capitalization_exceptions holds key->value pairs)" ) - items = tuple(entries) # materialize once; entries may be a generator + try: + items = tuple(entries) # materialize once; entries may be a generator + except TypeError: + raise TypeError( + f"Lexicon.{field_name} must be an iterable of strings, " + f"got {entries!r}" + ) from None normalized = set() for w in items: if not isinstance(w, str): @@ -93,6 +99,13 @@ def _normpairs( "iterable of (key, value) pairs, not a bare string" ) pairs = raw.items() if isinstance(raw, Mapping) else raw + try: + pairs = iter(pairs) + except TypeError: + raise TypeError( + "capitalization_exceptions must be a mapping or an " + f"iterable of (key, value) pairs, got {raw!r}" + ) from None deduped: dict[str, str] = {} for entry in pairs: # A 2-char str entry would unpack "ab" into ("a", "b") diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 9aa6d859..1a334269 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -6,8 +6,10 @@ from __future__ import annotations import dataclasses +from collections.abc import Iterable from dataclasses import dataclass, field from enum import Enum, StrEnum, auto +from typing import Any from nameparser._types import Role, _guarded_getstate, _guarded_setstate @@ -87,6 +89,20 @@ def _reject_bare_string_order(value: object) -> None: ) +def _require_iterable(value: Iterable[Any], field_name: str) -> Iterable[Any]: + # Probe with iter() so a non-iterable value (an int, a bool, ...) + # raises a message naming the field, matching the treatment + # patronymic_rules already gets, instead of a bare "'int' object is + # not iterable" surfacing from whatever tuple()/frozenset() call + # happens to run first. + try: + return iter(value) + except TypeError: + raise TypeError( + f"{field_name} must be an iterable, got {value!r}" + ) from None + + @dataclass(frozen=True, slots=True) class Policy: """The behavior switches a parser runs with: name order, @@ -153,7 +169,7 @@ class Policy: def __post_init__(self) -> None: _reject_bare_string_order(self.name_order) - order = tuple(self.name_order) + order = tuple(_require_iterable(self.name_order, "name_order")) for element in order: if not isinstance(element, Role): raise TypeError( @@ -203,7 +219,7 @@ def __post_init__(self) -> None: ) from None object.__setattr__(self, "patronymic_rules", frozenset(rules)) for pairs_name in ("nickname_delimiters", "maiden_delimiters"): - pairs = tuple(getattr(self, pairs_name)) + pairs = tuple(_require_iterable(getattr(self, pairs_name), pairs_name)) for pair in pairs: if (not isinstance(pair, tuple) or len(pair) != 2 or not all(isinstance(s, str) for s in pair)): @@ -232,7 +248,8 @@ def __post_init__(self) -> None: f"extra_suffix_delimiters must be an iterable of strings, " f"not a bare string: {self.extra_suffix_delimiters!r}" ) - delimiters = tuple(self.extra_suffix_delimiters) + delimiters = tuple(_require_iterable( + self.extra_suffix_delimiters, "extra_suffix_delimiters")) for d in delimiters: if not isinstance(d, str): raise TypeError( diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index ba82fff7..79bcb053 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -69,6 +69,21 @@ def test_lexicon_rejects_non_str_vocab_entries() -> None: Lexicon(titles={"Dr.", 42}) # type: ignore[arg-type] +def test_lexicon_rejects_non_iterable_vocab_field() -> None: + # A non-iterable (an int) used to fall through to tuple(entries) + # and surface as a bare "'int' object is not iterable" with no + # hint which field was wrong. + with pytest.raises(TypeError, match="titles.*iterable"): + Lexicon(titles=5) # type: ignore[arg-type] + with pytest.raises(TypeError, match="titles.*iterable"): + Lexicon.empty().add(titles=5) # type: ignore[arg-type] + + +def test_capitalization_exceptions_rejects_non_iterable() -> None: + with pytest.raises(TypeError, match="capitalization_exceptions"): + Lexicon(capitalization_exceptions=5) # type: ignore[arg-type] + + def test_entries_normalizing_to_empty_raise() -> None: # "." or "" is a data bug (stray split artifact, empty CSV cell); # dropping it silently would also let a future data-module typo diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 47047d55..11b7f69a 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -165,6 +165,22 @@ def test_extra_suffix_delimiters_validated_and_coerced() -> None: assert isinstance(hash(p), int) +def test_non_iterable_fields_name_the_offending_field() -> None: + # Non-iterables (an int, a bool) used to fall through to tuple()/ + # frozenset() and surface as a bare "'int' object is not iterable", + # with no hint which field was wrong -- unlike patronymic_rules, + # which has always named itself. Every iterable-typed Policy field + # gets the same treatment. + with pytest.raises(TypeError, match="name_order"): + Policy(name_order=5) # type: ignore[arg-type] + with pytest.raises(TypeError, match="nickname_delimiters"): + Policy(nickname_delimiters=5) # type: ignore[arg-type] + with pytest.raises(TypeError, match="maiden_delimiters"): + Policy(maiden_delimiters=5) # type: ignore[arg-type] + with pytest.raises(TypeError, match="extra_suffix_delimiters"): + Policy(extra_suffix_delimiters=5) # type: ignore[arg-type] + + def test_policy_patch_mirrors_policy_field_names() -> None: policy_fields = {f.name for f in dataclasses.fields(Policy)} patch_fields = {f.name for f in dataclasses.fields(PolicyPatch)} From 64751b8142c58311aac45bbb41764350193f0f37 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 18:37:57 -0700 Subject: [PATCH 209/275] test: pin the assemble() unassigned-role fallback as an invariant role = Role.GIVEN when a main-stream token reaches assemble() with no role set was previously untested -- assign/group always set a role in the real pipeline, so nothing exercised this last-resort branch. Construct the invariant violation directly (an unassigned-role WorkToken) so a future refactor can't silently change or lose this documented safety-net behavior (review finding on #288). --- nameparser/_pipeline/_assemble.py | 9 +++++++-- tests/v2/pipeline/test_assemble.py | 21 +++++++++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/nameparser/_pipeline/_assemble.py b/nameparser/_pipeline/_assemble.py index d3a6a169..42fcef7b 100644 --- a/nameparser/_pipeline/_assemble.py +++ b/nameparser/_pipeline/_assemble.py @@ -8,8 +8,13 @@ Structural tokens (dropped maiden markers) are omitted, like delimiter characters. A main-stream token that somehow reaches here with no role -takes Role.GIVEN -- parse is total over str (spec §5a) and must not -raise on content; the fallback is deliberately boring. +takes Role.GIVEN -- parse must never raise on the *content* of a name, +so the fallback is deliberately boring rather than an exception. This +should never fire in practice (assign/group must set a role on every +main-stream token); it exists as a last-resort safety net against a +pipeline bug, not as sanctioned behavior for real input -- see +test_assemble_falls_back_to_given_for_unassigned_role in +tests/v2/pipeline/test_assemble.py. """ from __future__ import annotations diff --git a/tests/v2/pipeline/test_assemble.py b/tests/v2/pipeline/test_assemble.py index ba667825..793d5db0 100644 --- a/tests/v2/pipeline/test_assemble.py +++ b/tests/v2/pipeline/test_assemble.py @@ -1,9 +1,9 @@ from nameparser._lexicon import Lexicon from nameparser._pipeline import run from nameparser._pipeline._assemble import assemble -from nameparser._pipeline._state import ParseState +from nameparser._pipeline._state import ParseState, WorkToken from nameparser._policy import Policy -from nameparser._types import AmbiguityKind, ParsedName +from nameparser._types import AmbiguityKind, ParsedName, Role, Span _LEX = Lexicon( titles=frozenset({"dr"}), @@ -52,6 +52,23 @@ def test_empty_parse_is_falsy() -> None: assert not _parse(" ") +def test_assemble_falls_back_to_given_for_unassigned_role() -> None: + # This should never happen through the real pipeline -- assign/group + # always set a role on every main-stream token before assemble runs. + # This test constructs the invariant violation directly to pin the + # documented last-resort behavior: assemble() must stay total over + # its input and never raise just because upstream left a role unset. + state = ParseState( + original="Jane", + lexicon=Lexicon.default(), + policy=Policy(), + tokens=(WorkToken("Jane", Span(0, 4), role=None),), + ) + pn = assemble(state) + assert pn.tokens[0].role is Role.GIVEN + assert pn.given == "Jane" + + def test_ambiguity_with_all_indices_dropped_is_omitted() -> None: # an ambiguity whose referent tokens were ALL dropped describes # nothing; emitting it hollow would mislead consumers. Born-empty From 85bee413af718550f869a8f102dfc5f1876e6c40 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 18:38:04 -0700 Subject: [PATCH 210/275] test: pin hand-duplicated regex/table copies to their config source The layering rule forbids the pipeline/_render from importing nameparser.config directly, so several patterns are copied by hand with "keep in sync by hand" comments -- nothing previously checked that promise. Tests may legally import both sides (test_layering.py's own convention), so add sync tests for every duplicated pattern/table, including the documented "initial" trailing-"?" deviation between config and the pipeline copies (review finding on #288). --- tests/v2/test_regex_sync.py | 73 +++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 tests/v2/test_regex_sync.py diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py new file mode 100644 index 00000000..7985501b --- /dev/null +++ b/tests/v2/test_regex_sync.py @@ -0,0 +1,73 @@ +"""Pins the pipeline's hand-duplicated regex/table copies to their +nameparser.config.regexes source of truth. + +The 2.0 layering rule forbids nameparser._pipeline/_render importing +nameparser.config directly, so several patterns and tables are copied +by hand into the modules that need them, each with a "keep in sync by +hand" comment. Nothing previously enforced that promise: if +config/regexes.py changed, the copies would silently diverge with no +CI signal. Tests may legally import both sides (test_layering.py's own +convention), so this module is where the promise gets checked. +""" +from nameparser.config import regexes as _config +from nameparser._pipeline import _assign, _post_rules, _tokenize, _vocab +from nameparser import _render + + +def test_emoji_ranges_match_config() -> None: + assert _tokenize._EMOJI_RANGES == _config._EMOJI_RANGES + + +def test_bidi_pattern_matches_config() -> None: + assert _tokenize._BIDI.pattern == _config.re_bidi.pattern + assert _tokenize._BIDI.flags == _config.re_bidi.flags + + +def test_period_not_at_end_matches_config() -> None: + source = _config.REGEXES["period_not_at_end"] + assert _vocab._PERIOD_NOT_AT_END.pattern == source.pattern + assert _vocab._PERIOD_NOT_AT_END.flags == source.flags + + +def test_period_abbreviation_matches_config() -> None: + source = _config.REGEXES["period_abbreviation"] + assert _assign._PERIOD_ABBREV.pattern == source.pattern + assert _assign._PERIOD_ABBREV.flags == source.flags + + +def test_roman_numeral_matches_config() -> None: + source = _config.REGEXES["roman_numeral"] + assert _assign._ROMAN.pattern == source.pattern + assert _assign._ROMAN.flags == source.flags + + +def test_patronymic_patterns_match_config() -> None: + pairs = ( + (_post_rules._EAST_SLAVIC, "east_slavic_patronymic"), + (_post_rules._EAST_SLAVIC_CYR, "east_slavic_patronymic_cyrillic"), + (_post_rules._TURKIC, "turkic_patronymic_marker"), + (_post_rules._TURKIC_CYR, "turkic_patronymic_marker_cyrillic"), + ) + for copy, key in pairs: + source = _config.REGEXES[key] + assert copy.pattern == source.pattern, key + assert copy.flags == source.flags, key + + +def test_initial_copies_agree_with_each_other_and_config() -> None: + # _vocab._INITIAL and _render._INITIAL are both v1's "initial" + # pattern minus its trailing "?" (documented at _render.py's + # _INITIAL definition: the two call sites always fullmatch a + # non-empty token, so the empty-string alternative is dropped on + # purpose). Assert both the internal-copy agreement and the exact, + # documented relationship to the config source, so a future edit to + # either side that breaks the relationship fails loudly here + # instead of silently drifting. + assert _vocab._INITIAL.pattern == _render._INITIAL.pattern + source = _config.REGEXES["initial"] + # config's pattern is the pipeline copy with an extra "?" spliced in + # just before the trailing "$", making the whole group optional. + trimmed = _vocab._INITIAL.pattern + reconstructed = trimmed[:-1] + "?" + trimmed[-1:] + assert source.pattern == reconstructed + assert source.flags == _vocab._INITIAL.flags From 26dd5da122b3ff97a48a789a0413c93d0839cf02 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 18:38:12 -0700 Subject: [PATCH 211/275] test: isolate pipeline-stage branches only hit via the full corpus Several branches were only reached through tests/v2/test_cases.py's full-pipeline corpus, so a regression scoped to one stage wouldn't be pinpointed by that stage's own test file: assign's SUFFIX_COMMA structure, segment's cosmetic-vs-structural empty-bucket handling (single trailing comma vs. a leading comma), group's extra_suffix_delimiters entry-splitting in a FAMILY_COMMA tail segment, and tokenize's interleaving of two simultaneously active extracted regions (review finding on #288). --- tests/v2/pipeline/test_assign.py | 13 +++++++++++++ tests/v2/pipeline/test_group.py | 30 ++++++++++++++++++++++++++++-- tests/v2/pipeline/test_segment.py | 19 +++++++++++++++++++ tests/v2/pipeline/test_tokenize.py | 19 +++++++++++++++++++ 4 files changed, 79 insertions(+), 2 deletions(-) diff --git a/tests/v2/pipeline/test_assign.py b/tests/v2/pipeline/test_assign.py index 88bb4352..b506b5a2 100644 --- a/tests/v2/pipeline/test_assign.py +++ b/tests/v2/pipeline/test_assign.py @@ -101,6 +101,19 @@ def test_suffix_comma_and_extra_segments() -> None: assert _by_role(out, Role.SUFFIX) == "Jr." +def test_suffix_comma_structure() -> None: + # Structure.SUFFIX_COMMA (segment.py): >1 word before the first + # comma AND every post-first segment is entirely lenient-suffix. + # ('Smith, John, Jr.' above only has ONE word before its first + # comma, so it is FAMILY_COMMA with a trailing suffix segment, not + # this branch.) Previously this assign() branch (tail=1) was only + # reached via the full-corpus test, not its own isolated case. + out = _assigned("John Smith, PhD") + assert _by_role(out, Role.GIVEN) == "John" + assert _by_role(out, Role.FAMILY) == "Smith" + assert _by_role(out, Role.SUFFIX) == "PhD" + + def test_trailing_suffix_run_no_comma() -> None: out = _assigned("John Jack Kennedy PhD MD") assert _by_role(out, Role.MIDDLE) == "Jack" diff --git a/tests/v2/pipeline/test_group.py b/tests/v2/pipeline/test_group.py index 182a603d..654fa800 100644 --- a/tests/v2/pipeline/test_group.py +++ b/tests/v2/pipeline/test_group.py @@ -22,8 +22,10 @@ # real vocabulary overlap; the subset rule only constrains particles. -def _grouped(text: str) -> ParseState: - state = ParseState(original=text, lexicon=_LEX, policy=Policy()) +def _grouped(text: str, policy: Policy | None = None, + lexicon: Lexicon | None = None) -> ParseState: + state = ParseState(original=text, lexicon=lexicon or _LEX, + policy=policy or Policy()) return group(classify(segment(tokenize(extract_delimited(state))))) @@ -128,6 +130,30 @@ def test_initials_do_not_count_as_rootnames_for_conjunction_carveout() -> None: assert _piece_texts(out) == [["J.", "Ruiz", "y", "Gomez"]] +def test_extra_suffix_delimiter_splits_tail_entries() -> None: + # v1 expand_suffix_delimiter parity (#191): a configured delimiter + # is transparent in a FAMILY_COMMA tail segment -- it separates + # suffix ENTRIES (each rendered independently) and is itself + # dropped, rather than becoming a suffix token or fusing the + # entries on either side into one "joined" run. + lex = Lexicon(suffix_acronyms=frozenset({"phd"}), + suffix_words=frozenset({"jr", "v", "md"})) + out = _grouped("Smith, John, V MD / PhD", + Policy(extra_suffix_delimiters=frozenset({"/"})), + lexicon=lex) + # one tail SEGMENT, split into three suffix pieces: "/" is dropped + # rather than surviving as its own piece + assert _piece_texts(out) == [["Smith"], ["John"], ["V", "MD", "PhD"]] + slash_idx = next(i for i, t in enumerate(out.tokens) if t.text == "/") + assert slash_idx in out.dropped + # "MD" continues the "V MD" entry (joined); "PhD" starts a fresh + # entry across the dropped delimiter, so it does NOT get "joined" + md_tok = next(t for t in out.tokens if t.text == "MD") + phd_tok = next(t for t in out.tokens if t.text == "PhD") + assert "joined" in md_tok.tags + assert "joined" not in phd_tok.tags + + def test_suffix_comma_name_segment_gets_no_additional_count() -> None: # v1 parity: additional_parts_count applies to FAMILY_COMMA parts only; # ', PhD' must not tip the single-letter-conjunction carve-out diff --git a/tests/v2/pipeline/test_segment.py b/tests/v2/pipeline/test_segment.py index 5aa5a9fe..3bf8de8c 100644 --- a/tests/v2/pipeline/test_segment.py +++ b/tests/v2/pipeline/test_segment.py @@ -78,6 +78,25 @@ def test_comma_only_input_is_no_comma_structure() -> None: assert out.segments == () +def test_single_trailing_comma_is_cosmetic_and_dropped() -> None: + # v1 parity: exactly ONE trailing comma is cosmetic and stripped + # (segment.py's "pop the trailing empty bucket" step), so + # 'Smith, John,' segments identically to 'Smith, John' -- unlike a + # genuinely empty INTERIOR bucket ('Doe,, Jr.'), which stays + # structural and keeps its position. + out = _segmented("Smith, John,") + assert out.structure is Structure.FAMILY_COMMA + assert [_texts(out, s) for s in out.segments] == [["Smith"], ["John"]] + + +def test_leading_comma_yields_empty_first_segment() -> None: + # A leading comma produces a non-trailing empty bucket, which is + # structural (not cosmetic) and keeps its position as segment 0. + out = _segmented(",John Smith") + assert out.structure is Structure.FAMILY_COMMA + assert [_texts(out, s) for s in out.segments] == [[], ["John", "Smith"]] + + def test_strict_comma_suffixes_veto_lenient_only_members() -> None: # lenient_comma_suffixes=False: the post-comma test drops back to # the strict predicate, so initial-shaped suffix words no longer diff --git a/tests/v2/pipeline/test_tokenize.py b/tests/v2/pipeline/test_tokenize.py index 3e94e184..4dc47965 100644 --- a/tests/v2/pipeline/test_tokenize.py +++ b/tests/v2/pipeline/test_tokenize.py @@ -52,6 +52,25 @@ def test_extracted_regions_are_skipped_and_tokenized_with_role() -> None: assert starts == sorted(starts) +def test_two_simultaneous_extracted_regions_interleave_correctly() -> None: + # tokenize's masked/extracted loops each run over ALL their regions + # before the final span-sort; every other test here exercises at + # most one masked region at a time. With two active extraction + # types (nickname parens AND a configured maiden bracket) on the + # same string, main-stream and both extracted regions must still + # interleave into the correct overall span order. + pol = Policy(maiden_delimiters=frozenset({("[", "]")})) + out = _tokenized("John (Jack) [Doe] Smith", pol) + assert [(t.text, t.role) for t in out.tokens] == [ + ("John", None), + ("Jack", Role.NICKNAME), + ("Doe", Role.MAIDEN), + ("Smith", None), + ] + starts = [t.span.start for t in out.tokens] + assert starts == sorted(starts) + + def test_comma_inside_extracted_region_is_not_an_offset() -> None: out = _tokenized('John "Jack, Jr" Kim') assert out.comma_offsets == () From 3ae385e2db2775b93c9b2494aa9d9177cc1851f7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 18:58:45 -0700 Subject: [PATCH 212/275] docs: clarify Ambiguity/ParsedName subset check is value-equality-based "always a subset" read as identity to a casual reader, but the constructor check uses `in` (Token's == equality): two distinct Token instances with identical text/span/role/tags satisfy it. Document the distinction at the field, the class docstring, and the check site (polish item from the #288 review). --- nameparser/_types.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/nameparser/_types.py b/nameparser/_types.py index e0f6ddb7..6e412330 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -258,8 +258,10 @@ class Ambiguity: kind: AmbiguityKind #: Human-readable specifics of this occurrence (wording unstable). detail: str - #: The tokens involved -- always a subset of the owning - #: ParsedName's tokens; may be empty (e.g. unbalanced-delimiter). + #: The tokens involved -- always a value-equal subset of the owning + #: ParsedName's tokens (checked with ==, not identity: two distinct + #: Token instances with identical text/span/role/tags satisfy this); + #: may be empty (e.g. unbalanced-delimiter). tokens: tuple[Token, ...] # in the class body so @dataclass(slots=True) keeps them @@ -299,10 +301,10 @@ class ParsedName: :meth:`render`, :meth:`initials`, :meth:`capitalized`, or ``str()``. Constructor-enforced invariants: spans ascending, non-overlapping, - in bounds of `original`; every Ambiguity's tokens are a subset of - `tokens`. Provenance semantics (text == original[span] for - parser-produced names) are documented, not enforced -- transforms - like replace() legitimately break them. + in bounds of `original`; every Ambiguity's tokens are a value-equal + subset of `tokens` (see Ambiguity.tokens). Provenance semantics + (text == original[span] for parser-produced names) are documented, + not enforced -- transforms like replace() legitimately break them. """ #: The input string exactly as passed to parse(). @@ -354,6 +356,9 @@ def __post_init__(self) -> None: prev_end = tok.span.end for amb in self.ambiguities: for tok in amb.tokens: + # `in` uses Token's value equality, not identity: this + # only guarantees a value-equal token exists in + # self.tokens, not that `tok` IS one of those objects. if tok not in self.tokens: raise ValueError( f"Ambiguity token {tok.text!r} is not a subset of " From 4cc75db0d4d2cd588ae16508ecf0d4bd2d795351 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 18:58:45 -0700 Subject: [PATCH 213/275] docs: signpost PolicyPatch's scalar-field validation asymmetry name_order and the union (set-valued) fields get an iter()/type probe in __post_init__; the four scalar override fields (middle_as_family, lenient_comma_suffixes, strip_emoji, strip_bidi) deliberately get none until apply_patch runs -- previously true only per the class docstring and the tests, with nothing at the loop itself flagging the omission as intentional rather than a gap (polish item from the #288 review). --- nameparser/_policy.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 1a334269..d07c849a 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -372,6 +372,15 @@ def __post_init__(self) -> None: f"{f.name} must be an iterable, got {value!r}{hint}" ) from None object.__setattr__(self, f.name, frozenset(value)) + # middle_as_family, lenient_comma_suffixes, strip_emoji, and + # strip_bidi are scalar (compose="override") fields and + # DELIBERATELY get no type check here, unlike name_order and + # the union fields above: a PolicyPatch(strip_emoji="off") is + # constructible, and only raises once apply_patch runs + # Policy.__post_init__'s bool check. This is the one place the + # module's eager-validation ethos doesn't apply -- see the + # class docstring ("Values are validated when the patch is + # applied... not at patch construction"). def apply_patch(policy: Policy, patch: PolicyPatch) -> Policy: From 66a49637f4ca10434fab5636c1e40dc2419e5896 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 18:58:45 -0700 Subject: [PATCH 214/275] docs: drop stale v1 parser.py line-number citations The prose in each comment already names the specific v1 logic being ported; the hardcoded line numbers (1285, 1313, 1333, 1368, 1390) add nothing on top of that and would silently go stale on the next edit to parser.py, with nothing to catch the drift (polish item from the #288 review). --- nameparser/_pipeline/_assign.py | 18 +++++++++--------- nameparser/_pipeline/_group.py | 3 +-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index a742bb66..bd2a086f 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -114,9 +114,9 @@ def _assign_main(seg_idx: int, state: ParseState, if not rest: return # v1 nickname rule (plan deviation #2): v1's p_len == 1 counted - # the WHOLE segment before any title peeling (parser.py:1285) -- - # 'Xyz. (Bud) Smith' has two pieces, so the title peel wins and - # Smith stays the given name (pinned live 2026-07-17) + # the WHOLE segment before any title peeling -- 'Xyz. (Bud) Smith' + # has two pieces, so the title peel wins and Smith stays the given + # name (pinned live 2026-07-17) if len(pieces) == 1 and len(rest) == 1 and has_nickname: _set_roles(tokens, pieces[rest[0]], Role.FAMILY) return @@ -173,10 +173,10 @@ def assign(state: ParseState) -> ParseState: # PARTICLE_OR_GIVEN is deliberately not emitted here: after a # comma the family is already fixed, so a leading given-position # particle is not meaningfully ambiguous. - # v1: "lastname part may have suffixes in it" (parser.py:1368) - # -- the first piece is always the family even if suffix-shaped; - # any later strict-suffix piece goes to SUFFIX per piece - # ('Smith Jr., John' -> family=Smith, suffix=Jr.) + # v1: "lastname part may have suffixes in it" -- the first + # piece is always the family even if suffix-shaped; any later + # strict-suffix piece goes to SUFFIX per piece ('Smith Jr., + # John' -> family=Smith, suffix=Jr.) fam_pieces = state.pieces[0] fam_tags = state.piece_tags[0] for k, piece in enumerate(fam_pieces): @@ -190,8 +190,8 @@ def assign(state: ParseState) -> ParseState: n = _peel_leading_titles(pieces, ptags, tokens) given_done = False for m in range(n, len(pieces)): - # v1 walk order (parser.py:1390): the first non-title - # piece is ALWAYS the given, before any suffix check -- + # v1 walk order: the first non-title piece is ALWAYS + # the given, before any suffix check -- # 'Hardman, RN - CRNA' keeps first='RN'. One deliberate # 2.0 deviation, classified fix(comma-family): when that # piece is the segment's ONLY piece and unambiguously diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 675a88ac..55a98647 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -194,8 +194,7 @@ def group(state: ParseState) -> ParseState: all_pieces: list[tuple[tuple[int, ...], ...]] = [] all_ptags: list[tuple[frozenset[str], ...]] = [] # v1 parity: additional_parts_count=1 applies only to FAMILY_COMMA - # parts (parser.py:1313, 1369); the SUFFIX_COMMA pre-comma segment - # gets 0 (parser.py:1333). + # parts; the SUFFIX_COMMA pre-comma segment gets 0. additional = 1 if state.structure is Structure.FAMILY_COMMA else 0 # v1 expand_suffix_delimiter parity (#191): tail segments (wholly # consumed as suffixes by assign) drop delimiter-core tokens, the From 371b093a97b16b7b914c9c56b810de79ca4bd52b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 19:21:37 -0700 Subject: [PATCH 215/275] docs: cross-link token/span mentions in concepts.rst to API reference Co-Authored-By: Claude Sonnet 5 --- docs/concepts.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 5533413d..9eb933fc 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -12,7 +12,7 @@ From string to name -------------------- Every parse follows the same path: the input string is split into -tokens, each token is assigned one of the seven roles — ``title``, +:class:`tokens `, each token is assigned one of the seven roles — ``title``, ``given``, ``middle``, ``family``, ``suffix``, ``nickname``, ``maiden`` — and every string you read off the result is computed from those tokens at read time. @@ -23,7 +23,7 @@ and ``Vega`` each carry the ``family`` role, which is why ``name.family`` returns ``"de la Vega"`` — the field is a view that joins the family-role tokens in order, not a stored string. -Each token also records where it came from. A span is a pair of +Each token also records where it came from. A :class:`~nameparser.Span` is a pair of character positions bounding the token in the original string: ``Dr.`` has span ``(0, 3)``, and ``name.original[0:3]`` is exactly ``"Dr."``. Internally, spans let the pipeline refer to a token by @@ -121,7 +121,7 @@ other names, so the parse records a ``particle-or-given`` ambiguity alongside its answer. You can inspect ``ambiguities`` to decide, case by case, whether your data needs a second look. -Tokens also carry tags — a second, independent label alongside their +:class:`Tokens ` also carry tags — a second, independent label alongside their role, recording how a token was classified rather than what part of the name it belongs to — but only a handful of them are part of the stable API: ``particle``, ``conjunction``, ``initial``, and ``joined``. From d238e869fd05c62703956bc15c40475e4475201f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 20:40:04 -0700 Subject: [PATCH 216/275] docs: sharpen Lexicon/Policy split and rendering-arguments examples - Retitle "three containers" section and swap the Policy example from name_order (which actually correlates with language for Chinese/ Hungarian/Vietnamese names) to nickname_delimiters, which doesn't. - Note that capitalization_exceptions keys match punctuation-normalized text, so "phd" also covers "Ph.D.". - Fix a stale nickname_delimiters example (guillemets are already in the default set) and drop an unneeded frozenset() wrapper. - Replace the rendering-arguments stub signatures with links to the real methods plus a worked, doctested example, including capitalized(force=True) for genuinely mixed-case input. Co-Authored-By: Claude Sonnet 5 --- docs/concepts.rst | 18 ++++++++++-------- docs/customize.rst | 44 ++++++++++++++++++++++++++++++++++---------- 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 9eb933fc..d5f5764d 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -41,8 +41,8 @@ else — tokens, spans, the rest of the roles — carried over unchanged. ``str()`` renders the default view; nothing about calling it mutates the value you called it on. -The three containers ---------------------- +Three containers, three concerns +-------------------------------- Every piece of nameparser configuration falls into exactly one of three places, and which one is decided by a single question: what does @@ -61,12 +61,14 @@ this setting vary with? ``render(spec)``, or a keyword to ``initials()``/``capitalized()``. "Dean" is a common given name, so it is not in the default titles -vocabulary — but in some data it is more common as a title. Which -reading is right is a fact about the language and domain the names -come from, not about any one dataset or report: that makes it a -:class:`~nameparser.Lexicon` entry. A CRM that always exports "Family, Given" strings is a fact -about that one data source, not about the language of the names in -it — that's a :class:`~nameparser.Policy`. One particular report +vocabulary. But in some data it is more common as a title. The right +reading is a fact about the domain the names come from: that makes it a +:class:`~nameparser.Lexicon` entry. A CRM that wraps nicknames in +square brackets — ``John [Johnny] Smith`` — instead of quotes is a +fact about that one data source's export format, not about the +language of the names in it — that's a +:class:`~nameparser.Policy` (``nickname_delimiters={('[', ']')}``). +One particular report wanting names formatted as "Family, Given" while every other consumer of the same parsed data wants "Given Family" is a fact about where the string is going next, decided at the moment you render it — that's a diff --git a/docs/customize.rst b/docs/customize.rst index 17369b43..44007198 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -31,6 +31,11 @@ maps a lowercase key to its exact-cased replacement (``"phd"`` → ``dataclasses.replace()`` instead: ``dataclasses.replace(lex, capitalization_exceptions=(("phd", "PhD"),))``. +The key is matched against the token with punctuation normalized away, +not against the raw text, so one ``"phd"`` entry covers ``"phd"``, +``"Phd"``, and ``"Ph.D."`` alike — you don't need a separate key for +each way a source might punctuate it. + Two fields — ``suffix_acronyms_ambiguous`` and ``particles_ambiguous`` — mark entries from ``suffix_acronyms`` and ``particles`` that are also plausible as ordinary name words on their own (an acronym suffix that @@ -115,7 +120,7 @@ listed below. .. doctest:: >>> from nameparser import Parser, Policy - >>> policy = Policy(maiden_delimiters=frozenset({("(", ")")})) + >>> policy = Policy(maiden_delimiters={("(", ")")}) >>> Parser(policy=policy).parse("Jane (Jones) Smith").maiden 'Jones' @@ -124,7 +129,7 @@ the specific intent — so listing a pair there automatically drops it from the effective ``nickname_delimiters`` set, and the one-liner above is the whole recipe. To *add* delimiters instead of rerouting them, build on the named default: -``nickname_delimiters=DEFAULT_NICKNAME_DELIMITERS | {("«", "»")}``. +``nickname_delimiters=DEFAULT_NICKNAME_DELIMITERS | {("[", "]")}``. .. _rendering-arguments: @@ -132,19 +137,38 @@ Presentation: rendering arguments ---------------------------------- Once a name is parsed, how it's displayed is a separate decision made -at the point of output, not baked into the parse: +at the point of output, not baked into the parse. Three methods on +:class:`~nameparser.ParsedName` cover it — see :doc:`modules` for full +signatures: + +* :meth:`~nameparser.ParsedName.render` fills a format spec from the + seven role fields. +* :meth:`~nameparser.ParsedName.initials` is the same idea narrowed to + first letters, with its own ``delimiter``/``separator`` arguments. +* :meth:`~nameparser.ParsedName.capitalized` returns a new, case-fixed + :class:`~nameparser.ParsedName` instead of a string. It only touches + input that's already single-case (all lower, all upper) unless you + pass ``force=True`` — mixed case is left alone by default on the + assumption that someone already capitalized it on purpose. -.. code-block:: python +.. doctest:: - def render(self, spec: str = "{title} {given} {middle} {family} {suffix}") -> str: ... - def initials(self, spec: str = "{given} {middle} {family}", - delimiter: str = ".", separator: str = " ") -> str: ... - def capitalized(self, lexicon: Lexicon | None = None, *, force: bool = False) -> ParsedName: ... + >>> from nameparser import parse + >>> name = parse("Dr. Juan Q. Xavier de la Vega III") + >>> name.render("{family}, {given} {middle}") + 'de la Vega, Juan Q. Xavier' + >>> name.initials(spec="{given}{middle}{family}", delimiter="", separator="") + 'JQXV' + >>> str(parse("DR. JUAN DE LA VEGA").capitalized()) + 'Dr. Juan de la Vega' + >>> str(parse("JuAn DE LA vEGA").capitalized()) + 'JuAn DE LA vEGA' + >>> str(parse("JuAn DE LA vEGA").capitalized(force=True)) + 'Juan de la Vega' Looking for v1's ``string_format``? It's the ``render(spec)`` argument now — pass your own format string per call instead of setting it once -on a shared config object. See :doc:`usage` for worked examples of all -three. +on a shared config object. Sharing a configured parser ---------------------------- From e7a5c4d660ba3a4d86ff379c6855fb2894d9ddc5 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 20:50:57 -0700 Subject: [PATCH 217/275] feat: default str()/render() shows nickname and maiden MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Derek's terminal find: str(parse('Von Johnson (smith)')) silently dropped the nickname. v1's default string_format always rendered '({nickname})'; the 2.0 default had lost it. The default spec is now '{title} {given} {middle} {family} {suffix} ({nickname}) née {maiden}' -- v1's nickname position restored, maiden appended with a marker that round-trips (parens re-extract as nickname, née is a maiden marker, pinned by a reparse assertion). The #254 collapse learns to drop the trailing orphaned 'née' (and the bare-née all-empty case); content 'Née' is untouched (case-exact match). The HumanName facade keeps v1's own string_format default. TDD: red first; differential exit 0 (facade rendering unchanged). Co-Authored-By: Claude Fable 5 --- docs/release_log.rst | 1 + docs/usage.rst | 8 ++++++++ nameparser/_render.py | 7 +++++++ nameparser/_types.py | 12 ++++++++++-- tests/v2/test_render.py | 19 +++++++++++++++++++ 5 files changed, 45 insertions(+), 2 deletions(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index 56b61400..44be99af 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -28,6 +28,7 @@ Release Log - Change suffix-delimiter rendering: with a custom ``Policy``/``Constants`` suffix delimiter configured (e.g. ``suffix_delimiter="/"``, ``"John Smith, RN/CRNA"``), 1.x split the token and rendered ``suffix="RN, CRNA"``; 2.0 keeps the no-space delimiter-core token whole (``suffix="RN/CRNA"``) -- role assignment is unchanged, only rendering differs (anti-#100, migration plan deviation 5; ``tests/v2/cases.py`` row ``suffix_delimiter_no_space_core``). Only fires with a non-default policy, so it does not appear in the (default-policy) differential corpus - Change ``comparison_key()``/``matches()`` (both APIs) to fold components with ``str.casefold()`` where 1.4 used ``str.lower()``: Unicode case-pair forms now compare equal -- ``"STRASSE"`` matches ``"Straße"``, and a Greek final-sigma variant matches its regular-sigma form. Strictly more permissive (anything 1.4 matched still matches); parse output is unaffected -- vocabulary normalization itself uses ``lower()``, v1-parity (``tests/v2/test_types.py`` row ``test_matches_casefolds_unicode_case_pairs``) - Change delimiter-overlap precedence in the 2.0 API: a pair listed in ``Policy.maiden_delimiters`` is dropped from the effective ``nickname_delimiters`` set (maiden wins), so ``Policy(maiden_delimiters={("(", ")")})`` alone routes parenthesized content to ``maiden`` -- no need to rebuild the nickname set. The default nickname set is now the public ``DEFAULT_NICKNAME_DELIMITERS`` constant. The 1.x facade keeps v1's nickname-wins precedence on overlap via a shim-side pre-subtraction, so no ``HumanName`` behavior changes (``tests/v2/cases.py`` row ``maiden_delimiters_win_when_shared``; ``tests/v2/test_config_shim.py`` row ``test_snapshot_overlap_keeps_v1_nickname_precedence``) + - Change the 2.0 API's default ``render()``/``str()`` spec to show every non-empty field: ``"{title} {given} {middle} {family} {suffix} ({nickname}) née {maiden}"``. v1's default already showed ``({nickname})``; 2.0 had dropped it (so ``str(parse("Von Johnson (smith)"))`` silently lost the nickname) and adds ``née {maiden}``. Both decorations round-trip -- parentheses re-extract as the nickname and ``née`` is a maiden marker. The ``HumanName`` facade keeps v1's own ``string_format`` default, unchanged (``tests/v2/test_render.py`` row ``test_default_str_includes_nickname_and_maiden``) - Recognize typographic nickname delimiters by default in BOTH APIs (closes #273): smart quotes (``“Jack”``), German/Polish low-high quotes (``„Hansi“``), Swedish right-right quotes (``”Ann”``), guillemets in both directions (``«Petit»``, ``»Hansi«``, inner spacing tolerated), CJK corner brackets (``「タロ」``/``『ハナ』``), and fullwidth parentheses. In 1.x these leaked into ``middle`` as literal text. Curly *single* quotes stay excluded (U+2019 is the apostrophe in "O’Connor" -- pinned by ``curly_apostrophe_stays_literal``). The v1 keyed idioms work on the new named sentinels (``smart_double_quotes``, ``guillemets``, ...). A delimiter character consumed by another pair's extraction no longer emits a spurious ``unbalanced-delimiter`` ambiguity (``tests/v2/cases.py`` rows ``*_nickname``; ``tests/v2/pipeline/test_extract.py``). Not present in the differential corpus Everything else in the 486-name differential corpus (built from the diff --git a/docs/usage.rst b/docs/usage.rst index e39fb4f7..782fafff 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -69,6 +69,14 @@ Nicknames and maiden names >>> parse("Jane Smith née Jones").maiden 'Jones' +Both fields appear in the default ``str()`` rendering — the nickname +in parentheses, the maiden name after ``née``: + +.. doctest:: + + >>> str(parse("Jane (Janie) Smith née Jones")) + 'Jane Smith (Janie) née Jones' + Comparing names ---------------- diff --git a/nameparser/_render.py b/nameparser/_render.py index ad236f6e..f4590aca 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -51,6 +51,13 @@ def _collapse(rendered: str) -> str: .replace(' ""', "")) rendered = _SPACE_BEFORE_COMMA.sub(",", rendered) rendered = _SPACES.sub(" ", rendered.strip()) + # the default spec's maiden decoration: an empty {maiden} leaves a + # trailing bare 'née' (lowercase, exactly the template's spelling + # -- a capitalized name 'Née' as content is untouched); on an + # all-empty parse the decoration IS the whole string + rendered = rendered.removesuffix(" née") + if rendered == "née": + rendered = "" if rendered and _COMMA_CHAR.fullmatch(rendered[-1]): rendered = rendered[:-1] return rendered.strip(", ") diff --git a/nameparser/_types.py b/nameparser/_types.py index 6e412330..636af22b 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -540,9 +540,17 @@ def matches(self, other: str | ParsedName, *, # tests/v2/test_layering.py), so these import at call time -- module # level stays internal-import-free. - def render(self, spec: str = "{title} {given} {middle} {family} {suffix}") -> str: + def render( + self, + spec: str = ("{title} {given} {middle} {family} {suffix} " + "({nickname}) née {maiden}"), + ) -> str: """Fill the str.format spec from the seven role fields and the - derived views; empty fields collapse (#254). Unknown keys raise + derived views; empty fields collapse (#254), including the + default spec's decorations -- '()' around an absent nickname + and a trailing orphaned 'née'. The default shows every + non-empty field and round-trips: parentheses re-extract as the + nickname and 'née' is a maiden marker. Unknown keys raise KeyError naming the valid fields.""" import nameparser._render as _render return _render.render(self, spec) diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index aedaaff5..774ff1cc 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -69,6 +69,25 @@ def test_render_empty_parse_is_empty_string() -> None: assert render(_pn("", []), "{title} {given} {middle} {family} {suffix}") == "" +def test_default_str_includes_nickname_and_maiden() -> None: + # Derek's terminal find (2026-07-19): str(parse('Von Johnson + # (smith)')) dropped the nickname. v1's default string_format + # always showed '({nickname})'; the 2.0 default restores it and + # adds 'née {maiden}' -- and both decorations round-trip: parens + # re-extract as nickname, née is a maiden marker. + from nameparser import parse + + assert str(parse("Von Johnson (smith)")) == "Von Johnson (smith)" + assert str(parse("Jane Smith née Jones")) == "Jane Smith née Jones" + both = parse("Jane (Janie) Smith née Jones") + assert str(both) == "Jane Smith (Janie) née Jones" + reparsed = parse(str(both)) + assert reparsed.nickname == "Janie" and reparsed.maiden == "Jones" + # no orphaned decoration when the fields are empty + assert str(parse("Dr. Juan Q. Xavier de la Vega III")) == ( + "Dr. Juan Q. Xavier de la Vega III") + + def test_parsedname_render_and_str_delegate() -> None: pn = _delavega() assert pn.render() == "Dr. Juan de la Vega III" From bf1141c25f8e6d0abd2fb2715d23129811ced8e8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 20:53:14 -0700 Subject: [PATCH 218/275] =?UTF-8?q?feat:=20default=20spec=20is=20Derek's?= =?UTF-8?q?=20chosen=20format=20=E2=80=94=20quoted=20nickname,=20parenthes?= =?UTF-8?q?ized=20maiden?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit '{title} {given} "{nickname}" {middle} {family} ({maiden}) {suffix}': nickname in the classic quoted position after the given name, maiden parenthesized after the family. The collapse's existing empty-'""' and empty-'()' rules cover both decorations, so the née trailing-strip special case is deleted. Trade-off documented: the quoted nickname round-trips exactly; parenthesized maiden re-parses as a nickname (custom 'née {maiden}' spec available for lossless round-trip). Co-Authored-By: Claude Fable 5 --- docs/release_log.rst | 2 +- docs/usage.rst | 5 +++-- nameparser/_render.py | 7 ------- nameparser/_types.py | 14 +++++++------- tests/v2/test_render.py | 18 +++++++++--------- 5 files changed, 20 insertions(+), 26 deletions(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index 44be99af..6333a5f1 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -28,7 +28,7 @@ Release Log - Change suffix-delimiter rendering: with a custom ``Policy``/``Constants`` suffix delimiter configured (e.g. ``suffix_delimiter="/"``, ``"John Smith, RN/CRNA"``), 1.x split the token and rendered ``suffix="RN, CRNA"``; 2.0 keeps the no-space delimiter-core token whole (``suffix="RN/CRNA"``) -- role assignment is unchanged, only rendering differs (anti-#100, migration plan deviation 5; ``tests/v2/cases.py`` row ``suffix_delimiter_no_space_core``). Only fires with a non-default policy, so it does not appear in the (default-policy) differential corpus - Change ``comparison_key()``/``matches()`` (both APIs) to fold components with ``str.casefold()`` where 1.4 used ``str.lower()``: Unicode case-pair forms now compare equal -- ``"STRASSE"`` matches ``"Straße"``, and a Greek final-sigma variant matches its regular-sigma form. Strictly more permissive (anything 1.4 matched still matches); parse output is unaffected -- vocabulary normalization itself uses ``lower()``, v1-parity (``tests/v2/test_types.py`` row ``test_matches_casefolds_unicode_case_pairs``) - Change delimiter-overlap precedence in the 2.0 API: a pair listed in ``Policy.maiden_delimiters`` is dropped from the effective ``nickname_delimiters`` set (maiden wins), so ``Policy(maiden_delimiters={("(", ")")})`` alone routes parenthesized content to ``maiden`` -- no need to rebuild the nickname set. The default nickname set is now the public ``DEFAULT_NICKNAME_DELIMITERS`` constant. The 1.x facade keeps v1's nickname-wins precedence on overlap via a shim-side pre-subtraction, so no ``HumanName`` behavior changes (``tests/v2/cases.py`` row ``maiden_delimiters_win_when_shared``; ``tests/v2/test_config_shim.py`` row ``test_snapshot_overlap_keeps_v1_nickname_precedence``) - - Change the 2.0 API's default ``render()``/``str()`` spec to show every non-empty field: ``"{title} {given} {middle} {family} {suffix} ({nickname}) née {maiden}"``. v1's default already showed ``({nickname})``; 2.0 had dropped it (so ``str(parse("Von Johnson (smith)"))`` silently lost the nickname) and adds ``née {maiden}``. Both decorations round-trip -- parentheses re-extract as the nickname and ``née`` is a maiden marker. The ``HumanName`` facade keeps v1's own ``string_format`` default, unchanged (``tests/v2/test_render.py`` row ``test_default_str_includes_nickname_and_maiden``) + - Change the 2.0 API's default ``render()``/``str()`` spec to show every non-empty field: ``'{title} {given} "{nickname}" {middle} {family} ({maiden}) {suffix}'``. v1's default already showed the nickname; 2.0 had dropped it (so ``str(parse("Von Johnson (smith)"))`` silently lost it) and never showed the maiden name. The quoted nickname round-trips exactly; the parenthesized maiden re-parses as a nickname (presentation over lossless round-trip -- use ``née {maiden}`` in a custom spec if you need the round-trip). The ``HumanName`` facade keeps v1's own ``string_format`` default, unchanged (``tests/v2/test_render.py`` row ``test_default_str_includes_nickname_and_maiden``) - Recognize typographic nickname delimiters by default in BOTH APIs (closes #273): smart quotes (``“Jack”``), German/Polish low-high quotes (``„Hansi“``), Swedish right-right quotes (``”Ann”``), guillemets in both directions (``«Petit»``, ``»Hansi«``, inner spacing tolerated), CJK corner brackets (``「タロ」``/``『ハナ』``), and fullwidth parentheses. In 1.x these leaked into ``middle`` as literal text. Curly *single* quotes stay excluded (U+2019 is the apostrophe in "O’Connor" -- pinned by ``curly_apostrophe_stays_literal``). The v1 keyed idioms work on the new named sentinels (``smart_double_quotes``, ``guillemets``, ...). A delimiter character consumed by another pair's extraction no longer emits a spurious ``unbalanced-delimiter`` ambiguity (``tests/v2/cases.py`` rows ``*_nickname``; ``tests/v2/pipeline/test_extract.py``). Not present in the differential corpus Everything else in the 486-name differential corpus (built from the diff --git a/docs/usage.rst b/docs/usage.rst index 782fafff..2842f531 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -70,12 +70,13 @@ Nicknames and maiden names 'Jones' Both fields appear in the default ``str()`` rendering — the nickname -in parentheses, the maiden name after ``née``: +quoted after the given name, the maiden name parenthesized after the +family name: .. doctest:: >>> str(parse("Jane (Janie) Smith née Jones")) - 'Jane Smith (Janie) née Jones' + 'Jane "Janie" Smith (Jones)' Comparing names ---------------- diff --git a/nameparser/_render.py b/nameparser/_render.py index f4590aca..ad236f6e 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -51,13 +51,6 @@ def _collapse(rendered: str) -> str: .replace(' ""', "")) rendered = _SPACE_BEFORE_COMMA.sub(",", rendered) rendered = _SPACES.sub(" ", rendered.strip()) - # the default spec's maiden decoration: an empty {maiden} leaves a - # trailing bare 'née' (lowercase, exactly the template's spelling - # -- a capitalized name 'Née' as content is untouched); on an - # all-empty parse the decoration IS the whole string - rendered = rendered.removesuffix(" née") - if rendered == "née": - rendered = "" if rendered and _COMMA_CHAR.fullmatch(rendered[-1]): rendered = rendered[:-1] return rendered.strip(", ") diff --git a/nameparser/_types.py b/nameparser/_types.py index 636af22b..0d78601c 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -542,16 +542,16 @@ def matches(self, other: str | ParsedName, *, def render( self, - spec: str = ("{title} {given} {middle} {family} {suffix} " - "({nickname}) née {maiden}"), + spec: str = ('{title} {given} "{nickname}" {middle} {family} ' + "({maiden}) {suffix}"), ) -> str: """Fill the str.format spec from the seven role fields and the derived views; empty fields collapse (#254), including the - default spec's decorations -- '()' around an absent nickname - and a trailing orphaned 'née'. The default shows every - non-empty field and round-trips: parentheses re-extract as the - nickname and 'née' is a maiden marker. Unknown keys raise - KeyError naming the valid fields.""" + default spec's decorations (empty '""' and '()' wrappers). + The default shows every non-empty field: the nickname quoted + after the given name, the maiden name parenthesized after the + family name. Unknown keys raise KeyError naming the valid + fields.""" import nameparser._render as _render return _render.render(self, spec) diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 774ff1cc..224df8cd 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -71,18 +71,18 @@ def test_render_empty_parse_is_empty_string() -> None: def test_default_str_includes_nickname_and_maiden() -> None: # Derek's terminal find (2026-07-19): str(parse('Von Johnson - # (smith)')) dropped the nickname. v1's default string_format - # always showed '({nickname})'; the 2.0 default restores it and - # adds 'née {maiden}' -- and both decorations round-trip: parens - # re-extract as nickname, née is a maiden marker. + # (smith)')) dropped the nickname; the default spec now shows all + # seven fields -- nickname quoted after the given name, maiden + # parenthesized after the family name (Derek's chosen format). + # The nickname decoration round-trips exactly; the maiden parens + # re-extract as a NICKNAME on reparse (documented trade-off: + # presentation over lossless round-trip). from nameparser import parse - assert str(parse("Von Johnson (smith)")) == "Von Johnson (smith)" - assert str(parse("Jane Smith née Jones")) == "Jane Smith née Jones" + assert str(parse("Von Johnson (smith)")) == 'Von "smith" Johnson' + assert str(parse("Jane Smith née Jones")) == "Jane Smith (Jones)" both = parse("Jane (Janie) Smith née Jones") - assert str(both) == "Jane Smith (Janie) née Jones" - reparsed = parse(str(both)) - assert reparsed.nickname == "Janie" and reparsed.maiden == "Jones" + assert str(both) == 'Jane "Janie" Smith (Jones)' # no orphaned decoration when the fields are empty assert str(parse("Dr. Juan Q. Xavier de la Vega III")) == ( "Dr. Juan Q. Xavier de la Vega III") From 020e5ad3bf0ab4ab12be8fb5c2aff75fa6aaec6a Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 23:05:15 -0700 Subject: [PATCH 219/275] fix: .get() on the regexes proxy is no longer swallowed by __getattr__ 1.4's #256 deprecation told users ".get() remains available for intentional soft access". On CONSTANTS.capitalization_exceptions that holds -- TupleManager subclasses dict, so .get() comes for free. The regexes proxy is not a dict, and its catch-all __getattr__ claimed the name `get` as a regex lookup, so the escape hatch the deprecation message pointed at raised AttributeError instead. Define get() explicitly, with dict.get semantics. Co-Authored-By: Claude Opus 4.8 --- nameparser/_config_shim.py | 6 ++++++ tests/v2/test_config_shim.py | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index 096df927..e9cd5b36 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -524,6 +524,12 @@ def __iter__(self) -> Iterator[str]: def keys(self) -> KeysView[str]: return self._regexes().keys() + def get(self, name: str, default: object = None) -> object: + # Defined explicitly because __getattr__ would otherwise claim + # `get` as a regex name. The sibling managers inherit this from + # dict; #256's deprecation text promised it on both. + return self._regexes().get(name, default) + def __setattr__(self, name: str, value: object) -> None: self._raise_readonly(name) diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index d267b529..ea5afad8 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -11,6 +11,7 @@ CONSTANTS, Constants, SetManager, TupleManager, _DelimiterManager, _RegexesProxy, _cached_parser, ) +from nameparser.config.regexes import EMPTY_REGEX, REGEXES _DATA_DIR = Path(__file__).parent / "data" @@ -195,6 +196,17 @@ def test_regexes_membership_iteration_and_deepcopy() -> None: assert isinstance(copy.deepcopy(r), _RegexesProxy) +def test_regexes_get_is_the_soft_access_escape_hatch() -> None: + # 1.4's #256 deprecation told users ".get() remains available for + # intentional soft access"; __getattr__ must not swallow `get` + # itself. capitalization_exceptions inherits it from dict -- the + # regexes proxy is not a dict, so it has to say so explicitly. + r = _RegexesProxy() + assert r.get("word") is REGEXES["word"] + assert r.get("typo") is None + assert r.get("typo", EMPTY_REGEX) is EMPTY_REGEX + + def test_constants_default_fields_present() -> None: c = Constants() assert "dr" in c.titles From d8f06646aa20f0c255db2e1f891102bd6031d6ad Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 23:05:15 -0700 Subject: [PATCH 220/275] Remove the dead regexes.no_vowels config (#268) Nothing in nameparser/, tests/, or docs/ referenced it -- it was dead in v1 and the 2.0 pipeline never looked at it either. Reading CONSTANTS.regexes.no_vowels now raises AttributeError like any other unknown key (#256). No dedicated test: asserting a key's absence from a config constant is the constant-content testing we avoid. The full suite passing with no residual references is the evidence. Co-Authored-By: Claude Opus 4.8 --- nameparser/config/regexes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nameparser/config/regexes.py b/nameparser/config/regexes.py index 17716df3..d2e7e26e 100644 --- a/nameparser/config/regexes.py +++ b/nameparser/config/regexes.py @@ -26,7 +26,6 @@ "double_quotes": re.compile(r'\"(.*?)\"'), "parenthesis": re.compile(r'\((.*?)\)'), "roman_numeral": re.compile(r'^(X|IX|IV|V?I{0,3})$', re.I), - "no_vowels": re.compile(r'^[^aeyiuo]+$', re.I), "period_not_at_end": re.compile(r'.*\..+$', re.I), "emoji": re_emoji, "bidi": re_bidi, From 08cfeb53f570b0da8c7705e110e2db7342f9824e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 18 Jul 2026 23:08:02 -0700 Subject: [PATCH 221/275] docs: rewrite the 2.0 release log new-API-first and complete the audit The 2.0.0 section documented only locale packs and a machine-generated behavior-diff draft. It never mentioned that 2.0 introduces a new API, and listed none of the removals -- a reader upgrading met the ma/do acronym data change before learning parse() exists. Restructure into: orientation paragraph, The 2.0 API, Breaking Changes, Behavior Changes, International name support, Compatibility layer, Command line, Documentation. Same shape 1.3.0 already uses, scaled up. Audited against the v2.0 milestone and all 218 branch commits. Adds the previously undocumented removals (#223, #243, #245, #255, #256, #257, #258, #261, #268, #279, #280), the new API surface, the Python 3.11 floor, the CLI rewrite, and the docs rewrite. Two removals are called out as silent breaks, since neither raises: HumanName == str flipping to False (#223), and the 1.3.1-documented CONSTANTS.regexes.bidi = False opt-out now raising. Drops the per-bullet cases.py row citations and differential-corpus qualifiers -- they serve the author, not the reader -- and folds the harness explanation into one closing note. Co-Authored-By: Claude Opus 4.8 --- docs/release_log.rst | 118 ++++++++++++++++++++++++++++++------------- 1 file changed, 84 insertions(+), 34 deletions(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index 6333a5f1..32eb133b 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -2,40 +2,90 @@ Release Log =========== * 2.0.0 - unreleased - **Locale packs (opt-in)** - - - Add ``nameparser.locales`` with the first two packs: ``locales.RU`` (East Slavic patronymic order) and ``locales.TR_AZ`` (Turkic patronymic markers). Packs are pure data folded in at ``parser_for(locales.RU)``; they compose (``parser_for(locales.RU, locales.TR_AZ)`` unions the rules) and are never auto-detected -- there is no reliable way to detect a name's language (closes the pack half of #270; #271/#272/#146 stay staged for 2.x) - - The CLI gains ``--locale CODE`` (``python -m nameparser --locale ru "..."``) - - Add non-Latin vocabulary to the default lexicon (#269): Cyrillic, Greek, Arabic and Hebrew titles, conjunctions, and name particles -- native-script entries cannot collide with Latin-script names. Deferred pending vetting: Cyrillic ``мл``/``ст`` suffixes and the bare Greek ``κ`` title (it collides with the initial+surname shape). Behavior note: ``محمد بن سلمان`` now chains ``بن`` onto the family name where 1.x read it as a middle name - - Add the Arabic-script bound given names (#269 follow-up): ``عبد``, the kunya pair ``أبو``/``ابو``, and ``أم``/``ام`` join the following word into the given name like their transliterations (``abdul``, ``abu``, ``umm``) always did -- ``عبد الرحمن محمد`` now parses given ``عبد الرحمن``, family ``محمد``, where 1.x split it into given + middle (``tests/v2/cases.py`` rows ``arabic_bound_given_*``). Not present in the differential corpus - - Add Hebrew honorifics, Hebrew post-nominals, and Devanagari titles (#269 follow-up): the Israeli honorifics ``גברת``, ``פרופ'``/``פרופ׳``, ``פרופסור``, ``עו"ד``/``עו״ד``, and ``הרב`` as plain titles; the post-nominals ``ז"ל``/``ז״ל`` and ``שליט"א``/``שליט״א`` as suffixes (both gershayim spellings each); and a new #269 script -- Devanagari ``श्री``, ``श्रीमती``, and ``डॉ`` (``डॉ.`` matches via edge-period normalization). Latin ``sri``/``shri`` deliberately NOT added (they collide with real given names; the native script cannot). Deferred: bare ``רב`` (ordinary word "many") and ``בר`` as a particle (Bar is a common modern given name). Not present in the differential corpus - - Add Arabic honorific titles and the conjunction ``و`` (#269 follow-up): the doctor/professor/hajj/sheikha/engineer forms (``الدكتور``/``الدكتورة``/``دكتور``/``دكتورة``, ``الأستاذ``/``الأستاذة``/``أستاذ``/``أستاذة``, ``الحاج``/``الحاجة``, ``الشيخة``, ``مهندس``) as given-name titles -- Arabic honorifics precede the given name, like ``الشيخ``. Deferred under the collision rule: bare ``سيد``/``شيخ``/``أمير``/``سلطان`` (all common given names), the ``د.`` abbreviation (bare ``د`` would swallow initials), and the Ottoman post-nominals ``باشا``/``بك``/``أفندي`` (survive as family names). Not present in the differential corpus - - **Behavior Changes (draft)** - - .. note:: - This section is a **draft**, generated from the v1-vs-2.0 - differential harness (``tools/differential/``) run against the - pre-M12 v1 test corpus (486 name strings). It will be replaced - by hand-written release notes before 2.0 ships; entries below - are grouped by the harness's classification and reference the - ``tests/v2/cases.py`` row (if any) that pins the new behavior. - - - Fix ``"Andrews, M.D."``-shaped input: the lone strict-suffix-or-title piece after a comma (e.g. ``"Smith, Dr."``, ``"Andrews, M.D."``) was routed to ``first`` in 1.x; 2.0 routes it to ``suffix``/``title`` instead, since the pre-comma piece is definitionally the family name (``tests/v2/cases.py`` rows ``family_comma_lone_suffix_piece``, ``family_comma_lone_title``; verified live against 1.4.0 -- 1/486 corpus names diff, all others parity) - - Fix a lone recognized trailing suffix (e.g. ``"Johnson PhD"``, ``"Mr. Johnson PhD"``) being routed to ``first``/``last`` in 1.x when no comma is present; 2.0 keeps a recognized suffix in ``suffix`` (``tests/v2/cases.py`` rows ``suffix_stays_suffix``, ``suffix_stays_suffix_title``; not present in the differential corpus -- no v1 test string exercises the bare two-token shape, so this is unverified against 1.4 live but pinned by the case table) - - Fix maiden-name markers (``née``/``nee``/``born``/``geb.``/``roz.``) being folded into ``middle``/``last`` in 1.x (e.g. ``"Jane Smith née Jones"`` → ``middle="Smith née"``); 2.0 recognizes the marker and routes the following name to the new ``maiden`` field (closes #274; ``tests/v2/cases.py`` row ``maiden_marker``; not present in the differential corpus) - - Data change: ``ma``/``do`` added to ``suffix_acronyms_ambiguous`` -- ambiguous acronyms count as suffixes only when written with periods, so a bare common surname (``"Jack Ma"``, ``"Anh Do"``) keeps its family name under 2.0's keep-recognized-suffixes routing, matching 1.4's output (``tests/v2/cases.py`` row ``ambiguous_surname_acronyms``). Side effect: parenthesized/quoted ``"(MA)"``/``"(DO)"`` (no periods) no longer escape to ``suffix`` the way 1.x did -- they now fall through to nickname parsing like any other ambiguous-acronym delimited content. Not present in the differential corpus - - Change suffix-delimiter rendering: with a custom ``Policy``/``Constants`` suffix delimiter configured (e.g. ``suffix_delimiter="/"``, ``"John Smith, RN/CRNA"``), 1.x split the token and rendered ``suffix="RN, CRNA"``; 2.0 keeps the no-space delimiter-core token whole (``suffix="RN/CRNA"``) -- role assignment is unchanged, only rendering differs (anti-#100, migration plan deviation 5; ``tests/v2/cases.py`` row ``suffix_delimiter_no_space_core``). Only fires with a non-default policy, so it does not appear in the (default-policy) differential corpus - - Change ``comparison_key()``/``matches()`` (both APIs) to fold components with ``str.casefold()`` where 1.4 used ``str.lower()``: Unicode case-pair forms now compare equal -- ``"STRASSE"`` matches ``"Straße"``, and a Greek final-sigma variant matches its regular-sigma form. Strictly more permissive (anything 1.4 matched still matches); parse output is unaffected -- vocabulary normalization itself uses ``lower()``, v1-parity (``tests/v2/test_types.py`` row ``test_matches_casefolds_unicode_case_pairs``) - - Change delimiter-overlap precedence in the 2.0 API: a pair listed in ``Policy.maiden_delimiters`` is dropped from the effective ``nickname_delimiters`` set (maiden wins), so ``Policy(maiden_delimiters={("(", ")")})`` alone routes parenthesized content to ``maiden`` -- no need to rebuild the nickname set. The default nickname set is now the public ``DEFAULT_NICKNAME_DELIMITERS`` constant. The 1.x facade keeps v1's nickname-wins precedence on overlap via a shim-side pre-subtraction, so no ``HumanName`` behavior changes (``tests/v2/cases.py`` row ``maiden_delimiters_win_when_shared``; ``tests/v2/test_config_shim.py`` row ``test_snapshot_overlap_keeps_v1_nickname_precedence``) - - Change the 2.0 API's default ``render()``/``str()`` spec to show every non-empty field: ``'{title} {given} "{nickname}" {middle} {family} ({maiden}) {suffix}'``. v1's default already showed the nickname; 2.0 had dropped it (so ``str(parse("Von Johnson (smith)"))`` silently lost it) and never showed the maiden name. The quoted nickname round-trips exactly; the parenthesized maiden re-parses as a nickname (presentation over lossless round-trip -- use ``née {maiden}`` in a custom spec if you need the round-trip). The ``HumanName`` facade keeps v1's own ``string_format`` default, unchanged (``tests/v2/test_render.py`` row ``test_default_str_includes_nickname_and_maiden``) - - Recognize typographic nickname delimiters by default in BOTH APIs (closes #273): smart quotes (``“Jack”``), German/Polish low-high quotes (``„Hansi“``), Swedish right-right quotes (``”Ann”``), guillemets in both directions (``«Petit»``, ``»Hansi«``, inner spacing tolerated), CJK corner brackets (``「タロ」``/``『ハナ』``), and fullwidth parentheses. In 1.x these leaked into ``middle`` as literal text. Curly *single* quotes stay excluded (U+2019 is the apostrophe in "O’Connor" -- pinned by ``curly_apostrophe_stays_literal``). The v1 keyed idioms work on the new named sentinels (``smart_double_quotes``, ``guillemets``, ...). A delimiter character consumed by another pair's extraction no longer emits a spurious ``unbalanced-delimiter`` ambiguity (``tests/v2/cases.py`` rows ``*_nickname``; ``tests/v2/pipeline/test_extract.py``). Not present in the differential corpus - - Everything else in the 486-name differential corpus (built from the - v1 test banks as of commit ``2d5d8c2``, pre-dating the M12 test - reconciliation) parses identically between nameparser 1.4.0 and - this working tree; see ``tools/differential/README.md`` for how to - reproduce the comparison. + nameparser 2.0 adds a new parsing API alongside ``HumanName``. + ``parse()`` returns an immutable ``ParsedName`` whose seven fields + are named for what they are (``given``, ``family``) rather than + where they sit in a Western name, configured by two frozen value + objects -- a ``Lexicon`` of vocabulary and a ``Policy`` of behavior + -- instead of a mutable global. ``HumanName`` keeps working: it is + now a compatibility facade over the same pipeline, and it stays + through 2.x. The removals below are the deprecations announced in + 1.3.0 and 1.4.0 coming due; if your code runs warning-free on + 1.4.0, most of them will not affect you. See :doc:`migrate` for the + field-by-field map. + + **The 2.0 API** + + - Add ``parse(text)``, returning an immutable ``ParsedName`` with seven fields -- ``title``, ``given``, ``middle``, ``family``, ``suffix``, ``nickname``, ``maiden`` -- plus the derived views ``given_names``, ``surnames``, ``family_base`` and ``family_particles``. Parsing is a pure function of the text, a ``Lexicon`` and a ``Policy``: nothing global is consulted and nothing is mutated + - Add ``Lexicon``, the frozen vocabulary object. ``Lexicon.default()`` is the shipped vocabulary and ``Lexicon.empty()`` is a blank one; ``add(**entries)``, ``remove(**entries)`` and field-wise ``|`` all return new instances. Its fields are ``titles``, ``given_name_titles``, ``suffix_acronyms``, ``suffix_words``, ``suffix_acronyms_ambiguous``, ``particles``, ``particles_ambiguous``, ``conjunctions``, ``bound_given_names``, ``maiden_markers`` and ``capitalization_exceptions`` + - Add ``Policy``, the frozen behavior object: ``name_order``, ``patronymic_rules``, ``middle_as_family``, ``nickname_delimiters``, ``maiden_delimiters``, ``extra_suffix_delimiters``, ``lenient_comma_suffixes``, ``strip_emoji`` and ``strip_bidi`` + - Add ``Parser``, a reusable parser bound to a lexicon and policy (``Parser(lexicon=..., policy=...).parse(text)``), and ``parser_for(*locales, base=None)``, which folds locale packs onto a base parser + - Add ``name_order`` and the constants ``GIVEN_FIRST``, ``FAMILY_FIRST`` and ``FAMILY_FIRST_GIVEN_LAST``, so a family-first name can be parsed as written rather than reordered by hand (closes the configuration half of #270) + - Add ``PatronymicRule`` with the members ``EAST_SLAVIC`` and ``TURKIC``. v1's single ``patronymic_name_order`` flag enabled both detectors at once; ``Policy(patronymic_rules=...)`` lets you enable either one alone + - Add ``PolicyPatch`` and the ``UNSET`` sentinel for partial policy deltas that compose -- set-valued fields union, scalar fields override with later winning. This is the mechanism locale packs are built from, and ``UNSET`` is only needed when you must distinguish "not set" from a real ``False`` or ``None`` + - Add ``Token``, ``Span`` and ``Role``: every field is backed by tokens carrying exact ``(start, end)`` offsets into the original string, reachable with ``tokens_for(Role.GIVEN)``. This replaces v1's ``*_list`` attributes and makes it possible to highlight or re-slice the input the parse came from + - Add ``Ambiguity`` and the ``AmbiguityKind`` enum, so a parse reports what it had to guess at instead of guessing silently. The kinds emitted today are ``particle-or-given``, ``unbalanced-delimiter`` and ``comma-structure``; ``order`` and ``suffix-or-nickname`` are reserved and not yet emitted + - Add ``ParsedName`` output and comparison methods: ``render(spec)``, ``initials()``, ``capitalized()`` (which returns a new value rather than mutating in place), ``as_dict()``, ``replace(**fields)``, ``matches()`` and ``comparison_key()`` + - Add ``Locale``, the public pack type. Writing your own needs no registration -- construct a ``Locale`` and pass it to ``parser_for()`` + - Ship a fully typed public API (PEP 561): the core modules are checked under strict mypy settings, and nameparser 2.0 has no runtime dependencies + + **Breaking Changes** + + - Raise the minimum Python to 3.11 and drop the last runtime dependency, ``typing_extensions`` (#257). Python 3.10 is no longer supported + - Remove ``HumanName.__eq__`` and ``__hash__`` (deprecated in 1.3.0, #223): instances now compare and hash by identity, so ``HumanName("John Smith") == "John Smith"`` is ``False`` where 1.x returned ``True``. **This changes result silently rather than raising** -- it is the one removal that can pass unnoticed into production. Use ``matches()`` to ask whether two names are the same, and ``comparison_key()`` as a dict key or sort key + - Remove every v1 parsing hook and subclass extension point: ``pre_process``, ``post_process``, ``parse_pieces``, ``parse_nicknames``, ``join_on_conjunctions``, the ``is_*`` predicates, ``cap_word``/``cap_piece``, ``handle_firstnames``, ``fix_phd`` and the rest. A subclass that overrides one gets a ``DeprecationWarning`` at construction naming the hooks, because the facade delegates to the core ``Parser`` and never calls them (closes #280). Customize through ``Lexicon``/``Policy`` instead + - Remove regex configuration: ``CONSTANTS.regexes`` is now a read-only proxy. Reads still work, but ``CONSTANTS.regexes.bidi = False``, item assignment and ``Constants(regexes=...)`` all raise ``TypeError``. **If you followed 1.3.1's advice to keep bidi marks with** ``CONSTANTS.regexes.bidi = False``, that opt-out is now ``Policy(strip_bidi=False)`` on the 2.0 API; the same applies to ``regexes.emoji`` and ``Policy(strip_emoji=False)`` + - Remove ``bytes`` input and the ``encoding`` argument (#245): passing ``bytes`` to ``HumanName`` or to a set manager raises ``TypeError`` with a decode hint, and ``SetManager.add_with_encoding()`` and ``DEFAULT_ENCODING`` are gone. Decode first, then use ``add()`` + - Remove ``SetManager.__call__`` (#243); iterate the manager or call ``set(manager)``. ``remove()`` of a missing member now raises ``KeyError`` like ``set.remove`` -- ``discard()`` is the ignore-missing form -- and the set operators ``|``, ``&``, ``-`` and ``^`` return a plain ``set`` + - Remove ``HumanName`` slice access and item assignment (#258): ``name[1:-3]`` and ``name['first'] = value`` raise ``TypeError``. String-key reads (``name['first']``) and iteration are unchanged; assign fields as plain attributes + - Remove ``Constants.empty_attribute_default`` (#255): empty fields are always ``''``. Assigning it raises ``TypeError``; a pickle carrying the key still loads, with the value ignored + - Remove ``constants=None`` (#261): both ``HumanName(..., constants=None)`` and ``hn.C = None`` raise ``TypeError``. Use ``Constants()`` for library defaults or ``CONSTANTS.copy()`` for a private snapshot + - Remove silent unknown-key access on the mapping managers (#256): ``CONSTANTS.capitalization_exceptions.typo`` and ``CONSTANTS.regexes.typo`` raise ``AttributeError`` naming the miss and the known keys, instead of returning ``None``/``EMPTY_REGEX``. ``.get()`` remains available on both for intentional soft access + - Remove support for ``Constants`` pickles written by nameparser 1.2.x or earlier (#279): loading one raises ``ValueError`` telling you to re-pickle under 1.3/1.4 first + - Remove the dead ``regexes.no_vowels`` pattern (#268) and the ``Constants.suffixes_prefixes_titles`` cached union; neither was read by the parser + - Change ``HumanName``'s ``*_list`` attributes to read-only properties: they remain readable snapshots, but ``name.first_list = [...]`` now raises ``AttributeError`` + - Change the ``Constants`` constructor to keyword-only; positional construction no longer works + + **Behavior Changes** + + - Recognize maiden-name markers -- ``née``, ``nee``, ``born``, ``geb.``, ``roz.`` and the Scandinavian participle forms -- and route the following name to the new ``maiden`` field. 1.x folded them into ``middle``/``last``, so ``"Jane Smith née Jones"`` parsed as ``middle="Smith née"`` (closes #274) + - Recognize typographic nickname delimiters by default in both APIs (closes #273): smart quotes (``“Jack”``), German and Polish low-high quotes (``„Hansi“``), Swedish right-right quotes (``”Ann”``), guillemets in either direction (``«Petit»``, ``»Hansi«``), CJK corner brackets (``「タロ」``, ``『ハナ』``) and fullwidth parentheses. In 1.x these leaked into ``middle`` as literal text. Curly *single* quotes stay excluded, because U+2019 is the apostrophe in "O’Connor" + - Fix a lone suffix or title after a comma being routed to ``first``: ``"Andrews, M.D."`` and ``"Smith, Dr."`` now fill ``suffix``/``title``, since the piece before the comma is definitionally the family name + - Fix a lone recognized trailing suffix with no comma being routed to ``first``/``last``: ``"Johnson PhD"`` and ``"Mr. Johnson PhD"`` now keep the suffix in ``suffix`` + - Fix a split ``"Ph. D."`` credential being read as two tokens; it now classifies as one suffix, replacing v1's ``fix_phd`` hook + - Fold a leading never-given particle into the family name. Note that ``Lexicon.particles_ambiguous`` is the **complement** of v1's ``non_first_name_prefixes``, not a rename -- it lists the particles that *may* double as a given name, where v1 listed the ones that may not. Copying a v1 customization across without inverting it silently reverses the behavior; see :doc:`migrate` + - Add ``ma`` and ``do`` to ``suffix_acronyms_ambiguous``, so ``"Jack Ma"`` and ``"Anh Do"`` keep their family names. Ambiguous acronyms count as suffixes only when written with periods; as a side effect, a parenthesized or quoted ``"(MA)"``/``"(DO)"`` now falls through to nickname parsing instead of escaping to ``suffix`` + - Change ``comparison_key()`` and ``matches()`` in both APIs to fold with ``str.casefold()`` where 1.4 used ``str.lower()``, so Unicode case pairs compare equal -- ``"STRASSE"`` matches ``"Straße"``, and a Greek final sigma matches its regular form. This is strictly more permissive: anything 1.4 matched still matches. Vocabulary normalization deliberately still uses ``lower()``, for v1 parity + - Change the 2.0 API's default ``render()``/``str()`` spec to show every non-empty field: ``'{title} {given} "{nickname}" {middle} {family} ({maiden}) {suffix}'``. The quoted nickname round-trips exactly; the parenthesized maiden re-parses as a nickname, a deliberate choice of presentation over lossless round-trip -- use ``née {maiden}`` in a custom spec if you need it to survive a reparse. ``HumanName`` keeps v1's own ``string_format`` default, unchanged + - Change delimiter-overlap precedence in the 2.0 API: a pair listed in ``Policy.maiden_delimiters`` is dropped from the effective nickname set, so ``Policy(maiden_delimiters={("(", ")")})`` alone routes parenthesized content to ``maiden``. The default nickname set is exported as ``DEFAULT_NICKNAME_DELIMITERS``. ``HumanName`` keeps v1's nickname-wins precedence, so no existing behavior changes + - Change suffix-delimiter rendering when a custom suffix delimiter is configured: for ``suffix_delimiter="/"`` and ``"John Smith, RN/CRNA"``, 1.x split the token and rendered ``suffix="RN, CRNA"`` where 2.0 keeps it whole as ``"RN/CRNA"``. Role assignment is unchanged; only rendering differs + + **International name support** + + - Add ``nameparser.locales`` with the first two packs: ``locales.RU`` (East Slavic patronymic order) and ``locales.TR_AZ`` (Turkic patronymic markers). Packs are pure data folded in at ``parser_for(locales.RU)``, they compose (``parser_for(locales.RU, locales.TR_AZ)`` unions the rules), and they are never auto-detected -- there is no reliable way to detect a name's language from the name alone. ``locales.available()`` and ``locales.get(code)`` look packs up by code; loading is lazy, so importing the package imports no pack (closes the pack half of #270; #271/#272/#146 stay staged for 2.x) + - Add non-Latin vocabulary to the default lexicon (#269): Cyrillic, Greek, Arabic and Hebrew titles, conjunctions and name particles. Native-script entries cannot collide with Latin-script names, which is what makes them safe to enable by default. Deferred pending vetting: the Cyrillic ``мл``/``ст`` suffixes and the bare Greek ``κ`` title, which collides with the initial-plus-surname shape. Behavior note: ``محمد بن سلمان`` now chains ``بن`` onto the family name where 1.x read it as a middle name + - Add Arabic-script bound given names (#269): ``عبد``, the kunya pair ``أبو``/``ابو``, and ``أم``/``ام`` join the following word into the given name exactly as their transliterations (``abdul``, ``abu``, ``umm``) always did, so ``عبد الرحمن محمد`` parses given ``عبد الرحمن``, family ``محمد`` where 1.x split it into given plus middle + - Add Arabic honorific titles and the conjunction ``و`` (#269): the doctor, professor, hajj, sheikha and engineer forms (``الدكتور``/``الدكتورة``/``دكتور``/``دكتورة``, ``الأستاذ``/``الأستاذة``/``أستاذ``/``أستاذة``, ``الحاج``/``الحاجة``, ``الشيخة``, ``مهندس``) as given-name titles, since Arabic honorifics precede the given name like ``الشيخ``. Deferred under the collision rule: bare ``سيد``/``شيخ``/``أمير``/``سلطان`` (all common given names), the ``د.`` abbreviation (bare ``د`` would swallow initials), and the Ottoman post-nominals ``باشا``/``بك``/``أفندي`` (which survive as family names) + - Add Hebrew honorifics and post-nominals, and Devanagari titles (#269): the Israeli honorifics ``גברת``, ``פרופ'``/``פרופ׳``, ``פרופסור``, ``עו"ד``/``עו״ד`` and ``הרב`` as titles; ``ז"ל``/``ז״ל`` and ``שליט"א``/``שליט״א`` as suffixes, in both gershayim spellings; and Devanagari ``श्री``, ``श्रीमती`` and ``डॉ``. Latin ``sri``/``shri`` were deliberately not added, because they collide with real given names where the native script cannot. Deferred: bare ``רב`` (an ordinary word meaning "many") and ``בר`` as a particle (Bar is a common modern given name) + + **Compatibility layer** + + - Reimplement ``HumanName`` as a facade over the 2.0 pipeline, and ``Constants`` as a shim resolving to a ``(Lexicon, Policy)`` snapshot with a shared parser cache. Fields, aggregates, mutation through ``name.C.titles.add(...)``, rendering defaults, ``capitalize()``, ``matches()``, ``comparison_key()``, iteration, ``as_dict()`` and pickling are all preserved, and ``nameparser.parser`` and ``nameparser.config`` remain importable. The compatibility layer ships through 2.x and is removed in 3.0 + - Note that ``CONSTANTS.capitalize_name`` and ``force_mixed_case_capitalization`` are still honored through the facade, but the 2.0 API never capitalizes during ``parse()`` -- call ``capitalized()`` when you want it + + **Command line** + + - Rewrite ``python -m nameparser`` over the 2.0 API with a real argument parser. It prints the parse plus its capitalized form and initials by default, takes ``--json`` to emit the fields as JSON (``python -m nameparser --json "Doe, John"``), takes ``--locale CODE`` to apply a pack, and exits with a usage message on bad arguments + + **Documentation** + + - Rewrite the documentation new-API-first: a new front page and README, ``usage.rst`` as a 2.0 getting-started tour, a principle-first ``customize.rst``, a reference split into the 2.0 API and the compatibility layer, and new pages for :doc:`concepts`, :doc:`locales` and :doc:`migrate` -- the last carrying full attribute and configuration maps from v1 names to 2.0 names (#262). The 1.x documentation remains online as the readthedocs ``stable`` build + + Parsing changes were checked against a 486-name differential corpus + built from the v1 test banks; everything not listed above parses + identically between 1.4.0 and 2.0. The harness ships in + ``tools/differential/`` -- see its README to reproduce the + comparison against your own names. * 1.4.0 - July 12, 2026 From 24aa7873024c4278297f68a460c8aa759f88f307 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 11:33:48 -0700 Subject: [PATCH 222/275] fix: pickled HumanName keeps multi-word suffix entries intact __setstate__ restored components by joining each pickled *_list into one string and letting replace() re-split it on whitespace. That destroyed the entry boundaries: suffix_list ["Ph. D."] came back as ["Ph.", "D."], and the suffix view's ", " join then rendered one credential as two. Build tokens per entry instead, tagging continuation words "joined" -- the inverse of _list_for's heal -- so list -> pickle -> list is the identity again. This was a regression against 1.4.0, not an inherited quirk: all eight affected corpus names round-trip unchanged under a live 1.4.0, and now match it byte for byte. 8 of the 486 differential-corpus names drifted before the fix (any multi-word suffix run in one comma segment: "Jr. MD", "Q.C. M.P.", "V Jr.", "MD PhD - FACS Fellow"); 0 drift after, checking str(), as_dict() and all six _list attributes. The differential harness could not have caught this -- it compares fresh v1 and 2.0 parses and never crosses a pickle boundary. Note the list SETTER (_set_field) splits the same way on purpose: 1.4.0 also renders hn.suffix = ["Ph. D.", "MD"] as "Ph., D., MD", so that path is v1 parity and is deliberately left alone. Co-Authored-By: Claude Opus 4.8 --- nameparser/_facade.py | 33 ++++++++++++++++++--------------- tests/v2/test_facade.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/nameparser/_facade.py b/nameparser/_facade.py index 0ea0083b..99de7ffa 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -31,7 +31,7 @@ from nameparser._config_shim import CONSTANTS, Constants, _cached_parser from nameparser._lexicon import _normalize from nameparser._parser import Parser -from nameparser._types import FOLDED_TAG, ParsedName, Role +from nameparser._types import FOLDED_TAG, ParsedName, Role, Token _V2_FIELD = {"first": "given", "last": "family"} # v1 name -> v2 name _V1_SPELLING = {v2: v1 for v1, v2 in _V2_FIELD.items()} @@ -662,18 +662,21 @@ def __setstate__(self, state: dict[str, Any]) -> None: self._suffix_delimiter = state.get("suffix_delimiter", defaults.suffix_delimiter) self._full_name = state.get("_full_name", "") - # components come back exactly as pickled (spec §2): synthetic - # tokens via replace(), never a re-parse. Known edge: replace() - # re-splits on whitespace without the "joined" tag, so joined-tag - # healing is lost for multi-word list elements -- v1's fix_phd - # suffix pickles as ["Ph. D."] but round-trips to ["Ph.", "D."], - # rendering the suffix as "Ph., D.". Classify in the differential - # harness / M12 if it surfaces. - parsed = ParsedName(original=str(state.get("original", "")), - tokens=(), ambiguities=()) - fields = {} + # Components come back exactly as pickled (spec §2): synthetic + # tokens, never a re-parse. Build them per *_list ENTRY rather + # than from one joined string -- an entry may hold several words + # ("Ph. D.", "Q.C. M.P."), and re-splitting the joined string on + # whitespace would promote each word to its own entry, which the + # suffix view then renders comma-separated ("Ph., D."). Marking + # continuation words "joined" is the inverse of _list_for's heal, + # so list -> pickle -> list is the identity v1 gave us. + tokens: list[Token] = [] for member in _MEMBERS: - values = state.get(f"{member}_list") or [] - fields[_V2_FIELD.get(member, member)] = " ".join(values) - self._parsed = parsed.replace( - **{k: v for k, v in fields.items() if v}) + role = Role(_V2_FIELD.get(member, member)) + for entry in state.get(f"{member}_list") or []: + for position, word in enumerate(entry.split()): + tokens.append(Token( + word, None, role, + frozenset({"joined"}) if position else frozenset())) + self._parsed = ParsedName( + original=str(state.get("original", "")), tokens=tuple(tokens)) diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index 0873d529..b18ba42d 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -307,6 +307,39 @@ def test_pickle_round_trip_preserves_components() -> None: assert loaded.C is CONSTANTS # shared sentinel restored +#: Every name in the 486-name differential corpus whose HumanName pickle +#: round-trip drifted before the __setstate__ element-boundary fix. Each +#: carries a multi-word suffix entry -- a "joined" continuation token -- +#: inside one comma segment. Verified against a live 1.4.0: all eight +#: round-trip unchanged there, so drift here is a regression, not an +#: inherited quirk. +_MULTIWORD_SUFFIX_NAMES = [ + "Andrew Perkins, Jr., Col. (Ret)", + "Clarke, Kenneth, Q.C. M.P.", + "Doe, John, MD PhD - FACS Fellow", + "Franklin Washington, Jr. MD", + "John Smith Ph. D.", + "John Smith, Ph. D.", + "John Smith, V Jr.", + "John Smith, V MD", +] + + +@pytest.mark.parametrize("raw", _MULTIWORD_SUFFIX_NAMES) +def test_pickle_round_trip_preserves_multiword_suffix_entries(raw: str) -> None: + # __setstate__ restores components from the pickled *_list values. + # Joining them into one string and letting replace() re-split on + # whitespace destroys the entry boundaries, and the suffix view's + # ", " join then renders one credential as two ("Ph." + "D." -> + # "Ph., D."). Entries must survive as entries. + n = HumanName(raw) + loaded = pickle.loads(pickle.dumps(n)) + assert loaded.suffix_list == n.suffix_list + assert loaded.suffix == n.suffix + assert str(loaded) == str(n) + assert loaded.as_dict() == n.as_dict() + + # -- Gap 2: parse_full_name() (v1's documented re-parse idiom) ------------- From 26d261ab3e626123f9ccc507a3bcc3a2fe0dcb42 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 11:50:20 -0700 Subject: [PATCH 223/275] docs: replace "Nothing breaks." with a check the reader can run The claim was contradicted by the 2.0 removals list. Worse, it was unfalsifiable -- it gave a reader no way to find out whether their own code was affected. Replace it with the actual promise: if your suite runs clean on 1.4 under -W error::DeprecationWarning, it runs on 2.0. Verified against a live 1.4.0 -- 11 of the 15 removals do warn there. Name the four that do not, in a new "Before you upgrade" section: regexes assignment (including the CONSTANTS.regexes.bidi = False recipe 1.3.1 itself recommended), *_list assignment, positional Constants(), and subclass parsing-hook overrides. All four raise or warn on contact, so the reader meets them immediately. Call out separately the one removal that changes results in silence: HumanName.__eq__ is gone, so name == "John Smith" is now False. It warned on 1.4 and says nothing on 2.0, making it the only way to get a wrong answer without being told. Also drop "or never" from the migration framing -- the compatibility layer is removed in 3.0, as the next sentence says. Co-Authored-By: Claude Opus 4.8 --- README.rst | 13 +++++++++---- docs/migrate.rst | 39 +++++++++++++++++++++++++++++++++------ 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index f2379c9a..4d8b7298 100644 --- a/README.rst +++ b/README.rst @@ -44,11 +44,16 @@ Learn more Coming from 1.x ---------------- -Nothing breaks. 2.0 keeps ``HumanName`` and ``CONSTANTS`` working exactly -as before — same imports, same attributes, same mutation API. See +``HumanName`` and ``CONSTANTS`` keep working in 2.0 — same imports, same +attributes, same mutation API. What 2.0 removes is the batch of +deprecations 1.3 and 1.4 announced, so if your test suite runs clean on +1.4 under ``python -W error::DeprecationWarning``, you are nearly done. +Two things that check will not catch: four removals 1.4 never warned +about, which all raise on contact, and one that changes results silently +— ``name == "John Smith"`` is now ``False``. `Migrating from HumanName `__ -for translating a v1 customization into the new API, whenever that's -convenient for you. +covers both, and translates a v1 customization into the new API whenever +that's convenient for you. See the `release log `__ for the full list of changes in the 2.0 series. diff --git a/docs/migrate.rst b/docs/migrate.rst index 48fe1472..b053a54d 100644 --- a/docs/migrate.rst +++ b/docs/migrate.rst @@ -1,13 +1,12 @@ Migrating from HumanName ========================= -Nothing breaks. 2.0 keeps ``HumanName`` and ``CONSTANTS`` working -exactly as before — same imports, same attributes, same mutation API -(``name.C.titles.add(...)``, ``name.first = "..."``, and so on). -Upgrading to 2.0 and migrating your code to the new -:class:`~nameparser.Parser`/:class:`~nameparser.Lexicon`/ +``HumanName`` and ``CONSTANTS`` keep working in 2.0 — same imports, +same attributes, same mutation API (``name.C.titles.add(...)``, +``name.first = "..."``, and so on). Upgrading to 2.0 and migrating your +code to the new :class:`~nameparser.Parser`/:class:`~nameparser.Lexicon`/ :class:`~nameparser.Policy` API are two separate decisions — you can do -the former today and the latter whenever it's convenient, or never. The +the former today and the latter whenever it's convenient. The compatibility layer (``HumanName`` and ``nameparser.config``) is removed in 3.0; that release is not scheduled. @@ -19,6 +18,34 @@ This page exists for the other direction: translating a v1 customization or a v1-shaped comparison into the 2.0 API, one row per old name. +Before you upgrade +------------------ + +What 2.0 removes is the batch of deprecations 1.3 and 1.4 announced. If +your test suite runs clean on 1.4 under ``python -W +error::DeprecationWarning``, it will run on 2.0 — with four exceptions +that 1.4 never warned about, each of which fails loudly the first time +you hit it: + +* ``CONSTANTS.regexes. = ...`` raises ``TypeError``. This includes + ``CONSTANTS.regexes.bidi = False``, the opt-out 1.3.1 recommended for + keeping bidirectional marks; the 2.0 spellings are + ``Policy(strip_bidi=False)`` and ``Policy(strip_emoji=False)`` +* assigning a ``*_list`` attribute (``name.first_list = [...]``) raises + ``AttributeError`` — the lists are read-only snapshots in 2.0 +* positional ``Constants(...)`` construction raises ``TypeError``; the + constructor is keyword-only +* a subclass overriding a v1 parsing hook (``pre_process``, + ``parse_pieces``, ``is_title``, and the rest) gets a + ``DeprecationWarning`` at construction naming the hooks it overrode, + because 2.0 delegates parsing to the core parser and never calls them + +One removal changes results without saying anything. ``HumanName`` no +longer defines ``__eq__``, so ``name == "John Smith"`` is now ``False`` +where 1.x returned ``True``. That one *did* warn on 1.4, but nothing +will tell you on 2.0. If you compare names anywhere, grep for ``==`` +before upgrading and move to ``matches()`` — see `Comparison`_. + Attribute map ------------- From 599c5e03dc1ec620a763f6f316a70091b8174ef4 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 11:59:35 -0700 Subject: [PATCH 224/275] docs: map capitalize() by its method signature, not its config constant Both capitalize rows were written from the v1 config module rather than the v1 method signature, which hid the mapping users actually need. The methods-table row lumped capitalize() together with force_mixed_case_capitalization in one cell, so the clean 1:1 -- capitalize(force=...) -> capitalized(force=...) -- was invisible. Split them and note that capitalized()'s optional first argument is a Lexicon. The config-table note said the constant "moves from stored config to a per-call argument", implying the per-call form is new. It is not: v1's capitalize(force=True) already existed and is what users wrote. What 2.0 drops is the stored global default that supplied `force` when the call omitted it. Say that instead. Swept the remaining rows for the same config-vs-call-argument confusion. None found: string_format, initials_format, initials_delimiter, initials_separator and suffix_delimiter were all constructor kwargs, instance attributes and CONSTANTS defaults in v1, but none was ever a method argument -- v1's initials() takes none, and __str__ reads self.string_format. capitalize(force=...) was the only real call argument, so those rows are accurate as written. Co-Authored-By: Claude Opus 4.8 --- docs/migrate.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/migrate.rst b/docs/migrate.rst index b053a54d..980d999e 100644 --- a/docs/migrate.rst +++ b/docs/migrate.rst @@ -110,10 +110,12 @@ Attribute map - Moves from a ``HumanName``/``Constants`` scalar to a ``Policy`` set field, so more than one custom delimiter can be active at once - * - ``capitalize()``, ``force_mixed_case_capitalization`` - - ``capitalized(lexicon, force=...)`` + * - ``capitalize(force=...)`` + - ``capitalized(force=...)`` - :meth:`~nameparser.ParsedName.capitalized` returns a new value - rather than mutating in place + rather than mutating in place. Its optional first argument takes + a :class:`~nameparser.Lexicon`, if you need custom + capitalization exceptions Side by side: @@ -214,7 +216,9 @@ rendering argument, where the 2.0 equivalent isn't config at all): - ``regexes.emoji = False`` becomes ``Policy(strip_emoji=False)`` * - ``force_mixed_case_capitalization`` - ``capitalized(force=True)`` - - Moves from stored config to a per-call argument + - The stored default is gone; pass ``force`` at the call site. + v1's ``capitalize(force=True)`` already worked that way — this + constant only supplied the default for calls that omitted it * - ``capitalize_name`` - *(no equivalent)* - 2.0 never capitalizes automatically during ``parse()``; call From 521076c290e9e162beb3e59962a5b63bde2d4c3a Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 12:14:56 -0700 Subject: [PATCH 225/275] docs: warn that the default render spec loses the maiden name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit usage.rst demonstrated str(parse("Jane (Janie) Smith née Jones")) as the happy path without noting that the parenthesized maiden name reads back as a nickname. Anyone doing parse -> store as text -> reparse (a common pipeline shape) silently converts maiden to nickname, and the docs led them there. Show the lossy reparse explicitly, then the fix: render "née {maiden}" when the string has to survive a round trip. The advice existed only in the 2.0.0 release log, which is the wrong home for a recipe -- read once at upgrade time, organized by version rather than by task. Cross-link it from customize.rst's rendering section, where a reader choosing a spec is making exactly this decision. Also correct the release log: the differential harness does not "ship" -- tools/differential/README.md says it is excluded from the wheel. Say "in the source repository" instead. Co-Authored-By: Claude Opus 4.8 --- docs/customize.rst | 6 ++++++ docs/release_log.rst | 7 ++++--- docs/usage.rst | 27 +++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index 44007198..7dfa3dbc 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -170,6 +170,12 @@ Looking for v1's ``string_format``? It's the ``render(spec)`` argument now — pass your own format string per call instead of setting it once on a shared config object. +A spec chooses what the output is *for*. The default is written for +display and does not survive a reparse — it parenthesizes the maiden +name, which reads back as a nickname. When the rendered string will be +parsed again, spell the marker out (``née {maiden}``) so the field +round-trips; see :ref:`the round-trip note in the tour `. + Sharing a configured parser ---------------------------- diff --git a/docs/release_log.rst b/docs/release_log.rst index 32eb133b..13799793 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -83,9 +83,10 @@ Release Log Parsing changes were checked against a 486-name differential corpus built from the v1 test banks; everything not listed above parses - identically between 1.4.0 and 2.0. The harness ships in - ``tools/differential/`` -- see its README to reproduce the - comparison against your own names. + identically between 1.4.0 and 2.0. The harness lives in + ``tools/differential/`` in the source repository (it is development + tooling, not part of the installed package) -- see its README to + reproduce the comparison against your own names. * 1.4.0 - July 12, 2026 diff --git a/docs/usage.rst b/docs/usage.rst index 2842f531..a2085ecb 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -78,6 +78,33 @@ family name: >>> str(parse("Jane (Janie) Smith née Jones")) 'Jane "Janie" Smith (Jones)' +.. _maiden-roundtrip: + +That default is built for display, and the two fields differ in what +survives it. The quoted nickname reparses as a nickname; the +parenthesized maiden name reparses as a *nickname* too, so a +parse-render-reparse round trip silently loses it: + +.. doctest:: + + >>> parse('Jane "Janie" Smith (Jones)').maiden + '' + >>> parse('Jane "Janie" Smith (Jones)').nickname + 'Janie Jones' + +If the rendered string has to survive a reparse — storing names as text +and reading them back, for instance — render the marker explicitly +instead of relying on the default: + +.. doctest:: + + >>> name = parse("Jane Smith née Jones") + >>> text = name.render("{given} {family} née {maiden}") + >>> text + 'Jane Smith née Jones' + >>> parse(text).maiden + 'Jones' + Comparing names ---------------- From 08dcc5186f0358ff5933db58c0165fa39ba9e286 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 12:16:34 -0700 Subject: [PATCH 226/275] docs: add worked examples for ambiguities, tokens/spans, and name_order These three had no executable example anywhere in the docs, despite being the features 2.0 is pitched on. Ambiguity and AmbiguityKind appeared in zero code blocks project-wide; Token.span only in prose; name_order (#270) as a single table row. The only working examples of tokens_for(), Token and Role lived in migrate.rst -- a page a reader new to nameparser never opens. usage.rst gains two sections: "When the parser had to guess" shows the ambiguities loop on "Van Johnson" and the StrEnum affordance (kind == "particle-or-given"), which nothing previously revealed, plus the practical use: a non-empty ambiguities is a routing signal for human review. "Tokens and spans" makes the provenance claim executable -- iterate tokens, then slice original[span.start:span.end] to recover the source text. concepts.rst asserted the (0, 3) offsets in prose; now they run. customize.rst gains a name_order example contrasting the default reading of "Nguyen Van Minh" with FAMILY_FIRST, and pins the documented comma override in a second block. Every block runs under sphinx-build -b doctest in CI, so a behavior change fails the build rather than rotting quietly. Co-Authored-By: Claude Opus 4.8 --- docs/customize.rst | 25 ++++++++++++++++++- docs/usage.rst | 61 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index 7dfa3dbc..42f213ef 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -117,9 +117,32 @@ listed below. - Excludes bidirectional control characters the same way. Defaults to ``True``. +``name_order`` is the one most likely to matter for non-Western data. +Positional input is assigned in the order you declare, so a +family-first name parses as written instead of needing to be +rearranged afterwards: + +.. doctest:: + + >>> from nameparser import Parser, Policy, FAMILY_FIRST, parse + >>> parse("Nguyen Van Minh").family # default GIVEN_FIRST + 'Van Minh' + >>> family_first = Parser(policy=Policy(name_order=FAMILY_FIRST)) + >>> name = family_first.parse("Nguyen Van Minh") + >>> name.family, name.given + ('Nguyen', 'Van Minh') + +An explicit comma still wins, on the reasoning that someone who wrote +one meant it — so the same parser reads ``"Thomas, John"`` as +family-then-given regardless of the configured order: + +.. doctest:: + + >>> family_first.parse("Thomas, John").family + 'Thomas' + .. doctest:: - >>> from nameparser import Parser, Policy >>> policy = Policy(maiden_delimiters={("(", ")")}) >>> Parser(policy=policy).parse("Jane (Jones) Smith").maiden 'Jones' diff --git a/docs/usage.rst b/docs/usage.rst index a2085ecb..85707462 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -19,7 +19,8 @@ A parsed name has seven fields: ``title``, ``given``, ``middle``, ``family``, ``suffix``, ``nickname``, and ``maiden``. Parsing never raises; unparseable input yields a :class:`~nameparser.ParsedName` with empty fields plus any ``ambiguities`` the parser noticed along the way -(see :doc:`concepts`). +(see `When the parser had to guess`_ below, and :doc:`concepts` for why +they exist). Aggregate views ---------------- @@ -120,6 +121,64 @@ the same name, allowing for order and case?" use ``matches()`` or >>> parse("JUAN DE LA VEGA").comparison_key() == parse("Juan de la Vega").comparison_key() True +When the parser had to guess +----------------------------- + +Some names have no single correct reading. ``"Van Johnson"`` could be +the given name ``Van``, or the family-name particle ``van``. 2.0 takes +the more likely reading and *records* the choice on ``ambiguities`` +rather than deciding silently: + +.. doctest:: + + >>> name = parse("Van Johnson") + >>> name.given, name.family + ('Van', 'Johnson') + >>> for a in name.ambiguities: + ... print(a.kind.value, "-", a.detail) + particle-or-given - leading 'Van' may be a family-name particle; read as a given name + +:class:`~nameparser.AmbiguityKind` members are their own string values, +so branching on a kind needs no import: + +.. doctest:: + + >>> name.ambiguities[0].kind == "particle-or-given" + True + >>> [t.text for t in name.ambiguities[0].tokens] + ['Van'] + +Most names report none. A non-empty ``ambiguities`` is a useful signal +for routing a record to human review instead of trusting it silently. + +Tokens and spans +------------------ + +The seven fields are the convenient view. Underneath, each one is +backed by tokens carrying their exact offsets into the original string, +so you can always get back to the text a field came from: + +.. doctest:: + + >>> name = parse("Juan de la Vega") + >>> for tok in name.tokens: + ... print(f"{tok.text!r:8} {tok.role.value:8} {tuple(tok.span)}") + 'Juan' given (0, 4) + 'de' family (5, 7) + 'la' family (8, 10) + 'Vega' family (11, 15) + >>> tok = name.tokens[1] + >>> name.original[tok.span.start:tok.span.end] + 'de' + +``tokens_for()`` narrows that to a single role: + +.. doctest:: + + >>> from nameparser import Role + >>> [t.text for t in name.tokens_for(Role.FAMILY)] + ['de', 'la', 'Vega'] + Correcting a parse -------------------- From 9be1df78fdf828b27666c2b7a7c36b97625e64f3 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 12:18:43 -0700 Subject: [PATCH 227/275] docs: show what the Lexicon fields actually do Three vocabulary features were explained in prose and never demonstrated, so the reader saw the principle without its instance. suffix_acronyms_ambiguous had thirteen lines of abstract prose ("they narrow how an existing entry is read when it appears alone") and no code. The shipped ma/do case makes it immediate: "Jack Ma" keeps its family name, "Jack M.A." reads as a suffix. That example existed only in the release log. capitalization_exceptions showed a constructor call with no output, so the payoff was invisible. Now chained end to end -- "jane smith dds" capitalizing to "Jane Smith Dds" by default, "Jane Smith DDS" with the exception added. This doubles as the only example of capitalized()'s lexicon argument, which appeared nowhere. Writing that example exposed a trap in the surrounding prose: the suggested dataclasses.replace(lex, capitalization_exceptions=(("phd", "PhD"),)) REPLACES the default pairs rather than extending them, dropping ii/iii/iv/md -- a reader following it literally gets "John Smith Phd". The example uses tuple(default...) + (...) and says why. Lexicon.remove() and | were both named in prose and never shown; add a line each. remove() uses "sir" rather than "dr", because "Dr." with a period is recognized structurally as an abbreviation and still parses as a title after removal -- which would have made a confusing example. Co-Authored-By: Claude Opus 4.8 --- docs/customize.rst | 52 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index 42f213ef..ed6f38ad 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -25,11 +25,48 @@ accepts a plain set of lowercase words, keyword by field name (``titles`` above; ``particles``, ``suffix_words``, and the rest work the same way) — see :doc:`modules` for the full field list. +Removing works the same way, and drops the word from recognition: + +.. doctest:: + + >>> lean = Lexicon.default().remove(titles={"sir"}) + >>> Parser(lexicon=lean).parse("Sir Robert Johns").title + '' + +Whole lexicons compose with ``|``, which unions field by field — handy +for keeping a shared house vocabulary separate from a per-source one +and combining them at parser construction: + +.. doctest:: + + >>> house = Lexicon.empty().add(titles={"dean"}) + >>> per_source = Lexicon.empty().add(titles={"provost"}) + >>> sorted((house | per_source).titles) + ['dean', 'provost'] + ``capitalization_exceptions`` is the one pair-valued field — each entry maps a lowercase key to its exact-cased replacement (``"phd"`` → ``"PhD"``), so it isn't a fit for ``add()``/``remove()``. Change it with -``dataclasses.replace()`` instead: ``dataclasses.replace(lex, -capitalization_exceptions=(("phd", "PhD"),))``. +``dataclasses.replace()`` instead, and pass the result to +``capitalized()``: + +.. doctest:: + + >>> import dataclasses + >>> from nameparser import parse + >>> str(parse("jane smith dds").capitalized()) + 'Jane Smith Dds' + >>> default = Lexicon.default() + >>> lex = dataclasses.replace( + ... default, + ... capitalization_exceptions=tuple(default.capitalization_exceptions) + ... + (("dds", "DDS"),)) + >>> str(parse("jane smith dds").capitalized(lex)) + 'Jane Smith DDS' + +Note the ``tuple(...) + ...``: assigning a bare ``(("dds", "DDS"),)`` +would *replace* the default exceptions rather than extend them, so +``"phd"`` and the rest would stop being fixed. The key is matched against the token with punctuation normalized away, not against the raw text, so one ``"phd"`` entry covers ``"phd"``, @@ -50,6 +87,17 @@ the first place: "Dean" is also a common given name, and a default that swallowed it as a title would misparse "Dean Martin" for everyone.) +``ma`` is a shipped example. It is both a credential and a common +surname, so it is listed in ``suffix_acronyms_ambiguous`` and counts as +a suffix only when written with periods: + +.. doctest:: + + >>> parse("Jack Ma").family + 'Ma' + >>> parse("Jack M.A.").suffix + 'M.A.' + Behavior: Policy ----------------- From 3edfe4e49772cade7431da499566d079dc6e08b2 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 12:21:24 -0700 Subject: [PATCH 228/275] docs: move stranded recipes out of the release log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Actionable knowledge had accumulated in release_log.rst, which readers consult once at upgrade time and never search for how-to. Relocate it to the task-oriented pages. migrate.rst "Before you upgrade" gains: - The pre-1.3.0 pickle constraint, called out as an ordering hazard. A Constants blob written by 1.2.x must be re-pickled ON 1.3/1.4; after upgrading, the code that could rewrite it is gone. Every other migration step is fixable afterwards, which is why this one cannot live in a changelog. - A replacement table for the warned removals (decode() for bytes, set(manager), discard(), Constants()/CONSTANTS.copy(), plain attribute assignment, .get()). The section previously assumed a clean -W error::DeprecationWarning run and offered nothing to the readers who most need it -- those who didn't get one. - A pointer to tools/differential/ for diffing 1.4 vs 2.0 over the reader's own corpus, noting it is source-repo tooling, not installed. migrate.rst also inlines the casefold example (STRASSE/Straße) and the two behavior-diff shapes worth grepping fixtures for, both of which previously redirected to the release log. locales.rst gains "What works without a pack". The page listed two packs, so a reader with Arabic or Hebrew data concluded there was no support -- while the five-script default vocabulary from #269 handles it out of the box and was documented nowhere but the changelog. Also records which entries are deliberately held back and how to add them. Writing that last example surfaced the reason to document it: adding an honorific to given_name_titles ALONE silently does nothing, because the field is a marker over titles rather than its own vocabulary. The doctest caught it; the text now says so. Co-Authored-By: Claude Opus 4.8 --- docs/locales.rst | 48 +++++++++++++++++++++++++++++++ docs/migrate.rst | 74 +++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 115 insertions(+), 7 deletions(-) diff --git a/docs/locales.rst b/docs/locales.rst index 97f2fdc2..6a91ef21 100644 --- a/docs/locales.rst +++ b/docs/locales.rst @@ -7,6 +7,54 @@ patronymics, Turkic patronymic markers, and so on. Packs apply only when you ask for one by name, and every pack makes the same promise: a pack never changes a name it doesn't declare. +What works without a pack +-------------------------- + +Most international names need no pack at all. The default vocabulary +covers five scripts — Latin, Cyrillic, Greek, Arabic and Hebrew, plus +Devanagari titles — so honorifics, conjunctions and name particles in +those scripts are recognized out of the box: + +.. doctest:: + + >>> from nameparser import parse + >>> name = parse("الشيخ محمد بن سلمان") + >>> name.title, name.given, name.family + ('الشيخ', 'محمد', 'بن سلمان') + >>> parse("عبد الرحمن محمد").given + 'عبد الرحمن' + +Native-script entries are safe to enable by default precisely because +they cannot collide with Latin-script names. That rules out the +reverse: transliterations like ``sri``/``shri`` are *not* in the +default vocabulary, because they are also ordinary given names in +Latin script. The same conservatism holds back a handful of native +entries that collide within their own script — bare ``سيد`` and +``شيخ`` (common given names), Hebrew ``רב`` (an ordinary word), the +Arabic ``د.`` abbreviation (it would swallow initials). + +If your data is homogeneous enough that a collision can't occur, the +reasoning behind the default doesn't apply to you — add the entry: + +.. doctest:: + + >>> from nameparser import Lexicon, Parser + >>> lex = Lexicon.default().add( + ... titles={"سيد"}, given_name_titles={"سيد"}) + >>> name = Parser(lexicon=lex).parse("سيد محمد") + >>> name.title, name.given + ('سيد', 'محمد') + +Both fields, because ``given_name_titles`` is a marker over ``titles`` +rather than a separate vocabulary: ``titles`` makes the word a title at +all, and listing it in ``given_name_titles`` says the honorific +precedes the *given* name — as Arabic ones do — so the word after it +isn't read as a family name. Adding it to ``given_name_titles`` alone +has no effect. + +A pack is for something different: a *structural* rule, like reordering +a patronymic, that vocabulary alone can't express. + Using a pack ------------ diff --git a/docs/migrate.rst b/docs/migrate.rst index 980d999e..206094f1 100644 --- a/docs/migrate.rst +++ b/docs/migrate.rst @@ -46,6 +46,46 @@ where 1.x returned ``True``. That one *did* warn on 1.4, but nothing will tell you on 2.0. If you compare names anywhere, grep for ``==`` before upgrading and move to ``matches()`` — see `Comparison`_. +One step has to happen *before* you upgrade, because the fix is only +available on the version you're leaving: a ``Constants`` pickle written +by nameparser 1.2.x or earlier must be re-pickled under 1.3 or 1.4. +2.0 refuses to load one, and by then the code that could rewrite it is +gone. + +If your suite did *not* run clean, these are the replacements for the +warned removals: + +.. list-table:: + :header-rows: 1 + :widths: 45 55 + + * - 1.4 spelling + - 2.0 replacement + * - ``HumanName(b"...")``, ``manager.add(b"...")`` + - Decode first: ``HumanName(raw.decode("utf-8"))`` + * - ``SetManager.add_with_encoding(b"...")`` + - ``add()``, on already-decoded ``str`` + * - ``manager()`` (calling a set manager) + - ``set(manager)``, or iterate it directly + * - ``manager.remove(missing)`` (was tolerant) + - ``discard()`` to ignore missing; ``remove()`` now raises + ``KeyError`` like ``set.remove`` + * - ``HumanName(..., constants=None)`` + - ``Constants()`` for library defaults, or ``CONSTANTS.copy()`` + for a private snapshot of the current shared config + * - ``name['first'] = value`` + - ``name.first = value`` + * - ``CONSTANTS.regexes.typo`` (returned ``None``) + - ``.get("typo")`` for intentional soft access; attribute access + now raises ``AttributeError`` naming the known keys + * - ``CONSTANTS.empty_attribute_default`` + - Gone; empty fields are always ``''`` + +Finally, if you want to see the difference on your own data rather than +ours, ``tools/differential/`` in the source repository diffs 1.4 and +2.0 parses over a corpus of names you supply. It is development +tooling, not part of the installed package — see its README. + Attribute map ------------- @@ -263,15 +303,35 @@ behavior: One behavior changed underneath both methods: components now fold with ``str.casefold()`` instead of ``str.lower()``, so more Unicode -case-pairs compare equal than did under 1.4 (see the 2.0.0 section of -:doc:`release_log` for the exact rule and examples). +case-pairs compare equal than did under 1.4. The change is strictly +more permissive — anything 1.4 matched still matches: + +.. doctest:: + + >>> parse("Anna STRASSE").matches("Anna Straße") + True Behavior changes ----------------- Beyond the API surface mapped above, a handful of parse *outputs* -differ between 1.4 and 2.0 for specific input shapes — comma-suffix -routing, maiden-marker detection, an ambiguous-acronym data change, and -one rendering difference under a custom suffix delimiter. These are -listed with their reasoning and test coverage in the 2.0.0 section of -:doc:`release_log`; they aren't repeated here. +differ between 1.4 and 2.0 for specific input shapes. The full list, +with reasoning, is in the 2.0.0 section of :doc:`release_log`. These +are the shapes worth grepping your own fixtures for, because a +recognized suffix or title now stays in its own field instead of +landing in ``first``/``last``: + +.. doctest:: + + >>> HumanName("Andrews, M.D.").last, HumanName("Andrews, M.D.").suffix + ('Andrews', 'M.D.') + >>> HumanName("Johnson PhD").first, HumanName("Johnson PhD").suffix + ('Johnson', 'PhD') + +Under 1.4 those read ``first="M.D."``/``last="Andrews"`` and +``first="Johnson"``/``last="PhD"`` respectively. Two more to check for: +a maiden marker now fills the ``maiden`` field rather than being folded +into ``middle``/``last`` (``"Jane Smith née Jones"``), and with a +custom suffix delimiter configured, a no-space delimiter group renders +whole (``"RN/CRNA"``) where 1.x split it (``"RN, CRNA"``) — the role +assignment is identical, only the rendered string differs. From d8feb3eb9d0f073beb23c810265d6bd5a3d82984 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 12:29:04 -0700 Subject: [PATCH 229/275] Reject given_name_titles entries that are not titles Lexicon guarded two of its three marker-over-base relationships: particles_ambiguous <= particles and suffix_acronyms_ambiguous <= suffix_acronyms both raised ValueError, while given_name_titles <= titles was unchecked. An entry present only in given_name_titles is never consulted by any rule, so the misconfiguration did nothing and said nothing -- the silent-degradation failure mode #256 was deprecated for. Found while writing the locales.rst example: the doctest failed because the honorific was added to given_name_titles alone. Fold all three checks into one _SUBSET_FIELDS loop; the messages are unchanged. v1 compatibility is preserved by intersecting at the shim boundary. v1 treats titles and first_name_titles as independent sets and only consults first_name_titles for a word already recognized as a title, so an entry missing from titles was unreachable in v1 too -- dropping it changes no v1 behavior while satisfying the invariant. Verified against a live 1.4.0 with a word absent from the defaults: first_name_titles alone yields no title on both 1.4 and the 2.0 facade; both fields yield the title on both. Without this the guard broke six compat tests, since Constants(titles=[...]) replaces titles wholesale while first_name_titles keeps its defaults. (An earlier check of the same question used "sheikh", which is already in the default TITLES, so it proved nothing -- the retest with a novel word reversed the conclusion.) Docs: customize.rst states the subset rule for all three pairs, and its remove() example moves from "sir" to "professor" -- "sir" is itself a given_name_title, so removing it now correctly raises. Co-Authored-By: Claude Opus 4.8 --- docs/customize.rst | 11 +++++++++-- docs/locales.rst | 4 ++-- nameparser/_config_shim.py | 12 ++++++++++-- nameparser/_lexicon.py | 32 +++++++++++++++++++------------- tests/v2/test_lexicon.py | 16 ++++++++++++++++ 5 files changed, 56 insertions(+), 19 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index ed6f38ad..1bd1fa7a 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -29,10 +29,17 @@ Removing works the same way, and drops the word from recognition: .. doctest:: - >>> lean = Lexicon.default().remove(titles={"sir"}) - >>> Parser(lexicon=lean).parse("Sir Robert Johns").title + >>> lean = Lexicon.default().remove(titles={"professor"}) + >>> Parser(lexicon=lean).parse("Professor Robert Johns").title '' +A few fields mark a subset of another — ``given_name_titles`` over +``titles``, ``particles_ambiguous`` over ``particles``, +``suffix_acronyms_ambiguous`` over ``suffix_acronyms``. Entries must +appear in the base field too, so add to both and remove from the marker +first; anything else raises ``ValueError`` naming the orphans rather +than leaving a marker entry that no rule will ever consult. + Whole lexicons compose with ``|``, which unions field by field — handy for keeping a shared house vocabulary separate from a per-source one and combining them at parser construction: diff --git a/docs/locales.rst b/docs/locales.rst index 6a91ef21..d246b768 100644 --- a/docs/locales.rst +++ b/docs/locales.rst @@ -49,8 +49,8 @@ Both fields, because ``given_name_titles`` is a marker over ``titles`` rather than a separate vocabulary: ``titles`` makes the word a title at all, and listing it in ``given_name_titles`` says the honorific precedes the *given* name — as Arabic ones do — so the word after it -isn't read as a family name. Adding it to ``given_name_titles`` alone -has no effect. +isn't read as a family name. Listing it in ``given_name_titles`` alone +raises ``ValueError`` rather than quietly doing nothing. A pack is for something different: a *structural* rule, like reordering a patronymic, that vocabulary alone can't express. diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index e9cd5b36..92ebafc6 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -933,11 +933,19 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: """ from nameparser.config.maiden_markers import MAIDEN_MARKERS acronyms = frozenset(self.suffix_acronyms) + titles = frozenset(self.titles) # keep in sync with _lexicon._default_lexicon() (pinned by the # default-Constants equality test in tests/v2/test_config_shim.py) lexicon = Lexicon( - titles=frozenset(self.titles), - given_name_titles=frozenset(self.first_name_titles), + titles=titles, + # intersect, same reasoning as suffix_acronyms_ambiguous + # below: v1 treats titles and first_name_titles as + # independent sets, and only consults first_name_titles for + # a word already recognized as a title -- so an entry + # missing from titles was never reachable in v1 either, and + # dropping it preserves v1 behavior exactly while + # satisfying Lexicon's subset invariant + given_name_titles=frozenset(self.first_name_titles) & titles, suffix_acronyms=acronyms, suffix_words=frozenset(self.suffix_not_acronyms), # intersect: Lexicon enforces ambiguous <= acronyms; v1 diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index efff1bbd..0a8a6ae2 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -23,6 +23,18 @@ "conjunctions", "bound_given_names", "maiden_markers", ) +#: (marker, base) pairs: each marker field narrows how entries of its +#: base vocabulary are read, and carries no vocabulary of its own. An +#: entry present in the marker but absent from the base is never +#: consulted by any rule, so it is rejected rather than silently +#: ignored -- a no-op configuration is the failure mode these fields +#: are most likely to be misused into. +_SUBSET_FIELDS = ( + ("particles_ambiguous", "particles"), + ("suffix_acronyms_ambiguous", "suffix_acronyms"), + ("given_name_titles", "titles"), +) + def _normalize(word: str) -> str: """Lowercase, strip whitespace and EDGE periods -- v1's lc() @@ -215,19 +227,13 @@ def __post_init__(self) -> None: canonical = _normpairs(self.capitalization_exceptions) object.__setattr__(self, "capitalization_exceptions", canonical) object.__setattr__(self, "_cap_map", MappingProxyType(dict(canonical))) - if not self.particles_ambiguous <= self.particles: - extra = ", ".join(sorted(self.particles_ambiguous - self.particles)) - raise ValueError( - f"particles_ambiguous must be a subset of particles; " - f"not in particles: {extra}" - ) - if not self.suffix_acronyms_ambiguous <= self.suffix_acronyms: - extra = ", ".join(sorted( - self.suffix_acronyms_ambiguous - self.suffix_acronyms)) - raise ValueError( - f"suffix_acronyms_ambiguous must be a subset of " - f"suffix_acronyms; not in suffix_acronyms: {extra}" - ) + for marker, base in _SUBSET_FIELDS: + orphans = getattr(self, marker) - getattr(self, base) + if orphans: + raise ValueError( + f"{marker} must be a subset of {base}; " + f"not in {base}: {', '.join(sorted(orphans))}" + ) # -- constructors ---------------------------------------------------- diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 79bcb053..250a3a71 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -227,3 +227,19 @@ def test_suffix_ambiguous_must_be_subset_of_acronyms() -> None: # depends on the membership tests agreeing about it with pytest.raises(ValueError, match="subset"): Lexicon(suffix_acronyms_ambiguous=frozenset({"ma"})) + + +def test_given_name_titles_must_be_subset_of_titles() -> None: + # third marker-over-base pair, same rule: given_name_titles marks + # which TITLES precede the given name rather than the family name, + # so an entry missing from titles is never consulted -- a silent + # no-op is exactly the failure mode the other two guards prevent + with pytest.raises(ValueError, match="subset"): + Lexicon(given_name_titles=frozenset({"sheikh"})) + + +def test_remove_orphaning_given_name_titles_raises() -> None: + lex = Lexicon(titles=frozenset({"sheikh"}), + given_name_titles=frozenset({"sheikh"})) + with pytest.raises(ValueError, match="subset"): + lex.remove(titles={"sheikh"}) From 418a210436e874219b9d49da562249b792952b0d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 12:49:38 -0700 Subject: [PATCH 230/275] Tell the caller how to fix a subset violation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The message stated the constraint and left the remedy to be inferred: "given_name_titles must be a subset of titles; not in titles: سيد". Since adding to both fields is the only thing the caller can have meant, the error should say so. given_name_titles marks a subset of titles; not in titles: سيد. Add them to titles as well — add(titles={...}, given_name_titles={...}) Considered auto-adding to the base field instead, and rejected it for 2.0: a typo in a marker field would silently mint new vocabulary (add(given_name_titles={"profesor"}) inventing a title), the same inference would have to extend to the other two marker pairs and to remove()'s cascade, and the direction is one-way -- strict now can relax in 2.1, loose now can never tighten. The message carries most of the ergonomic benefit at none of that cost. Parametrized over all three marker pairs, via constructor lambdas rather than **kwargs: Lexicon's signature is heterogeneous (capitalization_exceptions is pair-valued), so dynamic kwargs fail mypy, and tests/ is type-checked here. Co-Authored-By: Claude Opus 4.8 --- nameparser/_lexicon.py | 6 ++++-- tests/v2/test_lexicon.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 0a8a6ae2..75000e60 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -231,8 +231,10 @@ def __post_init__(self) -> None: orphans = getattr(self, marker) - getattr(self, base) if orphans: raise ValueError( - f"{marker} must be a subset of {base}; " - f"not in {base}: {', '.join(sorted(orphans))}" + f"{marker} marks a subset of {base}; " + f"not in {base}: {', '.join(sorted(orphans))}. " + f"Add them to {base} as well — " + f"add({base}={{...}}, {marker}={{...}})" ) # -- constructors ---------------------------------------------------- diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 250a3a71..5e3b5fa3 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -1,4 +1,5 @@ import dataclasses +from collections.abc import Callable import pytest @@ -238,6 +239,21 @@ def test_given_name_titles_must_be_subset_of_titles() -> None: Lexicon(given_name_titles=frozenset({"sheikh"})) +@pytest.mark.parametrize(("make", "base"), [ + (lambda w: Lexicon(given_name_titles=w), "titles"), + (lambda w: Lexicon(particles_ambiguous=w), "particles"), + (lambda w: Lexicon(suffix_acronyms_ambiguous=w), "suffix_acronyms"), +], ids=["given_name_titles", "particles_ambiguous", "suffix_acronyms_ambiguous"]) +def test_subset_error_names_the_fix( + make: Callable[[frozenset[str]], Lexicon], base: str +) -> None: + # the constraint alone leaves the reader to infer the remedy; every + # marker field's error should name the base field to add to, since + # adding to both is the only thing the caller can have meant + with pytest.raises(ValueError, match=f"Add them to {base} as well"): + make(frozenset({"zzqnovel"})) + + def test_remove_orphaning_given_name_titles_raises() -> None: lex = Lexicon(titles=frozenset({"sheikh"}), given_name_titles=frozenset({"sheikh"})) From 182249a05f7e29ccf846bc42d4655fab6c126309 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 13:08:06 -0700 Subject: [PATCH 231/275] Enforce the bound-given/never-given-particle invariant on Lexicon nameparser/config/prefixes.py asserts two invariants on its own data: NON_FIRST_NAME_PREFIXES stays a subset of PREFIXES, and stays disjoint from BOUND_FIRST_NAMES. Only the first had a 2.0 counterpart -- particles_ambiguous <= particles is checked on every Lexicon, while the disjointness rule guarded nameparser's shipped vocabulary and nothing a caller supplied. In 2.0's complement model the second reads as bound_given_names & particles <= particles_ambiguous "a bound given name that is also a particle must be one that may start a given name". A particle declared never-to-start-a-given-name cannot also bind one; before this, the contradiction resolved by discarding the never-given declaration at leading position, with no signal. Note the import-time assert only protects the shipped data, and vanishes under python -O. This check runs on every Lexicon. v1 compatibility, as with given_name_titles: v1 has no runtime guard, so a caller can add a never-given prefix to bound_first_names, and 1.4 then lets the bound rule win -- "dos Santos Silva" parses first="dos Santos". The shim reproduces that by promoting such a word to may-be-given rather than raising on a config v1 accepted. Verified byte-identical to a live 1.4.0 for both the leading and non-leading positions, and pinned by test_snapshot_keeps_a_bound_never_given_prefix_parseable -- the hole had no coverage, since no existing test built that combination. Co-Authored-By: Claude Opus 4.8 --- nameparser/_config_shim.py | 19 ++++++++++++++----- nameparser/_lexicon.py | 13 +++++++++++++ tests/v2/test_config_shim.py | 18 +++++++++++++++++- tests/v2/test_lexicon.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 6 deletions(-) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index 92ebafc6..202fc15c 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -934,6 +934,8 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: from nameparser.config.maiden_markers import MAIDEN_MARKERS acronyms = frozenset(self.suffix_acronyms) titles = frozenset(self.titles) + particles = frozenset(self.prefixes) + bound = frozenset(self.bound_first_names) # keep in sync with _lexicon._default_lexicon() (pinned by the # default-Constants equality test in tests/v2/test_config_shim.py) lexicon = Lexicon( @@ -953,13 +955,20 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: # ambiguous entry lingers (the entry simply stops mattering) suffix_acronyms_ambiguous=frozenset( self.suffix_acronyms_ambiguous) & acronyms, - particles=frozenset(self.prefixes), + particles=particles, # complement translation: v1 marks the never-given subset; - # v2 marks the may-be-given subset - particles_ambiguous=frozenset(self.prefixes) - - frozenset(self.non_first_name_prefixes), + # v2 marks the may-be-given subset. The trailing union keeps + # a config v1 accepted: prefixes.py asserts its own data has + # no word in both non_first_name_prefixes and + # bound_first_names, but nothing stops a caller adding one at + # runtime, and v1 then lets the bound rule win (leading "dos + # Santos Silva" parses first="dos Santos"). Treating such a + # word as may-be-given reproduces that instead of raising. + particles_ambiguous=( + particles - frozenset(self.non_first_name_prefixes)) + | (bound & particles), conjunctions=frozenset(self.conjunctions), - bound_given_names=frozenset(self.bound_first_names), + bound_given_names=bound, # v1 Constants has no manager for these (#274 is 2.0 # behavior); the data module is the only source maiden_markers=frozenset(MAIDEN_MARKERS), diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 75000e60..6901f14d 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -236,6 +236,19 @@ def __post_init__(self) -> None: f"Add them to {base} as well — " f"add({base}={{...}}, {marker}={{...}})" ) + # The v2 form of prefixes.py's NON_FIRST_NAME_PREFIXES-disjoint- + # from-BOUND_FIRST_NAMES assertion. That module guards its own + # data at import; this guards vocabulary a caller supplies. + contradictory = ( + self.bound_given_names & self.particles) - self.particles_ambiguous + if contradictory: + raise ValueError( + f"bound_given_names entries that are also particles must be " + f"in particles_ambiguous; not in particles_ambiguous: " + f"{', '.join(sorted(contradictory))}. A particle that never " + f"starts a given name cannot also bind one — add them to " + f"particles_ambiguous, or drop them from bound_given_names" + ) # -- constructors ---------------------------------------------------- diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index ea5afad8..f9d6a098 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -6,7 +6,7 @@ import pytest -from nameparser import GIVEN_FIRST, Lexicon, PatronymicRule, Policy +from nameparser import GIVEN_FIRST, HumanName, Lexicon, PatronymicRule, Policy from nameparser._config_shim import ( CONSTANTS, Constants, SetManager, TupleManager, _DelimiterManager, _RegexesProxy, _cached_parser, @@ -335,6 +335,22 @@ def test_v14_constants_blob_unpickles_into_shim() -> None: assert loaded.string_format == "{title} {first} {middle} {last} {suffix} ({nickname})" +def test_snapshot_keeps_a_bound_never_given_prefix_parseable() -> None: + # prefixes.py asserts its own data keeps non_first_name_prefixes + # disjoint from bound_first_names, but nothing stops a v1 caller + # adding one at runtime, and 1.4 accepts it -- letting the bound + # rule win, so "dos Santos Silva" parses first="dos Santos". + # Lexicon rejects that combination, so the shim promotes such a word + # to may-be-given rather than raising on config v1 allowed. + c = Constants() + assert "dos" in c.non_first_name_prefixes # baseline: never-given + c.bound_first_names.add("dos") + lexicon, _, _ = c._snapshot() # must not raise + assert "dos" in lexicon.particles_ambiguous + name = HumanName("dos Santos Silva", constants=c) + assert (name.first, name.last) == ("dos Santos", "Silva") + + def test_snapshot_field_translation() -> None: c = Constants() lexicon, policy, defaults = c._snapshot() diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 5e3b5fa3..9305f70c 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -254,6 +254,35 @@ def test_subset_error_names_the_fix( make(frozenset({"zzqnovel"})) +def test_bound_given_name_that_is_a_particle_must_be_ambiguous() -> None: + # nameparser/config/prefixes.py asserts this on its own data: + # NON_FIRST_NAME_PREFIXES stays disjoint from BOUND_FIRST_NAMES. In + # 2.0's complement model that reads as bound_given_names & particles + # <= particles_ambiguous. A particle declared never-to-start-a-given- + # name cannot simultaneously bind one; without the check, one of the + # two contradictory declarations is silently discarded. + with pytest.raises(ValueError, match="particles_ambiguous"): + Lexicon(particles=frozenset({"dos"}), + particles_ambiguous=frozenset(), + bound_given_names=frozenset({"dos"})) + + +def test_bound_given_name_may_overlap_an_ambiguous_particle() -> None: + # the shipped case: "abu" is both, and legitimately so + lex = Lexicon(particles=frozenset({"abu"}), + particles_ambiguous=frozenset({"abu"}), + bound_given_names=frozenset({"abu"})) + assert "abu" in lex.bound_given_names + # and a bound name that is not a particle at all is unconstrained + assert "abdul" in Lexicon(bound_given_names=frozenset({"abdul"}) + ).bound_given_names + + +def test_default_lexicon_satisfies_the_bound_particle_invariant() -> None: + d = Lexicon.default() + assert not (d.bound_given_names & d.particles) - d.particles_ambiguous + + def test_remove_orphaning_given_name_titles_raises() -> None: lex = Lexicon(titles=frozenset({"sheikh"}), given_name_titles=frozenset({"sheikh"})) From 022fd07c91f3ce6eb256469744ae2594ab67aa6c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 13:28:09 -0700 Subject: [PATCH 232/275] Guard the remaining config data invariants at import time prefixes.py asserts its own invariants at import so a bad edit fails immediately rather than drifting until a test happens to catch it. The other data modules had no such guard, and two of them had already drifted. Data fixes: - 'esq' was in both SUFFIX_ACRONYMS and SUFFIX_NOT_ACRONYMS. Those are the two halves of one distinction, not overlapping categories. _classify checks suffix_words first, so the acronym membership was unreachable -- removing it changes 0 of the 486 differential-corpus parses. Present in v1's data too; inherited, not introduced. - TITLES contained 'actor ' and 'television ' with trailing spaces, so "'actor' in TITLES" was False. Parsing was unaffected because both Lexicon and v1's SetManager normalize on ingest, which is exactly what let the typo survive. Also inherited from v1. Assertions added: - suffixes.py: SUFFIX_ACRONYMS_AMBIGUOUS <= SUFFIX_ACRONYMS, and SUFFIX_ACRONYMS disjoint from SUFFIX_NOT_ACRONYMS. - titles.py: FIRST_NAME_TITLES <= TITLES. True by construction today (TITLES = FIRST_NAME_TITLES | {...}), so it pins that against a future edit making TITLES standalone. - All six vocabulary modules plus CAPITALIZATION_EXCEPTIONS' keys: entries stored lowercase and whitespace-free. This is the check that found the TITLES typo. Deliberately NOT asserted: cross-category disjointness. TITLES and SUFFIX_ACRONYMS share 15 entries (lt, md, phd, cpt, ...) and every one is a legitimate homograph resolved by position, as are TITLES/PREFIXES and SUFFIX_ACRONYMS/PREFIXES. The predicates use .strip().lower() rather than _lexicon._normalize: config/ is the lower layer and _lexicon imports from it, so importing back would invert the layering. Each guard was verified to fire on a mutated copy of its own data -- an assertion that cannot fail is worth nothing. Note `assert` is stripped under `python -O`. These protect the shipped data during development; Lexicon re-checks the relationships at construction, which is what protects a caller's own vocabulary. Co-Authored-By: Claude Opus 4.8 --- nameparser/config/bound_first_names.py | 7 +++++++ nameparser/config/capitalization.py | 6 ++++++ nameparser/config/conjunctions.py | 7 +++++++ nameparser/config/maiden_markers.py | 7 +++++++ nameparser/config/prefixes.py | 5 +++++ nameparser/config/suffixes.py | 18 +++++++++++++++++- nameparser/config/titles.py | 20 ++++++++++++++++++-- 7 files changed, 67 insertions(+), 3 deletions(-) diff --git a/nameparser/config/bound_first_names.py b/nameparser/config/bound_first_names.py index d0602417..b2979f62 100644 --- a/nameparser/config/bound_first_names.py +++ b/nameparser/config/bound_first_names.py @@ -21,3 +21,10 @@ 'أم', # "umm" (mother of), hamza spelling 'ام', # "umm", hamza-less spelling } + + +# See prefixes.py: entries are looked up in normalized form, so a stray +# capital or trailing space makes one unreachable by direct membership +# test even though the parser's own ingest normalizes it away. +assert all(w == w.strip().lower() for w in BOUND_FIRST_NAMES), \ + "BOUND_FIRST_NAMES entries must be stored lowercase and whitespace-free" diff --git a/nameparser/config/capitalization.py b/nameparser/config/capitalization.py index fcd53763..57c442ab 100644 --- a/nameparser/config/capitalization.py +++ b/nameparser/config/capitalization.py @@ -8,3 +8,9 @@ """ Any pieces that are not capitalized by capitalizing the first letter. """ + + +# See prefixes.py: keys are looked up in normalized form. Only the keys -- +# the values are the exact-cased replacements, so they are cased on purpose. +assert all(k == k.strip().lower() for k in CAPITALIZATION_EXCEPTIONS), \ + "CAPITALIZATION_EXCEPTIONS keys must be stored lowercase and whitespace-free" diff --git a/nameparser/config/conjunctions.py b/nameparser/config/conjunctions.py index 52f01b34..b6d045ca 100644 --- a/nameparser/config/conjunctions.py +++ b/nameparser/config/conjunctions.py @@ -26,3 +26,10 @@ "of" and "the" are also include to facilitate joining multiple titles, e.g. "President of the United States". """ + + +# See prefixes.py: entries are looked up in normalized form, so a stray +# capital or trailing space makes one unreachable by direct membership +# test even though the parser's own ingest normalizes it away. +assert all(w == w.strip().lower() for w in CONJUNCTIONS), \ + "CONJUNCTIONS entries must be stored lowercase and whitespace-free" diff --git a/nameparser/config/maiden_markers.py b/nameparser/config/maiden_markers.py index 637028b1..8f8719a7 100644 --- a/nameparser/config/maiden_markers.py +++ b/nameparser/config/maiden_markers.py @@ -35,3 +35,10 @@ abbreviation "f." (collides with the initial "F." — only the full participles are safe). """ + + +# See prefixes.py: entries are looked up in normalized form, so a stray +# capital or trailing space makes one unreachable by direct membership +# test even though the parser's own ingest normalizes it away. +assert all(w == w.strip().lower() for w in MAIDEN_MARKERS), \ + "MAIDEN_MARKERS entries must be stored lowercase and whitespace-free" diff --git a/nameparser/config/prefixes.py b/nameparser/config/prefixes.py index efa92bac..7fa9caf0 100644 --- a/nameparser/config/prefixes.py +++ b/nameparser/config/prefixes.py @@ -126,3 +126,8 @@ "NON_FIRST_NAME_PREFIXES must stay a subset of PREFIXES" assert not (NON_FIRST_NAME_PREFIXES & BOUND_FIRST_NAMES), \ "NON_FIRST_NAME_PREFIXES must stay disjoint from BOUND_FIRST_NAMES" +# Entries are looked up in normalized form; a stray capital or trailing +# space makes one unreachable by direct membership test even though the +# parser's own ingest normalizes it away. +assert all(w == w.strip().lower() for w in PREFIXES), \ + "PREFIXES entries must be stored lowercase and whitespace-free" diff --git a/nameparser/config/suffixes.py b/nameparser/config/suffixes.py index 3582df0d..448437cd 100644 --- a/nameparser/config/suffixes.py +++ b/nameparser/config/suffixes.py @@ -391,7 +391,6 @@ 'emt-p', 'enp', 'erd', - 'esq', 'evp', 'faafp', 'faan', @@ -710,3 +709,20 @@ when matching against these pieces. """ + + +# Guard the invariants the docstrings above promise, so a future edit that +# breaks them fails at import time instead of silently drifting until a test +# happens to catch it (same rationale as prefixes.py). Note `assert` is +# stripped under `python -O`; Lexicon re-checks the relationships at +# construction, which is what protects a caller's own vocabulary. +assert SUFFIX_ACRONYMS_AMBIGUOUS <= SUFFIX_ACRONYMS, \ + "SUFFIX_ACRONYMS_AMBIGUOUS must stay a subset of SUFFIX_ACRONYMS" +assert not (SUFFIX_ACRONYMS & SUFFIX_NOT_ACRONYMS), \ + "a suffix is an acronym or a word, not both: " \ + f"{sorted(SUFFIX_ACRONYMS & SUFFIX_NOT_ACRONYMS)}" +# Entries are looked up in normalized form, so a stray capital or a +# trailing space makes an entry unreachable by direct membership test even +# though the parser's own ingest would paper over it. +assert all(w == w.strip().lower() for w in SUFFIX_ACRONYMS | SUFFIX_NOT_ACRONYMS), \ + "suffix entries must be stored lowercase and whitespace-free" diff --git a/nameparser/config/titles.py b/nameparser/config/titles.py index 0f1d9c86..c5469c56 100644 --- a/nameparser/config/titles.py +++ b/nameparser/config/titles.py @@ -89,7 +89,7 @@ 'academic', 'acolyte', 'activist', - 'actor ', + 'actor', 'actress', 'adept', 'adjutant', @@ -664,7 +664,7 @@ 'teacher', 'technical', 'technologist', - 'television ', + 'television', 'tenor', 'theater', 'theatre', @@ -770,3 +770,19 @@ 'श्रीमती', # Shrimati (Mrs.) 'डॉ', # Dr. abbreviation } + + +# Guard the invariants at import time, so a bad edit fails here instead of +# drifting silently until a test happens to catch it (see prefixes.py). +# The subset rule holds by construction today -- TITLES is defined as +# FIRST_NAME_TITLES | {...} -- so this pins it against a future edit that +# makes TITLES a standalone set. Lexicon enforces the same rule on +# caller-supplied vocabulary; `assert` is stripped under `python -O`. +assert FIRST_NAME_TITLES <= TITLES, \ + "FIRST_NAME_TITLES must stay a subset of TITLES" +# Entries are looked up in normalized form, so a stray capital or a +# trailing space makes an entry unreachable by direct membership test +# ('actor' in TITLES was False) even though the parser's own ingest +# normalizes and papers over it. TITLES covers FIRST_NAME_TITLES. +assert all(w == w.strip().lower() for w in TITLES), \ + "TITLES entries must be stored lowercase and whitespace-free" From 081a17c5f7a022eaad9a730519b0eb333556fd65 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 13:36:16 -0700 Subject: [PATCH 233/275] docs: record the vocabulary data corrections in the release log esq and the TITLES trailing spaces do not change any parse, so the entry says so plainly -- but membership tests against the exported constants do change, and "'actor' in TITLES" flipping from False to True is the kind of thing someone's code could depend on either way. Co-Authored-By: Claude Opus 4.8 --- docs/release_log.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release_log.rst b/docs/release_log.rst index 13799793..df172560 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -59,6 +59,7 @@ Release Log - Change the 2.0 API's default ``render()``/``str()`` spec to show every non-empty field: ``'{title} {given} "{nickname}" {middle} {family} ({maiden}) {suffix}'``. The quoted nickname round-trips exactly; the parenthesized maiden re-parses as a nickname, a deliberate choice of presentation over lossless round-trip -- use ``née {maiden}`` in a custom spec if you need it to survive a reparse. ``HumanName`` keeps v1's own ``string_format`` default, unchanged - Change delimiter-overlap precedence in the 2.0 API: a pair listed in ``Policy.maiden_delimiters`` is dropped from the effective nickname set, so ``Policy(maiden_delimiters={("(", ")")})`` alone routes parenthesized content to ``maiden``. The default nickname set is exported as ``DEFAULT_NICKNAME_DELIMITERS``. ``HumanName`` keeps v1's nickname-wins precedence, so no existing behavior changes - Change suffix-delimiter rendering when a custom suffix delimiter is configured: for ``suffix_delimiter="/"`` and ``"John Smith, RN/CRNA"``, 1.x split the token and rendered ``suffix="RN, CRNA"`` where 2.0 keeps it whole as ``"RN/CRNA"``. Role assignment is unchanged; only rendering differs + - Correct two errors in the shipped vocabulary data that 1.x carried: ``esq`` was listed as both a suffix acronym and a suffix word (it is a word), and ``actor`` and ``television`` had trailing spaces in ``TITLES``, so ``"actor" in TITLES`` was ``False``. **Parse output is unchanged** in both cases -- the parser normalizes entries on ingest and tests suffix words first, which is exactly what let the errors go unnoticed -- but code that tests membership against the exported constants directly will see corrected results. The data modules now assert these invariants at import time, alongside the ones ``prefixes.py`` already checked **International name support** From d0f887e8c8831c6c7d7f3d80af94610ead6945bf Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 13:54:10 -0700 Subject: [PATCH 234/275] docs: restore two warnings and three behaviors the v1 docs carried Comparing v1.4's docs against 2.0's found content that was lost rather than superseded. These are the corrections and the surprises. locales.rst promised "a pack never changes a name it doesn't declare", which reads as reassurance about names from other traditions. A pack declares a SHAPE, not a language: parser_for(locales.RU) reads "David Michael Abramovich" as given=Michael, family=David. v1 warned about this in bold; the rewrite replaced the warning with the implementer's view, in which the pack is behaving correctly. Restore the warning with the worked case, and make the promise sentence say what "declare" covers. customize.rst gains the wholesale-clear recipe. v1's one-liner (titles.clear()) is now a two-part operation, because emptying titles alone orphans given_name_titles and raises. It also over-promised: an empty vocabulary does not switch titles off, since a leading period-abbreviation is structural. usage.rst gains three behaviors, all verified intact in 2.0 and documented nowhere: - delimited content that is a known suffix, or ends in a period, reads as a suffix, not a nickname ("Andrew Perkins (MBA)"); and the exception to that, an ambiguous acronym standing alone, which keeps the nickname reading ("JEFFREY (JD) BRICKEN"). customize.rst teaches suffix_acronyms_ambiguous only via the periods rule (Jack Ma / M.A.), a different disambiguator, so this was unpredictable from the docs. - leading period-abbreviation titles, with the three bounds that keep them from swallowing ordinary names. This is what makes unfamiliar ranks work with no configuration, and it explains the customize.rst caveat above. - bool(name) to distinguish "nothing parsed" from "parsed to something". usage.rst said parsing never raises and stopped there. Co-Authored-By: Claude Opus 4.8 --- docs/customize.rst | 24 +++++++++++++++++ docs/locales.rst | 26 ++++++++++++++++++- docs/usage.rst | 65 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 113 insertions(+), 2 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index 1bd1fa7a..512dd220 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -40,6 +40,30 @@ appear in the base field too, so add to both and remove from the marker first; anything else raises ``ValueError`` naming the orphans rather than leaving a marker entry that no rule will ever consult. +That matters most when clearing a field wholesale. Emptying ``titles`` +alone orphans every ``given_name_titles`` entry, so both go together: + +.. doctest:: + + >>> d = Lexicon.default() + >>> lean = d.remove(titles=set(d.titles), + ... given_name_titles=set(d.given_name_titles)) + >>> Parser(lexicon=lean).parse("Hon Solo").given + 'Hon' + +Emptying the vocabulary does not switch titles off entirely, though. A +leading word that ends in a period is read as a title structurally, +without consulting ``titles`` at all — that is what lets unfamiliar +ranks and abbreviations work (see :ref:`abbreviated-titles`): + +.. doctest:: + + >>> bare = Parser(lexicon=Lexicon.empty()) + >>> bare.parse("Professor John Smith").title # vocabulary gone + '' + >>> bare.parse("Dr. John Smith").title # structural, stays + 'Dr.' + Whole lexicons compose with ``|``, which unions field by field — handy for keeping a shared house vocabulary separate from a per-source one and combining them at parser construction: diff --git a/docs/locales.rst b/docs/locales.rst index d246b768..8ab0b8d6 100644 --- a/docs/locales.rst +++ b/docs/locales.rst @@ -5,7 +5,10 @@ A locale pack is an opt-in bundle of policy — and, when a naming tradition needs it, vocabulary — for one specific pattern: East Slavic patronymics, Turkic patronymic markers, and so on. Packs apply only when you ask for one by name, and every pack makes the same promise: -a pack never changes a name it doesn't declare. +it never changes a name outside the shapes it declares. Read that +precisely — a shape is a pattern, not a language, so a pack can act on +a name from a tradition it was not written for. The warning under +`Using a pack`_ shows what that costs. What works without a pack -------------------------- @@ -116,6 +119,27 @@ Both shipped packs are policy-only — they carry no vocabulary of their own; see :doc:`concepts` for why that split (language vocabulary vs. behavior) is drawn where it is. +.. warning:: + + A pack declares a name *shape*, not a language, and it cannot tell + whose name it is looking at. Any surname that happens to end in a + patronymic suffix matches the East Slavic rule, including names + that are not East Slavic at all: + + .. doctest:: + + >>> ru = parser_for(locales.RU) + >>> name = ru.parse("David Michael Abramovich") + >>> name.given, name.family + ('Michael', 'David') + + The default parser reads that as given ``David``, family + ``Abramovich``. This is the trade the pack asks you to make, and it + is why packs are opt-in rather than automatic: enable one only for + data that is predominantly in the order it detects. If your input + mixes traditions, parse the subsets separately with different + parsers rather than enabling a pack over all of it. + Creating your own Locale ------------------------- diff --git a/docs/usage.rst b/docs/usage.rst index 85707462..e14312e0 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -20,7 +20,13 @@ A parsed name has seven fields: ``title``, ``given``, ``middle``, raises; unparseable input yields a :class:`~nameparser.ParsedName` with empty fields plus any ``ambiguities`` the parser noticed along the way (see `When the parser had to guess`_ below, and :doc:`concepts` for why -they exist). +they exist). A name with no fields set is falsy, which is how you tell +"nothing parsed" from "parsed to something": + +.. doctest:: + + >>> bool(parse("")), bool(parse(" ")), bool(parse("John")) + (False, False, True) Aggregate views ---------------- @@ -106,6 +112,63 @@ instead of relying on the default: >>> parse(text).maiden 'Jones' +Delimited content is not always a nickname. If what's inside is a known +suffix, or simply ends in a period, it is read as a suffix instead — +parenthesized credentials and retired ranks are far more common than +parenthesized nicknames that happen to be credentials: + +.. doctest:: + + >>> parse("Andrew Perkins (MBA)").suffix + 'MBA' + >>> parse("Andrew Perkins (Ret.)").suffix + 'Ret.' + +The exception to that exception is an *ambiguous* acronym — one that is +also a plausible name or set of initials. Standing alone inside +delimiters, it keeps the nickname reading: + +.. doctest:: + + >>> parse("JEFFREY (JD) BRICKEN").nickname + 'JD' + +``JD`` is in ``suffix_acronyms_ambiguous``; see :doc:`customize` for +what that field marks and how to add to it. + +.. _abbreviated-titles: + +Titles you didn't configure +----------------------------- + +A leading word that ends in a period is read as a title even when it is +in no vocabulary list, which is what lets unfamiliar ranks, honorifics +and abbreviations work without configuring anything: + +.. doctest:: + + >>> parse("Major. Dona Smith").title + 'Major.' + >>> parse("Foo. Xyz. John Smith").title # chains + 'Foo. Xyz.' + +The rule is bounded in three ways, so it doesn't swallow ordinary +names. Single initials are left alone, so are abbreviations with +interior periods, and the rule only applies to the leading run — the +same word after the given name is a middle name: + +.. doctest:: + + >>> parse("J. Smith").given + 'J.' + >>> parse("E.T. Jones").given + 'E.T.' + >>> parse("John Major. Smith").middle + 'Major.' + +Because this is structural rather than vocabulary-driven, emptying +``titles`` does not switch it off; see :doc:`customize`. + Comparing names ---------------- From b20e1cd2cdef89b054283c7ea4efc938e470db74 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 13:55:06 -0700 Subject: [PATCH 235/275] docs: explain how roles get assigned, and what input shapes work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit concepts.rst described the data model — string becomes tokens, tokens carry roles, fields are views — but never the decision procedure. v1's README did: a vocabulary layer claims words for what they are, wherever they sit, and a positional layer assigns whatever is left by where it sits. That framing is what customize.rst's Lexicon/Policy split rests on, so without it "which container does my setting go in?" is answerable only by consulting a table. It also carries three things v1 said and 2.0 had dropped: that an unrecognized word still gets a sensible role (the positional layer recognizes nothing), that title/suffix are really pre-nominal and post-nominal — "Dr." is a title before a name and a suffix after — and that parsing is deterministic, with no model and no training data, so a wrong parse is wrong reproducibly and can be fixed by configuration. usage.rst gains v1 README's "Supported Name Structures". Comma order was demonstrated once inside a matches() example and described once in a name_order table cell, so a reader could not learn that "Doe, John" works without stumbling onto it. Co-Authored-By: Claude Opus 4.8 --- docs/concepts.rst | 30 ++++++++++++++++++++++++++++++ docs/usage.rst | 22 ++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/docs/concepts.rst b/docs/concepts.rst index d5f5764d..a6ca4303 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -41,6 +41,36 @@ else — tokens, spans, the rest of the roles — carried over unchanged. ``str()`` renders the default view; nothing about calling it mutates the value you called it on. +Two layers decide the roles +---------------------------- + +Roles are not assigned in one pass. A vocabulary layer runs first and +claims words for what they *are*, wherever they sit: titles, particles, +conjunctions, recognized suffixes, and anything set off by nickname or +maiden delimiters. Titles chain, so ``"Asst. Vice Chancellor"`` is one +title; particles join forward, so ``de la`` attaches to ``Vega``. + +Whatever the vocabulary layer has not claimed is left to a positional +layer, which assigns purely by where a word sits: the first unclaimed +word is the given name, the last is the family name, and anything +between them is the middle name. ``name_order`` and an explicit comma +change what "first" and "last" mean here; nothing else does. + +This is the whole parser in two sentences, and it explains its +character. A word nameparser has never seen still gets a sensible role, +because the positional layer does not need to recognize anything. The +same word can play different parts in different places — ``Dr.`` is a +title before a name and a suffix after it, which is why the field names +``title`` and ``suffix`` are really "pre-nominal" and "post-nominal". +And nothing is statistical: there is no model and no training data, so +the same input always parses the same way, and a parse that is wrong is +wrong reproducibly, which is what makes it fixable by configuration. + +The split also tells you which container a setting belongs in, before +you look anything up: if you are teaching the parser a *word*, it goes +in the :class:`~nameparser.Lexicon`; if you are changing how unclaimed +words are *arranged*, it goes in the :class:`~nameparser.Policy`. + Three containers, three concerns -------------------------------- diff --git a/docs/usage.rst b/docs/usage.rst index e14312e0..4a0b1596 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -28,6 +28,28 @@ they exist). A name with no fields set is falsy, which is how you tell >>> bool(parse("")), bool(parse(" ")), bool(parse("John")) (False, False, True) +Input shapes +------------- + +Three arrangements are understood, and every piece of each is optional: + +1. ``Title First Middle Middle Last Suffix`` +2. ``Last, Title First Middle Middle[,] Suffix [, Suffix]`` +3. ``Title First Middle Last [,] Suffix [, Suffix]`` + +A comma before the given name is the signal for family-first order, so +the second form needs no configuration: + +.. doctest:: + + >>> parse("de la Vega, Juan Q. Xavier III").family + 'de la Vega' + >>> parse("Doe Jr., John").suffix + 'Jr.' + +For family-first input *without* a comma — common outside Europe — set +``name_order``; see :doc:`customize`. + Aggregate views ---------------- From b20059b5be1e13f219dc50a46bdea2811cdd2754 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 13:56:45 -0700 Subject: [PATCH 236/275] docs: show the customizations that only had table rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v1's customize.rst carried ~109 doctest lines; the 2.0 rewrite kept 31. Prose survived better than code did, leaving six capabilities described but never demonstrated. Each of these exists and works; only the example was missing. - bound_given_names had exactly one mention in all of 2.0's docs, a rename row in migrate.rst. That is the entire Arabic given-name feature, invisible to anyone not migrating from v1. Show the default, adding one, and emptying the set to switch it off; note the shipped set now includes Arabic-script spellings, not just transliterations. - particles_ambiguous gets the before/after that makes the flip concrete: "van Gogh" keeps given='van' because van may be a given name, "de Mesnil" has no given name at all because de may not. That second rule — a name starting with a never-given particle is entirely surname — was documented nowhere. - extra_suffix_delimiters, additive nickname delimiters, and the strip flags each get the unconfigured parse first. Seeing "Jane Smith, RN - CRNA" come back as given='RN', family='Jane Smith' is the reason a reader goes looking for the setting; the table row alone never conveyed that. The additive-delimiter example also warns about replacing the defaults, the same trap already flagged for capitalization_exceptions. - usage.rst gains v1's sorting recipe: family_base is the key you want for alphabetizing, since it drops the particles a phone book ignores. Co-Authored-By: Claude Opus 4.8 --- docs/customize.rst | 92 ++++++++++++++++++++++++++++++++++++++++++++++ docs/usage.rst | 10 +++++ 2 files changed, 102 insertions(+) diff --git a/docs/customize.rst b/docs/customize.rst index 512dd220..02bc91a4 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -129,6 +129,54 @@ a suffix only when written with periods: >>> parse("Jack M.A.").suffix 'M.A.' +``particles_ambiguous`` is the same idea for surname particles. A +particle listed there may also be a given name, so a name that starts +with one keeps its given name; a particle *not* listed there is never a +given name, so a name starting with it has no given name at all — the +whole thing is the surname: + +.. doctest:: + + >>> parse("van Gogh").given # 'van' may be a given name + 'van' + >>> parse("de Mesnil").given # 'de' may not + '' + >>> parse("de Mesnil").family + 'de Mesnil' + +If your data never uses ``Van`` as a given name, take it out of the +ambiguous set and leading ``van`` becomes part of the surname: + +.. doctest:: + + >>> lex = Lexicon.default().remove(particles_ambiguous={"van"}) + >>> Parser(lexicon=lex).parse("van Gogh").family + 'van Gogh' + +Bound given names +~~~~~~~~~~~~~~~~~~ + +``bound_given_names`` holds given-name prefixes that attach to the +following word to form one given name — ``abdul``, ``abu``, ``umm`` and +their Arabic-script spellings (``عبد``, ``أبو``, ``أم``) among them: + +.. doctest:: + + >>> parse("abdul salam ahmed salem").given + 'abdul salam' + +Add your own, or empty the set to switch the behavior off entirely: + +.. doctest:: + + >>> lex = Lexicon.default().add(bound_given_names={"mohamad"}) + >>> Parser(lexicon=lex).parse("mohamad salam ahmed salem").given + 'mohamad salam' + >>> d = Lexicon.default() + >>> off = d.remove(bound_given_names=set(d.bound_given_names)) + >>> Parser(lexicon=off).parse("abdul salam ahmed salem").given + 'abdul' + Behavior: Policy ----------------- @@ -226,6 +274,50 @@ family-then-given regardless of the configured order: >>> Parser(policy=policy).parse("Jane (Jones) Smith").maiden 'Jones' +To *add* a delimiter pair rather than reroute one, build on the +exported default — assigning a bare set replaces the built-in pairs +instead of extending them, the same trap as ``capitalization_exceptions``: + +.. doctest:: + + >>> from nameparser import DEFAULT_NICKNAME_DELIMITERS + >>> parse("Benjamin {Ben} Franklin").middle # not a pair by default + '{Ben}' + >>> policy = Policy( + ... nickname_delimiters=DEFAULT_NICKNAME_DELIMITERS | {("{", "}")}) + >>> Parser(policy=policy).parse("Benjamin {Ben} Franklin").nickname + 'Ben' + +``extra_suffix_delimiters`` handles sources that separate post-nominals +with something other than a comma. The default reading of such a name +is bad enough to be the reason you'd go looking: + +.. doctest:: + + >>> name = parse("Jane Smith, RN - CRNA") + >>> name.given, name.family, name.suffix + ('RN', 'Jane Smith', 'CRNA') + >>> policy = Policy(extra_suffix_delimiters={" - "}) + >>> name = Parser(policy=policy).parse("Jane Smith, RN - CRNA") + >>> name.given, name.family, name.suffix + ('Jane', 'Smith', 'RN, CRNA') + +The strip flags keep characters the parser removes by default. Note +what happens to an emoji you keep — it becomes a token like any other, +and lands in the middle name: + +.. doctest:: + + >>> str(parse("Sam 😊 Smith")) # stripped by default + 'Sam Smith' + >>> kept = Parser(policy=Policy(strip_emoji=False)).parse("Sam 😊 Smith") + >>> str(kept), kept.middle + ('Sam 😊 Smith', '😊') + +``strip_bidi=False`` does the same for invisible bidirectional control +characters, which is occasionally what you want when round-tripping +right-to-left text verbatim. + A pair routes to exactly one field, and ``maiden_delimiters`` states the specific intent — so listing a pair there automatically drops it from the effective ``nickname_delimiters`` set, and the one-liner diff --git a/docs/usage.rst b/docs/usage.rst index 4a0b1596..e6936f99 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -66,6 +66,16 @@ while ``given_names`` and ``surnames`` roll several fields together — the same sense in which a passport form asks for your "given names" as one blank that can hold more than one word. +``family_base`` is what you want for alphabetizing, since it drops the +particles a phone book ignores: + +.. doctest:: + + >>> names = [parse(s) for s in + ... ["Vincent van Gogh", "Juan de la Vega", "John Smith"]] + >>> [n.family_base for n in sorted(names, key=lambda n: n.family_base.lower())] + ['Gogh', 'Smith', 'Vega'] + Dicts and strings ------------------ From 4dd99c16ca2ab456cbc45fe9c50d56d0d10589c4 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 14:04:02 -0700 Subject: [PATCH 237/275] docs: name the customize sections after the reader's problem MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v1's customize.rst used task-shaped headings — "Removing a Title", "Don't Remove Emojis", "Bound First Names" — so a reader scanning for their problem found it. The 2.0 rewrite named its three sections after the architecture instead (Vocabulary / Behavior / Presentation), which teaches the model better but leaves nothing to scan: the word "emoji" appeared nowhere outside a table cell, and the Policy section ran 150 lines without a subheading. Keep the three architectural sections and add task-shaped subheadings under them, so the page works for both reading and lookup. All eight terms a reader would search for (emoji, title, order, suffix, nickname, maiden, case, lexicon) now appear in a heading. Also fixes a seam left by the previous commit: the original maiden/nickname precedence paragraph had ended up orphaned after the strip-flag examples, several screens below the doctest it explains, and its closing sentence duplicated the additive-delimiter example added alongside it. Moved back to introduce its own example, duplicate dropped. Co-Authored-By: Claude Opus 4.8 --- docs/customize.rst | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index 02bc91a4..70d0da4a 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -11,6 +11,9 @@ why the split is drawn there. Vocabulary: Lexicon -------------------- +Adding and removing words +~~~~~~~~~~~~~~~~~~~~~~~~~~ + .. doctest:: >>> from nameparser import Lexicon, Parser @@ -40,8 +43,12 @@ appear in the base field too, so add to both and remove from the marker first; anything else raises ``ValueError`` naming the orphans rather than leaving a marker entry that no rule will ever consult. -That matters most when clearing a field wholesale. Emptying ``titles`` -alone orphans every ``given_name_titles`` entry, so both go together: +Turning title detection off +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The subset rule matters most when clearing a field wholesale. Emptying +``titles`` alone orphans every ``given_name_titles`` entry, so the two +go together: .. doctest:: @@ -64,6 +71,9 @@ ranks and abbreviations work (see :ref:`abbreviated-titles`): >>> bare.parse("Dr. John Smith").title # structural, stays 'Dr.' +Combining two lexicons +~~~~~~~~~~~~~~~~~~~~~~~ + Whole lexicons compose with ``|``, which unions field by field — handy for keeping a shared house vocabulary separate from a per-source one and combining them at parser construction: @@ -75,6 +85,9 @@ and combining them at parser construction: >>> sorted((house | per_source).titles) ['dean', 'provost'] +Fixing the case of a particular word +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ``capitalization_exceptions`` is the one pair-valued field — each entry maps a lowercase key to its exact-cased replacement (``"phd"`` → ``"PhD"``), so it isn't a fit for ``add()``/``remove()``. Change it with @@ -104,6 +117,9 @@ not against the raw text, so one ``"phd"`` entry covers ``"phd"``, ``"Phd"``, and ``"Ph.D."`` alike — you don't need a separate key for each way a source might punctuate it. +Words that are also ordinary names +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Two fields — ``suffix_acronyms_ambiguous`` and ``particles_ambiguous`` — mark entries from ``suffix_acronyms`` and ``particles`` that are also plausible as ordinary name words on their own (an acronym suffix that @@ -244,6 +260,9 @@ listed below. - Excludes bidirectional control characters the same way. Defaults to ``True``. +Family-first name order +~~~~~~~~~~~~~~~~~~~~~~~~ + ``name_order`` is the one most likely to matter for non-Western data. Positional input is assigned in the order you declare, so a family-first name parses as written instead of needing to be @@ -268,6 +287,14 @@ family-then-given regardless of the configured order: >>> family_first.parse("Thomas, John").family 'Thomas' +Nicknames, maiden names, and brackets +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A delimiter pair routes to exactly one field, and ``maiden_delimiters`` +states the more specific intent — so listing a pair there drops it from +the effective ``nickname_delimiters`` set automatically, and the +one-liner is the whole recipe: + .. doctest:: >>> policy = Policy(maiden_delimiters={("(", ")")}) @@ -288,6 +315,9 @@ instead of extending them, the same trap as ``capitalization_exceptions``: >>> Parser(policy=policy).parse("Benjamin {Ben} Franklin").nickname 'Ben' +Suffixes not separated by commas +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ``extra_suffix_delimiters`` handles sources that separate post-nominals with something other than a comma. The default reading of such a name is bad enough to be the reason you'd go looking: @@ -302,6 +332,9 @@ is bad enough to be the reason you'd go looking: >>> name.given, name.family, name.suffix ('Jane', 'Smith', 'RN, CRNA') +Keeping emoji and control characters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + The strip flags keep characters the parser removes by default. Note what happens to an emoji you keep — it becomes a token like any other, and lands in the middle name: @@ -318,13 +351,6 @@ and lands in the middle name: characters, which is occasionally what you want when round-tripping right-to-left text verbatim. -A pair routes to exactly one field, and ``maiden_delimiters`` states -the specific intent — so listing a pair there automatically drops it -from the effective ``nickname_delimiters`` set, and the one-liner -above is the whole recipe. To *add* delimiters instead of rerouting -them, build on the named default: -``nickname_delimiters=DEFAULT_NICKNAME_DELIMITERS | {("[", "]")}``. - .. _rendering-arguments: Presentation: rendering arguments From a6bd04cfbee9bf47785e90353adb9fdac6c78f6b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 14:12:08 -0700 Subject: [PATCH 238/275] docs: input shapes use the 2.0 field names The three patterns were lifted from v1's README verbatim, v1 vocabulary included, so a page teaching given/family described its input shapes in first/middle/last. That undercuts the rename the API is built on. Restoring the names also recovered detail the first pass had trimmed: v1's forms distinguish a comma that separates the family name (form 2, family-first) from a comma that only sets off suffixes (form 3, still given-then-family). That distinction is the whole reason both forms are listed, so it is now stated and pinned with a third doctest. The nickname and pre-comma suffix slots are back in the patterns too, the latter matching the "Doe Jr., John" example already there. Checked the rest of the 2.0 docs for the same slip; this was the only occurrence outside migrate.rst, where v1 names are the subject. Co-Authored-By: Claude Opus 4.8 --- docs/usage.rst | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index e6936f99..10aae706 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -33,19 +33,23 @@ Input shapes Three arrangements are understood, and every piece of each is optional: -1. ``Title First Middle Middle Last Suffix`` -2. ``Last, Title First Middle Middle[,] Suffix [, Suffix]`` -3. ``Title First Middle Last [,] Suffix [, Suffix]`` +1. ``Title Given "Nickname" Middle Middle Family Suffix`` +2. ``Family [Suffix], Title Given (Nickname) Middle Middle[,] Suffix [, Suffix]`` +3. ``Title Given Middle Family [Suffix], Suffix [, Suffix]`` -A comma before the given name is the signal for family-first order, so -the second form needs no configuration: +The last two differ in what the comma is doing. In form 2 it separates +the family name from the rest, so the family name comes first; in form +3 it only sets off suffixes, and the name before it is still +given-then-family: .. doctest:: - >>> parse("de la Vega, Juan Q. Xavier III").family + >>> parse("de la Vega, Juan Q. Xavier III").family # form 2 'de la Vega' - >>> parse("Doe Jr., John").suffix + >>> parse("Doe Jr., John").suffix # form 2, suffix before the comma 'Jr.' + >>> parse("John Doe, Jr.").family # form 3 + 'Doe' For family-first input *without* a comma — common outside Europe — set ``name_order``; see :doc:`customize`. From b23b08f4caae38e006f5781eb87f1b13e08856df Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 14:18:22 -0700 Subject: [PATCH 239/275] docs: explain which words attach to their neighbors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Input shapes" says where the fields sit; nothing said which words merge into one field rather than standing alone. Between them those are most of what decides a parse, so the new section sits next to it. Organized by which field the word pulls into rather than by what kind of word it is. The fields are a closed list the reader has just learned — seven of them — whereas the word kinds (particles, bound given names, conjunctions, maiden markers, title chains, suffix runs) are an open-ended taxonomy with no natural stopping point, which is what makes the set feel incomprehensible when listed that way. The one genuinely position-sensitive behavior gets its own paragraph instead of a mostly-empty column: a particle at the start of a name has no surname to attach to yet, so it either becomes the given name or turns the whole name into a surname, depending on whether it can double as a given name. Explaining that by cause turns two arbitrary-looking outcomes into one rule with two branches. Table examples are prose, not doctests, so CI cannot catch a wrong one — each was verified by hand against the parser. The conjunction and leading-particle claims are pinned as doctests. Co-Authored-By: Claude Opus 4.8 --- docs/usage.rst | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/docs/usage.rst b/docs/usage.rst index 10aae706..b2e6ee1d 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -54,6 +54,67 @@ given-then-family: For family-first input *without* a comma — common outside Europe — set ``name_order``; see :doc:`customize`. +Words that attach to their neighbors +-------------------------------------- + +Input shapes tell you where the fields sit. This tells you which words +merge into one field instead of standing alone — between them, that is +most of what decides a parse. + +Most words stand alone. A few pull a neighbor in, and the thing worth +knowing is which field they pull into: + +.. list-table:: + :header-rows: 1 + :widths: 14 42 44 + + * - Pulls into + - Example + - Words + * - ``family`` + - ``Juan de la Vega`` → ``de la Vega`` + - Particles (``de``, ``van``, ``von``, ``bin``), and conjunctions + inside a surname (``y``) + * - ``given`` + - ``abdul salam ahmed`` → ``abdul salam`` + - Bound given names (``abdul``, ``abu``, ``umm``, ``عبد``) + * - ``title`` + - ``Asst. Vice Chancellor John Smith`` → ``Asst. Vice Chancellor`` + - Consecutive titles chain + * - ``suffix`` + - ``John Smith PhD MD`` → ``PhD, MD`` + - Consecutive suffixes + * - ``maiden`` + - ``Jane Smith née Jones`` → ``Jones`` + - Maiden markers (``née``, ``geb.``) + +A conjunction joins whatever sits on both sides of it, so it can pull +two given names together as easily as two surnames: + +.. doctest:: + + >>> parse("John and Jane Smith").given + 'John and Jane' + >>> parse("Juan de la Vega y Rodriguez").family + 'de la Vega y Rodriguez' + +Position matters in exactly one place: the start of a name. A particle +there has no surname to attach to yet, so it either becomes the given +name or turns the whole name into a surname, depending on whether it is +one that can double as a given name: + +.. doctest:: + + >>> parse("van Gogh").given # 'van' can be a given name + 'van' + >>> parse("de Mesnil").given # 'de' cannot + '' + >>> parse("de Mesnil").family + 'de Mesnil' + +:doc:`customize` covers how to change which words are in each of these +sets, including which particles may double as given names. + Aggregate views ---------------- From 1e7676176b75c26402ea379505c91cc5b9ec8ac6 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 14:28:23 -0700 Subject: [PATCH 240/275] Label the CLI output sections; document the tussenvoegsel sort MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default output printed three things, two of them unlabeled ParsedName reprs — the parse and the capitalized view. For input that is already correctly cased those two are byte-identical, so the CLI looked like it was printing the same thing twice, and there was no way to tell which block was which. The third line was already labeled "Initials:", so the inconsistency was internal to the command. Label all three. usage.rst gains an intro to that section framing the CLI as the quick way to check how a string parses — trying a variation of a name that came out wrong, or a locale pack before wiring it into code — plus the real output, which the section previously showed only for --json. The family_base example gains its reason for existing. #130 asked for the surname split specifically for tussenvoegsels, the Dutch particle that directories file a name *under* rather than *by*. Naming the convention gives the searchable term, and the contrast doctest shows what the split avoids: sorting on family files van Gogh under "v" and de la Vega under "d". Co-Authored-By: Claude Opus 4.8 --- docs/usage.rst | 42 ++++++++++++++++++++++++++++++++++++++++-- nameparser/__main__.py | 5 +++++ tests/v2/test_cli.py | 11 +++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index b2e6ee1d..3c69bdcf 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -131,8 +131,10 @@ while ``given_names`` and ``surnames`` roll several fields together — the same sense in which a passport form asks for your "given names" as one blank that can hold more than one word. -``family_base`` is what you want for alphabetizing, since it drops the -particles a phone book ignores: +``family_base`` is the one to sort on. Dutch and Belgian directories +file a name under the base surname and ignore the *tussenvoegsel* — the +``van``, ``de`` or ``van der`` in front of it — and the same convention +applies wherever particles are common: .. doctest:: @@ -141,6 +143,14 @@ particles a phone book ignores: >>> [n.family_base for n in sorted(names, key=lambda n: n.family_base.lower())] ['Gogh', 'Smith', 'Vega'] +Sorting on ``family`` instead files those under "d", "S" and "v", which +is the problem the split exists to solve: + +.. doctest:: + + >>> [n.family for n in sorted(names, key=lambda n: n.family.lower())] + ['de la Vega', 'Smith', 'van Gogh'] + Dicts and strings ------------------ @@ -358,6 +368,34 @@ everything else carried over. Command line ------------ +``python -m nameparser`` parses one name and prints what it found. It +is the quickest way to see how a particular string comes out — worth +reaching for when a name parsed unexpectedly and you want to check a +variation of it, or when trying a locale pack before wiring one into +code: + +:: + + $ python -m nameparser "dr. juan de la vega iii" + Parsed: + + Capitalized: + + Initials: j. v. + +``--json`` prints the fields as a single line instead, which is the +form to pipe somewhere: + :: $ python -m nameparser --json "Doe, John" diff --git a/nameparser/__main__.py b/nameparser/__main__.py index fd1dde53..b569951c 100644 --- a/nameparser/__main__.py +++ b/nameparser/__main__.py @@ -33,7 +33,12 @@ def main(argv: list[str] | None = None) -> int: if args.json: print(json.dumps(n.as_dict(), ensure_ascii=False)) return 0 + # Label each section: the two reprs are byte-identical for input + # that is already correctly cased, so without labels there is no + # telling which is the parse and which is the capitalized view. + print("Parsed:") print(repr(n)) + print("Capitalized:") print(repr(n.capitalized())) print("Initials:", n.initials()) return 0 diff --git a/tests/v2/test_cli.py b/tests/v2/test_cli.py index 861e25d6..5b4811f3 100644 --- a/tests/v2/test_cli.py +++ b/tests/v2/test_cli.py @@ -15,6 +15,17 @@ def test_cli_prints_repr_capitalized_and_initials() -> None: assert "Initials:" in out +def test_cli_labels_each_section() -> None: + # two of the three outputs were unlabeled ParsedName reprs, and for + # already-cased input they are byte-identical -- the reader cannot + # tell which is the parse and which is the capitalized view + out = _run("Dr. Juan de la Vega III").stdout + assert out.count(" None: proc = _run("John Smith", "--json") data = json.loads(proc.stdout) From 261c9c9f6946046eeeac44073545085f460b659a Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 14:36:37 -0700 Subject: [PATCH 241/275] docs: retitle usage.rst "Using the parser" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The page has roughly tripled — 414 lines across 13 sections — since it was named. "Getting started" now sets the wrong expectation in both directions: a newcomer expects something short, and a reader returning to look up how to sort by surname or what ambiguities does never thinks to check a page called Getting started. "Using the parser" also aligns the title with the URL it has always had (usage.html), and echoes v1's "Using the HumanName Parser" for readers coming back. Dropping HumanName from it on purpose: 2.0's entry point is parse(), and naming the compatibility layer in the title of the main tour would point newcomers at the wrong API. The README already showed the strain — it linked "Getting started" and then glossed it as "the full tour". Gloss updated too; its list of contents predated input shapes, attaching words, ambiguities and tokens. Co-Authored-By: Claude Opus 4.8 --- README.rst | 2 +- docs/release_log.rst | 2 +- docs/usage.rst | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 4d8b7298..e33b3a4b 100644 --- a/README.rst +++ b/README.rst @@ -36,7 +36,7 @@ for combining or splitting them further. Learn more ---------- -* `Getting started `__ — the full tour: parsing, aggregates, dicts, rendering, dedup +* `Using the parser `__ — the full tour: input shapes, aggregates, rendering, comparison, ambiguities, tokens * `Customizing the parser `__ — vocabulary, behavior, and presentation * `Locale packs `__ — opt-in bundles for East Slavic patronymics, Turkic markers, and more * There's also a CLI: ``python -m nameparser --json "Doe, John"`` diff --git a/docs/release_log.rst b/docs/release_log.rst index df172560..616c6e74 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -80,7 +80,7 @@ Release Log **Documentation** - - Rewrite the documentation new-API-first: a new front page and README, ``usage.rst`` as a 2.0 getting-started tour, a principle-first ``customize.rst``, a reference split into the 2.0 API and the compatibility layer, and new pages for :doc:`concepts`, :doc:`locales` and :doc:`migrate` -- the last carrying full attribute and configuration maps from v1 names to 2.0 names (#262). The 1.x documentation remains online as the readthedocs ``stable`` build + - Rewrite the documentation new-API-first: a new front page and README, ``usage.rst`` as a tour of the 2.0 API, a principle-first ``customize.rst``, a reference split into the 2.0 API and the compatibility layer, and new pages for :doc:`concepts`, :doc:`locales` and :doc:`migrate` -- the last carrying full attribute and configuration maps from v1 names to 2.0 names (#262). The 1.x documentation remains online as the readthedocs ``stable`` build Parsing changes were checked against a 486-name differential corpus built from the v1 test banks; everything not listed above parses diff --git a/docs/usage.rst b/docs/usage.rst index 3c69bdcf..800e0622 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1,5 +1,5 @@ -Getting started -=============== +Using the parser +================ Requires Python 3.11+. ``pip install nameparser`` From 5f1f6ac5e4a8fad3c0a0d3844acf12483b2b8edd Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 14:44:49 -0700 Subject: [PATCH 242/275] docs: name the subject in the configuration heading "Three containers, three concerns" never said what was in three containers, so a reader scanning the page could not tell the section was about configuration. The chime is memorable once you have read it and invisible before that. "Configuration lives in three containers" adds the subject, keeps the compact subject-verb shape of its neighbor "Two layers decide the roles", and echoes the section's own opening sentence. Deliberately not converting concepts.rst headings to the task-shaped style used in customize.rst -- one page explains, the other is looked things up in. Also fixes the page intro, which still enumerated four ideas and promised "these four ideas". Adding the two-layer section earlier today made it five; the enumeration never mentioned how tokens get their roles. Co-Authored-By: Claude Opus 4.8 --- docs/concepts.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index a6ca4303..1bdc90b0 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -3,10 +3,11 @@ How the parser works ``parse()`` turns a name string into a :class:`~nameparser.ParsedName`. This page explains the model behind -that call: how a string becomes tokens and tokens become fields, -where configuration lives and why it is split the way it is, why -parsers are plain values, and what happens when a name is genuinely -ambiguous. The task pages all build on these four ideas. +that call: how a string becomes tokens and tokens become fields, how +those tokens get their roles, where configuration lives and why it is +split the way it is, why parsers are plain values, and what happens +when a name is genuinely ambiguous. The task pages all build on these +five ideas. From string to name -------------------- @@ -71,8 +72,8 @@ you look anything up: if you are teaching the parser a *word*, it goes in the :class:`~nameparser.Lexicon`; if you are changing how unclaimed words are *arranged*, it goes in the :class:`~nameparser.Policy`. -Three containers, three concerns --------------------------------- +Configuration lives in three containers +---------------------------------------- Every piece of nameparser configuration falls into exactly one of three places, and which one is decided by a single question: what does From d4dd8a184b2395a1b6d938098d8fb0f4dfa6e5f1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 15:06:51 -0700 Subject: [PATCH 243/275] Restore 'esq' to SUFFIX_ACRONYMS; it was never a duplicate 022fd07 removed 'esq' from SUFFIX_ACRONYMS because it was also in SUFFIX_NOT_ACRONYMS, and added an assert requiring the two sets to be disjoint. Both were wrong: the sets are matched with DIFFERENT normalization. The word test strips only edge periods; the acronym test strips all of them. So "Esq" matches only through the word set and "E.S.Q." only through the acronym set, and an entry in both is covering two spellings rather than repeating itself. The removal was a v1-compat regression that also lost the family name: "John Smith E.S.Q." 1.4.0: suffix='E.S.Q.' last='Smith' 022fd07: suffix='' family='E.S.Q.' The 486-name differential corpus missed it because no corpus name uses the multi-dot spelling, and the reasoning I checked instead -- "the word branch is tested first, so the acronym membership is unreachable" -- was true only for the spellings the word branch can normalize. Pinned by a case-table row so the two-spelling coverage cannot be "cleaned up" again. Replaced the bad assert with one that holds and has a real payoff: SUFFIX_ACRONYMS_AMBIGUOUS must not intersect SUFFIX_NOT_ACRONYMS. suffix_as_written ORs the two branches, so a word membership silently bypasses the period gate the ambiguous set exists to impose -- listing 'ma' in suffix_words would make "Jack Ma" lose its family name. Release log corrected: that bullet claimed parse output was unchanged for both data edits. It was true only for the TITLES trailing spaces. Co-Authored-By: Claude Opus 4.8 --- docs/release_log.rst | 2 +- nameparser/config/suffixes.py | 20 +++++++++++++++++--- tests/v2/cases.py | 7 +++++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index 616c6e74..931324ae 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -59,7 +59,7 @@ Release Log - Change the 2.0 API's default ``render()``/``str()`` spec to show every non-empty field: ``'{title} {given} "{nickname}" {middle} {family} ({maiden}) {suffix}'``. The quoted nickname round-trips exactly; the parenthesized maiden re-parses as a nickname, a deliberate choice of presentation over lossless round-trip -- use ``née {maiden}`` in a custom spec if you need it to survive a reparse. ``HumanName`` keeps v1's own ``string_format`` default, unchanged - Change delimiter-overlap precedence in the 2.0 API: a pair listed in ``Policy.maiden_delimiters`` is dropped from the effective nickname set, so ``Policy(maiden_delimiters={("(", ")")})`` alone routes parenthesized content to ``maiden``. The default nickname set is exported as ``DEFAULT_NICKNAME_DELIMITERS``. ``HumanName`` keeps v1's nickname-wins precedence, so no existing behavior changes - Change suffix-delimiter rendering when a custom suffix delimiter is configured: for ``suffix_delimiter="/"`` and ``"John Smith, RN/CRNA"``, 1.x split the token and rendered ``suffix="RN, CRNA"`` where 2.0 keeps it whole as ``"RN/CRNA"``. Role assignment is unchanged; only rendering differs - - Correct two errors in the shipped vocabulary data that 1.x carried: ``esq`` was listed as both a suffix acronym and a suffix word (it is a word), and ``actor`` and ``television`` had trailing spaces in ``TITLES``, so ``"actor" in TITLES`` was ``False``. **Parse output is unchanged** in both cases -- the parser normalizes entries on ingest and tests suffix words first, which is exactly what let the errors go unnoticed -- but code that tests membership against the exported constants directly will see corrected results. The data modules now assert these invariants at import time, alongside the ones ``prefixes.py`` already checked + - Correct a long-standing typo in the shipped vocabulary that 1.x carried: ``actor`` and ``television`` had trailing spaces in ``TITLES``, so ``"actor" in TITLES`` was ``False``. **Parse output is unchanged** -- the parser normalizes entries on ingest, which is exactly what let the typo go unnoticed -- but code that tests membership against the exported constants directly will see corrected results. The data modules now assert their invariants at import time, alongside the ones ``prefixes.py`` already checked; entries must be stored lowercase and whitespace-free, so this class of typo now fails the build **International name support** diff --git a/nameparser/config/suffixes.py b/nameparser/config/suffixes.py index 448437cd..48dd9464 100644 --- a/nameparser/config/suffixes.py +++ b/nameparser/config/suffixes.py @@ -391,6 +391,11 @@ 'emt-p', 'enp', 'erd', + # Also in SUFFIX_NOT_ACRONYMS, and NOT redundant: the word test + # strips only edge periods while the acronym test strips all of + # them, so the multi-dot spelling "E.S.Q." matches only here while + # bare "Esq" matches only there. + 'esq', 'evp', 'faafp', 'faan', @@ -718,9 +723,18 @@ # construction, which is what protects a caller's own vocabulary. assert SUFFIX_ACRONYMS_AMBIGUOUS <= SUFFIX_ACRONYMS, \ "SUFFIX_ACRONYMS_AMBIGUOUS must stay a subset of SUFFIX_ACRONYMS" -assert not (SUFFIX_ACRONYMS & SUFFIX_NOT_ACRONYMS), \ - "a suffix is an acronym or a word, not both: " \ - f"{sorted(SUFFIX_ACRONYMS & SUFFIX_NOT_ACRONYMS)}" +# NOT asserted: disjointness of SUFFIX_ACRONYMS and SUFFIX_NOT_ACRONYMS. +# The two sets are matched with different normalization -- the word test +# strips only edge periods, the acronym test strips all of them -- so an +# entry in both is covering two spellings, not duplicated. 'esq' matches +# "Esq" only as a word and "E.S.Q." only as an acronym. +# DO assert that an ambiguous acronym is not also a plain suffix word: +# suffix_as_written ORs the two branches, so the word membership would +# bypass the period gate the ambiguous set exists to impose. +assert not (SUFFIX_ACRONYMS_AMBIGUOUS & SUFFIX_NOT_ACRONYMS), \ + "an ambiguous acronym must not also be a suffix word (the word " \ + "branch bypasses its period gate): " \ + f"{sorted(SUFFIX_ACRONYMS_AMBIGUOUS & SUFFIX_NOT_ACRONYMS)}" # Entries are looked up in normalized form, so a stray capital or a # trailing space makes an entry unreachable by direct membership test even # though the parser's own ingest would paper over it. diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 004638aa..559502b8 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -67,6 +67,13 @@ def __post_init__(self) -> None: {"given": "John", "family": "Smith", "suffix": "Esq"}, notes="the suffix_words branch of the delimited-content " "escape (v1 parity, pinned live 2026-07-17)"), + Case("suffix_acronym_multidot_spelling", "John Smith E.S.Q.", + {"given": "John", "family": "Smith", "suffix": "E.S.Q."}, + notes="'esq' is in BOTH suffix_acronyms and suffix_words on " + "purpose, and the two are not redundant: the word test " + "strips only EDGE periods, the acronym test strips all " + "of them, so only the acronym membership matches the " + "multi-dot spelling (v1 parity, pinned live 2026-07-19)"), Case("bound_given_whole_segment", "salem, abdul salam", {"given": "abdul salam", "family": "salem"}, notes="v1 joins bound given names freely in the post-comma " From 0c642dc1d9e6b78d8ba62b6837070267e2cdc03a Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 15:09:40 -0700 Subject: [PATCH 244/275] Check given_name_titles per word, not per entry d8feb3e added given_name_titles <= titles to _SUBSET_FIELDS. That is the wrong shape for the field: _post_rules space-joins a multi-token title run and looks the JOINED string up, so "grand duke" is a legitimate entry whose WORDS are titles while the phrase itself is not. The invariant made multi-word honorifics inexpressible in the 2.0 API, and the matching shim intersection silently dropped them, reclassifying the given name: Constants: titles += {grand, duke}, first_name_titles += {grand duke} "Grand Duke John" 1.4.0: first='John' d8feb3e: last='John' Check the words instead. The original footgun -- add(given_name_titles= {"sheikh"}) doing nothing -- still raises, because 'sheikh'.split() is not in titles either. The shim now drops only entries with a non-title word, which are the ones v1 could not reach. Two more fixes to the same validation block: - _SUBSET_FIELDS carried one rationale ("an orphan is never consulted") for all its pairs, and it was false for both survivors: an orphan particles_ambiguous entry makes _assign emit a spurious ambiguity, and an orphan suffix_acronyms_ambiguous entry makes _vocab treat the word as a period-gated suffix. Each pair now carries its own reason, and the reason appears in the error. - The remedy was "Add them to {base} as well", which inverts the intent of a caller who was REMOVING from the base -- telling them to add back what they just removed. Now names both directions. And adds the invariant that guard should have been: an ambiguous suffix acronym must not also be a suffix word. suffix_as_written ORs the two branches, so the word membership bypasses the period gate the ambiguous set exists to impose -- add(suffix_words={"ma"}) was accepted and made "Jack Ma" lose its family name. Co-Authored-By: Claude Opus 4.8 --- nameparser/_config_shim.py | 18 +++++----- nameparser/_lexicon.py | 65 ++++++++++++++++++++++++++++-------- tests/v2/test_config_shim.py | 25 ++++++++++++++ tests/v2/test_lexicon.py | 61 +++++++++++++++++++++++---------- 4 files changed, 131 insertions(+), 38 deletions(-) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index 202fc15c..f9f76dd3 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -940,14 +940,16 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: # default-Constants equality test in tests/v2/test_config_shim.py) lexicon = Lexicon( titles=titles, - # intersect, same reasoning as suffix_acronyms_ambiguous - # below: v1 treats titles and first_name_titles as - # independent sets, and only consults first_name_titles for - # a word already recognized as a title -- so an entry - # missing from titles was never reachable in v1 either, and - # dropping it preserves v1 behavior exactly while - # satisfying Lexicon's subset invariant - given_name_titles=frozenset(self.first_name_titles) & titles, + # Drop only the entries v1 could never reach. v1 looks + # first_name_titles up on the JOINED title string, so a + # multi-word entry ("grand duke") is reachable when its + # words are titles even though the phrase is not -- keep + # those. An entry with a non-title word cannot match in v1 + # either, so dropping it preserves v1 exactly while + # satisfying Lexicon's per-word check. + given_name_titles=frozenset( + e for e in self.first_name_titles + if set(e.split()) <= titles), suffix_acronyms=acronyms, suffix_words=frozenset(self.suffix_not_acronyms), # intersect: Lexicon enforces ambiguous <= acronyms; v1 diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 6901f14d..84fb5466 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -23,16 +23,27 @@ "conjunctions", "bound_given_names", "maiden_markers", ) -#: (marker, base) pairs: each marker field narrows how entries of its -#: base vocabulary are read, and carries no vocabulary of its own. An -#: entry present in the marker but absent from the base is never -#: consulted by any rule, so it is rejected rather than silently -#: ignored -- a no-op configuration is the failure mode these fields -#: are most likely to be misused into. +#: (marker, base, why) triples. Each marker narrows how entries of its +#: base vocabulary are read and carries no vocabulary of its own, so an +#: entry outside the base is a configuration mistake -- but the mistake +#: differs per pair, and the reason is recorded here rather than +#: generalized, because an orphan is NOT simply inert: +#: +#: * particles_ambiguous: _assign keys on the tag alone, so an orphan +#: makes the parse emit a spurious particle-or-given ambiguity. +#: * suffix_acronyms_ambiguous: _vocab returns True on the ambiguous +#: set before testing suffix_acronyms, so an orphan silently turns a +#: word into a period-gated suffix. +#: +#: given_name_titles is deliberately NOT here: it is looked up against +#: the space-joined title run, not per token, so a multi-word phrase is +#: a legitimate entry whose words -- not whose phrase -- must be +#: titles. It gets its own per-word check below. _SUBSET_FIELDS = ( - ("particles_ambiguous", "particles"), - ("suffix_acronyms_ambiguous", "suffix_acronyms"), - ("given_name_titles", "titles"), + ("particles_ambiguous", "particles", + "an orphan emits a spurious particle-or-given ambiguity"), + ("suffix_acronyms_ambiguous", "suffix_acronyms", + "an orphan silently becomes a period-gated suffix"), ) @@ -227,15 +238,31 @@ def __post_init__(self) -> None: canonical = _normpairs(self.capitalization_exceptions) object.__setattr__(self, "capitalization_exceptions", canonical) object.__setattr__(self, "_cap_map", MappingProxyType(dict(canonical))) - for marker, base in _SUBSET_FIELDS: + for marker, base, why in _SUBSET_FIELDS: orphans = getattr(self, marker) - getattr(self, base) if orphans: raise ValueError( f"{marker} marks a subset of {base}; " - f"not in {base}: {', '.join(sorted(orphans))}. " - f"Add them to {base} as well — " - f"add({base}={{...}}, {marker}={{...}})" + f"not in {base}: {', '.join(sorted(orphans))} " + f"({why}). Add them to {base}, or drop them from " + f"{marker}" ) + # given_name_titles is matched against the space-joined title + # run, so check its WORDS rather than its entries: "grand duke" + # is reachable when 'grand' and 'duke' are titles, and requiring + # the phrase itself would make multi-word honorifics + # inexpressible (and silently reclassify names through the v1 + # facade, where such entries are legal). + title_words = {w for e in self.given_name_titles for w in e.split()} + orphans = title_words - self.titles + if orphans: + raise ValueError( + f"every word of a given_name_titles entry must be in " + f"titles; not in titles: {', '.join(sorted(orphans))}. " + f"A word that is not a title is never read as one, so " + f"the entry would have no effect. Add them to titles, " + f"or drop the entry from given_name_titles" + ) # The v2 form of prefixes.py's NON_FIRST_NAME_PREFIXES-disjoint- # from-BOUND_FIRST_NAMES assertion. That module guards its own # data at import; this guards vocabulary a caller supplies. @@ -249,6 +276,18 @@ def __post_init__(self) -> None: f"starts a given name cannot also bind one — add them to " f"particles_ambiguous, or drop them from bound_given_names" ) + # suffix_as_written ORs the acronym and word branches, so a word + # membership bypasses the period gate the ambiguous set exists + # to impose: listing 'ma' in suffix_words would make "Jack Ma" + # read as a suffix and lose the family name. + gate_bypassed = self.suffix_acronyms_ambiguous & self.suffix_words + if gate_bypassed: + raise ValueError( + f"an ambiguous suffix acronym must not also be a suffix " + f"word; in both: {', '.join(sorted(gate_bypassed))}. The " + f"word branch matches without periods, which bypasses the " + f"period gate suffix_acronyms_ambiguous exists to impose" + ) # -- constructors ---------------------------------------------------- diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index f9d6a098..eaebcba2 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -335,6 +335,31 @@ def test_v14_constants_blob_unpickles_into_shim() -> None: assert loaded.string_format == "{title} {first} {middle} {last} {suffix} ({nickname})" +def test_snapshot_keeps_a_multi_word_first_name_title() -> None: + # v1 looks first_name_titles up on the joined title string, so a + # multi-word entry is reachable with only its WORDS in titles. + # Verified on a live 1.4.0: "Grand Duke John" -> first='John'. + # Intersecting the entry away made the lookup miss and silently + # reclassified the given name as a family name. + c = Constants() + c.titles.add("grand", "duke") + c.first_name_titles.add("grand duke") + lexicon, _, _ = c._snapshot() + assert "grand duke" in lexicon.given_name_titles + name = HumanName("Grand Duke John", constants=c) + assert (name.title, name.first, name.last) == ("Grand Duke", "John", "") + + +def test_snapshot_drops_an_unreachable_first_name_title() -> None: + # the single-word case stays dropped: an entry whose word is not a + # title is unreachable in v1 too, so removing it preserves v1 + c = Constants() + c.first_name_titles.add("zzqnotatitle") + lexicon, _, _ = c._snapshot() + assert "zzqnotatitle" not in lexicon.given_name_titles + assert HumanName("zzqnotatitle John Smith", constants=c).title == "" + + def test_snapshot_keeps_a_bound_never_given_prefix_parseable() -> None: # prefixes.py asserts its own data keeps non_first_name_prefixes # disjoint from bound_first_names, but nothing stops a v1 caller diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 9305f70c..0259aa04 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -230,27 +230,44 @@ def test_suffix_ambiguous_must_be_subset_of_acronyms() -> None: Lexicon(suffix_acronyms_ambiguous=frozenset({"ma"})) -def test_given_name_titles_must_be_subset_of_titles() -> None: - # third marker-over-base pair, same rule: given_name_titles marks - # which TITLES precede the given name rather than the family name, - # so an entry missing from titles is never consulted -- a silent - # no-op is exactly the failure mode the other two guards prevent - with pytest.raises(ValueError, match="subset"): +def test_given_name_titles_may_hold_a_multi_word_phrase() -> None: + # given_name_titles is looked up against the SPACE-JOINED title run + # (_pipeline/_post_rules.py), not per token, so "grand duke" is a + # legitimate entry. Its words must be titles -- that is what makes + # the run a title in the first place -- but the phrase itself is + # not, and must not be required to be. + lex = Lexicon(titles=frozenset({"grand", "duke"}), + given_name_titles=frozenset({"grand duke"})) + assert "grand duke" in lex.given_name_titles + + +def test_given_name_titles_words_must_be_titles() -> None: + # a word that is not a title is never read as one, so the entry + # would silently do nothing -- the failure mode the guards prevent. + # Checked per WORD, not per entry, so multi-word phrases stay legal + # (see test_given_name_titles_may_hold_a_multi_word_phrase). + with pytest.raises(ValueError, match="every word of a given_name_titles"): Lexicon(given_name_titles=frozenset({"sheikh"})) + with pytest.raises(ValueError, match="duke"): + Lexicon(titles=frozenset({"grand"}), + given_name_titles=frozenset({"grand duke"})) -@pytest.mark.parametrize(("make", "base"), [ - (lambda w: Lexicon(given_name_titles=w), "titles"), - (lambda w: Lexicon(particles_ambiguous=w), "particles"), - (lambda w: Lexicon(suffix_acronyms_ambiguous=w), "suffix_acronyms"), -], ids=["given_name_titles", "particles_ambiguous", "suffix_acronyms_ambiguous"]) +@pytest.mark.parametrize(("make", "marker", "base"), [ + (lambda w: Lexicon(particles_ambiguous=w), + "particles_ambiguous", "particles"), + (lambda w: Lexicon(suffix_acronyms_ambiguous=w), + "suffix_acronyms_ambiguous", "suffix_acronyms"), +], ids=["particles_ambiguous", "suffix_acronyms_ambiguous"]) def test_subset_error_names_the_fix( - make: Callable[[frozenset[str]], Lexicon], base: str + make: Callable[[frozenset[str]], Lexicon], marker: str, base: str ) -> None: - # the constraint alone leaves the reader to infer the remedy; every - # marker field's error should name the base field to add to, since - # adding to both is the only thing the caller can have meant - with pytest.raises(ValueError, match=f"Add them to {base} as well"): + # the constraint alone leaves the reader to infer the remedy, and + # the remedy is direction-dependent: a caller who was ADDING wants + # to add to the base, one who was REMOVING wants to drop from the + # marker. Naming both keeps the message right either way. + with pytest.raises(ValueError, match=f"Add them to {base}, or drop " + f"them from {marker}"): make(frozenset({"zzqnovel"})) @@ -286,5 +303,15 @@ def test_default_lexicon_satisfies_the_bound_particle_invariant() -> None: def test_remove_orphaning_given_name_titles_raises() -> None: lex = Lexicon(titles=frozenset({"sheikh"}), given_name_titles=frozenset({"sheikh"})) - with pytest.raises(ValueError, match="subset"): + with pytest.raises(ValueError, match="every word of a given_name_titles"): lex.remove(titles={"sheikh"}) + + +def test_remove_error_offers_the_removal_remedy_too() -> None: + # a caller removing from the base does not want to be told to add + # it back; the message names both directions + lex = Lexicon(particles=frozenset({"van"}), + particles_ambiguous=frozenset({"van"})) + with pytest.raises(ValueError, + match="drop them from particles_ambiguous"): + lex.remove(particles={"van"}) From bd87bb90c420cef7a0cb16d1a994409c8c416afa Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 15:10:22 -0700 Subject: [PATCH 245/275] Re-validate Lexicon on unpickle __setstate__ checked the field LAYOUT and then hand-assigned every value, so a state dict could load a Lexicon that no constructor would produce: invariants unchecked, entries unnormalized, a plain set on a frozenset field. The layout guard exists to catch version skew, but the likeliest skew is same-name/flipped-meaning rather than a changed field list -- particles_ambiguous inverted sense between v1's never-given set and v2's may-be-given set without changing its name. A 2.0-alpha pickle carrying the old sense passed the name check and loaded silently inverted. Call __post_init__ after restoring state. That re-runs normalization and all four invariant checks, and rebuilds _cap_map for free -- the hand-rolled rebuild it replaces was already duplicating that line. Co-Authored-By: Claude Opus 4.8 --- nameparser/_lexicon.py | 13 ++++++++++--- tests/v2/test_lexicon.py | 23 +++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 84fb5466..bbd8d4bd 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -355,9 +355,16 @@ def __setstate__(self, state: dict[str, object]) -> None: ) for name, value in state.items(): object.__setattr__(self, name, value) - object.__setattr__( - self, "_cap_map", - MappingProxyType(dict(self.capitalization_exceptions))) + # Re-run construction validation rather than trusting the blob. + # The layout check above catches SHAPE skew; this catches + # CONTENT skew, which is likelier -- particles_ambiguous flipped + # meaning between v1's never-given set and v2's may-be-given set + # without changing its name, so an old pickle would otherwise + # load a semantically inverted lexicon in silence. It also + # normalizes, so a hand-built state dict cannot smuggle in + # unnormalized entries or a plain set. Rebuilds _cap_map for + # free, which this used to do by hand. + self.__post_init__() def __or__(self, other: Lexicon) -> Lexicon: if not isinstance(other, Lexicon): diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 0259aa04..97688ef5 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -307,6 +307,29 @@ def test_remove_orphaning_given_name_titles_raises() -> None: lex.remove(titles={"sheikh"}) +def test_unpickling_revalidates_invariants() -> None: + # the field-layout guard catches SHAPE skew, but the likeliest skew + # is same-name/flipped-meaning: particles_ambiguous inverted sense + # between v1's never-given set and v2's may-be-given set. A state + # dict that violates an invariant must not load in silence. + lex = Lexicon(particles=frozenset({"van"}), + particles_ambiguous=frozenset({"van"})) + state = lex.__getstate__() + state["particles"] = frozenset() # orphans particles_ambiguous + forged = Lexicon.__new__(Lexicon) + with pytest.raises(ValueError, match="particles_ambiguous"): + forged.__setstate__(state) + + +def test_unpickling_normalizes_entries() -> None: + lex = Lexicon(titles=frozenset({"dr"})) + state = lex.__getstate__() + state["titles"] = {"Sir ", "DR."} # unnormalized, and a set + restored = Lexicon.__new__(Lexicon) + restored.__setstate__(state) + assert restored.titles == frozenset({"sir", "dr"}) + + def test_remove_error_offers_the_removal_remedy_too() -> None: # a caller removing from the base does not want to be told to add # it back; the message names both directions From d69cb5939ca7cee65c4dc433a603430fd337eae6 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 15:11:09 -0700 Subject: [PATCH 246/275] Complete the read surface on the regexes proxy 020e5ad fixed .get() being swallowed by _RegexesProxy.__getattr__ but stopped at the one method that had a deprecation promise attached. items(), values() and len() had the identical failure -- v1's regexes was a dict subclass, this proxy is not, and the catch-all __getattr__ claims every unimplemented mapping name as a regex lookup: CONSTANTS.regexes.items() -> AttributeError: no regex named 'items' That is the exact confusing message the get() fix was written to eliminate, and items()/values() are the next most likely v1 idioms after get(). Spell out the read half of the mapping surface, and say in the comment why a proxy has to. Co-Authored-By: Claude Opus 4.8 --- nameparser/_config_shim.py | 21 +++++++++++++++++---- tests/v2/test_config_shim.py | 11 +++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index f9f76dd3..cc987fcd 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -22,7 +22,8 @@ import functools import warnings from collections.abc import ( - Callable, Iterable, Iterator, KeysView, Mapping, Set, + Callable, ItemsView, Iterable, Iterator, KeysView, Mapping, Set, + ValuesView, ) from typing import NamedTuple, Self @@ -524,12 +525,24 @@ def __iter__(self) -> Iterator[str]: def keys(self) -> KeysView[str]: return self._regexes().keys() + # Defined explicitly because __getattr__ would otherwise claim each + # of these names as a regex lookup and raise "no regex named + # 'items'". The sibling managers inherit the whole surface from + # dict; this proxy has to spell out the read half it supports. + # #256's deprecation text promised .get() on both managers. + def get(self, name: str, default: object = None) -> object: - # Defined explicitly because __getattr__ would otherwise claim - # `get` as a regex name. The sibling managers inherit this from - # dict; #256's deprecation text promised it on both. return self._regexes().get(name, default) + def items(self) -> ItemsView[str, object]: + return self._regexes().items() + + def values(self) -> ValuesView[object]: + return self._regexes().values() + + def __len__(self) -> int: + return len(self._regexes()) + def __setattr__(self, name: str, value: object) -> None: self._raise_readonly(name) diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index eaebcba2..d87cb033 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -196,6 +196,17 @@ def test_regexes_membership_iteration_and_deepcopy() -> None: assert isinstance(copy.deepcopy(r), _RegexesProxy) +def test_regexes_supports_the_v1_read_only_mapping_surface() -> None: + # v1's regexes was a dict subclass. The proxy is not, and its + # catch-all __getattr__ claims every unimplemented mapping method + # as a regex name -- the same failure that made .get() raise + # "no regex named 'get'". Cover the rest of the read surface. + r = _RegexesProxy() + assert dict(r.items())["word"] is REGEXES["word"] + assert REGEXES["word"] in list(r.values()) + assert len(r) == len(REGEXES) + + def test_regexes_get_is_the_soft_access_escape_hatch() -> None: # 1.4's #256 deprecation told users ".get() remains available for # intentional soft access"; __getattr__ must not swallow `get` From 53c6442dd13412e8998b8b7aea7fc1e4f5509f50 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 15:11:58 -0700 Subject: [PATCH 247/275] Pin the two-piece deviation in the bound-particle translation 182249a's comment claimed the particles_ambiguous union "reproduces" v1 for a config that puts a word in both bound_first_names and non_first_name_prefixes. It reproduces two of three shapes. v1's join has a reserve_last guard, so with only two pieces it does not fire: "dos Santos" 1.4.0: last='dos Santos' here: given='dos' v1's rule is piece-count dependent; a static vocabulary set cannot express that, so the deviation is irreducible without reintroducing the raise on a config v1 accepted. Narrow the claim to what the code does and pin the divergent case as a deviation, so it is recorded rather than discovered later as a bug. Co-Authored-By: Claude Opus 4.8 --- nameparser/_config_shim.py | 13 ++++++++++++- tests/v2/test_config_shim.py | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index cc987fcd..e9a5900c 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -978,7 +978,18 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: # bound_first_names, but nothing stops a caller adding one at # runtime, and v1 then lets the bound rule win (leading "dos # Santos Silva" parses first="dos Santos"). Treating such a - # word as may-be-given reproduces that instead of raising. + # word as may-be-given reproduces that rather than raising. + # + # KNOWN DEVIATION, pinned by + # test_bound_never_given_prefix_deviates_on_two_pieces: v1's + # join has a reserve_last guard, so with only two pieces it + # does NOT fire and the word stays never-given ("dos Santos" + # -> last="dos Santos"). Promotion here is unconditional, so + # that case reads given="dos", family="Santos". v1's rule is + # piece-count dependent and a static vocabulary set cannot + # express it; the alternative is raising on a config v1 + # accepted, which is worse. Only reachable via a runtime + # config the shipped data forbids. particles_ambiguous=( particles - frozenset(self.non_first_name_prefixes)) | (bound & particles), diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index d87cb033..e30fbb4e 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -371,6 +371,20 @@ def test_snapshot_drops_an_unreachable_first_name_title() -> None: assert HumanName("zzqnotatitle John Smith", constants=c).title == "" +def test_bound_never_given_prefix_deviates_on_two_pieces() -> None: + # Pinned DEVIATION, not parity. v1's bound-given join has a + # reserve_last guard, so with only two pieces it does not fire and + # the word stays never-given: 1.4.0 gives first='', last='dos + # Santos'. The shim promotes such a word to may-be-given + # unconditionally, because v1's rule is piece-count dependent and a + # static vocabulary set cannot express that. Three-piece parity is + # the case that matters and is covered below; this records the cost. + c = Constants() + c.bound_first_names.add("dos") + name = HumanName("dos Santos", constants=c) + assert (name.first, name.last) == ("dos", "Santos") + + def test_snapshot_keeps_a_bound_never_given_prefix_parseable() -> None: # prefixes.py asserts its own data keeps non_first_name_prefixes # disjoint from bound_first_names, but nothing stops a v1 caller From 42b3ad68a62f3b10295621af4c516f7c34688de6 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 15:12:55 -0700 Subject: [PATCH 248/275] docs: correct five factual errors found in review Each verified against a live 1.4.0. - release_log: assigning empty_attribute_default raises AttributeError, not TypeError. - release_log + migrate: only the mapping manager's AttributeError lists the known keys; the regexes proxy says "no regex named 'typo'". The two pages had also disagreed with each other. - migrate: v1's CONSTANTS.regexes.typo returned EMPTY_REGEX, not None. None was the capitalization_exceptions behavior. - README + migrate: "four removals ... which all raise on contact" was false for one of the four. A subclass overriding a parsing hook gets a DeprecationWarning and parsing proceeds -- and DeprecationWarning is suppressed by default outside __main__, so it is the opposite of raising on contact. This matters: it is the one pre-upgrade check item a reader could reasonably skip. - release_log: the "Smith, Dr." example claimed a title was being routed to first in 1.x. It was not -- 1.4.0 already gave title='Dr.'. What moved is the pre-comma piece, first -> family. The bullet now describes the change that happened instead of one that didn't. Co-Authored-By: Claude Opus 4.8 --- README.rst | 4 ++-- docs/migrate.rst | 10 ++++++---- docs/release_log.rst | 6 +++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index e33b3a4b..9e792882 100644 --- a/README.rst +++ b/README.rst @@ -49,8 +49,8 @@ attributes, same mutation API. What 2.0 removes is the batch of deprecations 1.3 and 1.4 announced, so if your test suite runs clean on 1.4 under ``python -W error::DeprecationWarning``, you are nearly done. Two things that check will not catch: four removals 1.4 never warned -about, which all raise on contact, and one that changes results silently -— ``name == "John Smith"`` is now ``False``. +about (three raise on contact, the fourth only warns), and one that +changes results silently — ``name == "John Smith"`` is now ``False``. `Migrating from HumanName `__ covers both, and translates a v1 customization into the new API whenever that's convenient for you. diff --git a/docs/migrate.rst b/docs/migrate.rst index 206094f1..4b031e90 100644 --- a/docs/migrate.rst +++ b/docs/migrate.rst @@ -24,8 +24,8 @@ Before you upgrade What 2.0 removes is the batch of deprecations 1.3 and 1.4 announced. If your test suite runs clean on 1.4 under ``python -W error::DeprecationWarning``, it will run on 2.0 — with four exceptions -that 1.4 never warned about, each of which fails loudly the first time -you hit it: +that 1.4 never warned about. The first three raise the first time you +hit them; the fourth only warns, so read it carefully: * ``CONSTANTS.regexes. = ...`` raises ``TypeError``. This includes ``CONSTANTS.regexes.bidi = False``, the opt-out 1.3.1 recommended for @@ -75,9 +75,11 @@ warned removals: for a private snapshot of the current shared config * - ``name['first'] = value`` - ``name.first = value`` - * - ``CONSTANTS.regexes.typo`` (returned ``None``) + * - ``CONSTANTS.regexes.typo`` (returned ``EMPTY_REGEX``) - ``.get("typo")`` for intentional soft access; attribute access - now raises ``AttributeError`` naming the known keys + now raises ``AttributeError``. The same applies to + ``capitalization_exceptions``, whose error also lists the known + keys * - ``CONSTANTS.empty_attribute_default`` - Gone; empty fields are always ``''`` diff --git a/docs/release_log.rst b/docs/release_log.rst index 931324ae..8bd08692 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -38,9 +38,9 @@ Release Log - Remove ``bytes`` input and the ``encoding`` argument (#245): passing ``bytes`` to ``HumanName`` or to a set manager raises ``TypeError`` with a decode hint, and ``SetManager.add_with_encoding()`` and ``DEFAULT_ENCODING`` are gone. Decode first, then use ``add()`` - Remove ``SetManager.__call__`` (#243); iterate the manager or call ``set(manager)``. ``remove()`` of a missing member now raises ``KeyError`` like ``set.remove`` -- ``discard()`` is the ignore-missing form -- and the set operators ``|``, ``&``, ``-`` and ``^`` return a plain ``set`` - Remove ``HumanName`` slice access and item assignment (#258): ``name[1:-3]`` and ``name['first'] = value`` raise ``TypeError``. String-key reads (``name['first']``) and iteration are unchanged; assign fields as plain attributes - - Remove ``Constants.empty_attribute_default`` (#255): empty fields are always ``''``. Assigning it raises ``TypeError``; a pickle carrying the key still loads, with the value ignored + - Remove ``Constants.empty_attribute_default`` (#255): empty fields are always ``''``. Assigning it raises ``AttributeError``; a pickle carrying the key still loads, with the value ignored - Remove ``constants=None`` (#261): both ``HumanName(..., constants=None)`` and ``hn.C = None`` raise ``TypeError``. Use ``Constants()`` for library defaults or ``CONSTANTS.copy()`` for a private snapshot - - Remove silent unknown-key access on the mapping managers (#256): ``CONSTANTS.capitalization_exceptions.typo`` and ``CONSTANTS.regexes.typo`` raise ``AttributeError`` naming the miss and the known keys, instead of returning ``None``/``EMPTY_REGEX``. ``.get()`` remains available on both for intentional soft access + - Remove silent unknown-key access on the mapping managers (#256): ``CONSTANTS.capitalization_exceptions.typo`` and ``CONSTANTS.regexes.typo`` raise ``AttributeError`` naming the miss, instead of returning ``None``/``EMPTY_REGEX`` (the mapping manager also lists the known keys). ``.get()`` remains available on both for intentional soft access - Remove support for ``Constants`` pickles written by nameparser 1.2.x or earlier (#279): loading one raises ``ValueError`` telling you to re-pickle under 1.3/1.4 first - Remove the dead ``regexes.no_vowels`` pattern (#268) and the ``Constants.suffixes_prefixes_titles`` cached union; neither was read by the parser - Change ``HumanName``'s ``*_list`` attributes to read-only properties: they remain readable snapshots, but ``name.first_list = [...]`` now raises ``AttributeError`` @@ -50,7 +50,7 @@ Release Log - Recognize maiden-name markers -- ``née``, ``nee``, ``born``, ``geb.``, ``roz.`` and the Scandinavian participle forms -- and route the following name to the new ``maiden`` field. 1.x folded them into ``middle``/``last``, so ``"Jane Smith née Jones"`` parsed as ``middle="Smith née"`` (closes #274) - Recognize typographic nickname delimiters by default in both APIs (closes #273): smart quotes (``“Jack”``), German and Polish low-high quotes (``„Hansi“``), Swedish right-right quotes (``”Ann”``), guillemets in either direction (``«Petit»``, ``»Hansi«``), CJK corner brackets (``「タロ」``, ``『ハナ』``) and fullwidth parentheses. In 1.x these leaked into ``middle`` as literal text. Curly *single* quotes stay excluded, because U+2019 is the apostrophe in "O’Connor" - - Fix a lone suffix or title after a comma being routed to ``first``: ``"Andrews, M.D."`` and ``"Smith, Dr."`` now fill ``suffix``/``title``, since the piece before the comma is definitionally the family name + - Fix the pre-comma piece being routed to ``first`` when everything after the comma is a suffix or title: ``"Andrews, M.D."`` now reads family ``Andrews`` / suffix ``M.D.`` where 1.x read given ``M.D.`` / family ``Andrews``, and ``"Smith, Dr."`` moves ``Smith`` from ``first`` to ``family`` (the title was already correct in 1.x). The piece before a comma is definitionally the family name - Fix a lone recognized trailing suffix with no comma being routed to ``first``/``last``: ``"Johnson PhD"`` and ``"Mr. Johnson PhD"`` now keep the suffix in ``suffix`` - Fix a split ``"Ph. D."`` credential being read as two tokens; it now classifies as one suffix, replacing v1's ``fix_phd`` hook - Fold a leading never-given particle into the family name. Note that ``Lexicon.particles_ambiguous`` is the **complement** of v1's ``non_first_name_prefixes``, not a rename -- it lists the particles that *may* double as a given name, where v1 listed the ones that may not. Copying a v1 customization across without inverting it silently reverses the behavior; see :doc:`migrate` From 02e9726f1b0b8ad25db4b8265cfe1ac502dbbc5b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 15:14:56 -0700 Subject: [PATCH 249/275] Close the remaining review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Test gaps: - __setstate__ now rejects a bare str component list. A str is iterable, so the per-entry rebuild split its CHARACTERS and produced a plausible-looking wrong name ("John" -> first "J o h n") rather than failing at the load site. Only reachable from foreign or hand-built state, since v1 stored lists. - Added a test that restores a FOREIGN state dict carrying a multi-word suffix entry. The parametrized round-trips all go through v2's own __getstate__, which is self-consistent by construction, so none of them exercised the 1.4-produced-blob case the fix exists for. It passes as written; it was simply untested. - Added a test importing all seven guarded config modules. An import-time assert only runs if its module is imported, and maiden_markers is lazy -- its guard fired only incidentally, whenever some other test happened to build a default Lexicon. Docs and comments: - locales.rst: a pack's Lexicon fragment is validated standalone, before the union onto the base, so a fragment that marks a word must also carry it. Latent today (both shipped packs are policy-only) but a trap for the next pack author. - titles.py: the normalization comment was past tense about a bug fixed in the same commit, so a later reader would find the claim false and not know if it was stale. Restated as the hypothetical, which reads correctly forever. - _config_shim.py: name the drift-guard test instead of describing it. Not changed: the "spec §2" citations flagged as dangling (the specs directory is gitignored). They predate this branch and appear four times in _facade.py; fixing one would be inconsistent, and fixing all is its own change. Co-Authored-By: Claude Opus 4.8 --- docs/locales.rst | 8 ++++++++ nameparser/_config_shim.py | 4 ++-- nameparser/_facade.py | 14 +++++++++++++- nameparser/config/titles.py | 7 ++++--- tests/v2/test_contracts.py | 16 ++++++++++++++++ tests/v2/test_facade.py | 27 +++++++++++++++++++++++++++ 6 files changed, 70 insertions(+), 6 deletions(-) diff --git a/docs/locales.rst b/docs/locales.rst index 8ab0b8d6..e4897ef2 100644 --- a/docs/locales.rst +++ b/docs/locales.rst @@ -151,6 +151,14 @@ patch: every field defaults to :data:`~nameparser.UNSET` (leave it alone) instead of to a concrete value, so a pack only ever states what it changes. +The ``policy`` half works that way, but the ``lexicon`` half does not. +A pack's :class:`~nameparser.Lexicon` is a complete value in its own +right and is validated on its own, before it is unioned onto the base +— so a fragment that marks a word must also carry the word it marks. +To make an existing base title precede the given name, restate the +title in the fragment rather than listing it in ``given_name_titles`` +alone. Both shipped packs are policy-only, so neither hits this. + .. doctest:: >>> from nameparser import Lexicon, Locale, PolicyPatch, parser_for diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index e9a5900c..b76d0338 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -949,8 +949,8 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: titles = frozenset(self.titles) particles = frozenset(self.prefixes) bound = frozenset(self.bound_first_names) - # keep in sync with _lexicon._default_lexicon() (pinned by the - # default-Constants equality test in tests/v2/test_config_shim.py) + # keep in sync with _lexicon._default_lexicon() (pinned by + # tests/v2/test_config_shim.py::test_snapshot_field_translation) lexicon = Lexicon( titles=titles, # Drop only the entries v1 could never reach. v1 looks diff --git a/nameparser/_facade.py b/nameparser/_facade.py index 99de7ffa..dde82a40 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -673,7 +673,19 @@ def __setstate__(self, state: dict[str, Any]) -> None: tokens: list[Token] = [] for member in _MEMBERS: role = Role(_V2_FIELD.get(member, member)) - for entry in state.get(f"{member}_list") or []: + entries = state.get(f"{member}_list") or [] + # A bare str is iterable, so without this the loop below + # would split its CHARACTERS and build a plausible-looking + # wrong name ("John" -> first "J o h n"). v1 stored lists, + # so this only guards foreign or hand-built state -- but it + # fails at the load site instead of much later. + if isinstance(entries, str): + raise TypeError( + f"{member}_list must be a list of strings, not a str " + f"({entries!r}); this pickle was not written by " + f"nameparser" + ) + for entry in entries: for position, word in enumerate(entry.split()): tokens.append(Token( word, None, role, diff --git a/nameparser/config/titles.py b/nameparser/config/titles.py index c5469c56..3b8d3e9c 100644 --- a/nameparser/config/titles.py +++ b/nameparser/config/titles.py @@ -781,8 +781,9 @@ assert FIRST_NAME_TITLES <= TITLES, \ "FIRST_NAME_TITLES must stay a subset of TITLES" # Entries are looked up in normalized form, so a stray capital or a -# trailing space makes an entry unreachable by direct membership test -# ('actor' in TITLES was False) even though the parser's own ingest -# normalizes and papers over it. TITLES covers FIRST_NAME_TITLES. +# trailing space would make an entry unreachable by direct membership +# test -- 'actor ' with a trailing space makes "'actor' in TITLES" +# False -- even though the parser's own ingest normalizes and papers +# over it. TITLES covers FIRST_NAME_TITLES. assert all(w == w.strip().lower() for w in TITLES), \ "TITLES entries must be stored lowercase and whitespace-free" diff --git a/tests/v2/test_contracts.py b/tests/v2/test_contracts.py index be4381b7..c25f58de 100644 --- a/tests/v2/test_contracts.py +++ b/tests/v2/test_contracts.py @@ -66,3 +66,19 @@ def test_every_stable_tag_has_a_trigger(tag: str) -> None: pn = parse(text) tagged = next(t for t in pn.tokens if t.text == token_text) assert tag in tagged.tags + + +def test_every_guarded_config_module_is_imported() -> None: + """Import-time asserts only run if the module is imported. + + Five of the seven guarded config modules are pulled in by ``import + nameparser``, but ``maiden_markers`` is lazy (imported inside + ``_snapshot``/``_default_lexicon``), so its guard fired only + incidentally, whenever some other test happened to build a default + Lexicon. Importing them all here makes every guard unconditional. + """ + import importlib + + for name in ("titles", "suffixes", "prefixes", "bound_first_names", + "conjunctions", "maiden_markers", "capitalization"): + importlib.import_module(f"nameparser.config.{name}") diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index b18ba42d..dddf7a62 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -307,6 +307,33 @@ def test_pickle_round_trip_preserves_components() -> None: assert loaded.C is CONSTANTS # shared sentinel restored +def test_setstate_rejects_a_bare_string_component_list() -> None: + # a genuine v1 pickle stores lists, but __setstate__ reads foreign + # state: a bare str is iterable, so entry.split() over its + # CHARACTERS silently produced a plausible-looking wrong name + # ("John" -> first "J o h n") instead of failing at the load site + n = HumanName("John Smith") + state = n.__getstate__() + state["first_list"] = "John" + loaded = HumanName.__new__(HumanName) + with pytest.raises(TypeError, match="first_list"): + loaded.__setstate__(state) + + +def test_setstate_restores_a_foreign_multi_word_suffix_entry() -> None: + # the parametrized round-trip tests above go through v2's own + # __getstate__, which is self-consistent by construction. The case + # the fix actually protects is a 1.4-produced blob carrying a + # multi-word suffix entry, so build that state directly. + n = HumanName("John Smith") + state = n.__getstate__() + state["suffix_list"] = ["Ph. D."] + loaded = HumanName.__new__(HumanName) + loaded.__setstate__(state) + assert loaded.suffix_list == ["Ph. D."] + assert loaded.suffix == "Ph. D." + + #: Every name in the 486-name differential corpus whose HumanName pickle #: round-trip drifted before the __setstate__ element-boundary fix. Each #: carries a multi-word suffix entry -- a "joined" continuation token -- From 23ee08e861b16a8f69b099d38873cd17b1036047 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 15:43:14 -0700 Subject: [PATCH 250/275] Drop the given_name_titles invariant; translate instead of filtering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A second review of the remediation found the per-word rule was wrong too, in the opposite direction from the whole-entry rule it replaced. Both are now gone. The lookup key is the space-joined run of Role.TITLE tokens, built by the PARSE rather than drawn from this vocabulary. It can contain a multi-word TITLES member ("chargé d'affaires" is one, and the per-word rule rejected it), and it can contain a conjunction, because 'and' in "Sir and Dame Alex" is tagged Role.TITLE. No static relation over these sets decides reachability. Two attempts each rejected working configs, while the condition they guarded -- an entry nothing consults -- is inert. Guarding a harmless case cost three broken configurations, so the guard goes. The shim now TRANSLATES rather than filters. v1 joins the raw title run and applies lc(), which strips only the whole string's edge periods, so an interior word keeps its own ("lt. col"); v2 normalizes per token and then joins ("lt col"). Re-folding per word converts between them. Filtering had silently swapped given and family for every multi-word honorific containing an abbreviation or a conjunction. Verified against a live 1.4.0: "Grand Duke John", "Lt. Col. Smith" and "Sir and Dame Alex" now all match. Also fixed, all from the same review: - _normalize now strips to a fixed point. strip().strip(".") left periods-around-whitespace half done ('. a .' -> ' a ' -> 'a'), so any value re-normalized later changed under its owner -- which the unpickle revalidation had just made reachable. - __setstate__ RAISES on unnormalized state instead of quietly correcting it. Silently rewriting caller data on load would have made it a fourth undocumented transformation. - The suffix_acronyms_ambiguous/suffix_words invariant kept, but the shim now subtracts, so a v1-legal config translates instead of raising at parse time naming a field (suffix_words) that does not exist on v1's Constants. - The facade's setstate guard covers Mapping and bytes and checks entry types, not just bare str. A dict silently loaded its keys. - _RegexesProxy gained copy(), and reports dict mutators as unsupported rather than as missing regexes. Co-Authored-By: Claude Opus 4.8 --- nameparser/_config_shim.py | 59 ++++++++++++++++++++++-------- nameparser/_facade.py | 27 +++++++++----- nameparser/_lexicon.py | 71 +++++++++++++++++++++++++----------- tests/v2/test_config_shim.py | 51 ++++++++++++++++++++++---- tests/v2/test_facade.py | 19 ++++++++++ tests/v2/test_lexicon.py | 70 ++++++++++++++++++++++++++--------- 6 files changed, 225 insertions(+), 72 deletions(-) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index b76d0338..45f85aa3 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -27,7 +27,7 @@ ) from typing import NamedTuple, Self -from nameparser._lexicon import Lexicon +from nameparser._lexicon import Lexicon, _normalize from nameparser._parser import Parser from nameparser._policy import PatronymicRule, Policy from nameparser.util import lc @@ -505,9 +505,22 @@ def _regexes() -> Mapping[str, object]: from nameparser.config.regexes import REGEXES return REGEXES + #: dict methods this read-only proxy deliberately does not carry. + #: Without this, __getattr__ reports them as missing regexes and + #: sends the reader hunting for a vocabulary entry. + _DICT_ONLY = frozenset({ + "pop", "popitem", "setdefault", "update", "clear", "fromkeys", + }) + def __getattr__(self, name: str) -> object: if name.startswith("_"): raise AttributeError(name) + if name in _RegexesProxy._DICT_ONLY: + raise AttributeError( + f"{name!r} is not supported on CONSTANTS.regexes: it is a " + f"read-only view in 2.0, and parsing behavior is configured " + f"through named Policy flags" + ) try: return self._regexes()[name] except KeyError: @@ -543,6 +556,12 @@ def values(self) -> ValuesView[object]: def __len__(self) -> int: return len(self._regexes()) + def copy(self) -> dict[str, object]: + # v1's regexes was a dict subclass, so copy() returned a plain + # mutable dict. Mutating the copy never affected parsing there + # either, so this is a faithful read-only carry-over. + return dict(self._regexes()) + def __setattr__(self, name: str, value: object) -> None: self._raise_readonly(name) @@ -953,23 +972,33 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: # tests/v2/test_config_shim.py::test_snapshot_field_translation) lexicon = Lexicon( titles=titles, - # Drop only the entries v1 could never reach. v1 looks - # first_name_titles up on the JOINED title string, so a - # multi-word entry ("grand duke") is reachable when its - # words are titles even though the phrase is not -- keep - # those. An entry with a non-title word cannot match in v1 - # either, so dropping it preserves v1 exactly while - # satisfying Lexicon's per-word check. + # TRANSLATE, do not filter. The two versions build the same + # lookup key differently: v1 joins the raw title run and + # then applies lc(), which strips only the whole string's + # edge periods, so an interior word keeps its own ("lt. + # col"). v2 normalizes each token and then joins ("lt col"). + # Re-folding per word converts a v1 entry into the v2 + # spelling; filtering instead dropped every multi-word + # honorific containing an abbreviation or a conjunction and + # silently swapped given and family. given_name_titles=frozenset( - e for e in self.first_name_titles - if set(e.split()) <= titles), + " ".join(_normalize(w) for w in e.split()) + for e in self.first_name_titles), suffix_acronyms=acronyms, suffix_words=frozenset(self.suffix_not_acronyms), - # intersect: Lexicon enforces ambiguous <= acronyms; v1 - # behaves the same when an acronym is deleted but its - # ambiguous entry lingers (the entry simply stops mattering) - suffix_acronyms_ambiguous=frozenset( - self.suffix_acronyms_ambiguous) & acronyms, + # Intersect with acronyms: Lexicon enforces ambiguous <= + # acronyms; v1 behaves the same when an acronym is deleted + # but its ambiguous entry lingers (the entry stops + # mattering). Subtract suffix_words for the mirror reason: + # v1's is_suffix ORs the acronym and word branches, so + # adding an ambiguous acronym to suffix_not_acronyms simply + # ungates it. Lexicon forbids that overlap because the word + # branch bypasses the period gate -- so drop it from the + # ambiguous set, leaving a plain acronym plus a word, which + # is exactly what v1 ends up doing. + suffix_acronyms_ambiguous=( + frozenset(self.suffix_acronyms_ambiguous) + & acronyms) - frozenset(self.suffix_not_acronyms), particles=particles, # complement translation: v1 marks the never-given subset; # v2 marks the may-be-given subset. The trailing union keeps diff --git a/nameparser/_facade.py b/nameparser/_facade.py index dde82a40..bb7875c3 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -8,7 +8,7 @@ import dataclasses import warnings -from collections.abc import Iterator +from collections.abc import Iterator, Mapping from typing import Any # Import order matters here -- breaks a real import cycle. nameparser. @@ -674,18 +674,25 @@ def __setstate__(self, state: dict[str, Any]) -> None: for member in _MEMBERS: role = Role(_V2_FIELD.get(member, member)) entries = state.get(f"{member}_list") or [] - # A bare str is iterable, so without this the loop below - # would split its CHARACTERS and build a plausible-looking - # wrong name ("John" -> first "J o h n"). v1 stored lists, - # so this only guards foreign or hand-built state -- but it - # fails at the load site instead of much later. - if isinstance(entries, str): + # Everything here is iterable but shreds differently: a str + # yields characters ("John" -> first "J o h n"), a Mapping + # yields keys only, bytes yields ints. v1 stored lists, so + # this only guards foreign or hand-built state -- but it + # names the field at the load site instead of failing + # opaquely later, or not at all. + if isinstance(entries, (str, bytes, Mapping)): raise TypeError( - f"{member}_list must be a list of strings, not a str " - f"({entries!r}); this pickle was not written by " - f"nameparser" + f"{member}_list must be a list of strings, not " + f"{type(entries).__name__} ({entries!r}); this " + f"pickle was not written by nameparser" ) for entry in entries: + if not isinstance(entry, str): + raise TypeError( + f"{member}_list entries must be strings, got " + f"{entry!r}; this pickle was not written by " + f"nameparser" + ) for position, word in enumerate(entry.split()): tokens.append(Token( word, None, role, diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index bbd8d4bd..8fa6c055 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -12,6 +12,7 @@ from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from types import MappingProxyType +from typing import cast #: Vocabulary set fields, in declaration order. add()/remove() operate #: on exactly these and reject capitalization_exceptions (its entries @@ -61,8 +62,19 @@ def _normalize(word: str) -> str: while lower() applies Unicode SpecialCasing contextually and keeps both as authored. This function is the single fold for storage AND match-time lookups, so matching stays symmetric either way; lower() - is what v1's lc() used, preserving which cross-spellings match.""" - return word.lower().strip().strip(".") + is what v1's lc() used, preserving which cross-spellings match. + + Strips to a FIXED POINT. A single strip().strip(".") leaves + periods-around-whitespace half done ('. a .' -> ' a '), so a value + that is re-normalized later -- on unpickle, or by a second add() -- + would change under its owner. v1 never re-normalized, so this only + matters now that storage and match-time share one fold.""" + word = word.lower() + while True: + stripped = word.strip().strip(".") + if stripped == word: + return word + word = stripped def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: @@ -247,22 +259,17 @@ def __post_init__(self) -> None: f"({why}). Add them to {base}, or drop them from " f"{marker}" ) - # given_name_titles is matched against the space-joined title - # run, so check its WORDS rather than its entries: "grand duke" - # is reachable when 'grand' and 'duke' are titles, and requiring - # the phrase itself would make multi-word honorifics - # inexpressible (and silently reclassify names through the v1 - # facade, where such entries are legal). - title_words = {w for e in self.given_name_titles for w in e.split()} - orphans = title_words - self.titles - if orphans: - raise ValueError( - f"every word of a given_name_titles entry must be in " - f"titles; not in titles: {', '.join(sorted(orphans))}. " - f"A word that is not a title is never read as one, so " - f"the entry would have no effect. Add them to titles, " - f"or drop the entry from given_name_titles" - ) + # NOT validated: given_name_titles against titles. The lookup + # key is the space-joined run of Role.TITLE tokens, which is + # built by the parse rather than drawn from this vocabulary -- + # it can hold multi-word entries ("chargé d'affaires" is itself + # a TITLES member) and conjunctions ("sir and dame", where 'and' + # is tagged Role.TITLE). No static relation over these sets can + # decide reachability, and two attempts at one each rejected + # working configurations. An unreachable entry here is inert: + # nothing consults it, nothing misparses. That is the cheap + # failure, and guarding it proved the expensive one. + # # The v2 form of prefixes.py's NON_FIRST_NAME_PREFIXES-disjoint- # from-BOUND_FIRST_NAMES assertion. That module guards its own # data at import; this guards vocabulary a caller supplies. @@ -355,16 +362,36 @@ def __setstate__(self, state: dict[str, object]) -> None: ) for name, value in state.items(): object.__setattr__(self, name, value) + # cast is safe: __post_init__ below runs _normset over each of + # these and raises for anything that is not an iterable of str, + # so the values are only read as such once it has returned + given = {name: cast("Iterable[str]", state[name]) + for name in _VOCAB_FIELDS} # Re-run construction validation rather than trusting the blob. # The layout check above catches SHAPE skew; this catches # CONTENT skew, which is likelier -- particles_ambiguous flipped # meaning between v1's never-given set and v2's may-be-given set # without changing its name, so an old pickle would otherwise - # load a semantically inverted lexicon in silence. It also - # normalizes, so a hand-built state dict cannot smuggle in - # unnormalized entries or a plain set. Rebuilds _cap_map for - # free, which this used to do by hand. + # load a semantically inverted lexicon in silence. Rebuilds + # _cap_map for free, which this used to do by hand. self.__post_init__() + # __post_init__ also normalizes, and quietly accepting a + # rewritten value would make this a place where caller data + # changes without a word. A pickle this library wrote is always + # normalized already (_normalize converges), so a difference + # here means the state came from somewhere else -- say so while + # the offending entries can still be named. + drifted = sorted( + f"{name}: {', '.join(sorted(frozenset(given[name]) - getattr(self, name)))}" + for name in _VOCAB_FIELDS + if frozenset(given[name]) != getattr(self, name) + ) + if drifted: + raise ValueError( + "incompatible Lexicon pickle: entries are not normalized " + f"({'; '.join(drifted)}); this state was not written by " + "nameparser" + ) def __or__(self, other: Lexicon) -> Lexicon: if not isinstance(other, Lexicon): diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index e30fbb4e..e095deb2 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -205,6 +205,17 @@ def test_regexes_supports_the_v1_read_only_mapping_surface() -> None: assert dict(r.items())["word"] is REGEXES["word"] assert REGEXES["word"] in list(r.values()) assert len(r) == len(REGEXES) + assert r.copy() == dict(REGEXES) # v1's dict.copy() + + +def test_regexes_unsupported_method_is_not_reported_as_a_missing_regex() -> None: + # "no regex named 'pop'" sends the reader looking for a vocabulary + # entry when the truth is the proxy does not implement a mutator + r = _RegexesProxy() + with pytest.raises(AttributeError, match="not supported"): + r.pop + with pytest.raises(AttributeError, match="no regex named"): + r.definitely_not_a_regex def test_regexes_get_is_the_soft_access_escape_hatch() -> None: @@ -361,14 +372,40 @@ def test_snapshot_keeps_a_multi_word_first_name_title() -> None: assert (name.title, name.first, name.last) == ("Grand Duke", "John", "") -def test_snapshot_drops_an_unreachable_first_name_title() -> None: - # the single-word case stays dropped: an entry whose word is not a - # title is unreachable in v1 too, so removing it preserves v1 +@pytest.mark.parametrize(("entry", "text"), [ + # v1's lookup key is lc(joined-title-run), and lc strips only the + # EDGE periods of the whole string, so an interior word keeps its + # period. v2 normalizes per token and then joins, so the entry has + # to be TRANSLATED between the two spellings, not filtered out. + ("lt. col", "Lt. Col. Smith"), + # v1 joins titles across conjunctions, and v2 tags the conjunction + # Role.TITLE too, so the joined key contains it in both versions + ("sir and dame", "Sir and Dame Alex"), +]) +def test_snapshot_translates_a_multi_word_first_name_title( + entry: str, text: str +) -> None: c = Constants() - c.first_name_titles.add("zzqnotatitle") - lexicon, _, _ = c._snapshot() - assert "zzqnotatitle" not in lexicon.given_name_titles - assert HumanName("zzqnotatitle John Smith", constants=c).title == "" + c.titles.add("lt", "col") + c.first_name_titles.add(entry) + name = HumanName(text, constants=c) + assert name.first and not name.last, f"{entry!r} was not honored" + + +def test_snapshot_ungates_an_ambiguous_acronym_added_as_a_suffix_word() -> None: + # v1's is_suffix ORs the acronym and word branches, so adding an + # ambiguous acronym to suffix_not_acronyms simply ungates it: 1.4.0 + # accepts the config and reads "John Smith jd" with suffix='jd'. + # Lexicon forbids the overlap (the word branch would bypass the + # period gate), so translate rather than let the raise through -- + # dropping it from the ambiguous set leaves a plain acronym plus a + # word, which is v1's effective behavior. + c = Constants() + c.suffix_not_acronyms.add("jd") + lexicon, _, _ = c._snapshot() # must not raise + assert "jd" not in lexicon.suffix_acronyms_ambiguous + name = HumanName("John Smith jd", constants=c) + assert (name.first, name.last, name.suffix) == ("John", "Smith", "jd") def test_bound_never_given_prefix_deviates_on_two_pieces() -> None: diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index dddf7a62..8de1d1ee 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -307,6 +307,25 @@ def test_pickle_round_trip_preserves_components() -> None: assert loaded.C is CONSTANTS # shared sentinel restored +@pytest.mark.parametrize("bad", [ + "John", # str: iterating yields characters + {"John": 1}, # Mapping: iterating yields keys only + b"John", # bytes: iterating yields ints + ["John", None], # a non-str entry +]) +def test_setstate_rejects_malformed_component_lists(bad: object) -> None: + # _normset on the Lexicon side rejects str AND Mapping AND checks + # entry types; this guard covered only str, so a dict silently + # loaded its keys and bytes/None died later with an opaque + # AttributeError instead of naming the field + n = HumanName("John Smith") + state = n.__getstate__() + state["first_list"] = bad + loaded = HumanName.__new__(HumanName) + with pytest.raises(TypeError, match="first_list"): + loaded.__setstate__(state) + + def test_setstate_rejects_a_bare_string_component_list() -> None: # a genuine v1 pickle stores lists, but __setstate__ reads foreign # state: a bare str is iterable, so entry.split() over its diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 97688ef5..6a6a74da 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -1,9 +1,10 @@ import dataclasses +import pickle from collections.abc import Callable import pytest -from nameparser._lexicon import Lexicon +from nameparser._lexicon import Lexicon, _normalize def test_entries_are_normalized_at_construction() -> None: @@ -241,16 +242,21 @@ def test_given_name_titles_may_hold_a_multi_word_phrase() -> None: assert "grand duke" in lex.given_name_titles -def test_given_name_titles_words_must_be_titles() -> None: - # a word that is not a title is never read as one, so the entry - # would silently do nothing -- the failure mode the guards prevent. - # Checked per WORD, not per entry, so multi-word phrases stay legal - # (see test_given_name_titles_may_hold_a_multi_word_phrase). - with pytest.raises(ValueError, match="every word of a given_name_titles"): - Lexicon(given_name_titles=frozenset({"sheikh"})) - with pytest.raises(ValueError, match="duke"): - Lexicon(titles=frozenset({"grand"}), - given_name_titles=frozenset({"grand duke"})) +def test_given_name_titles_is_not_constrained_against_titles() -> None: + # Deliberately unvalidated. The lookup key is the joined run of + # Role.TITLE tokens, which the PARSE builds -- it can hold a + # multi-word TITLES member, or a conjunction that is title-tagged + # ("sir and dame"). No static relation over these sets decides + # reachability, and an unreachable entry is inert rather than + # harmful, so nothing is rejected here. + from nameparser.config.titles import TITLES + + multi = next(t for t in TITLES if " " in t) # "chargé d'affaires" + assert Lexicon(given_name_titles=frozenset({multi})).given_name_titles + assert Lexicon(titles=frozenset({"sir", "dame"}), + given_name_titles=frozenset({"sir and dame"})) + # and an entry nothing will ever consult is accepted, not rejected + assert Lexicon(given_name_titles=frozenset({"zzqnotatitle"})) @pytest.mark.parametrize(("make", "marker", "base"), [ @@ -300,11 +306,26 @@ def test_default_lexicon_satisfies_the_bound_particle_invariant() -> None: assert not (d.bound_given_names & d.particles) - d.particles_ambiguous -def test_remove_orphaning_given_name_titles_raises() -> None: +def test_removing_a_title_leaves_its_given_name_marker_alone() -> None: + # the mirror of the above: removing from titles does not raise, and + # the now-inert marker entry simply stops being reachable lex = Lexicon(titles=frozenset({"sheikh"}), given_name_titles=frozenset({"sheikh"})) - with pytest.raises(ValueError, match="every word of a given_name_titles"): - lex.remove(titles={"sheikh"}) + lean = lex.remove(titles={"sheikh"}) + assert lean.titles == frozenset() + assert lean.given_name_titles == frozenset({"sheikh"}) + + +@pytest.mark.parametrize("word", [ + "dr.", " Dr. ", ". a .", ". .", "..x..", " .b. ", +]) +def test_normalize_reaches_a_fixed_point(word: str) -> None: + # strip() then strip(".") leaves periods-around-whitespace half + # done: '. a .' -> ' a ' -> 'a'. Anything that stores a normalized + # value and later re-normalizes it (unpickling, a second add) then + # sees the value change under it, so the fold has to converge. + once = _normalize(word) + assert _normalize(once) == once def test_unpickling_revalidates_invariants() -> None: @@ -321,13 +342,26 @@ def test_unpickling_revalidates_invariants() -> None: forged.__setstate__(state) -def test_unpickling_normalizes_entries() -> None: +def test_unpickling_rejects_unnormalized_entries() -> None: + # a genuine pickle always holds normalized entries, so state that + # does not is foreign. Correcting it silently would make the load + # path a fourth place where caller data is rewritten without a + # word; say so instead, at the site that can still name the entry. lex = Lexicon(titles=frozenset({"dr"})) state = lex.__getstate__() - state["titles"] = {"Sir ", "DR."} # unnormalized, and a set + state["titles"] = {"Sir ", "DR."} restored = Lexicon.__new__(Lexicon) - restored.__setstate__(state) - assert restored.titles == frozenset({"sir", "dr"}) + with pytest.raises(ValueError, match="not normalized"): + restored.__setstate__(state) + + +def test_unpickling_a_real_pickle_is_the_identity() -> None: + # the corollary: anything this library produced round-trips + # unchanged, including entries whose normalized form needed more + # than one strip pass to converge + for lex in (Lexicon.default(), Lexicon.empty(), + Lexicon(titles=frozenset({". a ."}))): + assert pickle.loads(pickle.dumps(lex)) == lex def test_remove_error_offers_the_removal_remedy_too() -> None: From 59040a25d7a1f2c03cdf3e46b7ca541d3dc6f82a Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 15:52:16 -0700 Subject: [PATCH 251/275] Translate only the first_name_titles entries v1 could match The translation added in 23ee08e handled the reachable cases correctly but widened the degenerate ones, in both directions. v1's lookup key is lc(" ".join(title_list)) -- always single-spaced, never empty. So: - An entry holding a whitespace run ("grand duke") could never match there. Translating it collapsed the run and started matching, turning an inert entry into a behavior change. - An entry that is all periods (".", "..") folds to "" under per-word normalization, which _normset rejects -- so a config 1.4.0 accepts and ignores raised ValueError at parse time, naming a field that does not exist on v1's Constants. The same complaint 23ee08e leveled at the suffix_words error, reintroduced two lines away. Filter to entries v1's key shape can produce, then drop any that fold away. Both verified against a live 1.4.0. Two comment corrections from the same review: - the suffix_acronyms_ambiguous subtraction claimed parity with v1; it trades a raise for the two-piece deviation already documented below it ("Jack Ma" is last='Ma' in v1, suffix='Ma' here), so say that rather than claiming equivalence. - __setstate__ said "not written by nameparser" about state that an earlier commit on this branch could legitimately have written, before _normalize converged. Now "not written by this version". Co-Authored-By: Claude Opus 4.8 --- nameparser/_config_shim.py | 20 ++++++++++++++++---- nameparser/_lexicon.py | 2 +- tests/v2/test_config_shim.py | 17 +++++++++++++++++ 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index 45f85aa3..3e9e4fd3 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -981,9 +981,18 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: # spelling; filtering instead dropped every multi-word # honorific containing an abbreviation or a conjunction and # silently swapped given and family. + # Only entries v1 could actually match: its key is the + # joined title run, so always single-spaced and never + # empty. An entry holding a whitespace run was inert there + # (translating it would start matching), and one that folds + # away entirely would trip _normset's empty-entry check on + # a config v1 simply ignored. given_name_titles=frozenset( - " ".join(_normalize(w) for w in e.split()) - for e in self.first_name_titles), + t for t in ( + " ".join(_normalize(w) for w in e.split()) + for e in self.first_name_titles + if e == " ".join(e.split()) + ) if t), suffix_acronyms=acronyms, suffix_words=frozenset(self.suffix_not_acronyms), # Intersect with acronyms: Lexicon enforces ambiguous <= @@ -994,8 +1003,11 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: # adding an ambiguous acronym to suffix_not_acronyms simply # ungates it. Lexicon forbids that overlap because the word # branch bypasses the period gate -- so drop it from the - # ambiguous set, leaving a plain acronym plus a word, which - # is exactly what v1 ends up doing. + # ambiguous set, leaving a plain acronym plus a word. That + # trades a raise for the same two-piece deviation noted + # below: v1's reserve_last guard keeps the final piece as a + # surname ("Jack Ma" -> last='Ma'), and the ungated word + # branch here has none, so it reads suffix='Ma'. suffix_acronyms_ambiguous=( frozenset(self.suffix_acronyms_ambiguous) & acronyms) - frozenset(self.suffix_not_acronyms), diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 8fa6c055..5bdb3ab6 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -390,7 +390,7 @@ def __setstate__(self, state: dict[str, object]) -> None: raise ValueError( "incompatible Lexicon pickle: entries are not normalized " f"({'; '.join(drifted)}); this state was not written by " - "nameparser" + "this version of nameparser" ) def __or__(self, other: Lexicon) -> Lexicon: diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index e095deb2..40f3e221 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -372,6 +372,23 @@ def test_snapshot_keeps_a_multi_word_first_name_title() -> None: assert (name.title, name.first, name.last) == ("Grand Duke", "John", "") +@pytest.mark.parametrize("entry", [".", "..", "grand duke", "grand\tduke"]) +def test_snapshot_drops_first_name_titles_v1_could_not_match(entry: str) -> None: + # v1's key is lc(" ".join(title_list)) -- always single-spaced, and + # never empty -- so an entry with a whitespace run, or one that is + # all periods, could never match there. Translating it anyway would + # either widen behavior (the double-space entry starts matching) or + # raise (the all-period entry normalizes to empty and _normset + # rejects it). 1.4.0 accepts all of these and leaves them inert. + c = Constants() + c.titles.add("grand", "duke") + c.first_name_titles.add(entry) + lexicon, _, _ = c._snapshot() # must not raise + assert "" not in lexicon.given_name_titles + name = HumanName("Grand Duke John", constants=c) + assert (name.first, name.last) == ("", "John") + + @pytest.mark.parametrize(("entry", "text"), [ # v1's lookup key is lc(joined-title-run), and lc strips only the # EDGE periods of the whole string, so an interior word keeps its From dcb102f2f409baca06685ef941d84e0a44b960a8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 15:55:48 -0700 Subject: [PATCH 252/275] Ignore, don't ungate, an ambiguous acronym added as a suffix word MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The subtraction in 23ee08e traded a raise for something worse. It dropped the word from suffix_acronyms_ambiguous, which ungates it -- and v1 does NOT ungate. v1's is_suffix already accepts an ambiguous acronym through the acronym branch, and reserve_last keeps it as the surname, so adding it to suffix_not_acronyms is INERT there: c.suffix_not_acronyms.add("ma"); "Jack Ma" 1.4.0: first='Jack' last='Ma' (same as unconfigured) 23ee08e: first='Jack' last='' suffix='Ma' A person with no family name, silently -- from a change whose stated purpose was avoiding a loud error. Subtract from suffix_words instead, which reproduces v1's inertness. The test that was supposed to cover this used "John Smith jd", where v1 yields suffix='jd' with or without the config, so it passed on a case the translation does not decide. Rewritten to "Jack Ma", where it does. Also from the same review: - The justification for removing the given_name_titles guard cited "chargé d'affaires" as a multi-word TITLES member that would be rejected. Neither version ever tags it -- v2 builds its key per token, so a space-bearing vocabulary entry cannot appear in one. The conjunction case ("sir and dame", where 'and' is tagged Role.TITLE) is real and sufficient on its own; the false premise is struck from the comment and from the test that pinned it. - A stale comment still pointed at the per-word check deleted in the same commit. - The unpickle drift check covered ten fields and silently re-canonicalized the eleventh; capitalization_exceptions now raises like the rest. Every configuration broken across this session's four attempts now matches a live 1.4.0: grand duke, lt. col, sir and dame, ma-as-word, all-period entry, and E.S.Q. Co-Authored-By: Claude Opus 4.8 --- nameparser/_config_shim.py | 29 +++++++++++++++-------------- nameparser/_lexicon.py | 32 +++++++++++++++++++------------- tests/v2/test_config_shim.py | 28 ++++++++++++++++------------ tests/v2/test_lexicon.py | 26 ++++++++++++++++++-------- 4 files changed, 68 insertions(+), 47 deletions(-) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index 3e9e4fd3..af45f542 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -968,6 +968,7 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: titles = frozenset(self.titles) particles = frozenset(self.prefixes) bound = frozenset(self.bound_first_names) + ambiguous_acronyms = frozenset(self.suffix_acronyms_ambiguous) & acronyms # keep in sync with _lexicon._default_lexicon() (pinned by # tests/v2/test_config_shim.py::test_snapshot_field_translation) lexicon = Lexicon( @@ -994,23 +995,23 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: if e == " ".join(e.split()) ) if t), suffix_acronyms=acronyms, - suffix_words=frozenset(self.suffix_not_acronyms), + # Drop any ambiguous acronym from the word set rather than + # the other way round. Lexicon forbids the overlap because + # the word branch bypasses the period gate, and adding an + # ambiguous acronym to suffix_not_acronyms is INERT in v1 + # anyway: is_suffix already accepts it via the acronym + # branch, and reserve_last keeps it as the surname. So + # ignoring the addition reproduces v1 ("Jack Ma" keeps + # last='Ma'), where dropping it from the AMBIGUOUS set + # instead ungated the word and lost the family name -- + # a silent misparse worse than the raise it avoided. + suffix_words=frozenset( + self.suffix_not_acronyms) - ambiguous_acronyms, # Intersect with acronyms: Lexicon enforces ambiguous <= # acronyms; v1 behaves the same when an acronym is deleted # but its ambiguous entry lingers (the entry stops - # mattering). Subtract suffix_words for the mirror reason: - # v1's is_suffix ORs the acronym and word branches, so - # adding an ambiguous acronym to suffix_not_acronyms simply - # ungates it. Lexicon forbids that overlap because the word - # branch bypasses the period gate -- so drop it from the - # ambiguous set, leaving a plain acronym plus a word. That - # trades a raise for the same two-piece deviation noted - # below: v1's reserve_last guard keeps the final piece as a - # surname ("Jack Ma" -> last='Ma'), and the ungated word - # branch here has none, so it reads suffix='Ma'. - suffix_acronyms_ambiguous=( - frozenset(self.suffix_acronyms_ambiguous) - & acronyms) - frozenset(self.suffix_not_acronyms), + # mattering). + suffix_acronyms_ambiguous=ambiguous_acronyms, particles=particles, # complement translation: v1 marks the never-given subset; # v2 marks the may-be-given subset. The trailing union keeps diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 5bdb3ab6..fd971f92 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -36,10 +36,9 @@ #: set before testing suffix_acronyms, so an orphan silently turns a #: word into a period-gated suffix. #: -#: given_name_titles is deliberately NOT here: it is looked up against -#: the space-joined title run, not per token, so a multi-word phrase is -#: a legitimate entry whose words -- not whose phrase -- must be -#: titles. It gets its own per-word check below. +#: given_name_titles is deliberately NOT here and has no check of its +#: own -- see the note in __post_init__ for why every attempt at one +#: rejected working configurations. _SUBSET_FIELDS = ( ("particles_ambiguous", "particles", "an orphan emits a spurious particle-or-given ambiguity"), @@ -260,15 +259,16 @@ def __post_init__(self) -> None: f"{marker}" ) # NOT validated: given_name_titles against titles. The lookup - # key is the space-joined run of Role.TITLE tokens, which is - # built by the parse rather than drawn from this vocabulary -- - # it can hold multi-word entries ("chargé d'affaires" is itself - # a TITLES member) and conjunctions ("sir and dame", where 'and' - # is tagged Role.TITLE). No static relation over these sets can - # decide reachability, and two attempts at one each rejected - # working configurations. An unreachable entry here is inert: - # nothing consults it, nothing misparses. That is the cheap - # failure, and guarding it proved the expensive one. + # key is the space-joined run of Role.TITLE tokens, built by the + # parse rather than drawn from this vocabulary, and a + # conjunction inside a run is itself tagged Role.TITLE -- so + # "sir and dame" is a matchable key whose middle word is in + # conjunctions, not titles. A whole-entry check rejected + # multi-word entries; a per-word check rejected that one. An + # unreachable entry here is inert: nothing consults it, nothing + # misparses. That is the cheap failure, and guarding it proved + # the expensive one -- three working configurations broken + # across two attempts. Do not add a third. # # The v2 form of prefixes.py's NON_FIRST_NAME_PREFIXES-disjoint- # from-BOUND_FIRST_NAMES assertion. That module guards its own @@ -386,6 +386,12 @@ def __setstate__(self, state: dict[str, object]) -> None: for name in _VOCAB_FIELDS if frozenset(given[name]) != getattr(self, name) ) + # the pair field too, or ten fields raise and the eleventh is + # quietly re-canonicalized (keys normalized, sorted, deduped) + given_pairs = cast("Iterable[tuple[str, str]]", + state["capitalization_exceptions"]) + if tuple(given_pairs) != self.capitalization_exceptions: + drifted.append("capitalization_exceptions") if drifted: raise ValueError( "incompatible Lexicon pickle: entries are not normalized " diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index 40f3e221..c033b4aa 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -409,20 +409,24 @@ def test_snapshot_translates_a_multi_word_first_name_title( assert name.first and not name.last, f"{entry!r} was not honored" -def test_snapshot_ungates_an_ambiguous_acronym_added_as_a_suffix_word() -> None: - # v1's is_suffix ORs the acronym and word branches, so adding an - # ambiguous acronym to suffix_not_acronyms simply ungates it: 1.4.0 - # accepts the config and reads "John Smith jd" with suffix='jd'. - # Lexicon forbids the overlap (the word branch would bypass the - # period gate), so translate rather than let the raise through -- - # dropping it from the ambiguous set leaves a plain acronym plus a - # word, which is v1's effective behavior. +def test_snapshot_ignores_an_ambiguous_acronym_added_as_a_suffix_word() -> None: + # Adding an ambiguous acronym to suffix_not_acronyms is INERT in + # v1: is_suffix already accepts it through the acronym branch, and + # reserve_last keeps it as the surname anyway. 1.4.0 with this + # config parses "Jack Ma" exactly as it does without it. + # + # Lexicon forbids the overlap, since the word branch would bypass + # the period gate. Dropping the word from the AMBIGUOUS set instead + # (the reverse subtraction) ungates it and loses the family name + # entirely -- worse than the raise it was meant to avoid. Drop it + # from suffix_words instead, which reproduces v1's inertness. c = Constants() - c.suffix_not_acronyms.add("jd") + c.suffix_not_acronyms.add("ma") lexicon, _, _ = c._snapshot() # must not raise - assert "jd" not in lexicon.suffix_acronyms_ambiguous - name = HumanName("John Smith jd", constants=c) - assert (name.first, name.last, name.suffix) == ("John", "Smith", "jd") + assert "ma" in lexicon.suffix_acronyms_ambiguous + assert "ma" not in lexicon.suffix_words + name = HumanName("Jack Ma", constants=c) + assert (name.first, name.last, name.suffix) == ("Jack", "Ma", "") def test_bound_never_given_prefix_deviates_on_two_pieces() -> None: diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 6a6a74da..33ffc303 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -244,15 +244,12 @@ def test_given_name_titles_may_hold_a_multi_word_phrase() -> None: def test_given_name_titles_is_not_constrained_against_titles() -> None: # Deliberately unvalidated. The lookup key is the joined run of - # Role.TITLE tokens, which the PARSE builds -- it can hold a - # multi-word TITLES member, or a conjunction that is title-tagged - # ("sir and dame"). No static relation over these sets decides - # reachability, and an unreachable entry is inert rather than + # Role.TITLE tokens, which the PARSE builds, and a conjunction + # inside a run is title-tagged too -- so "sir and dame" is a + # matchable key whose middle word is a conjunction, not a title. + # A per-word check rejected it; a whole-entry check rejected + # multi-word entries. An unreachable entry is inert rather than # harmful, so nothing is rejected here. - from nameparser.config.titles import TITLES - - multi = next(t for t in TITLES if " " in t) # "chargé d'affaires" - assert Lexicon(given_name_titles=frozenset({multi})).given_name_titles assert Lexicon(titles=frozenset({"sir", "dame"}), given_name_titles=frozenset({"sir and dame"})) # and an entry nothing will ever consult is accepted, not rejected @@ -355,6 +352,19 @@ def test_unpickling_rejects_unnormalized_entries() -> None: restored.__setstate__(state) +def test_unpickling_rejects_uncanonical_capitalization_exceptions() -> None: + # the pair field gets the same treatment as the vocab sets: it is + # re-canonicalized by __post_init__ (normalized keys, sorted, + # deduped), so accepting a rewritten value would leave one field + # silently corrected while ten raise + lex = Lexicon(capitalization_exceptions=(("phd", "PhD"),)) + state = lex.__getstate__() + state["capitalization_exceptions"] = ((" PhD. ", "Ph.D."), ("aa", "AA")) + restored = Lexicon.__new__(Lexicon) + with pytest.raises(ValueError, match="not normalized"): + restored.__setstate__(state) + + def test_unpickling_a_real_pickle_is_the_identity() -> None: # the corollary: anything this library produced round-trips # unchanged, including entries whose normalized form needed more From 234b35f71f624b02285b7b1d9cfad2af2bc507e6 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 15:56:52 -0700 Subject: [PATCH 253/275] Record this session's hard-won conventions in AGENTS.md Four rounds of review found the same class of mistake repeatedly: inferring an invariant from data that looked inconsistent, then editing the data or config to fit. Two of the notes below are anti-cleanup warnings for exactly the fences that got removed. Gotchas: - Comparing against v1 needs PYTHONSAFEPATH=1 and a directory outside the worktree, or --isolated still imports the branch as "v1" and every comparison reports parity. This produces false confidence rather than an error, which is the worst failure mode a verification method can have. - 'esq' is in both suffix sets on purpose; the branches normalize differently, so it is coverage of two spellings, not duplication. Deleting it lost the family name for "John Smith E.S.Q.". - Lexicon.given_name_titles is deliberately unvalidated, with the reason: a conjunction inside a title run is itself tagged Role.TITLE, so no static relation over the vocabulary decides reachability. Two guards, three broken configurations. - _normalize must reach a fixed point, now that storage and match-time share one fold and unpickling re-validates. 2.0 conventions: - Invariants guard harm, not no-ops. The criterion that would have prevented most of the above: construct the config it forbids and check what actually breaks. - The shim translates; it never raises on a config v1 accepted and never silently changes the parse. Documents all four transformations with their v1-reachability arguments, and the testing rule that would have caught the last bug -- test the case the translation DECIDES, not one where both branches agree. Co-Authored-By: Claude Opus 4.8 --- AGENTS.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index eca650d2..925f7602 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,6 +119,8 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. Sanctioned deviation, facade layer only: `HumanName` and the shim `Constants` organize by v1 concern groups (`# -- render defaults --`, `# -- config / parsing --`, `# -- fields --`, ..., dunders and pickle last) — the classes mirror v1's own surface and die in 3.0; the canonical order still binds every core type. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). +- **Invariants guard harm, not no-ops**: add a constructor check when violating it produces a *wrong parse*, not when it produces *nothing*. A false positive costs a working configuration; a true positive on an inert condition costs the user nothing, so that trade is never worth taking. `suffix_acronyms_ambiguous ∩ suffix_words` is guarded because the overlap loses a family name; `given_name_titles` is not, because an unreachable entry is simply never consulted (see Gotchas). Before adding one, construct the config it forbids and check what actually breaks. +- **The shim TRANSLATES; it never raises on a config v1 accepted, and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Four exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), and `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`). When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). - **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. - **Pickling**: v2 types must round-trip (`Parser` will be picklable by construction, and it holds a `Lexicon`). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, and values are deliberately NOT re-validated (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` keeps its own copy of the guard (layering) plus the `mappingproxy` slot rebuild; a new unpicklable slot type needs the same treatment plus a round-trip test. @@ -164,6 +166,14 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **`suffix_not_acronyms` vs `is_an_initial` tension** — single-letter roman numeral suffixes (`i`, `v`) are in `suffix_not_acronyms` but also match the `is_an_initial` regex (single uppercase letter), so `is_suffix()` rejects them. The lenient test lives in `is_suffix_lenient()`, which accepts `suffix_not_acronyms` members unconditionally and is only safe in unambiguous positions: (1) suffix-comma detection uses it via `are_suffixes_after_comma()`; (2) lastname-comma post-comma parsing uses it inline, only when `nxt is None` and `len(parts)==2` (no `parts[2]` suffix segment). See issues #136, #144. +**Comparing against v1 needs `PYTHONSAFEPATH=1` AND a directory outside the worktree** — `uv run --isolated --no-project --with 'nameparser==1.4.0'` still puts the checkout's `nameparser/` ahead of the pinned wheel on `sys.path`, so the "v1" side silently imports the branch and every comparison reports parity. Run it as `cd && PYTHONSAFEPATH=1 uv run --isolated --no-project --with 'nameparser==1.4.0' python -c "..."`. This produces false confidence rather than an error, so it invalidates results without ever looking wrong. + +**`esq` is in both `SUFFIX_ACRONYMS` and `SUFFIX_NOT_ACRONYMS` on purpose — do not "deduplicate" it** — the two branches normalize differently (see the asymmetry gotcha above): the word test strips only edge periods, the acronym test strips all of them. `Esq` matches only through the word set, `E.S.Q.` only through the acronym set. Removing either membership drops a spelling and, for the acronym one, loses the family name (`"John Smith E.S.Q."` → `family='E.S.Q.'`). Pinned by the `suffix_acronym_multidot_spelling` case-table row. The disjointness that IS asserted is `SUFFIX_ACRONYMS_AMBIGUOUS ∩ SUFFIX_NOT_ACRONYMS`, which is a different claim: `suffix_as_written` ORs the branches, so a word membership bypasses the period gate the ambiguous set exists to impose. + +**`Lexicon.given_name_titles` is deliberately unvalidated against `titles` — do not add a check** — the lookup key is the space-joined run of `Role.TITLE` tokens, built by the parse, and a conjunction inside a run is itself tagged `Role.TITLE`, so `"sir and dame"` is a matchable key whose middle word lives in `conjunctions`. A whole-entry check rejected multi-word entries; a per-word check rejected that one. No static relation over the vocabulary sets decides reachability. An unreachable entry is inert — nothing consults it and nothing misparses — so the condition being "guarded" costs the user nothing while each guard cost a working configuration. If a diagnostic is wanted, it must be non-blocking (a property, a repr note, a warning), never a raise. + +**`_normalize` must reach a fixed point** — storage and match-time share the one fold, and `Lexicon.__setstate__` re-validates, so a value that changes on re-normalization changes under its owner. `strip().strip(".")` alone is not idempotent (`'. a .'` → `' a '` → `'a'`). The loop is the fix; keep any new stripping inside it. + **Expected-failure tests use `@pytest.mark.xfail`** — the conftest parametrized fixture breaks `@unittest.expectedFailure`; always use `@pytest.mark.xfail` instead. **`lc()` strips leading and trailing periods** — `'M.D.'` → `'m.d'`, not `'md'` (interior periods are preserved). Exception keys in `capitalization_exceptions` are dot-free, so lookups must also try `.replace('.', '')`. From 8147ac6f2867906198c6aa0084348997dc674c85 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 16:12:16 -0700 Subject: [PATCH 254/275] An ambiguous acronym is a suffix when the name can spare it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v1 reads "John Smith MA" as suffix='MA' and "Jack MA" as last='MA'. That is not inconsistency — it is evidence-weighing. With only two pieces, "one of them is a credential" is the less likely reading, so the ambiguous acronym stays the surname; with three, peeling it still leaves a given and a family name, so the credential reading wins. 2.0 had generalized the period gate over every position, making a bare ambiguous acronym never a suffix, so "John Smith MA" came out family='MA', middle='Smith'. That was an unclassified divergence, and the wrong call: MA after a full name really is more likely a degree. Root cause is the same shape as the rest of this session's bugs. In v1 suffix_acronyms_ambiguous has exactly ONE use site -- parse_nicknames' delimited-content escape, deciding whether "(JD)" is a nickname or a suffix. It is a delimiter-path disambiguator, not a vocabulary property, and 2.0 promoted it to a universal rule. Narrow it to what it can decide: peel a bare ambiguous acronym only when at least two name pieces remain. This is v1's reserve_last restricted to the ambiguous set -- 2.0 still peels UNambiguous suffixes when nothing is left ("Smith PhD" -> suffix, a classified fix), because there the vocabulary is not in doubt. Verified against a live 1.4.0: 9 of 10 shapes now identical, the tenth being that classified PhD fix. Four case-table rows pin both halves of the rule, and the release-log bullet now describes it instead of claiming periods are the only path. Case rows use uppercase "MA" where a suffix is expected, so the intended reading is legible without running the parser. (Case itself is not yet an input to the decision; it could reasonably become one.) Co-Authored-By: Claude Opus 4.8 --- docs/release_log.rst | 2 +- nameparser/_pipeline/_assign.py | 13 +++++++++++++ tests/v2/cases.py | 26 +++++++++++++++++++------- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index 8bd08692..12a113f5 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -54,7 +54,7 @@ Release Log - Fix a lone recognized trailing suffix with no comma being routed to ``first``/``last``: ``"Johnson PhD"`` and ``"Mr. Johnson PhD"`` now keep the suffix in ``suffix`` - Fix a split ``"Ph. D."`` credential being read as two tokens; it now classifies as one suffix, replacing v1's ``fix_phd`` hook - Fold a leading never-given particle into the family name. Note that ``Lexicon.particles_ambiguous`` is the **complement** of v1's ``non_first_name_prefixes``, not a rename -- it lists the particles that *may* double as a given name, where v1 listed the ones that may not. Copying a v1 customization across without inverting it silently reverses the behavior; see :doc:`migrate` - - Add ``ma`` and ``do`` to ``suffix_acronyms_ambiguous``, so ``"Jack Ma"`` and ``"Anh Do"`` keep their family names. Ambiguous acronyms count as suffixes only when written with periods; as a side effect, a parenthesized or quoted ``"(MA)"``/``"(DO)"`` now falls through to nickname parsing instead of escaping to ``suffix`` + - Add ``ma`` and ``do`` to ``suffix_acronyms_ambiguous``, the set of post-nominals that are also ordinary surnames. An entry there is read as a credential when the name can spare it — written with periods (``"John Smith M.A."``), or when removing it still leaves a given *and* a family name (``"John Smith MA"`` → suffix ``MA``). With only two pieces to go around, the surname reading wins instead: ``"Jack Ma"`` and ``"Anh Do"`` keep their family names. As a side effect, a parenthesized or quoted ``"(MA)"``/``"(DO)"`` now falls through to nickname parsing rather than escaping to ``suffix``, since inside delimiters the nickname reading is the plausible one - Change ``comparison_key()`` and ``matches()`` in both APIs to fold with ``str.casefold()`` where 1.4 used ``str.lower()``, so Unicode case pairs compare equal -- ``"STRASSE"`` matches ``"Straße"``, and a Greek final sigma matches its regular form. This is strictly more permissive: anything 1.4 matched still matches. Vocabulary normalization deliberately still uses ``lower()``, for v1 parity - Change the 2.0 API's default ``render()``/``str()`` spec to show every non-empty field: ``'{title} {given} "{nickname}" {middle} {family} ({maiden}) {suffix}'``. The quoted nickname round-trips exactly; the parenthesized maiden re-parses as a nickname, a deliberate choice of presentation over lossless round-trip -- use ``née {maiden}`` in a custom spec if you need it to survive a reparse. ``HumanName`` keeps v1's own ``string_format`` default, unchanged - Change delimiter-overlap precedence in the 2.0 API: a pair listed in ``Policy.maiden_delimiters`` is dropped from the effective nickname set, so ``Policy(maiden_delimiters={("(", ")")})`` alone routes parenthesized content to ``maiden``. The default nickname set is exported as ``DEFAULT_NICKNAME_DELIMITERS``. ``HumanName`` keeps v1's nickname-wins precedence, so no existing behavior changes diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index bd2a086f..335e456f 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -136,6 +136,19 @@ def _assign_main(seg_idx: int, state: ParseState, and "initial" not in tokens[pieces[rest[k - 2]][0]].tags): k -= 1 continue + # A bare ambiguous acronym ("MA", not "M.A.") is a credential + # only when peeling it still leaves a given AND a family name. + # With two pieces, "one of them is a credential" is the less + # likely reading, so it stays the family name -- "Jack MA" is a + # person, "John Smith MA" is a person with a degree. This is + # v1's reserve_last narrowed to the ambiguous set: 2.0 + # deliberately peels UNambiguous suffixes even when nothing is + # left ("Smith PhD" -> suffix, a classified fix), because there + # the vocabulary is not in doubt. + if (k - 1 >= 2 and len(piece) == 1 + and "vocab:suffix-ambiguous" in tokens[piece[0]].tags): + k -= 1 + continue break name_pieces, suffix_pieces = rest[:k], rest[k:] if not name_pieces and suffix_pieces: diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 559502b8..8224c9c4 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -85,13 +85,25 @@ def __post_init__(self) -> None: notes="v1 PREPENDED middle_list to last_list; folded tokens " "carry vocab:folded-middle and the family view orders " "them first (spans cannot reorder)"), - Case("ambiguous_surname_acronyms", "Jack Ma", - {"given": "Jack", "family": "Ma"}, - notes="'ma'/'do' joined suffix_acronyms_ambiguous: common " - "surnames need periods to read as credentials (v1 " - "parity restored; data change, flag for release log)"), - Case("ambiguous_surname_acronym_with_suffix", "Jack Ma Jr", - {"given": "Jack", "family": "Ma", "suffix": "Jr"}), + Case("ambiguous_surname_acronyms", "Jack MA", + {"given": "Jack", "family": "MA"}, + notes="'ma'/'do' joined suffix_acronyms_ambiguous: with only " + "two pieces, 'one of them is a credential' is the less " + "likely reading, so the ambiguous acronym stays the " + "family name (v1 parity via its reserve_last)"), + Case("ambiguous_surname_acronym_with_suffix", "Jack MA Jr", + {"given": "Jack", "family": "MA", "suffix": "Jr"}, + notes="'Jr' peels first; 'MA' would then be the only piece " + "left beside the given name, so it stays family"), + Case("ambiguous_acronym_is_a_suffix_when_a_family_name_remains", + "John Smith MA", + {"given": "John", "family": "Smith", "suffix": "MA"}, + notes="the other half of the same rule: three pieces means " + "peeling 'MA' still leaves given+family, so the " + "credential reading wins (v1 parity)"), + Case("ambiguous_acronym_suffix_with_middle", "John Q Smith MA", + {"given": "John", "middle": "Q", "family": "Smith", + "suffix": "MA"}), Case("initial_shaped_not_conjunction", "john e. smith", {"given": "john", "middle": "e.", "family": "smith"}, notes="v1 is_conjunction excludes initials at classify too"), From ed71eb42e453225ea25f03bf5a11eaf0f39b3288 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 16:32:16 -0700 Subject: [PATCH 255/275] Report the ambiguous-suffix readings instead of guessing silently particles_ambiguous emitted PARTICLE_OR_GIVEN; suffix_acronyms_ambiguous emitted nothing, for the structurally identical situation -- a word in an ambiguous vocabulary where the parser picks one of two readings. "parse('derek gulbranson MA').ambiguities" was empty even though the parser had just chosen credential over surname. Two kinds now emit: SUFFIX_OR_FAMILY (new) at the peel in _assign, in BOTH directions -- "John Smith MA" reports that MA was read as a suffix, "Jack MA" that it was read as the family name. Both are guesses. SUFFIX_OR_NICKNAME (reserved since the core API landed) at classify, for delimited content the vocabulary cannot settle: "(MBA)" escapes to suffix on vocabulary alone, so "(JD)" keeping the nickname reading was a silent coin-flip. Emitted at classify rather than at the escape in extract, which runs before tokenize and so has no token index to point at. The emission has to be at the DECISION site, not on tag presence. In "Joao da Silva do Amaral de Souza" the token 'do' carries vocab:suffix-ambiguous but sits mid-name and is never at the peel boundary -- no choice was made, so nothing is reported. Same reasoning excludes "Ma, Jack", where a comma fixes the family before the question arises: that mirrors the existing rule that PARTICLE_OR_GIVEN is not emitted on the FAMILY_COMMA path. An ambiguity is a property of a decision, not of a token. Corpus impact is ~1%: 5 of 486 names. test_every_ambiguity_kind_has_a_registered_trigger caught the new kind immediately and the case table's exact-ambiguities assertion caught all six affected rows -- both guards did their job. Co-Authored-By: Claude Opus 4.8 --- docs/release_log.rst | 2 +- docs/usage.rst | 21 ++++++++++++++++ nameparser/_pipeline/_assign.py | 23 +++++++++++++++-- nameparser/_pipeline/_classify.py | 23 +++++++++++++++-- nameparser/_types.py | 11 +++++++-- tests/v2/cases.py | 13 +++++++--- tests/v2/test_contracts.py | 4 +-- tests/v2/test_parser.py | 41 +++++++++++++++++++++++++++++++ 8 files changed, 126 insertions(+), 12 deletions(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index 12a113f5..9c5b872f 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -24,7 +24,7 @@ Release Log - Add ``PatronymicRule`` with the members ``EAST_SLAVIC`` and ``TURKIC``. v1's single ``patronymic_name_order`` flag enabled both detectors at once; ``Policy(patronymic_rules=...)`` lets you enable either one alone - Add ``PolicyPatch`` and the ``UNSET`` sentinel for partial policy deltas that compose -- set-valued fields union, scalar fields override with later winning. This is the mechanism locale packs are built from, and ``UNSET`` is only needed when you must distinguish "not set" from a real ``False`` or ``None`` - Add ``Token``, ``Span`` and ``Role``: every field is backed by tokens carrying exact ``(start, end)`` offsets into the original string, reachable with ``tokens_for(Role.GIVEN)``. This replaces v1's ``*_list`` attributes and makes it possible to highlight or re-slice the input the parse came from - - Add ``Ambiguity`` and the ``AmbiguityKind`` enum, so a parse reports what it had to guess at instead of guessing silently. The kinds emitted today are ``particle-or-given``, ``unbalanced-delimiter`` and ``comma-structure``; ``order`` and ``suffix-or-nickname`` are reserved and not yet emitted + - Add ``Ambiguity`` and the ``AmbiguityKind`` enum, so a parse reports what it had to guess at instead of guessing silently. The kinds emitted today are ``particle-or-given``, ``suffix-or-family``, ``suffix-or-nickname``, ``unbalanced-delimiter`` and ``comma-structure``; ``order`` is reserved and not yet emitted. The two suffix kinds cover the post-nominals that are also ordinary words: ``"John Smith MA"`` reports that ``MA`` was read as a credential rather than a surname, and ``"JEFFREY (JD) BRICKEN"`` that the delimited ``JD`` was read as a nickname rather than a suffix. A reading the vocabulary settles on its own — ``"John Smith M.A."``, ``"Andrew Perkins (MBA)"`` — is not a guess and reports nothing - Add ``ParsedName`` output and comparison methods: ``render(spec)``, ``initials()``, ``capitalized()`` (which returns a new value rather than mutating in place), ``as_dict()``, ``replace(**fields)``, ``matches()`` and ``comparison_key()`` - Add ``Locale``, the public pack type. Writing your own needs no registration -- construct a ``Locale`` and pass it to ``parser_for()`` - Ship a fully typed public API (PEP 561): the core modules are checked under strict mypy settings, and nameparser 2.0 has no runtime dependencies diff --git a/docs/usage.rst b/docs/usage.rst index 800e0622..2e1ebc60 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -318,6 +318,27 @@ so branching on a kind needs no import: >>> [t.text for t in name.ambiguities[0].tokens] ['Van'] +The post-nominals that double as ordinary surnames report the same way. +``MA`` after a full name is read as a credential, but after a single +given name it stays the surname — either way the choice is recorded: + +.. doctest:: + + >>> parse("John Smith MA").suffix + 'MA' + >>> [a.kind.value for a in parse("John Smith MA").ambiguities] + ['suffix-or-family'] + >>> parse("Jack MA").family + 'MA' + +A reading the vocabulary settles on its own is not a guess and reports +nothing — periods make ``M.A.`` unambiguously a credential: + +.. doctest:: + + >>> parse("John Smith M.A.").ambiguities + () + Most names report none. A non-empty ``ambiguities`` is a useful signal for routing a record to human review instead of trusting it silently. diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 335e456f..bb7187d3 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -124,6 +124,9 @@ def _assign_main(seg_idx: int, state: ParseState, # every piece is a strict suffix (v1's are_suffixes tail rule, with # the roman-numeral special: a final roman numeral after a # non-initial piece is a suffix) + # (piece, reading taken, reading declined) when the peel had to + # resolve a bare ambiguous acronym; both directions are guesses + ambiguous_pick: tuple[tuple[int, ...], str, str] | None = None k = len(rest) while k > 0: piece = pieces[rest[k - 1]] @@ -145,10 +148,18 @@ def _assign_main(seg_idx: int, state: ParseState, # deliberately peels UNambiguous suffixes even when nothing is # left ("Smith PhD" -> suffix, a classified fix), because there # the vocabulary is not in doubt. - if (k - 1 >= 2 and len(piece) == 1 - and "vocab:suffix-ambiguous" in tokens[piece[0]].tags): + bare_ambiguous = (len(piece) == 1 + and "vocab:suffix-ambiguous" in tokens[piece[0]].tags) + if bare_ambiguous and k - 1 >= 2: + ambiguous_pick = (piece, "a suffix", "an ordinary surname") k -= 1 continue + if bare_ambiguous and k >= 2: + # not peeled, so it stays the last name piece -- the family + # name under every order. (k < 2 means it is the only piece + # left and lands in the given position, which is not the + # fork this reports.) + ambiguous_pick = (piece, "a family name", "a post-nominal") break name_pieces, suffix_pieces = rest[:k], rest[k:] if not name_pieces and suffix_pieces: @@ -159,6 +170,14 @@ def _assign_main(seg_idx: int, state: ParseState, _set_roles(tokens, pieces[piece_idx], roles[pos]) for piece_idx in suffix_pieces: _set_roles(tokens, pieces[piece_idx], Role.SUFFIX) + if ambiguous_pick is not None: + piece, taken, declined = ambiguous_pick + ambiguities.append(PendingAmbiguity( + AmbiguityKind.SUFFIX_OR_FAMILY, + f"{tokens[piece[0]].text!r} written without periods is both " + f"a post-nominal and a surname; read as {taken} rather than " + f"{declined}", + tuple(piece))) # leading ambiguous particle read as a name (#121 surfaced) if name_pieces: head = pieces[name_pieces[0]] diff --git a/nameparser/_pipeline/_classify.py b/nameparser/_pipeline/_classify.py index 317be8f4..78bd988d 100644 --- a/nameparser/_pipeline/_classify.py +++ b/nameparser/_pipeline/_classify.py @@ -19,7 +19,10 @@ import dataclasses from nameparser._lexicon import _normalize -from nameparser._pipeline._state import ParseState, WorkToken +from nameparser._pipeline._state import ( + ParseState, PendingAmbiguity, WorkToken, +) +from nameparser._types import AmbiguityKind, Role from nameparser._pipeline._vocab import ( is_initial, period_joined_vocab, suffix_as_written, ) @@ -73,4 +76,20 @@ def classify(state: ParseState) -> ParseState: tokens = tuple( dataclasses.replace(t, tags=_tags_for(t, state)) for t in state.tokens) - return dataclasses.replace(state, tokens=tokens) + # Delimited content whose vocabulary cannot settle it: extract's + # escape sends an UNambiguous suffix straight through ("(MBA)" -> + # suffix) and keeps everything else as a nickname, so an AMBIGUOUS + # acronym in there was a coin the parser had to call. Reported here + # rather than at the escape itself, which runs before tokenize and + # so has no token index to point at. + ambiguities = list(state.ambiguities) + for i, token in enumerate(tokens): + if (token.role is Role.NICKNAME + and "vocab:suffix-ambiguous" in token.tags): + ambiguities.append(PendingAmbiguity( + AmbiguityKind.SUFFIX_OR_NICKNAME, + f"delimited {token.text!r} is also a post-nominal; read " + f"as a nickname rather than a suffix", + (i,))) + return dataclasses.replace(state, tokens=tokens, + ambiguities=tuple(ambiguities)) diff --git a/nameparser/_types.py b/nameparser/_types.py index 0d78601c..c1aaa5f2 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -231,9 +231,16 @@ class AmbiguityKind(StrEnum): #: two-word name under a non-default name_order). Not yet emitted; #: planned for 2.x. ORDER = "order" - #: Reserved: a trailing word reads plausibly as either a suffix or - #: a nickname. Not yet emitted; planned for 2.x. + #: Delimited content is an ambiguous suffix acronym, so it reads + #: plausibly as either a post-nominal or a nickname -- "JEFFREY + #: (JD) BRICKEN" keeps the nickname reading, where the + #: unambiguous "(MBA)" escapes to suffix on vocabulary alone. SUFFIX_OR_NICKNAME = "suffix-or-nickname" + #: An ambiguous suffix acronym written without periods, which is + #: also an ordinary surname -- "John Smith MA" reads MA as a + #: post-nominal because a family name remains, "Jack MA" reads it + #: as the family name because none would. + SUFFIX_OR_FAMILY = "suffix-or-family" #: A leading ambiguous particle was read as a given name -- "Van #: Johnson" parses given="Van", but "Van" is also a family-name #: particle in other names. diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 8224c9c4..f2059869 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -87,23 +87,27 @@ def __post_init__(self) -> None: "them first (spans cannot reorder)"), Case("ambiguous_surname_acronyms", "Jack MA", {"given": "Jack", "family": "MA"}, + ambiguities=("suffix-or-family",), notes="'ma'/'do' joined suffix_acronyms_ambiguous: with only " "two pieces, 'one of them is a credential' is the less " "likely reading, so the ambiguous acronym stays the " "family name (v1 parity via its reserve_last)"), Case("ambiguous_surname_acronym_with_suffix", "Jack MA Jr", {"given": "Jack", "family": "MA", "suffix": "Jr"}, + ambiguities=("suffix-or-family",), notes="'Jr' peels first; 'MA' would then be the only piece " "left beside the given name, so it stays family"), Case("ambiguous_acronym_is_a_suffix_when_a_family_name_remains", "John Smith MA", {"given": "John", "family": "Smith", "suffix": "MA"}, + ambiguities=("suffix-or-family",), notes="the other half of the same rule: three pieces means " "peeling 'MA' still leaves given+family, so the " "credential reading wins (v1 parity)"), Case("ambiguous_acronym_suffix_with_middle", "John Q Smith MA", {"given": "John", "middle": "Q", "family": "Smith", - "suffix": "MA"}), + "suffix": "MA"}, + ambiguities=("suffix-or-family",)), Case("initial_shaped_not_conjunction", "john e. smith", {"given": "john", "middle": "e.", "family": "smith"}, notes="v1 is_conjunction excludes initials at classify too"), @@ -399,11 +403,14 @@ def __post_init__(self) -> None: Case("whitespace", " ", {}), Case("bare_ambiguous_acronym", "John Ed", {"given": "John", "family": "Ed"}, - notes="'ed' is an ambiguous acronym; bare form is a name (C1)"), + ambiguities=("suffix-or-family",), + notes="'ed' is an ambiguous acronym; bare form is a name (C1), " + "and the parse reports which reading it took"), Case("comma_ambiguous_acronym", "Smith, Ed", {"given": "Ed", "family": "Smith"}), Case("ambiguous_acronym_with_suffix", "John Ed III", - {"given": "John", "family": "Ed", "suffix": "III"}), + {"given": "John", "family": "Ed", "suffix": "III"}, + ambiguities=("suffix-or-family",)), Case("phd_split", "John Ph. D.", {"given": "John", "suffix": "Ph. D."}, notes="v1 fix_phd; healed via the stable 'joined' tag"), diff --git a/tests/v2/test_contracts.py b/tests/v2/test_contracts.py index c25f58de..6b6b495b 100644 --- a/tests/v2/test_contracts.py +++ b/tests/v2/test_contracts.py @@ -11,10 +11,10 @@ AmbiguityKind.PARTICLE_OR_GIVEN: "Van Johnson", AmbiguityKind.UNBALANCED_DELIMITER: 'Jon "Nick Smith', AmbiguityKind.COMMA_STRUCTURE: "Smith, John, Extra, Jr.", + AmbiguityKind.SUFFIX_OR_NICKNAME: "JEFFREY (JD) BRICKEN", + AmbiguityKind.SUFFIX_OR_FAMILY: "John Smith MA", # no emitter yet -- arrives with locale-pack order detection (2.x) AmbiguityKind.ORDER: None, - # no emitter yet -- arrives with suffix/nickname refinement (2.x) - AmbiguityKind.SUFFIX_OR_NICKNAME: None, } diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 2ed3477b..cf3eca02 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -170,3 +170,44 @@ def test_phd_split_mid_name_is_a_suffix() -> None: assert pn.suffix == "Ph. D." assert pn.family == "Smith" assert pn.middle == "" + + +def test_ambiguous_acronym_reports_the_reading_it_took() -> None: + # 'ma' is both a post-nominal and a surname, so whichever way the + # peel resolves it is a guess -- the same shape as the leading + # ambiguous particle that already reports PARTICLE_OR_GIVEN + took_suffix = parse("John Smith MA") + assert took_suffix.suffix == "MA" + assert [a.kind for a in took_suffix.ambiguities] == \ + [AmbiguityKind.SUFFIX_OR_FAMILY] + assert [t.text for t in took_suffix.ambiguities[0].tokens] == ["MA"] + + took_family = parse("Jack MA") + assert took_family.family == "MA" + assert [a.kind for a in took_family.ambiguities] == \ + [AmbiguityKind.SUFFIX_OR_FAMILY] + + +@pytest.mark.parametrize("text", [ + "John Smith M.A.", # periods decide it; no guess + "Ma, Jack", # a comma fixes the family + "Joao da Silva do Amaral de Souza", # 'do' mid-name, never at the peel + "John Smith PhD", # unambiguous vocabulary +]) +def test_no_suffix_ambiguity_when_nothing_was_guessed(text: str) -> None: + assert [a for a in parse(text).ambiguities + if a.kind is AmbiguityKind.SUFFIX_OR_FAMILY] == [] + + +def test_delimited_ambiguous_acronym_reports_suffix_or_nickname() -> None: + # inside delimiters the competing readings are suffix and nickname: + # "(MBA)" is unambiguously a credential and escapes to suffix, while + # "(JD)" could be either, so it keeps the nickname reading -- a + # guess, and until now a silent one + n = parse("JEFFREY (JD) BRICKEN") + assert n.nickname == "JD" + assert [a.kind for a in n.ambiguities] == \ + [AmbiguityKind.SUFFIX_OR_NICKNAME] + assert [t.text for t in n.ambiguities[0].tokens] == ["JD"] + # the unambiguous one decided on vocabulary, so it is not a guess + assert parse("Andrew Perkins (MBA)").ambiguities == () From 49c67a2dbec86e6327b64d1a13998f898b057726 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 16:38:09 -0700 Subject: [PATCH 256/275] Document that an ambiguity records a decision, not a word The constraint that shaped the two new emitters is not obvious from either side, and ORDER is still reserved -- whoever wires it next needs the rule, and users need to know why a name containing an ambiguous word can report nothing. Three places, each for its own audience: - concepts.rst, in the Honest ambiguity essay: an empty ambiguities means the parse faced no fork, not that every word was unambiguous. Uses the two cases that surprised me -- 'do' mid-name in "Joao da Silva do Amaral de Souza", where nothing chooses, and "Ma, Jack", where the comma settles it before the question arises. - AmbiguityKind's docstring, which is what modules.rst renders into the API reference: the same point in two sentences, for a reader who never opens the concepts page. - AGENTS.md: the implementer rule. Emit at the decision site, never by scanning for a vocab:*-ambiguous tag; report both directions of a two-way fork; register a trigger in _AMBIGUITY_TRIGGERS and pin the kinds in the case table. Notes that the decision site already holds the token index and detail text a tag scan would have to rebuild. Co-Authored-By: Claude Opus 4.8 --- AGENTS.md | 1 + docs/concepts.rst | 10 ++++++++++ nameparser/_types.py | 8 +++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 925f7602..fe0f5376 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,6 +119,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. Sanctioned deviation, facade layer only: `HumanName` and the shim `Constants` organize by v1 concern groups (`# -- render defaults --`, `# -- config / parsing --`, `# -- fields --`, ..., dunders and pickle last) — the classes mirror v1's own surface and die in 3.0; the canonical order still binds every core type. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). +- **Ambiguities are emitted at the DECISION site**: an `Ambiguity` records a fork the parse had to call, not a token that sits in an ambiguous vocabulary. Emit where the branch is taken — the trailing-suffix peel in `_assign`, the delimiter escape's follow-up in `classify` — never by scanning for a `vocab:*-ambiguous` tag. The same tagged token is a genuine fork in one position and unremarkable in another (`do` mid-name in "Joao da Silva do Amaral de Souza" chooses nothing), and structure often settles the question before it arises, which is why `PARTICLE_OR_GIVEN` is deliberately not emitted on the `FAMILY_COMMA` path and `SUFFIX_OR_FAMILY` is not emitted for "Ma, Jack". The decision site also has the token index and the detail text in hand, which the tag scan would have to reconstruct. Report BOTH directions of a two-way fork — "John Smith MA" (read as a suffix) and "Jack MA" (read as the family name) are equally guesses. Every kind needs a trigger in `tests/v2/test_contracts.py::_AMBIGUITY_TRIGGERS` (an explicit `None`, strict-xfail, while reserved), and case-table rows pin expected kinds exactly, so a new emitter shows up in both immediately. - **Invariants guard harm, not no-ops**: add a constructor check when violating it produces a *wrong parse*, not when it produces *nothing*. A false positive costs a working configuration; a true positive on an inert condition costs the user nothing, so that trade is never worth taking. `suffix_acronyms_ambiguous ∩ suffix_words` is guarded because the overlap loses a family name; `given_name_titles` is not, because an unreachable entry is simply never consulted (see Gotchas). Before adding one, construct the config it forbids and check what actually breaks. - **The shim TRANSLATES; it never raises on a config v1 accepted, and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Four exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), and `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`). When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). diff --git a/docs/concepts.rst b/docs/concepts.rst index 1bdc90b0..249834b7 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -154,6 +154,16 @@ other names, so the parse records a ``particle-or-given`` ambiguity alongside its answer. You can inspect ``ambiguities`` to decide, case by case, whether your data needs a second look. +An ambiguity records a *decision*, not a word. The same token in a +different position may present no fork at all: ``do`` is in the +ambiguous post-nominal vocabulary, but in ``"Joao da Silva do Amaral +de Souza"`` it sits mid-name, where nothing has to choose between +readings — so nothing is recorded. A comma can settle the question +before it arises, too: ``"Ma, Jack"`` fixes the family name, so the +credential reading never comes up, while ``"John Smith MA"`` has to +call it and says so. An empty ``ambiguities`` means the parse faced no +fork, not that every word in it was unambiguous. + :class:`Tokens ` also carry tags — a second, independent label alongside their role, recording how a token was classified rather than what part of the name it belongs to — but only a handful of them are part of the diff --git a/nameparser/_types.py b/nameparser/_types.py index c1aaa5f2..854c5e2e 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -225,7 +225,13 @@ class AmbiguityKind(StrEnum): """The stable vocabulary of :class:`Ambiguity` kinds. A StrEnum: members ARE their string values, so ``kind == "particle-or-given"`` compares directly. New kinds may be added in minor releases; - existing values never change meaning.""" + existing values never change meaning. + + A kind names a FORK THE PARSE HAD TO CALL, not a word that could be + read two ways. The same token elsewhere in a name may present no + choice at all and is then reported by nothing -- an empty + ``ambiguities`` means the parse faced no fork, not that every word + was unambiguous.""" #: Reserved: the name's field order itself is uncertain (e.g. a #: two-word name under a non-default name_order). Not yet emitted; From d993db0fd930bdf0e0009a2029e87fa8f100cb3c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 16:45:46 -0700 Subject: [PATCH 257/275] Report unmatched closing delimiters too AmbiguityKind.UNBALANCED_DELIMITER has always documented itself as "a delimiter opened without closing (or closed without opening)", but only the first half existed. The scan is opener-driven -- it finds an open and looks rightward for its close -- so a close with nothing to its left was never in the search space: 'Jon "Nick Smith' -> unbalanced-delimiter 'John Smith)' -> (nothing) No design reason, just the direction of the walk. Both signal the same malformed input. Sweep for boundary-valid closes that no match consumed, reusing _close_ok -- which is what keeps apostrophes out of it. Verified on the 486-name corpus: "O'connor", "O'B.", "Queen's" all have close_ok=False because the apostrophe is mid-word, so none of them fire. Offsets already reported as unmatched OPENS are skipped, so a symmetric delimiter that satisfies both boundary tests still yields exactly one ambiguity ('John " Smith'). One corpus name newly reports: "Mari' Aube'", two word-final apostrophes. That is arguably correct -- a trailing apostrophe really is ambiguous between punctuation and a closing quote -- and the ambiguity is advisory, so the parse is unchanged (given "Mari'", family "Aube'"). Worth knowing it will be commoner on transliterated Arabic/Hebrew/Slavic data, where word-final apostrophes are ordinary. Found by auditing the existing emitters against the decision-site rule documented in the previous commit. Co-Authored-By: Claude Opus 4.8 --- nameparser/_pipeline/_extract.py | 27 +++++++++++++++++++--- tests/v2/pipeline/test_extract.py | 37 +++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/nameparser/_pipeline/_extract.py b/nameparser/_pipeline/_extract.py index 634fb233..77ab63d6 100644 --- a/nameparser/_pipeline/_extract.py +++ b/nameparser/_pipeline/_extract.py @@ -174,9 +174,30 @@ def extract_delimited(state: ParseState) -> ParseState: # An unmatched-open candidate whose character was consumed by a # later successful match (the bulk pass above runs ahead of the # main scan) is literal content there, not a dangling delimiter. - ambiguities = tuple( + reported = {offset for offset, _ in unbalanced} + ambiguities = [ a for offset, a in unbalanced - if not _overlaps(Span(offset, offset + 1), masked)) + if not _overlaps(Span(offset, offset + 1), masked)] + # The scan above is opener-driven: it searches for an open and then + # looks rightward for its close, so a close with no open to its + # LEFT is never in its search space. Sweep for those separately -- + # they signal the same malformed input, and the kind's contract has + # always covered them ("opened without closing, or closed without + # opening"). Same boundary test as the matched path, which is what + # keeps the apostrophe in "O'connor" out of it. + for _, open_, close in order: + start = 0 + while (j := text.find(close, start)) != -1: + start = j + 1 + if (j in reported # already an open + or not _close_ok(text, j, len(close)) + or _overlaps(Span(j, j + len(close)), masked)): + continue + reported.add(j) + ambiguities.append(PendingAmbiguity( + AmbiguityKind.UNBALANCED_DELIMITER, + f"unmatched {close!r} at offset {j}; treated as " + f"literal text")) return dataclasses.replace( state, extracted=tuple(extracted), masked=tuple(masked), - ambiguities=state.ambiguities + ambiguities) + ambiguities=state.ambiguities + tuple(ambiguities)) diff --git a/tests/v2/pipeline/test_extract.py b/tests/v2/pipeline/test_extract.py index 6f07c810..64c277f0 100644 --- a/tests/v2/pipeline/test_extract.py +++ b/tests/v2/pipeline/test_extract.py @@ -1,5 +1,8 @@ import dataclasses +import pytest + +from nameparser import parse from nameparser._lexicon import Lexicon from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._state import ParseState @@ -179,3 +182,37 @@ def test_nested_delimiters_inner_scan_order_pins() -> None: # flow into the extracted span verbatim -- v1 parity, deliberate out = extract_delimited(_state('John ("Jack") Kim')) assert out.extracted == ((Role.NICKNAME, Span(6, 12)),) + + +@pytest.mark.parametrize(("text", "char"), [ + ("John Smith)", ")"), + ("John Jack) Smith", ")"), + ('John Smith"', '"'), + ("John Smith」", "」"), # a #273 typographic pair +]) +def test_unmatched_close_is_reported(text: str, char: str) -> None: + # the scan is opener-driven, so a close with no open to its left was + # never in its search space and went unreported -- even though the + # AmbiguityKind docstring promises "or closed without opening" + kinds = [a.kind for a in parse(text).ambiguities] + assert kinds == [AmbiguityKind.UNBALANCED_DELIMITER] + assert char in parse(text).ambiguities[0].detail + + +@pytest.mark.parametrize("text", [ + "Brian O'connor", # apostrophe mid-word, not a delimiter + "O'B. John Smith", + "The Queen's Bench", + "John (Jack) Smith", # matched pair + 'John "Jack" Smith', +]) +def test_no_spurious_unmatched_close(text: str) -> None: + assert [a for a in parse(text).ambiguities + if a.kind is AmbiguityKind.UNBALANCED_DELIMITER] == [] + + +def test_unmatched_open_is_not_double_reported_as_a_close() -> None: + # a symmetric delimiter can satisfy both boundary tests; it must + # still produce exactly one ambiguity + assert len(parse('John " Smith').ambiguities) == 1 + assert len(parse('Jon "Nick Smith').ambiguities) == 1 From 3f0b754ba1ab87f96abcae4042e55317bfdddcea Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 16:55:05 -0700 Subject: [PATCH 258/275] A word-final apostrophe is not an unbalanced delimiter The close sweep added in d993db0 flagged "Mari' Aube'". It should not: an ambiguity means the part could REASONABLY read two ways where it sits, and a "'" after a word character cannot -- it is an apostrophe. "'" is the only delimiter character that occurs inside and at the end of real name parts, and the least likely of them to mark a nickname. The same position with a quote IS ambiguous, because quotes do not appear inside names, so the carve-out is deliberately this one character: "Mari' Aube'" -> 0 ambiguities 'Mari" Aube"' -> 2 ambiguities #273 excluded the curly apostrophe from the delimiter set outright for this reason. The straight one has to stay a delimiter (v1's quoted_word), so it gets the narrower treatment here. The carve-out is about what PRECEDES the character, so an apostrophe opening a quote is untouched: "John 'Jack Smith" still reports an unmatched open. Also allows a preceding period, for initials ("O'B. Smith'"). Corpus back to 0 of 486 reporting, where the naive close sweep had 1. Co-Authored-By: Claude Opus 4.8 --- nameparser/_pipeline/_extract.py | 19 +++++++++++++++++++ tests/v2/pipeline/test_extract.py | 23 +++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/nameparser/_pipeline/_extract.py b/nameparser/_pipeline/_extract.py index 77ab63d6..a08ffa1e 100644 --- a/nameparser/_pipeline/_extract.py +++ b/nameparser/_pipeline/_extract.py @@ -61,6 +61,24 @@ def _close_ok(text: str, j: int, width: int) -> bool: return k >= len(text) or text[k].isspace() or text[k] in COMMA_CHARS +def _apostrophe_after_a_word(text: str, j: int, close: str) -> bool: + """A "'" directly after a word character is an apostrophe, not a + dangling close quote -- "Mari' Aube'", "Ali Baba'". + + An ambiguity means the part could REASONABLY read two ways where it + sits, and this one cannot: "'" is the only delimiter character that + occurs inside and at the end of real name parts, and it is the + least likely of them to mark a nickname. The same position with a + quote IS ambiguous, since quotes do not appear inside names, so the + carve-out is deliberately this one character. #273 excluded the + curly apostrophe from the delimiter set outright for the same + reason; the straight one has to stay a delimiter (v1's + quoted_word), so it is handled here instead. + """ + return close == "'" and j > 0 and (text[j - 1].isalnum() + or text[j - 1] == ".") + + def _overlaps(span: Span, taken: list[Span]) -> bool: return any(span.start < t.end and t.start < span.end for t in taken) @@ -191,6 +209,7 @@ def extract_delimited(state: ParseState) -> ParseState: start = j + 1 if (j in reported # already an open or not _close_ok(text, j, len(close)) + or _apostrophe_after_a_word(text, j, close) or _overlaps(Span(j, j + len(close)), masked)): continue reported.add(j) diff --git a/tests/v2/pipeline/test_extract.py b/tests/v2/pipeline/test_extract.py index 64c277f0..74c12892 100644 --- a/tests/v2/pipeline/test_extract.py +++ b/tests/v2/pipeline/test_extract.py @@ -199,6 +199,29 @@ def test_unmatched_close_is_reported(text: str, char: str) -> None: assert char in parse(text).ambiguities[0].detail +def test_word_final_apostrophe_is_not_an_unbalanced_delimiter() -> None: + # ambiguity means the part could REASONABLY read two ways where it + # sits. A "'" after a word character is an apostrophe -- it is the + # one delimiter character that occurs inside and at the end of real + # name parts, and the least likely to mark a nickname. The same + # position with a quote IS ambiguous, because quotes do not appear + # inside names. (#273 excluded the curly apostrophe from the + # delimiters entirely for this reason; the straight one is a + # delimiter, so it needs the carve-out here instead.) + assert parse("Mari' Aube'").ambiguities == () + assert parse("Ali Baba'").ambiguities == () + quoted = parse('Mari" Aube"') + assert [a.kind for a in quoted.ambiguities] == \ + [AmbiguityKind.UNBALANCED_DELIMITER] * 2 + + +def test_apostrophe_after_a_space_is_still_reported() -> None: + # the carve-out is about what PRECEDES the character: an apostrophe + # opening a quote is still an unmatched open when nothing closes it + assert [a.kind for a in parse("John 'Jack Smith").ambiguities] == \ + [AmbiguityKind.UNBALANCED_DELIMITER] + + @pytest.mark.parametrize("text", [ "Brian O'connor", # apostrophe mid-word, not a delimiter "O'B. John Smith", From cc7c6a82032e271f1e197cb608a23f8dd641bbb7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 16:58:28 -0700 Subject: [PATCH 259/275] Point UNBALANCED_DELIMITER at the token it landed in Last gap from the emitter audit. The kind carried no tokens, so the ambiguity was locatable only by parsing an offset back out of its detail string -- which is prose, not an API. The stray character does survive into a token ('"Nick', 'Smith)'); nothing was pointing at it. extract_delimited runs before tokens exist, so PendingAmbiguity gains an `origin` character offset that tokenize resolves to the containing token's index once the stream is built. Stages after tokenize set indices directly and leave origin None. Every emitted kind now carries its tokens: 'Jon "Nick Smith' unbalanced-delimiter ['"Nick'] 'John Smith)' unbalanced-delimiter ['Smith)'] 'Van Johnson' particle-or-given ['Van'] 'John Smith MA' suffix-or-family ['MA'] 'JEFFREY (JD) BRICKEN' suffix-or-nickname ['JD'] 'Smith, John, Extra, Jr.' comma-structure ['Extra'] The docstring's "May carry no tokens" is narrowed to what remains true: a character inside a masked region belongs to no token. test_stage_field_ownership caught tokenize writing a field it did not own before -- fixed, and classify's entry corrected in the same pass, since its SUFFIX_OR_NICKNAME emitter had the same omission and only passed because no case-table row triggers it yet. Co-Authored-By: Claude Opus 4.8 --- nameparser/_pipeline/_extract.py | 3 ++- nameparser/_pipeline/_state.py | 9 ++++++++- nameparser/_pipeline/_tokenize.py | 16 +++++++++++++++- nameparser/_types.py | 6 ++++-- tests/v2/pipeline/test_extract.py | 17 +++++++++++++++++ tests/v2/pipeline/test_state.py | 10 ++++++++-- 6 files changed, 54 insertions(+), 7 deletions(-) diff --git a/nameparser/_pipeline/_extract.py b/nameparser/_pipeline/_extract.py index a08ffa1e..9188cdee 100644 --- a/nameparser/_pipeline/_extract.py +++ b/nameparser/_pipeline/_extract.py @@ -104,6 +104,7 @@ def _unmatched(open_: str, offset: int) -> tuple[int, PendingAmbiguity]: return (offset, PendingAmbiguity( AmbiguityKind.UNBALANCED_DELIMITER, f"unmatched {open_!r} at offset {offset}; treated as literal text", + origin=offset, )) @@ -216,7 +217,7 @@ def extract_delimited(state: ParseState) -> ParseState: ambiguities.append(PendingAmbiguity( AmbiguityKind.UNBALANCED_DELIMITER, f"unmatched {close!r} at offset {j}; treated as " - f"literal text")) + f"literal text", origin=j)) return dataclasses.replace( state, extracted=tuple(extracted), masked=tuple(masked), ambiguities=state.ambiguities + tuple(ambiguities)) diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py index 30d5e5f0..367ccb65 100644 --- a/nameparser/_pipeline/_state.py +++ b/nameparser/_pipeline/_state.py @@ -47,11 +47,18 @@ class Structure(Enum): @dataclass(frozen=True, slots=True) class PendingAmbiguity: """An ambiguity recorded mid-pipeline by token INDEX; assemble - materializes real Ambiguity objects over the final tokens.""" + materializes real Ambiguity objects over the final tokens. + + ``origin`` is for the one stage that runs BEFORE tokens exist: + extract_delimited knows only a character offset, so it records that + and tokenize resolves it to the containing token's index. Stages + after tokenize set ``indices`` directly and leave ``origin`` None. + """ kind: AmbiguityKind detail: str indices: tuple[int, ...] = () + origin: int | None = None @dataclass(frozen=True, slots=True) diff --git a/nameparser/_pipeline/_tokenize.py b/nameparser/_pipeline/_tokenize.py index 8351e8a7..2a3a5758 100644 --- a/nameparser/_pipeline/_tokenize.py +++ b/nameparser/_pipeline/_tokenize.py @@ -90,5 +90,19 @@ def tokenize(state: ParseState) -> ParseState: _tokenize_region(state, inner.start, inner.end, role, False, tokens, commas) tokens.sort(key=lambda t: t.span) + # extract_delimited runs before tokens exist, so its ambiguities + # carry a character offset instead of an index. Resolve them now + # that the stray character has landed in a token ('"Nick', 'Smith)') + # -- without this the ambiguity is locatable only by parsing the + # offset back out of its detail string. An offset inside a masked + # region belongs to no token; those keep an empty tuple, which the + # kind's contract already allows. + ambiguities = tuple( + a if a.origin is None else dataclasses.replace( + a, indices=tuple( + i for i, t in enumerate(tokens) + if t.span.start <= a.origin < t.span.end)) + for a in state.ambiguities) return dataclasses.replace(state, tokens=tuple(tokens), - comma_offsets=tuple(sorted(commas))) + comma_offsets=tuple(sorted(commas)), + ambiguities=ambiguities) diff --git a/nameparser/_types.py b/nameparser/_types.py index 854c5e2e..1875a532 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -252,8 +252,10 @@ class AmbiguityKind(StrEnum): #: particle in other names. PARTICLE_OR_GIVEN = "particle-or-given" #: A nickname/maiden delimiter opened without closing (or closed - #: without opening); the text was kept as literal name content. - #: May carry no tokens. + #: without opening); the text was kept as literal name content, so + #: the tokens are the one the stray character ended up inside. Rare + #: exception: a character that lands in no token at all (inside a + #: masked region) leaves the tuple empty. UNBALANCED_DELIMITER = "unbalanced-delimiter" #: More comma-separated segments than any recognized name shape; #: the parse is best-effort over the extra segments. diff --git a/tests/v2/pipeline/test_extract.py b/tests/v2/pipeline/test_extract.py index 74c12892..20885d80 100644 --- a/tests/v2/pipeline/test_extract.py +++ b/tests/v2/pipeline/test_extract.py @@ -239,3 +239,20 @@ def test_unmatched_open_is_not_double_reported_as_a_close() -> None: # still produce exactly one ambiguity assert len(parse('John " Smith').ambiguities) == 1 assert len(parse('Jon "Nick Smith').ambiguities) == 1 + + +@pytest.mark.parametrize(("text", "expected"), [ + ('Jon "Nick Smith', '"Nick'), + ("John (Jack Smith", "(Jack"), + ("John Smith)", "Smith)"), + ("John Jack) Smith", "Jack)"), +]) +def test_unbalanced_delimiter_points_at_its_token( + text: str, expected: str +) -> None: + # the kind was documented as possibly carrying no tokens, but the + # stray character does survive into one -- without it the ambiguity + # is locatable only by parsing an offset back out of the detail + # string, which is not an API + (amb,) = parse(text).ambiguities + assert [t.text for t in amb.tokens] == [expected] diff --git a/tests/v2/pipeline/test_state.py b/tests/v2/pipeline/test_state.py index 4b0c0423..83b0e7fe 100644 --- a/tests/v2/pipeline/test_state.py +++ b/tests/v2/pipeline/test_state.py @@ -46,9 +46,15 @@ def test_stage_field_ownership() -> None: ownership = { "extract_delimited": {"extracted", "masked", "ambiguities"}, - "tokenize": {"tokens", "comma_offsets"}, + # tokenize also rewrites ambiguities: extract_delimited runs + # before tokens exist, so its UNBALANCED_DELIMITER entries carry + # a character offset that tokenize resolves to a token index + "tokenize": {"tokens", "comma_offsets", "ambiguities"}, "segment": {"segments", "structure", "ambiguities"}, - "classify": {"tokens"}, + # classify also emits SUFFIX_OR_NICKNAME: the delimiter escape + # that decides it lives in extract_delimited, which has no token + # index to point at, so the report is raised here instead + "classify": {"tokens", "ambiguities"}, "group": {"tokens", "pieces", "piece_tags", "dropped"}, "assign": {"tokens", "ambiguities"}, "post_rules": {"tokens"}, From 81ea9c1784113aa8b2fb6dccfc287ee6103f1d15 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 19:16:20 -0700 Subject: [PATCH 260/275] Fix three defects in the new ambiguity emitters All three are mine, from the previous round, found by review. 1. Only one SUFFIX_OR_FAMILY survived per segment. ambiguous_pick was a single slot the peel loop overwrote, so "John Smith MA JD" made two coin-flips and reported one -- defeating the point of reporting. Collect them instead. The sibling SUFFIX_OR_NICKNAME emitter already did this correctly, so it was an oversight, not a design. 2. The detail string was wrong under two of three name orders. The comment claimed the unpeeled piece "stays the last name piece -- the family name under every order"; _name_positions maps a two-piece name to [FAMILY, GIVEN] under FAMILY_FIRST, so it is the GIVEN name there and the text said otherwise. Ambiguity.detail is public, so this misinformed callers. Now reads the role back after assignment. 3. Origin resolution was quadratic, and so was the check it fed. Resolving each ambiguity's offset rescanned every token, and ParsedName.__post_init__'s subset check then did `tok not in self.tokens` per referenced token -- O(ambiguities x tokens) twice over. Reachable because the new closer sweep can emit one ambiguity per delimiter character. Bisect over the (already sorted) token starts, and hash the token tuple once: ') ' * N before after N=1600 182 ms 12 ms N=3200 -- 25 ms (linear) The subset check was pre-existing but only became reachable at scale once ambiguities started carrying tokens. Verified: differential harness 486 names, 2 intentional diffs, 0 unexplained. The review separately confirmed the 8147ac6 peel change against 304 generated shapes under all three name orders -- 86 changed, all 86 from disagreeing with v1 to matching it, 0 regressions. Co-Authored-By: Claude Opus 4.8 --- nameparser/_pipeline/_assign.py | 37 +++++++++++++++++++------------ nameparser/_pipeline/_tokenize.py | 21 ++++++++++++++---- nameparser/_types.py | 12 +++++++--- tests/v2/test_parser.py | 22 ++++++++++++++++++ 4 files changed, 71 insertions(+), 21 deletions(-) diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index bb7187d3..3a510a5b 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -124,9 +124,12 @@ def _assign_main(seg_idx: int, state: ParseState, # every piece is a strict suffix (v1's are_suffixes tail rule, with # the roman-numeral special: a final roman numeral after a # non-initial piece is a suffix) - # (piece, reading taken, reading declined) when the peel had to - # resolve a bare ambiguous acronym; both directions are guesses - ambiguous_pick: tuple[tuple[int, ...], str, str] | None = None + # every bare ambiguous acronym the peel had to resolve -- one + # coin-flip each, in either direction, so this collects rather than + # overwrites. The role each ends up with is read back after + # assignment, since which role "not peeled" means depends on + # name_order. + ambiguous_picks: list[tuple[int, ...]] = [] k = len(rest) while k > 0: piece = pieces[rest[k - 1]] @@ -151,15 +154,15 @@ def _assign_main(seg_idx: int, state: ParseState, bare_ambiguous = (len(piece) == 1 and "vocab:suffix-ambiguous" in tokens[piece[0]].tags) if bare_ambiguous and k - 1 >= 2: - ambiguous_pick = (piece, "a suffix", "an ordinary surname") + ambiguous_picks.append(piece) k -= 1 continue if bare_ambiguous and k >= 2: - # not peeled, so it stays the last name piece -- the family - # name under every order. (k < 2 means it is the only piece - # left and lands in the given position, which is not the - # fork this reports.) - ambiguous_pick = (piece, "a family name", "a post-nominal") + # not peeled, so it stays the last NAME piece -- which role + # that is depends on name_order, so the detail reads it back + # below. (k < 2 means it is the only piece left, which is + # not the fork this reports.) + ambiguous_picks.append(piece) break name_pieces, suffix_pieces = rest[:k], rest[k:] if not name_pieces and suffix_pieces: @@ -170,13 +173,19 @@ def _assign_main(seg_idx: int, state: ParseState, _set_roles(tokens, pieces[piece_idx], roles[pos]) for piece_idx in suffix_pieces: _set_roles(tokens, pieces[piece_idx], Role.SUFFIX) - if ambiguous_pick is not None: - piece, taken, declined = ambiguous_pick + for piece in ambiguous_picks: + token = tokens[piece[0]] + # every pick was assigned a role just above; the fallback only + # keeps the wording sane if a future path reports before then + role = token.role.value if token.role is not None else "name" + taken, declined = ( + ("a suffix", "a name part") if token.role is Role.SUFFIX + else (f"a {role} name", "a post-nominal")) ambiguities.append(PendingAmbiguity( AmbiguityKind.SUFFIX_OR_FAMILY, - f"{tokens[piece[0]].text!r} written without periods is both " - f"a post-nominal and a surname; read as {taken} rather than " - f"{declined}", + f"{token.text!r} written without periods is both a " + f"post-nominal and an ordinary name; read as {taken} " + f"rather than {declined}", tuple(piece))) # leading ambiguous particle read as a name (#121 surfaced) if name_pieces: diff --git a/nameparser/_pipeline/_tokenize.py b/nameparser/_pipeline/_tokenize.py index 2a3a5758..7ea86fac 100644 --- a/nameparser/_pipeline/_tokenize.py +++ b/nameparser/_pipeline/_tokenize.py @@ -18,6 +18,7 @@ """ from __future__ import annotations +import bisect import dataclasses import re @@ -97,11 +98,23 @@ def tokenize(state: ParseState) -> ParseState: # offset back out of its detail string. An offset inside a masked # region belongs to no token; those keep an empty tuple, which the # kind's contract already allows. + # Bisect rather than rescan: the closer sweep can emit one + # ambiguity per delimiter character, so a linear scan per ambiguity + # is quadratic on pathological input (') ' * 1600 spent 180ms here, + # against 9ms before the sweep existed). Spans are non-overlapping + # and sorted just above, so the candidate is the last token whose + # start is <= the offset. + starts = [t.span.start for t in tokens] + + def _containing(offset: int) -> tuple[int, ...]: + i = bisect.bisect_right(starts, offset) - 1 + if i >= 0 and offset < tokens[i].span.end: + return (i,) + return () + ambiguities = tuple( - a if a.origin is None else dataclasses.replace( - a, indices=tuple( - i for i, t in enumerate(tokens) - if t.span.start <= a.origin < t.span.end)) + a if a.origin is None + else dataclasses.replace(a, indices=_containing(a.origin)) for a in state.ambiguities) return dataclasses.replace(state, tokens=tuple(tokens), comma_offsets=tuple(sorted(commas)), diff --git a/nameparser/_types.py b/nameparser/_types.py index 1875a532..e6751af4 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -369,12 +369,18 @@ def __post_init__(self) -> None: f"offset {prev_end}" ) prev_end = tok.span.end + # Hash once rather than rescanning the tuple per referenced + # token: a name can carry an ambiguity per token (a string of + # stray delimiters does), and the linear form made construction + # quadratic in their product. Set membership uses the same value + # equality the tuple scan did -- Token is frozen and hashable. + known = set(self.tokens) if self.ambiguities else () for amb in self.ambiguities: for tok in amb.tokens: - # `in` uses Token's value equality, not identity: this - # only guarantees a value-equal token exists in + # membership is by Token's value equality, not identity: + # this only guarantees a value-equal token exists in # self.tokens, not that `tok` IS one of those objects. - if tok not in self.tokens: + if tok not in known: raise ValueError( f"Ambiguity token {tok.text!r} is not a subset of " f"this ParsedName's tokens" diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index cf3eca02..5090b376 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -211,3 +211,25 @@ def test_delimited_ambiguous_acronym_reports_suffix_or_nickname() -> None: assert [t.text for t in n.ambiguities[0].tokens] == ["JD"] # the unambiguous one decided on vocabulary, so it is not a guess assert parse("Andrew Perkins (MBA)").ambiguities == () + + +def test_every_ambiguous_acronym_in_a_name_is_reported() -> None: + # one coin-flip per acronym: a single-slot record dropped all but + # the last, which defeats the point of reporting at all + n = parse("John Smith MA JD") + assert n.suffix == "MA, JD" + assert [a.kind for a in n.ambiguities] == \ + [AmbiguityKind.SUFFIX_OR_FAMILY] * 2 + assert sorted(t.text for a in n.ambiguities for t in a.tokens) == \ + ["JD", "MA"] + + +def test_ambiguous_acronym_detail_names_the_role_it_got() -> None: + # the unpeeled piece is the last NAME piece, which is the family + # name only under GIVEN_FIRST -- FAMILY_FIRST puts it in given, so + # the detail has to follow the role actually assigned + fam_first = Parser(policy=Policy(name_order=FAMILY_FIRST)) + n = fam_first.parse("Jack MA") + assert (n.family, n.given) == ("Jack", "MA") + assert "given name" in n.ambiguities[0].detail + assert "family name" not in n.ambiguities[0].detail From 2ff9189ee67a3e192616bb4882aca46a07519dc0 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 19:36:15 -0700 Subject: [PATCH 261/275] Report both branches of every fork the parser reports on Option 2 from the review. Three silent forks now report, and the documented claim that made them defects is corrected. Renamed SUFFIX_OR_FAMILY -> SUFFIX_OR_NAME. The old name asserted a role that is only right under one of the three name orders: FAMILY_FIRST puts the declined reading in GIVEN, and the roman-numeral and comma forks decline a MIDDLE. I had already fixed the detail string for this reason; the kind name had the same bug. Generalizing it also means one kind covers every suffix-versus-name-part fork instead of accreting one per cause. Newly reported: - The trailing roman numeral. "John Smith V" takes V as a suffix where "John Smith B" makes B the family name -- V/X/I are ordinary middle initials, so that is a call, not a fact. - PARTICLE_OR_GIVEN's missing branch. "Van Johnson" reported that Van was read as a given name; "Dr. Van Johnson" made the OPPOSITE call silently, because a leading title shifts Van off index 0, the prefix chain claims it, and that branch is taken in _group while the emitter lived in _assign. Both stages now report the side they decide. The group emitter records at the merge site rather than matching the merged shape -- the first attempt keyed on "multi-token piece whose head is particle-ambiguous" and fired on "Joao da Silva do Amaral de Souza" (mid-name 'do', no fork) and on the Arabic kunya (a bound-given merge, which becomes the GIVEN name). Same decision-not-token lesson, one level in. Docs corrected: concepts.rst and the AmbiguityKind docstring both said an empty ambiguities means the parse faced no fork. That was a universal claim about the whole parser made on the evidence of three emitters, and these findings falsify it. Now: an empty tuple means none of the LISTED forks came up, coverage grows over releases, and a non-empty tuple is the signal -- an empty one is not a guarantee. AGENTS.md gains the cross-stage rule: if a fork's branches are taken in different stages, each needs the emitter, the ownership map must list ambiguities for each, and it passes vacuously until a case row exercises the path. Verified: 486-name corpus, 2 intentional diffs, 0 unexplained. Co-Authored-By: Claude Opus 4.8 --- AGENTS.md | 2 +- docs/concepts.rst | 11 ++++++-- docs/release_log.rst | 2 +- docs/usage.rst | 2 +- nameparser/_pipeline/_assign.py | 6 ++++- nameparser/_pipeline/_group.py | 42 +++++++++++++++++++++++++----- nameparser/_types.py | 24 ++++++++++------- tests/v2/cases.py | 25 +++++++++++++----- tests/v2/pipeline/test_state.py | 6 ++++- tests/v2/test_contracts.py | 2 +- tests/v2/test_parser.py | 46 ++++++++++++++++++++++++++++++--- 11 files changed, 133 insertions(+), 35 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index fe0f5376..ccbf845a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,7 +119,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. Sanctioned deviation, facade layer only: `HumanName` and the shim `Constants` organize by v1 concern groups (`# -- render defaults --`, `# -- config / parsing --`, `# -- fields --`, ..., dunders and pickle last) — the classes mirror v1's own surface and die in 3.0; the canonical order still binds every core type. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). -- **Ambiguities are emitted at the DECISION site**: an `Ambiguity` records a fork the parse had to call, not a token that sits in an ambiguous vocabulary. Emit where the branch is taken — the trailing-suffix peel in `_assign`, the delimiter escape's follow-up in `classify` — never by scanning for a `vocab:*-ambiguous` tag. The same tagged token is a genuine fork in one position and unremarkable in another (`do` mid-name in "Joao da Silva do Amaral de Souza" chooses nothing), and structure often settles the question before it arises, which is why `PARTICLE_OR_GIVEN` is deliberately not emitted on the `FAMILY_COMMA` path and `SUFFIX_OR_FAMILY` is not emitted for "Ma, Jack". The decision site also has the token index and the detail text in hand, which the tag scan would have to reconstruct. Report BOTH directions of a two-way fork — "John Smith MA" (read as a suffix) and "Jack MA" (read as the family name) are equally guesses. Every kind needs a trigger in `tests/v2/test_contracts.py::_AMBIGUITY_TRIGGERS` (an explicit `None`, strict-xfail, while reserved), and case-table rows pin expected kinds exactly, so a new emitter shows up in both immediately. +- **Ambiguities are emitted at the DECISION site**: an `Ambiguity` records a fork the parse had to call, not a token that sits in an ambiguous vocabulary. Emit where the branch is taken — the trailing-suffix peel in `_assign`, the delimiter escape's follow-up in `classify` — never by scanning for a `vocab:*-ambiguous` tag. The same tagged token is a genuine fork in one position and unremarkable in another (`do` mid-name in "Joao da Silva do Amaral de Souza" chooses nothing), and structure often settles the question before it arises, which is why `PARTICLE_OR_GIVEN` is deliberately not emitted on the `FAMILY_COMMA` path and `SUFFIX_OR_NAME` is not emitted for "Ma, Jack". The decision site also has the token index and the detail text in hand, which the tag scan would have to reconstruct. **If a fork's two branches are taken in DIFFERENT stages, every one of them needs the emitter** -- `PARTICLE_OR_GIVEN` is decided in `_assign` when the ambiguous particle stays a lone leading piece and in `_group` when a title shifts it off index 0 and the prefix chain claims it, so both report; for two years only the first did. The stage-ownership map in `tests/v2/pipeline/test_state.py` must list `ambiguities` for each such stage, and it passes vacuously until a case row exercises the path, so add the row too. Report BOTH directions of a two-way fork — "John Smith MA" (read as a suffix) and "Jack MA" (read as the family name) are equally guesses. Every kind needs a trigger in `tests/v2/test_contracts.py::_AMBIGUITY_TRIGGERS` (an explicit `None`, strict-xfail, while reserved), and case-table rows pin expected kinds exactly, so a new emitter shows up in both immediately. - **Invariants guard harm, not no-ops**: add a constructor check when violating it produces a *wrong parse*, not when it produces *nothing*. A false positive costs a working configuration; a true positive on an inert condition costs the user nothing, so that trade is never worth taking. `suffix_acronyms_ambiguous ∩ suffix_words` is guarded because the overlap loses a family name; `given_name_titles` is not, because an unreachable entry is simply never consulted (see Gotchas). Before adding one, construct the config it forbids and check what actually breaks. - **The shim TRANSLATES; it never raises on a config v1 accepted, and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Four exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), and `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`). When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). diff --git a/docs/concepts.rst b/docs/concepts.rst index 249834b7..1e4a2f47 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -161,8 +161,15 @@ de Souza"`` it sits mid-name, where nothing has to choose between readings — so nothing is recorded. A comma can settle the question before it arises, too: ``"Ma, Jack"`` fixes the family name, so the credential reading never comes up, while ``"John Smith MA"`` has to -call it and says so. An empty ``ambiguities`` means the parse faced no -fork, not that every word in it was unambiguous. +call it and says so. + +So an empty ``ambiguities`` does not mean every word was unambiguous — +it means none of the forks the parser *reports on* came up. The kinds +it reports are listed in :class:`~nameparser.AmbiguityKind`, and they +are the ones where both readings are genuinely common in real names. +Coverage grows over releases; a name that reports nothing today may +report something later, so treat a non-empty ``ambiguities`` as a +signal worth acting on rather than an empty one as a guarantee. :class:`Tokens ` also carry tags — a second, independent label alongside their role, recording how a token was classified rather than what part of diff --git a/docs/release_log.rst b/docs/release_log.rst index 9c5b872f..724db8d7 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -24,7 +24,7 @@ Release Log - Add ``PatronymicRule`` with the members ``EAST_SLAVIC`` and ``TURKIC``. v1's single ``patronymic_name_order`` flag enabled both detectors at once; ``Policy(patronymic_rules=...)`` lets you enable either one alone - Add ``PolicyPatch`` and the ``UNSET`` sentinel for partial policy deltas that compose -- set-valued fields union, scalar fields override with later winning. This is the mechanism locale packs are built from, and ``UNSET`` is only needed when you must distinguish "not set" from a real ``False`` or ``None`` - Add ``Token``, ``Span`` and ``Role``: every field is backed by tokens carrying exact ``(start, end)`` offsets into the original string, reachable with ``tokens_for(Role.GIVEN)``. This replaces v1's ``*_list`` attributes and makes it possible to highlight or re-slice the input the parse came from - - Add ``Ambiguity`` and the ``AmbiguityKind`` enum, so a parse reports what it had to guess at instead of guessing silently. The kinds emitted today are ``particle-or-given``, ``suffix-or-family``, ``suffix-or-nickname``, ``unbalanced-delimiter`` and ``comma-structure``; ``order`` is reserved and not yet emitted. The two suffix kinds cover the post-nominals that are also ordinary words: ``"John Smith MA"`` reports that ``MA`` was read as a credential rather than a surname, and ``"JEFFREY (JD) BRICKEN"`` that the delimited ``JD`` was read as a nickname rather than a suffix. A reading the vocabulary settles on its own — ``"John Smith M.A."``, ``"Andrew Perkins (MBA)"`` — is not a guess and reports nothing + - Add ``Ambiguity`` and the ``AmbiguityKind`` enum, so a parse reports what it had to guess at instead of guessing silently. The kinds emitted today are ``particle-or-given``, ``suffix-or-name``, ``suffix-or-nickname``, ``unbalanced-delimiter`` and ``comma-structure``; ``order`` is reserved and not yet emitted. The two suffix kinds cover the post-nominals that are also ordinary words: ``"John Smith MA"`` reports that ``MA`` was read as a credential rather than a surname, and ``"JEFFREY (JD) BRICKEN"`` that the delimited ``JD`` was read as a nickname rather than a suffix. A reading the vocabulary settles on its own — ``"John Smith M.A."``, ``"Andrew Perkins (MBA)"`` — is not a guess and reports nothing - Add ``ParsedName`` output and comparison methods: ``render(spec)``, ``initials()``, ``capitalized()`` (which returns a new value rather than mutating in place), ``as_dict()``, ``replace(**fields)``, ``matches()`` and ``comparison_key()`` - Add ``Locale``, the public pack type. Writing your own needs no registration -- construct a ``Locale`` and pass it to ``parser_for()`` - Ship a fully typed public API (PEP 561): the core modules are checked under strict mypy settings, and nameparser 2.0 has no runtime dependencies diff --git a/docs/usage.rst b/docs/usage.rst index 2e1ebc60..cbf39630 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -327,7 +327,7 @@ given name it stays the surname — either way the choice is recorded: >>> parse("John Smith MA").suffix 'MA' >>> [a.kind.value for a in parse("John Smith MA").ambiguities] - ['suffix-or-family'] + ['suffix-or-name'] >>> parse("Jack MA").family 'MA' diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 3a510a5b..22618279 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -140,6 +140,10 @@ def _assign_main(seg_idx: int, state: ParseState, if (k == len(rest) and k >= 2 and len(piece) == 1 and _ROMAN.match(tokens[piece[0]].text) and "initial" not in tokens[pieces[rest[k - 2]][0]].tags): + # a trailing single letter is a name part unless it happens + # to be a roman numeral -- and V/X/I are ordinary middle + # initials, so taking it as a suffix is a call, not a fact + ambiguous_picks.append(piece) k -= 1 continue # A bare ambiguous acronym ("MA", not "M.A.") is a credential @@ -182,7 +186,7 @@ def _assign_main(seg_idx: int, state: ParseState, ("a suffix", "a name part") if token.role is Role.SUFFIX else (f"a {role} name", "a post-nominal")) ambiguities.append(PendingAmbiguity( - AmbiguityKind.SUFFIX_OR_FAMILY, + AmbiguityKind.SUFFIX_OR_NAME, f"{token.text!r} written without periods is both a " f"post-nominal and an ordinary name; read as {taken} " f"rather than {declined}", diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 55a98647..64ad3bcb 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -23,11 +23,13 @@ from collections.abc import Sequence, Set from enum import IntEnum -from nameparser._pipeline._state import ParseState, Structure, WorkToken +from nameparser._pipeline._state import ( + ParseState, PendingAmbiguity, Structure, WorkToken, +) from nameparser._pipeline._vocab import D as _D from nameparser._pipeline._vocab import PH as _PH from nameparser._pipeline._vocab import delimiter_cores -from nameparser._types import Role +from nameparser._types import AmbiguityKind, Role # the credential-pair regexes live in _vocab (shared with segment) @@ -87,9 +89,13 @@ def _is_rootname(piece: Sequence[int], ptags: Set[str], def _group_segment(seg: tuple[int, ...], additional: int, tokens: Sequence[WorkToken], bound_join: BoundJoin = BoundJoin.STRICT, - ) -> tuple[list[Piece], list[set[str]]]: + ) -> tuple[list[Piece], list[set[str]], list[int]]: pieces: list[Piece] = [[i] for i in seg] ptags: list[set[str]] = [set() for _ in seg] + # token indices where an ambiguous particle was chained into the + # family name from the position that would otherwise be the given + # name -- the other branch of the fork _assign reports (see group()) + particle_forks: list[int] = [] def title(k: int) -> bool: return _is_title_piece(pieces[k], ptags[k], tokens) @@ -168,6 +174,10 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), j += 1 while j < len(pieces) and not prefix(j) and not suffix(j): j += 1 + if (all(title(x) for x in range(k)) + and "vocab:particle-ambiguous" + in tokens[pieces[k][0]].tags): + particle_forks.append(pieces[k][0]) merge(k, j, drop={"prefix"}) k += 1 # bound given names: the first non-title piece joins the next @@ -185,12 +195,13 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), if not title(k) and not suffix(k)) if non_suffix >= bound_join: merge(first_name_k, first_name_k + 2) - return pieces, ptags + return pieces, ptags, particle_forks def group(state: ParseState) -> ParseState: tokens = list(state.tokens) dropped = list(state.dropped) + ambiguities = list(state.ambiguities) all_pieces: list[tuple[tuple[int, ...], ...]] = [] all_ptags: list[tuple[frozenset[str], ...]] = [] # v1 parity: additional_parts_count=1 applies only to FAMILY_COMMA @@ -209,8 +220,8 @@ def group(state: ParseState) -> ParseState: else BoundJoin.DISABLED) else: bound_join = BoundJoin.STRICT - pieces, ptags = _group_segment(seg, additional, tokens, - bound_join) + pieces, ptags, particle_forks = _group_segment( + seg, additional, tokens, bound_join) if tail_start is not None and seg_idx >= tail_start: # v1 renders each tail COMMA SEGMENT as one suffix entry # ('Smith, V MD' -> suffix 'V MD'); a delimiter core inside @@ -271,6 +282,23 @@ def group(state: ParseState) -> ParseState: ptags[m:j] = [] all_pieces.append(tuple(tuple(p) for p in pieces)) all_ptags.append(tuple(frozenset(t) for t in ptags)) + # The other half of PARTICLE_OR_GIVEN. _assign reports the fork + # when an ambiguous particle stays a lone leading piece ("Van + # Johnson" -> given). The prefix chain above takes the opposite + # branch whenever a title shifts it off index 0 ("Dr. Van + # Johnson" -> family "Van Johnson"), and that branch lives in + # this stage, so it has to report here -- a fork whose two sides + # are decided in different stages needs an emitter in each. + # Suppressed after a family comma for the same reason _assign + # suppresses it there: the family name is already fixed. + if not family_comma: + for i in particle_forks: + ambiguities.append(PendingAmbiguity( + AmbiguityKind.PARTICLE_OR_GIVEN, + f"{tokens[i].text!r} was chained into the family " + f"name; it is also a given name in other names", + (i,))) return dataclasses.replace( state, tokens=tuple(tokens), pieces=tuple(all_pieces), - piece_tags=tuple(all_ptags), dropped=tuple(dropped)) + piece_tags=tuple(all_ptags), dropped=tuple(dropped), + ambiguities=tuple(ambiguities)) diff --git a/nameparser/_types.py b/nameparser/_types.py index e6751af4..62ad1245 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -228,10 +228,12 @@ class AmbiguityKind(StrEnum): existing values never change meaning. A kind names a FORK THE PARSE HAD TO CALL, not a word that could be - read two ways. The same token elsewhere in a name may present no - choice at all and is then reported by nothing -- an empty - ``ambiguities`` means the parse faced no fork, not that every word - was unambiguous.""" + read two ways: the same token elsewhere in a name may present no + choice at all and is then reported by nothing. An empty + ``ambiguities`` therefore means none of the forks listed here came + up -- not that the parse was certain of everything. Coverage grows + over releases, so a non-empty tuple is a signal to act on; an empty + one is not a guarantee.""" #: Reserved: the name's field order itself is uncertain (e.g. a #: two-word name under a non-default name_order). Not yet emitted; @@ -242,11 +244,15 @@ class AmbiguityKind(StrEnum): #: (JD) BRICKEN" keeps the nickname reading, where the #: unambiguous "(MBA)" escapes to suffix on vocabulary alone. SUFFIX_OR_NICKNAME = "suffix-or-nickname" - #: An ambiguous suffix acronym written without periods, which is - #: also an ordinary surname -- "John Smith MA" reads MA as a - #: post-nominal because a family name remains, "Jack MA" reads it - #: as the family name because none would. - SUFFIX_OR_FAMILY = "suffix-or-family" + #: A trailing word reads plausibly as either a post-nominal or an + #: ordinary name part. Covers an ambiguous acronym written without + #: periods ("John Smith MA" takes MA as a credential because a + #: family name remains; "Jack MA" keeps it as the name because none + #: would) and a trailing roman numeral, which is a suffix where any + #: other single letter would be a name ("John Smith V" vs "John + #: Smith B"). Which name part was declined depends on position and + #: ``name_order``, so ``detail`` names it rather than the kind. + SUFFIX_OR_NAME = "suffix-or-name" #: A leading ambiguous particle was read as a given name -- "Van #: Johnson" parses given="Van", but "Van" is also a family-name #: particle in other names. diff --git a/tests/v2/cases.py b/tests/v2/cases.py index f2059869..8673952f 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -87,27 +87,34 @@ def __post_init__(self) -> None: "them first (spans cannot reorder)"), Case("ambiguous_surname_acronyms", "Jack MA", {"given": "Jack", "family": "MA"}, - ambiguities=("suffix-or-family",), + ambiguities=("suffix-or-name",), notes="'ma'/'do' joined suffix_acronyms_ambiguous: with only " "two pieces, 'one of them is a credential' is the less " "likely reading, so the ambiguous acronym stays the " "family name (v1 parity via its reserve_last)"), Case("ambiguous_surname_acronym_with_suffix", "Jack MA Jr", {"given": "Jack", "family": "MA", "suffix": "Jr"}, - ambiguities=("suffix-or-family",), + ambiguities=("suffix-or-name",), notes="'Jr' peels first; 'MA' would then be the only piece " "left beside the given name, so it stays family"), Case("ambiguous_acronym_is_a_suffix_when_a_family_name_remains", "John Smith MA", {"given": "John", "family": "Smith", "suffix": "MA"}, - ambiguities=("suffix-or-family",), + ambiguities=("suffix-or-name",), notes="the other half of the same rule: three pieces means " "peeling 'MA' still leaves given+family, so the " "credential reading wins (v1 parity)"), Case("ambiguous_acronym_suffix_with_middle", "John Q Smith MA", {"given": "John", "middle": "Q", "family": "Smith", "suffix": "MA"}, - ambiguities=("suffix-or-family",)), + ambiguities=("suffix-or-name",)), + Case("titled_ambiguous_particle_chains", "Dr. Van Johnson", + {"title": "Dr.", "family": "Van Johnson"}, + ambiguities=("particle-or-given",), + notes="the other branch of 'Van Johnson': a leading title " + "shifts Van off the given position, the prefix chain " + "fires, and the fork is reported from group rather than " + "assign (v1 parity on the fields)"), Case("initial_shaped_not_conjunction", "john e. smith", {"given": "john", "middle": "e.", "family": "smith"}, notes="v1 is_conjunction excludes initials at classify too"), @@ -357,7 +364,11 @@ def __post_init__(self) -> None: Case("mr_and_mrs", "Mr. and Mrs. John Smith", {"title": "Mr. and Mrs.", "given": "John", "family": "Smith"}), Case("roman_suffix", "John Smith V", - {"given": "John", "family": "Smith", "suffix": "V"}), + {"given": "John", "family": "Smith", "suffix": "V"}, + ambiguities=("suffix-or-name",), + notes="a trailing single letter is a name part unless it is a " + "roman numeral; V/X/I are also ordinary middle initials, " + "so the reading is reported"), Case("initial_not_suffix", "John V. Smith", {"given": "John", "middle": "V.", "family": "Smith"}), Case("lenient_after_comma", "John Ingram, V", @@ -403,14 +414,14 @@ def __post_init__(self) -> None: Case("whitespace", " ", {}), Case("bare_ambiguous_acronym", "John Ed", {"given": "John", "family": "Ed"}, - ambiguities=("suffix-or-family",), + ambiguities=("suffix-or-name",), notes="'ed' is an ambiguous acronym; bare form is a name (C1), " "and the parse reports which reading it took"), Case("comma_ambiguous_acronym", "Smith, Ed", {"given": "Ed", "family": "Smith"}), Case("ambiguous_acronym_with_suffix", "John Ed III", {"given": "John", "family": "Ed", "suffix": "III"}, - ambiguities=("suffix-or-family",)), + ambiguities=("suffix-or-name",)), Case("phd_split", "John Ph. D.", {"given": "John", "suffix": "Ph. D."}, notes="v1 fix_phd; healed via the stable 'joined' tag"), diff --git a/tests/v2/pipeline/test_state.py b/tests/v2/pipeline/test_state.py index 83b0e7fe..3b3775de 100644 --- a/tests/v2/pipeline/test_state.py +++ b/tests/v2/pipeline/test_state.py @@ -55,7 +55,11 @@ def test_stage_field_ownership() -> None: # that decides it lives in extract_delimited, which has no token # index to point at, so the report is raised here instead "classify": {"tokens", "ambiguities"}, - "group": {"tokens", "pieces", "piece_tags", "dropped"}, + # group also emits PARTICLE_OR_GIVEN: the prefix chain takes + # the particle branch of a fork whose given branch _assign + # takes, so each stage reports the side it decides + "group": {"tokens", "pieces", "piece_tags", "dropped", + "ambiguities"}, "assign": {"tokens", "ambiguities"}, "post_rules": {"tokens"}, } diff --git a/tests/v2/test_contracts.py b/tests/v2/test_contracts.py index 6b6b495b..0cd24164 100644 --- a/tests/v2/test_contracts.py +++ b/tests/v2/test_contracts.py @@ -12,7 +12,7 @@ AmbiguityKind.UNBALANCED_DELIMITER: 'Jon "Nick Smith', AmbiguityKind.COMMA_STRUCTURE: "Smith, John, Extra, Jr.", AmbiguityKind.SUFFIX_OR_NICKNAME: "JEFFREY (JD) BRICKEN", - AmbiguityKind.SUFFIX_OR_FAMILY: "John Smith MA", + AmbiguityKind.SUFFIX_OR_NAME: "John Smith MA", # no emitter yet -- arrives with locale-pack order detection (2.x) AmbiguityKind.ORDER: None, } diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 5090b376..1b218546 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -179,13 +179,13 @@ def test_ambiguous_acronym_reports_the_reading_it_took() -> None: took_suffix = parse("John Smith MA") assert took_suffix.suffix == "MA" assert [a.kind for a in took_suffix.ambiguities] == \ - [AmbiguityKind.SUFFIX_OR_FAMILY] + [AmbiguityKind.SUFFIX_OR_NAME] assert [t.text for t in took_suffix.ambiguities[0].tokens] == ["MA"] took_family = parse("Jack MA") assert took_family.family == "MA" assert [a.kind for a in took_family.ambiguities] == \ - [AmbiguityKind.SUFFIX_OR_FAMILY] + [AmbiguityKind.SUFFIX_OR_NAME] @pytest.mark.parametrize("text", [ @@ -196,7 +196,7 @@ def test_ambiguous_acronym_reports_the_reading_it_took() -> None: ]) def test_no_suffix_ambiguity_when_nothing_was_guessed(text: str) -> None: assert [a for a in parse(text).ambiguities - if a.kind is AmbiguityKind.SUFFIX_OR_FAMILY] == [] + if a.kind is AmbiguityKind.SUFFIX_OR_NAME] == [] def test_delimited_ambiguous_acronym_reports_suffix_or_nickname() -> None: @@ -219,7 +219,7 @@ def test_every_ambiguous_acronym_in_a_name_is_reported() -> None: n = parse("John Smith MA JD") assert n.suffix == "MA, JD" assert [a.kind for a in n.ambiguities] == \ - [AmbiguityKind.SUFFIX_OR_FAMILY] * 2 + [AmbiguityKind.SUFFIX_OR_NAME] * 2 assert sorted(t.text for a in n.ambiguities for t in a.tokens) == \ ["JD", "MA"] @@ -233,3 +233,41 @@ def test_ambiguous_acronym_detail_names_the_role_it_got() -> None: assert (n.family, n.given) == ("Jack", "MA") assert "given name" in n.ambiguities[0].detail assert "family name" not in n.ambiguities[0].detail + + +def test_trailing_roman_numeral_reports_the_fork() -> None: + # a trailing single letter is a name part unless it happens to be a + # roman numeral, in which case it is silently reclassified -- and + # V/X/I are common middle initials, so this is a real coin-flip + numeral = parse("John Smith V") + assert numeral.suffix == "V" + assert [a.kind for a in numeral.ambiguities] == \ + [AmbiguityKind.SUFFIX_OR_NAME] + assert [t.text for t in numeral.ambiguities[0].tokens] == ["V"] + # the same shape with a non-numeral letter faces no fork + assert parse("John Smith B").ambiguities == () + # and a numeral after an initial is not treated as a suffix at all + assert parse("John Q. V").ambiguities == () + + +def test_ambiguous_particle_reports_both_branches_of_its_fork() -> None: + # "Van Johnson" reads Van as a given name and says so. A leading + # title shifts Van off index 0, the prefix-chain merge fires, and + # Van becomes a particle instead -- the SAME fork, called the other + # way. The two branches are taken in different stages (_assign vs + # _group), so only the one with an emitter used to report. + given_reading = parse("Van Johnson") + assert given_reading.given == "Van" + assert [a.kind for a in given_reading.ambiguities] == \ + [AmbiguityKind.PARTICLE_OR_GIVEN] + + particle_reading = parse("Dr. Van Johnson") + assert particle_reading.family == "Van Johnson" + assert [a.kind for a in particle_reading.ambiguities] == \ + [AmbiguityKind.PARTICLE_OR_GIVEN] + assert [t.text for t in particle_reading.ambiguities[0].tokens] == ["Van"] + + +def test_unambiguous_particle_chain_reports_nothing() -> None: + # 'de' is never a given name, so merging it is not a fork + assert parse("Dr. de la Vega").ambiguities == () From de264af1eff9fb6c9103010171c62bb2a20a548f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 20:01:32 -0700 Subject: [PATCH 262/275] Only report the particle fork when the chain actually claims something Third instance of the same mistake in this emitter, and the same root cause both previous times: keyed on a SHAPE rather than the DECISION. When the piece after an ambiguous particle is a suffix, the chain's inner scan never advances, so merge(k, k+1) folds a piece into itself. Nothing is chained and the particle stays a lone leading piece -- but the record was appended before the merge, keyed only on "is a prefix" plus the ambiguous tag. Result: 'Dr. Van Jr.' -> given='Van' AMB "'Van' was chained into the family name" The detail asserted the opposite of what happened, for all 39 particles_ambiguous members. For do/freiherr/st the token lands in the TITLE instead. Worse, the no-op leaves exactly the shape _assign triggers on, so 36 of 39 double-reported the same token from two stages. `j > k + 1` is the discriminator -- it distinguishes a chain that claimed a following piece from one that claimed nothing -- and it fixes both defects, since the no-op was the only overlap between the two emitters' domains. They now partition cleanly. Also fixed the detail's wording. _group runs before assignment, so it cannot know which field the chained piece lands in: "Dr. Van Johnson de la Cruz" puts it in GIVEN, not family. This is the same defect the role-readback cured in _assign one commit earlier -- the lesson did not carry to the new emitter, and _group cannot read a role back, so the text now describes the decision rather than guessing an outcome. A "Dr. Van Jr." case row would have caught both criticals; 1539 tests passed with them present because the only titled-particle test used an UNAMBIGUOUS particle, pinning vocabulary instead of the decision. Verified: 0 spurious reports across all 39 ambiguous particles; corpus 486 names, 2 intentional diffs, 0 unexplained. Co-Authored-By: Claude Opus 4.8 --- nameparser/_pipeline/_group.py | 14 +++++++++++--- tests/v2/cases.py | 7 +++++++ tests/v2/test_parser.py | 27 +++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 64ad3bcb..dcc8fdb7 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -174,7 +174,14 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), j += 1 while j < len(pieces) and not prefix(j) and not suffix(j): j += 1 - if (all(title(x) for x in range(k)) + # j > k + 1 is what makes this a DECISION rather than a + # shape: when the next piece is a suffix the inner scan + # never advances, merge(k, k+1) folds a piece into itself, + # and the particle stays a lone leading piece -- nothing + # was chained, and _assign reports that case instead. + # Without this the two emitters both fire on the same token. + if (j > k + 1 + and all(title(x) for x in range(k)) and "vocab:particle-ambiguous" in tokens[pieces[k][0]].tags): particle_forks.append(pieces[k][0]) @@ -295,8 +302,9 @@ def group(state: ParseState) -> ParseState: for i in particle_forks: ambiguities.append(PendingAmbiguity( AmbiguityKind.PARTICLE_OR_GIVEN, - f"{tokens[i].text!r} was chained into the family " - f"name; it is also a given name in other names", + f"{tokens[i].text!r} was chained onto the following " + f"name piece; it is also a given name in other " + f"names", (i,))) return dataclasses.replace( state, tokens=tuple(tokens), pieces=tuple(all_pieces), diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 8673952f..ebc43830 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -115,6 +115,13 @@ def __post_init__(self) -> None: "shifts Van off the given position, the prefix chain " "fires, and the fork is reported from group rather than " "assign (v1 parity on the fields)"), + Case("titled_ambiguous_particle_no_op_chain", "Dr. Van Jr.", + {"title": "Dr.", "given": "Van", "suffix": "Jr."}, + notes="the piece after the particle is a suffix, so the chain " + "scan never advances and the merge is a no-op -- nothing " + "was chained, so there is no fork to report (the emitter " + "fired here for all 39 ambiguous particles, and _assign " + "double-reported the same token)"), Case("initial_shaped_not_conjunction", "john e. smith", {"given": "john", "middle": "e.", "family": "smith"}, notes="v1 is_conjunction excludes initials at classify too"), diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 1b218546..9df9a09f 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -271,3 +271,30 @@ def test_ambiguous_particle_reports_both_branches_of_its_fork() -> None: def test_unambiguous_particle_chain_reports_nothing() -> None: # 'de' is never a given name, so merging it is not a fork assert parse("Dr. de la Vega").ambiguities == () + + +@pytest.mark.parametrize("text", [ + "Dr. Van Jr.", # the piece after the particle is a suffix, so + "Dr. Van MD", # the chain scan never advances and merge() is + "Dr. Do Jr.", # a no-op -- nothing was chained, no fork taken +]) +def test_no_op_prefix_chain_is_not_a_fork(text: str) -> None: + assert parse(text).ambiguities == () + + +def test_a_fork_is_reported_by_exactly_one_stage() -> None: + # the no-op merge left the particle a lone leading piece, which is + # _assign's trigger, so both stages reported the same token + n = parse("Dr. Van Jr Smith") + assert n.given == "Van" + assert len(n.ambiguities) == 1 + + +def test_chained_particle_detail_does_not_claim_a_role() -> None: + # _group runs before assignment, so it cannot know which field the + # chained piece lands in -- "Dr. Van Johnson de la Cruz" puts it in + # GIVEN. The detail must describe the decision, not guess a role. + n = parse("Dr. Van Johnson de la Cruz") + assert n.given == "Van Johnson" + (amb,) = n.ambiguities + assert "family name" not in amb.detail From 30eb33879eb7020bc1ae60929f0423e337338a2d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 20:07:36 -0700 Subject: [PATCH 263/275] Give each SUFFIX_OR_NAME branch its own detail, and stop over-claiming Two findings from the review. The kind has one name and two causes, and they were sharing one template. The acronym branch turns on whether periods are written; the numeral branch turns on the letter being a numeral at all. Sharing the first produced: 'V' written without periods is both a post-nominal and an ordinary name; read as a suffix rather than a name part "written without periods" is the MA/M.A. distinction, which does not exist for V, and the flattened "a name part" hid that the declined reading is specifically a middle initial. The kind's own docstring says detail names what was declined; it could not even tell you which branch fired. Each branch now writes its own. And the documented claim is narrowed, for the second time. The review found a THIRD silent site for the roman-numeral fork -- the SUFFIX_COMMA tail, decided in _segment by is_suffix_lenient and stamped in _assign's blanket tail loop, so it is neither of the two comma exceptions already documented. "John Smith, V" reports nothing. Rather than chase coverage a third time, the claim no longer depends on it. It said an empty tuple means "none of the forks listed here came up", which is only true if every listed kind is emitted everywhere its fork occurs -- a property I cannot verify and have now been wrong about twice. It now says reporting is deliberately partial, names the comma paths as the known quiet ones, and states the only thing that holds regardless: a non-empty tuple is a signal, an empty one is not a guarantee. Verified: corpus 486 names, 2 intentional diffs, 0 unexplained. Co-Authored-By: Claude Opus 4.8 --- docs/concepts.rst | 14 ++++++------- nameparser/_pipeline/_assign.py | 35 +++++++++++++++++++++------------ nameparser/_types.py | 10 +++++----- tests/v2/test_parser.py | 13 ++++++++++++ 4 files changed, 47 insertions(+), 25 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 1e4a2f47..fb01a0ec 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -163,13 +163,13 @@ before it arises, too: ``"Ma, Jack"`` fixes the family name, so the credential reading never comes up, while ``"John Smith MA"`` has to call it and says so. -So an empty ``ambiguities`` does not mean every word was unambiguous — -it means none of the forks the parser *reports on* came up. The kinds -it reports are listed in :class:`~nameparser.AmbiguityKind`, and they -are the ones where both readings are genuinely common in real names. -Coverage grows over releases; a name that reports nothing today may -report something later, so treat a non-empty ``ambiguities`` as a -signal worth acting on rather than an empty one as a guarantee. +An empty ``ambiguities`` is therefore not a certificate of certainty. +Reporting is deliberately partial: :class:`~nameparser.AmbiguityKind` +lists the forks worth flagging, and even those are not reported +everywhere they occur — the comma paths stay quiet on purpose, since a +comma usually settles the structure before the question arises. +Coverage grows over releases. Treat a non-empty ``ambiguities`` as a +signal to act on; do not read an empty one as a guarantee. :class:`Tokens ` also carry tags — a second, independent label alongside their role, recording how a token was classified rather than what part of diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 22618279..338bbe44 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -129,7 +129,10 @@ def _assign_main(seg_idx: int, state: ParseState, # overwrites. The role each ends up with is read back after # assignment, since which role "not peeled" means depends on # name_order. - ambiguous_picks: list[tuple[int, ...]] = [] + # (piece, cause) -- one kind, two causes, and they need different + # wording: the acronym branch turns on whether periods are written, + # the numeral branch on the letter being a numeral at all + ambiguous_picks: list[tuple[tuple[int, ...], str]] = [] k = len(rest) while k > 0: piece = pieces[rest[k - 1]] @@ -143,7 +146,7 @@ def _assign_main(seg_idx: int, state: ParseState, # a trailing single letter is a name part unless it happens # to be a roman numeral -- and V/X/I are ordinary middle # initials, so taking it as a suffix is a call, not a fact - ambiguous_picks.append(piece) + ambiguous_picks.append((piece, "numeral")) k -= 1 continue # A bare ambiguous acronym ("MA", not "M.A.") is a credential @@ -158,7 +161,7 @@ def _assign_main(seg_idx: int, state: ParseState, bare_ambiguous = (len(piece) == 1 and "vocab:suffix-ambiguous" in tokens[piece[0]].tags) if bare_ambiguous and k - 1 >= 2: - ambiguous_picks.append(piece) + ambiguous_picks.append((piece, "acronym")) k -= 1 continue if bare_ambiguous and k >= 2: @@ -166,7 +169,7 @@ def _assign_main(seg_idx: int, state: ParseState, # that is depends on name_order, so the detail reads it back # below. (k < 2 means it is the only piece left, which is # not the fork this reports.) - ambiguous_picks.append(piece) + ambiguous_picks.append((piece, "acronym")) break name_pieces, suffix_pieces = rest[:k], rest[k:] if not name_pieces and suffix_pieces: @@ -177,20 +180,26 @@ def _assign_main(seg_idx: int, state: ParseState, _set_roles(tokens, pieces[piece_idx], roles[pos]) for piece_idx in suffix_pieces: _set_roles(tokens, pieces[piece_idx], Role.SUFFIX) - for piece in ambiguous_picks: + for piece, cause in ambiguous_picks: token = tokens[piece[0]] # every pick was assigned a role just above; the fallback only # keeps the wording sane if a future path reports before then role = token.role.value if token.role is not None else "name" - taken, declined = ( - ("a suffix", "a name part") if token.role is Role.SUFFIX - else (f"a {role} name", "a post-nominal")) + if cause == "numeral": + detail = ( + f"{token.text!r} is a roman numeral, so it reads as a " + f"generational suffix; any other single letter there " + f"would be a middle initial") + else: + taken, declined = ( + ("a suffix", "a name part") if token.role is Role.SUFFIX + else (f"a {role} name", "a post-nominal")) + detail = ( + f"{token.text!r} written without periods is both a " + f"post-nominal and an ordinary name; read as {taken} " + f"rather than {declined}") ambiguities.append(PendingAmbiguity( - AmbiguityKind.SUFFIX_OR_NAME, - f"{token.text!r} written without periods is both a " - f"post-nominal and an ordinary name; read as {taken} " - f"rather than {declined}", - tuple(piece))) + AmbiguityKind.SUFFIX_OR_NAME, detail, tuple(piece))) # leading ambiguous particle read as a name (#121 surfaced) if name_pieces: head = pieces[name_pieces[0]] diff --git a/nameparser/_types.py b/nameparser/_types.py index 62ad1245..da3433cc 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -229,11 +229,11 @@ class AmbiguityKind(StrEnum): A kind names a FORK THE PARSE HAD TO CALL, not a word that could be read two ways: the same token elsewhere in a name may present no - choice at all and is then reported by nothing. An empty - ``ambiguities`` therefore means none of the forks listed here came - up -- not that the parse was certain of everything. Coverage grows - over releases, so a non-empty tuple is a signal to act on; an empty - one is not a guarantee.""" + choice at all and is then reported by nothing. Reporting is also + partial -- a kind listed here is not necessarily emitted everywhere + its fork occurs (the comma paths stay quiet by design), and coverage + grows over releases. A non-empty tuple is a signal to act on; an + empty one is not a guarantee of certainty.""" #: Reserved: the name's field order itself is uncertain (e.g. a #: two-word name under a non-default name_order). Not yet emitted; diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 9df9a09f..3f3ef1a9 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -298,3 +298,16 @@ def test_chained_particle_detail_does_not_claim_a_role() -> None: assert n.given == "Van Johnson" (amb,) = n.ambiguities assert "family name" not in amb.detail + + +def test_each_suffix_or_name_branch_describes_itself() -> None: + # one kind, two causes: the acronym branch turns on periods, the + # roman-numeral branch turns on the letter being a numeral. Sharing + # one template made "V written without periods" -- a distinction + # that does not exist for V -- and hid which branch fired. + acronym = parse("John Smith MA").ambiguities[0].detail + assert "periods" in acronym and "post-nominal" in acronym + + numeral = parse("John Smith V").ambiguities[0].detail + assert "periods" not in numeral + assert "numeral" in numeral and "initial" in numeral From e8c3a46f7c24587d030349eb51ce16ef9414047a Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 20:26:30 -0700 Subject: [PATCH 264/275] AGENTS.md: what makes a fork worth reporting, and three sharper rules The ambiguity convention added earlier covered where to emit and how to test presence. Six review rounds and Derek's scope call exposed four gaps, three of them things I got wrong AFTER writing the rule. - A kind is worth adding only if a reader would hesitate too. The existing rule said where to emit, never whether a fork deserves a kind at all. "Smith, John V" reads as a middle initial to anyone, so reporting it is noise that teaches callers to ignore the field -- reachability of the second branch is necessary, not sufficient. This is what stopped the feature growing indefinitely; every reviewer kept finding more reachable forks, and reachability was the wrong measure. - A branch that runs but changes nothing is not a decision. merge(k, j) executes when j == k + 1, folding a piece into itself; keying on "the code got here" reported a fork for all 39 ambiguous particles on "Dr. Van Jr." and double-reported with _assign. Third instance of the same mistake, the last one after the rule was written down. - Pin the decision, not the vocabulary. The only titled-particle test used an UNAMBIGUOUS particle, so it walked the right code path and proved nothing about the branch under test -- two criticals passed 1539 tests. - Document the positive direction of a partial property. "A non-empty ambiguities is a signal" is checkable; "an empty one means no fork occurred" is a universal negative needing exhaustive verification. It was written twice and falsified twice, at sites I had not audited. Co-Authored-By: Claude Opus 4.8 --- AGENTS.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ccbf845a..cf297933 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,11 +119,12 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. Sanctioned deviation, facade layer only: `HumanName` and the shim `Constants` organize by v1 concern groups (`# -- render defaults --`, `# -- config / parsing --`, `# -- fields --`, ..., dunders and pickle last) — the classes mirror v1's own surface and die in 3.0; the canonical order still binds every core type. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). -- **Ambiguities are emitted at the DECISION site**: an `Ambiguity` records a fork the parse had to call, not a token that sits in an ambiguous vocabulary. Emit where the branch is taken — the trailing-suffix peel in `_assign`, the delimiter escape's follow-up in `classify` — never by scanning for a `vocab:*-ambiguous` tag. The same tagged token is a genuine fork in one position and unremarkable in another (`do` mid-name in "Joao da Silva do Amaral de Souza" chooses nothing), and structure often settles the question before it arises, which is why `PARTICLE_OR_GIVEN` is deliberately not emitted on the `FAMILY_COMMA` path and `SUFFIX_OR_NAME` is not emitted for "Ma, Jack". The decision site also has the token index and the detail text in hand, which the tag scan would have to reconstruct. **If a fork's two branches are taken in DIFFERENT stages, every one of them needs the emitter** -- `PARTICLE_OR_GIVEN` is decided in `_assign` when the ambiguous particle stays a lone leading piece and in `_group` when a title shifts it off index 0 and the prefix chain claims it, so both report; for two years only the first did. The stage-ownership map in `tests/v2/pipeline/test_state.py` must list `ambiguities` for each such stage, and it passes vacuously until a case row exercises the path, so add the row too. Report BOTH directions of a two-way fork — "John Smith MA" (read as a suffix) and "Jack MA" (read as the family name) are equally guesses. Every kind needs a trigger in `tests/v2/test_contracts.py::_AMBIGUITY_TRIGGERS` (an explicit `None`, strict-xfail, while reserved), and case-table rows pin expected kinds exactly, so a new emitter shows up in both immediately. +- **Ambiguities are emitted at the DECISION site**: an `Ambiguity` records a fork the parse had to call, not a token that sits in an ambiguous vocabulary. Emit where the branch is taken — the trailing-suffix peel in `_assign`, the delimiter escape's follow-up in `classify` — never by scanning for a `vocab:*-ambiguous` tag. The same tagged token is a genuine fork in one position and unremarkable in another (`do` mid-name in "Joao da Silva do Amaral de Souza" chooses nothing). **A branch that runs but changes nothing is not a decision either** -- the prefix chain's `merge(k, j)` executes even when `j == k + 1`, folding a piece into itself, and keying on "the code got here" reported a fork for all 39 ambiguous particles on "Dr. Van Jr.", where the particle stayed the GIVEN name and `_assign` reported the same token again. Check that the branch actually claimed something (`j > k + 1`) before recording. Structure also structure often settles the question before it arises, which is why `PARTICLE_OR_GIVEN` is deliberately not emitted on the `FAMILY_COMMA` path and `SUFFIX_OR_NAME` is not emitted for "Ma, Jack". The decision site also has the token index and the detail text in hand, which the tag scan would have to reconstruct. **If a fork's two branches are taken in DIFFERENT stages, every one of them needs the emitter** -- `PARTICLE_OR_GIVEN` is decided in `_assign` when the ambiguous particle stays a lone leading piece and in `_group` when a title shifts it off index 0 and the prefix chain claims it, so both report; for two years only the first did. The stage-ownership map in `tests/v2/pipeline/test_state.py` must list `ambiguities` for each such stage, and it passes vacuously until a case row exercises the path, so add the row too. Report BOTH directions of a two-way fork — "John Smith MA" (read as a suffix) and "Jack MA" (read as the family name) are equally guesses. Every kind needs a trigger in `tests/v2/test_contracts.py::_AMBIGUITY_TRIGGERS` (an explicit `None`, strict-xfail, while reserved), and case-table rows pin expected kinds exactly, so a new emitter shows up in both immediately. **Pin the decision, not the vocabulary**: the only titled-particle test used an UNAMBIGUOUS particle, so it walked the right code path and proved nothing about the branch under test -- two criticals passed 1539 tests. A row contrasting the two readings ("John Smith V" against "John Smith B") is what makes an emitter's absence meaningful. +- **A kind is worth adding only if a reader would hesitate too**: the test is not "does the code take a branch" but whether a person reading that input would genuinely be unsure. "Smith, John V" reads as a middle initial to anyone -- the comma settles it -- so reporting it would be noise that teaches callers to ignore the field, which costs more than the missing report. Reachability of the second branch is necessary, not sufficient. Prefer leaving a fork silent and documenting the omission (see the comma paths in concepts.rst) over emitting on input nobody finds ambiguous. - **Invariants guard harm, not no-ops**: add a constructor check when violating it produces a *wrong parse*, not when it produces *nothing*. A false positive costs a working configuration; a true positive on an inert condition costs the user nothing, so that trade is never worth taking. `suffix_acronyms_ambiguous ∩ suffix_words` is guarded because the overlap loses a family name; `given_name_titles` is not, because an unreachable entry is simply never consulted (see Gotchas). Before adding one, construct the config it forbids and check what actually breaks. - **The shim TRANSLATES; it never raises on a config v1 accepted, and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Four exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), and `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`). When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). -- **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. +- **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. **Document the positive direction of a partial property**: "a non-empty `ambiguities` is a signal to act on" is checkable, while "an empty one means no fork occurred" is a universal negative needing exhaustive verification -- that claim was written twice and falsified twice, at sites the author had not audited. - **Pickling**: v2 types must round-trip (`Parser` will be picklable by construction, and it holds a `Lexicon`). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, and values are deliberately NOT re-validated (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` keeps its own copy of the guard (layering) plus the `mappingproxy` slot rebuild; a new unpicklable slot type needs the same treatment plus a round-trip test. - **One sanctioned global**: the (future) cached default `Parser`. Lazily cached FROZEN singletons (`Lexicon.default()`'s `functools.cache`, the future default parser) are constants, not state; any second piece of module-level MUTABLE state requires amending the conventions doc, on purpose, in review. Sanctioned exceptions, facade layer only: `_config_shim.CONSTANTS` (the v1 shared singleton, mutable by design) and `_facade._WARNED_SUBCLASSES` (the once-per-subclass hook-warning dedup set) — both deleted with the layer in 3.0. - **Tests**: all v2 tests live in the `tests/v2/` package (its `conftest.py` overrides the v1 dual-run fixture — v2 code never reads shared `CONSTANTS`), one test module per source module plus cross-cutting `test_reprs.py` and `test_layering.py`, names stating behavior. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. From 4e02fa4ebc451f56cb99c41257147174755a9213 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 20:48:51 -0700 Subject: [PATCH 265/275] Bisect the mask overlap test; the close sweep made it quadratic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _overlaps scanned the whole masked list, and the unmatched-close sweep added last week calls it once per delimiter character found. On input with many matched pairs that is O(hits x pairs): 400 pairs spent 5.35ms here against 2.52ms before the sweep existed, growing 2.55x per doubling. masked is sorted and non-overlapping by construction -- the scan is position-driven, so every later match starts at or after the previous one's end -- which makes this the same bisect the origin resolution in _tokenize already uses, for the same reason. Scaling is linear again (1.9x per doubling, 800 pairs in 5.2ms). Two smaller wins in the same sweep: iterate distinct closes (the defaults list '”' twice, so the text was scanned 11 times for 10 closes), and reuse _unmatched() rather than restating its detail template inline. Also guard the origin resolution on there being an origin to resolve -- it was building starts and a closure on every parse for the overwhelmingly common no-ambiguity case. Co-Authored-By: Claude Opus 4.8 --- nameparser/_pipeline/_extract.py | 76 +++++++++++++++++++------------ nameparser/_pipeline/_tokenize.py | 24 ++++++---- 2 files changed, 61 insertions(+), 39 deletions(-) diff --git a/nameparser/_pipeline/_extract.py b/nameparser/_pipeline/_extract.py index 9188cdee..d343586f 100644 --- a/nameparser/_pipeline/_extract.py +++ b/nameparser/_pipeline/_extract.py @@ -29,6 +29,7 @@ """ from __future__ import annotations +import bisect import dataclasses import functools @@ -61,26 +62,39 @@ def _close_ok(text: str, j: int, width: int) -> bool: return k >= len(text) or text[k].isspace() or text[k] in COMMA_CHARS -def _apostrophe_after_a_word(text: str, j: int, close: str) -> bool: - """A "'" directly after a word character is an apostrophe, not a - dangling close quote -- "Mari' Aube'", "Ali Baba'". - - An ambiguity means the part could REASONABLY read two ways where it - sits, and this one cannot: "'" is the only delimiter character that - occurs inside and at the end of real name parts, and it is the - least likely of them to mark a nickname. The same position with a - quote IS ambiguous, since quotes do not appear inside names, so the - carve-out is deliberately this one character. #273 excluded the - curly apostrophe from the delimiter set outright for the same - reason; the straight one has to stay a delimiter (v1's - quoted_word), so it is handled here instead. - """ - return close == "'" and j > 0 and (text[j - 1].isalnum() - or text[j - 1] == ".") - - -def _overlaps(span: Span, taken: list[Span]) -> bool: - return any(span.start < t.end and t.start < span.end for t in taken) +# Delimiters that also occur INSIDE and at the end of real name parts, +# so a dangling one is literal rather than unbalanced. Only the straight +# apostrophe qualifies: quotes do not appear inside names, so the same +# position with a quote genuinely is ambiguous. #273 dropped the curly +# apostrophe from the defaults outright for this reason; the straight +# one has to stay a delimiter (v1's quoted_word), so it is carved out +# here instead. Deliberately not widened to a configured delimiter set. +WORD_INTERNAL_DELIMITERS = frozenset({"'"}) + + +def _word_internal(text: str, j: int, close: str) -> bool: + """A word-internal delimiter directly after a word character is part + of the word, not a dangling close -- "Mari' Aube'", "Ali Baba'".""" + return (close in WORD_INTERNAL_DELIMITERS and j > 0 + and (text[j - 1].isalnum() or text[j - 1] == ".")) + + +def _overlaps(span: Span, taken: list[Span], starts: list[int]) -> bool: + """`taken` sorted and non-overlapping (both hold by construction), + `starts` its start offsets. Bisect rather than scan: the closer + sweep tests one span per delimiter character found, so a linear + probe is quadratic in the number of matched pairs (400 pairs spent + 5.4ms here, against 2.5ms before the sweep existed). Same idiom, and + the same reason, as the origin resolution in _tokenize.""" + i = bisect.bisect_right(starts, span.start) - 1 + # the only candidates are the last span starting at or before us and + # its successor -- anything earlier ends before it, anything later + # starts after us + for k in (i, i + 1): + if 0 <= k < len(taken) and span.start < taken[k].end and ( + taken[k].start < span.end): + return True + return False @functools.lru_cache(maxsize=128) @@ -193,10 +207,13 @@ def extract_delimited(state: ParseState) -> ParseState: # An unmatched-open candidate whose character was consumed by a # later successful match (the bulk pass above runs ahead of the # main scan) is literal content there, not a dangling delimiter. + # offsets already claimed as unbalanced, by an open above or by a + # close in the sweep below -- either way, do not report one twice reported = {offset for offset, _ in unbalanced} + mask_starts = [s.start for s in masked] ambiguities = [ a for offset, a in unbalanced - if not _overlaps(Span(offset, offset + 1), masked)] + if not _overlaps(Span(offset, offset + 1), masked, mask_starts)] # The scan above is opener-driven: it searches for an open and then # looks rightward for its close, so a close with no open to its # LEFT is never in its search space. Sweep for those separately -- @@ -204,20 +221,21 @@ def extract_delimited(state: ParseState) -> ParseState: # always covered them ("opened without closing, or closed without # opening"). Same boundary test as the matched path, which is what # keeps the apostrophe in "O'connor" out of it. - for _, open_, close in order: + # Distinct closes only: the defaults list '”' twice (from both + # ('“','”') and ('”','”')), and a repeat can only rediscover + # offsets the first pass already handled. + for close in sorted({c for _, _, c in order}): start = 0 while (j := text.find(close, start)) != -1: start = j + 1 - if (j in reported # already an open + if (j in reported or not _close_ok(text, j, len(close)) - or _apostrophe_after_a_word(text, j, close) - or _overlaps(Span(j, j + len(close)), masked)): + or _word_internal(text, j, close) + or _overlaps(Span(j, j + len(close)), masked, + mask_starts)): continue reported.add(j) - ambiguities.append(PendingAmbiguity( - AmbiguityKind.UNBALANCED_DELIMITER, - f"unmatched {close!r} at offset {j}; treated as " - f"literal text", origin=j)) + ambiguities.append(_unmatched(close, j)[1]) return dataclasses.replace( state, extracted=tuple(extracted), masked=tuple(masked), ambiguities=state.ambiguities + tuple(ambiguities)) diff --git a/nameparser/_pipeline/_tokenize.py b/nameparser/_pipeline/_tokenize.py index 7ea86fac..ac19b163 100644 --- a/nameparser/_pipeline/_tokenize.py +++ b/nameparser/_pipeline/_tokenize.py @@ -104,18 +104,22 @@ def tokenize(state: ParseState) -> ParseState: # against 9ms before the sweep existed). Spans are non-overlapping # and sorted just above, so the candidate is the last token whose # start is <= the offset. - starts = [t.span.start for t in tokens] + # Nothing to resolve on the overwhelmingly common no-ambiguity + # parse, and building starts + the closure is not free. + ambiguities = state.ambiguities + if any(a.origin is not None for a in ambiguities): + starts = [t.span.start for t in tokens] - def _containing(offset: int) -> tuple[int, ...]: - i = bisect.bisect_right(starts, offset) - 1 - if i >= 0 and offset < tokens[i].span.end: - return (i,) - return () + def _containing(offset: int) -> tuple[int, ...]: + i = bisect.bisect_right(starts, offset) - 1 + if i >= 0 and offset < tokens[i].span.end: + return (i,) + return () - ambiguities = tuple( - a if a.origin is None - else dataclasses.replace(a, indices=_containing(a.origin)) - for a in state.ambiguities) + ambiguities = tuple( + a if a.origin is None + else dataclasses.replace(a, indices=_containing(a.origin)) + for a in ambiguities) return dataclasses.replace(state, tokens=tuple(tokens), comma_offsets=tuple(sorted(commas)), ambiguities=ambiguities) From d13bf5cf4838a24fbb7364e6ede67b1915e87890 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 20:49:02 -0700 Subject: [PATCH 266/275] Fold given_name_titles per word, so abbreviated entries match The lookup key was built in three places and one of them disagreed. post_rules joins normalized title tokens ('lt col'); the v1 shim translates first_name_titles the same way; but _normset stored native 2.0 entries with _normalize per whole entry, which leaves interior periods alone. So Lexicon(given_name_titles={'lt. col'}) stored a key the matcher can never build, and the entry silently matched nothing -- while the identical config through the Constants shim worked. Give the rule one home: _title_key(words) in _lexicon, called by all three. This is the field with no validation by design (two guards were tried and both broke working configs), which makes a silent no-op its most likely failure -- worth removing the one instance we control. Strictly widening: 'lt col' folds to itself, so no config that works today changes; entries with interior periods or repeated spaces start working instead of being inert. Co-Authored-By: Claude Opus 4.8 --- nameparser/_config_shim.py | 7 +++-- nameparser/_lexicon.py | 40 +++++++++++++++++++++-------- nameparser/_pipeline/_post_rules.py | 4 +-- tests/v2/test_lexicon.py | 18 +++++++++++++ 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index af45f542..05451c11 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -27,7 +27,7 @@ ) from typing import NamedTuple, Self -from nameparser._lexicon import Lexicon, _normalize +from nameparser._lexicon import Lexicon, _title_key from nameparser._parser import Parser from nameparser._policy import PatronymicRule, Policy from nameparser.util import lc @@ -965,14 +965,13 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: """ from nameparser.config.maiden_markers import MAIDEN_MARKERS acronyms = frozenset(self.suffix_acronyms) - titles = frozenset(self.titles) particles = frozenset(self.prefixes) bound = frozenset(self.bound_first_names) ambiguous_acronyms = frozenset(self.suffix_acronyms_ambiguous) & acronyms # keep in sync with _lexicon._default_lexicon() (pinned by # tests/v2/test_config_shim.py::test_snapshot_field_translation) lexicon = Lexicon( - titles=titles, + titles=frozenset(self.titles), # TRANSLATE, do not filter. The two versions build the same # lookup key differently: v1 joins the raw title run and # then applies lc(), which strips only the whole string's @@ -990,7 +989,7 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: # a config v1 simply ignored. given_name_titles=frozenset( t for t in ( - " ".join(_normalize(w) for w in e.split()) + _title_key(e.split()) for e in self.first_name_titles if e == " ".join(e.split()) ) if t), diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index fd971f92..3b9eb878 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -76,6 +76,18 @@ def _normalize(word: str) -> str: word = stripped +def _title_key(words: Iterable[str]) -> str: + """The given_name_titles lookup key for a run of title words. + + A multi-word title is matched as one key ('lt col'), so the fold has + to run per word and rejoin -- _normalize on the whole phrase would + leave interior periods. Defined once because it is built at match + time (post_rules) and at translation time (the v1 facade's + first_name_titles), and a divergence between the two fails silently: + the entry simply stops matching.""" + return " ".join(_normalize(w) for w in words) + + def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: # Reject a bare str before iterating: iterating "dr" would silently # yield the single characters {'d', 'r'} -- the set(str) footgun on @@ -106,7 +118,14 @@ def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: raise TypeError( f"Lexicon.{field_name} entries must be strings, got {w!r}" ) - n = _normalize(w) + # given_name_titles is the one field whose entries are matched as + # a multi-word run, so it folds per word: stored as the same key + # post_rules builds, or 'lt. col' would be kept verbatim and + # never match anything (a silent no-op on the config surface). + # Every other field holds single words, where the two folds + # agree. + n = _title_key(w.split()) if field_name == "given_name_titles" \ + else _normalize(w) # "." or "" is a data bug (stray split artifact, empty CSV # cell); dropping it silently would also let a data-module typo # vanish instead of failing CI. @@ -362,11 +381,6 @@ def __setstate__(self, state: dict[str, object]) -> None: ) for name, value in state.items(): object.__setattr__(self, name, value) - # cast is safe: __post_init__ below runs _normset over each of - # these and raises for anything that is not an iterable of str, - # so the values are only read as such once it has returned - given = {name: cast("Iterable[str]", state[name]) - for name in _VOCAB_FIELDS} # Re-run construction validation rather than trusting the blob. # The layout check above catches SHAPE skew; this catches # CONTENT skew, which is likelier -- particles_ambiguous flipped @@ -381,11 +395,15 @@ def __setstate__(self, state: dict[str, object]) -> None: # normalized already (_normalize converges), so a difference # here means the state came from somewhere else -- say so while # the offending entries can still be named. - drifted = sorted( - f"{name}: {', '.join(sorted(frozenset(given[name]) - getattr(self, name)))}" - for name in _VOCAB_FIELDS - if frozenset(given[name]) != getattr(self, name) - ) + # One pass per field, in _VOCAB_FIELDS declaration order. cast is + # safe: __post_init__ just ran _normset over each of these and + # raised for anything that was not an iterable of str. + drifted = [] + for name in _VOCAB_FIELDS: + lost = frozenset( + cast("Iterable[str]", state[name])) - getattr(self, name) + if lost: + drifted.append(f"{name}: {', '.join(sorted(lost))}") # the pair field too, or ten fields raise and the eleventh is # quietly re-canonicalized (keys normalized, sorted, deduped) given_pairs = cast("Iterable[tuple[str, str]]", diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index cca58918..fc96fb12 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -32,7 +32,7 @@ import dataclasses import re -from nameparser._lexicon import _normalize +from nameparser._lexicon import _title_key from nameparser._pipeline._state import ParseState, Structure, WorkToken from nameparser._policy import PatronymicRule from nameparser._types import FOLDED_TAG, Role @@ -71,7 +71,7 @@ def post_rules(state: ParseState) -> ParseState: # rule 1: title + lone given -> family (v1 handle_firstnames) if titles and givens and not middles and not families and not others: - joined = " ".join(_normalize(tokens[i].text) for i in titles) + joined = _title_key(tokens[i].text for i in titles) if joined not in state.lexicon.given_name_titles: for i in givens: _retag(tokens, i, Role.FAMILY) diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 33ffc303..68efd62f 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -4,6 +4,7 @@ import pytest +from nameparser import Parser from nameparser._lexicon import Lexicon, _normalize @@ -242,6 +243,23 @@ def test_given_name_titles_may_hold_a_multi_word_phrase() -> None: assert "grand duke" in lex.given_name_titles +@pytest.mark.parametrize("spelling", ["lt. col", "lt col", "Lt. Col."]) +def test_given_name_titles_folds_per_word_so_abbreviations_match( + spelling: str) -> None: + # The key is built per word and rejoined, so an abbreviated entry + # has to fold the same way the parse folds the title run. Storing + # 'lt. col' verbatim kept the interior period and matched nothing -- + # a silent no-op on the config surface, the failure this field is + # most prone to (it has no validation by design). + base = Lexicon.default() + lex = dataclasses.replace( + base, + titles=base.titles | {"lt", "col"}, + given_name_titles=base.given_name_titles | {spelling}) + assert "lt col" in lex.given_name_titles + assert Parser(lexicon=lex).parse("Lt. Col. Smith").given == "Smith" + + def test_given_name_titles_is_not_constrained_against_titles() -> None: # Deliberately unvalidated. The lookup key is the joined run of # Role.TITLE tokens, which the PARSE builds, and a conjunction From 8b917421824761530f6b9a267fd786475dd99def Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 20:49:14 -0700 Subject: [PATCH 267/275] Report each fork where it is decided _group_segment widened its return to a 3-tuple to carry particle forks out to the caller, which then wrote the message ~110 lines from the branch that earned it, behind a suppression flag the caller had already computed. _assign_main solves the same problem by taking the ambiguity list as an out-parameter; do that here too. The message now sits with the comment explaining why the fork exists, and one of the two family-comma suppression comments goes away. Same treatment for _assign's roman-numeral fork: its wording depends on nothing assigned later, so it emits at its trigger rather than riding a (piece, cause) tag through to a post-loop branch. That leaves ambiguous_picks holding one cause, which is what made the tag necessary. Also: collapse the two adjacent bare_ambiguous tests into one (the shared "either way, record the fork" was invisible between them), drop the unreachable role fallback in favour of an assert that says the invariant out loud, and put the cheap selective tag test before the per-piece title() scan in the group guard. No parse output changes: 1549 tests, and the differential corpus holds at 486 names / 2 intentional diffs / 0 unexplained. Co-Authored-By: Claude Opus 4.8 --- nameparser/_pipeline/_assign.py | 63 +++++++++++++++------------------ nameparser/_pipeline/_group.py | 61 ++++++++++++++++--------------- nameparser/_pipeline/_state.py | 12 ++++--- 3 files changed, 68 insertions(+), 68 deletions(-) diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 338bbe44..266e7d54 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -126,13 +126,11 @@ def _assign_main(seg_idx: int, state: ParseState, # non-initial piece is a suffix) # every bare ambiguous acronym the peel had to resolve -- one # coin-flip each, in either direction, so this collects rather than - # overwrites. The role each ends up with is read back after - # assignment, since which role "not peeled" means depends on - # name_order. - # (piece, cause) -- one kind, two causes, and they need different - # wording: the acronym branch turns on whether periods are written, - # the numeral branch on the letter being a numeral at all - ambiguous_picks: list[tuple[tuple[int, ...], str]] = [] + # overwrites. Deferred to after assignment because the wording reads + # the role back, and which role "not peeled" means depends on + # name_order. (The roman-numeral fork needs no such deferral and is + # reported at its trigger below.) + ambiguous_picks: list[tuple[int, ...]] = [] k = len(rest) while k > 0: piece = pieces[rest[k - 1]] @@ -146,7 +144,12 @@ def _assign_main(seg_idx: int, state: ParseState, # a trailing single letter is a name part unless it happens # to be a roman numeral -- and V/X/I are ordinary middle # initials, so taking it as a suffix is a call, not a fact - ambiguous_picks.append((piece, "numeral")) + ambiguities.append(PendingAmbiguity( + AmbiguityKind.SUFFIX_OR_NAME, + f"{tokens[piece[0]].text!r} is a roman numeral, so it " + f"reads as a generational suffix; any other single " + f"letter there would be a middle initial", + tuple(piece))) k -= 1 continue # A bare ambiguous acronym ("MA", not "M.A.") is a credential @@ -160,16 +163,13 @@ def _assign_main(seg_idx: int, state: ParseState, # the vocabulary is not in doubt. bare_ambiguous = (len(piece) == 1 and "vocab:suffix-ambiguous" in tokens[piece[0]].tags) - if bare_ambiguous and k - 1 >= 2: - ambiguous_picks.append((piece, "acronym")) - k -= 1 - continue + # k < 2 means it is the only piece left, which is not the fork + # this reports. if bare_ambiguous and k >= 2: - # not peeled, so it stays the last NAME piece -- which role - # that is depends on name_order, so the detail reads it back - # below. (k < 2 means it is the only piece left, which is - # not the fork this reports.) - ambiguous_picks.append((piece, "acronym")) + ambiguous_picks.append(tuple(piece)) + if k >= 3: # peeling still leaves given + family + k -= 1 + continue break name_pieces, suffix_pieces = rest[:k], rest[k:] if not name_pieces and suffix_pieces: @@ -180,26 +180,19 @@ def _assign_main(seg_idx: int, state: ParseState, _set_roles(tokens, pieces[piece_idx], roles[pos]) for piece_idx in suffix_pieces: _set_roles(tokens, pieces[piece_idx], Role.SUFFIX) - for piece, cause in ambiguous_picks: + for piece in ambiguous_picks: + # every pick is in rest, so the loops above just gave it a role token = tokens[piece[0]] - # every pick was assigned a role just above; the fallback only - # keeps the wording sane if a future path reports before then - role = token.role.value if token.role is not None else "name" - if cause == "numeral": - detail = ( - f"{token.text!r} is a roman numeral, so it reads as a " - f"generational suffix; any other single letter there " - f"would be a middle initial") - else: - taken, declined = ( - ("a suffix", "a name part") if token.role is Role.SUFFIX - else (f"a {role} name", "a post-nominal")) - detail = ( - f"{token.text!r} written without periods is both a " - f"post-nominal and an ordinary name; read as {taken} " - f"rather than {declined}") + assert token.role is not None + taken, declined = ( + ("a suffix", "a name part") if token.role is Role.SUFFIX + else (f"a {token.role.value} name", "a post-nominal")) ambiguities.append(PendingAmbiguity( - AmbiguityKind.SUFFIX_OR_NAME, detail, tuple(piece))) + AmbiguityKind.SUFFIX_OR_NAME, + f"{token.text!r} written without periods is both a " + f"post-nominal and an ordinary name; read as {taken} " + f"rather than {declined}", + piece)) # leading ambiguous particle read as a name (#121 surfaced) if name_pieces: head = pieces[name_pieces[0]] diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index dcc8fdb7..dd2aa483 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -89,13 +89,15 @@ def _is_rootname(piece: Sequence[int], ptags: Set[str], def _group_segment(seg: tuple[int, ...], additional: int, tokens: Sequence[WorkToken], bound_join: BoundJoin = BoundJoin.STRICT, - ) -> tuple[list[Piece], list[set[str]], list[int]]: + ambiguities: list[PendingAmbiguity] | None = None, + ) -> tuple[list[Piece], list[set[str]]]: pieces: list[Piece] = [[i] for i in seg] ptags: list[set[str]] = [set() for _ in seg] - # token indices where an ambiguous particle was chained into the - # family name from the position that would otherwise be the given - # name -- the other branch of the fork _assign reports (see group()) - particle_forks: list[int] = [] + # Out-parameter, same shape as _assign_main: forks are reported + # where they are decided. A caller that passes None (or a throwaway + # list) suppresses reporting -- see group() for when that applies. + if ambiguities is None: + ambiguities = [] def title(k: int) -> bool: return _is_title_piece(pieces[k], ptags[k], tokens) @@ -174,17 +176,33 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), j += 1 while j < len(pieces) and not prefix(j) and not suffix(j): j += 1 + # The other half of PARTICLE_OR_GIVEN. _assign reports the + # fork when an ambiguous particle stays a lone leading piece + # ("Van Johnson" -> given); the chain here takes the + # opposite branch whenever a title shifts it off index 0 + # ("Dr. Van Johnson" -> family "Van Johnson"). A fork whose + # two sides are decided in different stages needs an emitter + # in each. + # # j > k + 1 is what makes this a DECISION rather than a # shape: when the next piece is a suffix the inner scan # never advances, merge(k, k+1) folds a piece into itself, # and the particle stays a lone leading piece -- nothing # was chained, and _assign reports that case instead. # Without this the two emitters both fire on the same token. + # (Tag test first: it is a set lookup and almost no name has + # an ambiguous particle, while title() is a call per piece.) if (j > k + 1 - and all(title(x) for x in range(k)) and "vocab:particle-ambiguous" - in tokens[pieces[k][0]].tags): - particle_forks.append(pieces[k][0]) + in tokens[pieces[k][0]].tags + and all(title(x) for x in range(k))): + i = pieces[k][0] + ambiguities.append(PendingAmbiguity( + AmbiguityKind.PARTICLE_OR_GIVEN, + f"{tokens[i].text!r} was chained onto the following " + f"name piece; it is also a given name in other " + f"names", + (i,))) merge(k, j, drop={"prefix"}) k += 1 # bound given names: the first non-title piece joins the next @@ -202,7 +220,7 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), if not title(k) and not suffix(k)) if non_suffix >= bound_join: merge(first_name_k, first_name_k + 2) - return pieces, ptags, particle_forks + return pieces, ptags def group(state: ParseState) -> ParseState: @@ -227,8 +245,12 @@ def group(state: ParseState) -> ParseState: else BoundJoin.DISABLED) else: bound_join = BoundJoin.STRICT - pieces, ptags, particle_forks = _group_segment( - seg, additional, tokens, bound_join) + # Suppressed after a family comma for the same reason _assign + # suppresses it there: the family name is already fixed, so + # there is no fork left to report. + pieces, ptags = _group_segment( + seg, additional, tokens, bound_join, + None if family_comma else ambiguities) if tail_start is not None and seg_idx >= tail_start: # v1 renders each tail COMMA SEGMENT as one suffix entry # ('Smith, V MD' -> suffix 'V MD'); a delimiter core inside @@ -289,23 +311,6 @@ def group(state: ParseState) -> ParseState: ptags[m:j] = [] all_pieces.append(tuple(tuple(p) for p in pieces)) all_ptags.append(tuple(frozenset(t) for t in ptags)) - # The other half of PARTICLE_OR_GIVEN. _assign reports the fork - # when an ambiguous particle stays a lone leading piece ("Van - # Johnson" -> given). The prefix chain above takes the opposite - # branch whenever a title shifts it off index 0 ("Dr. Van - # Johnson" -> family "Van Johnson"), and that branch lives in - # this stage, so it has to report here -- a fork whose two sides - # are decided in different stages needs an emitter in each. - # Suppressed after a family comma for the same reason _assign - # suppresses it there: the family name is already fixed. - if not family_comma: - for i in particle_forks: - ambiguities.append(PendingAmbiguity( - AmbiguityKind.PARTICLE_OR_GIVEN, - f"{tokens[i].text!r} was chained onto the following " - f"name piece; it is also a given name in other " - f"names", - (i,))) return dataclasses.replace( state, tokens=tuple(tokens), pieces=tuple(all_pieces), piece_tags=tuple(all_ptags), dropped=tuple(dropped), diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py index 367ccb65..3feeacb0 100644 --- a/nameparser/_pipeline/_state.py +++ b/nameparser/_pipeline/_state.py @@ -68,11 +68,13 @@ class ParseState: extract_delimited -> extracted/masked; tokenize -> tokens (span- sorted)/comma_offsets; segment -> segments/structure; classify -> token tags; group -> pieces/piece_tags/dropped AND maiden token - roles; assign/post_rules -> the remaining token roles; ambiguities are - recorded by extract/segment/assign only (pinned by the ownership - test). Post-group, segments may retain indices of - dropped tokens -- assign iterates pieces, never segments. This - ownership map is pinned by tests/v2/pipeline/test_state.py.""" + roles; assign/post_rules -> the remaining token roles. Ambiguities + are recorded by every stage that DECIDES one -- extract (resolved to + a token index by tokenize), segment, classify, group, and assign -- + since a fork whose branches are taken in different stages needs an + emitter in each. Post-group, segments may retain indices of dropped + tokens -- assign iterates pieces, never segments. This ownership map + is pinned by tests/v2/pipeline/test_state.py.""" original: str lexicon: Lexicon From cea2a285915533a0c533a22df1d242624e36d02e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 20:49:27 -0700 Subject: [PATCH 268/275] Collapse seven copies of the config normalization assert Every vocabulary module carried the same all(w == w.strip().lower()) assert under a near-verbatim three-line comment, four of them differing only in the set name. One helper, assert_normalized(name, entries), states the rationale once and names the offending entries when it fires (the old copies only said that something was wrong). The fold stays the weaker w.strip().lower() rather than _lexicon's _normalize: entries like 'esq.' are legitimate data here, and config cannot import _lexicon anyway -- _lexicon imports these constants. The relationship asserts (FIRST_NAME_TITLES <= TITLES, the SUFFIX_ACRONYMS_AMBIGUOUS pair, the deliberate non-disjointness note) stay in the modules that own them; those encode facts about the data rather than hygiene. Co-Authored-By: Claude Opus 4.8 --- nameparser/config/_invariants.py | 27 ++++++++++++++++++++++++++ nameparser/config/bound_first_names.py | 8 +++----- nameparser/config/capitalization.py | 8 ++++---- nameparser/config/conjunctions.py | 8 +++----- nameparser/config/maiden_markers.py | 8 +++----- nameparser/config/prefixes.py | 7 ++----- nameparser/config/suffixes.py | 8 +++----- nameparser/config/titles.py | 11 ++++------- 8 files changed, 49 insertions(+), 36 deletions(-) create mode 100644 nameparser/config/_invariants.py diff --git a/nameparser/config/_invariants.py b/nameparser/config/_invariants.py new file mode 100644 index 00000000..98969132 --- /dev/null +++ b/nameparser/config/_invariants.py @@ -0,0 +1,27 @@ +"""Import-time hygiene checks shared by the vocabulary data modules. + +Every constant here is looked up in normalized form, so a stray capital +or a surrounding space makes an entry unreachable by a direct membership +test -- ``'actor ' in TITLES`` is False -- even though the parser's own +ingest (:func:`nameparser._lexicon._normalize`) normalizes it away and +papers over the typo. Checking at import turns a silently-inert entry +into an immediate failure. + +Deliberately a weaker fold than ``_normalize``, which also strips edge +periods: entries like ``'esq.'`` are legitimate data here, and this +module cannot import ``_lexicon`` anyway (``_lexicon`` imports these +constants). The relationship checks between constants stay in the +modules that own them -- those encode facts about the data, not hygiene. +""" +from __future__ import annotations + +from collections.abc import Iterable + + +def assert_normalized(name: str, entries: Iterable[str]) -> None: + """Assert every entry is stored lowercase and free of edge whitespace.""" + offenders = sorted(w for w in entries if w != w.strip().lower()) + assert not offenders, ( + f"{name} entries must be stored lowercase and whitespace-free; " + f"offending: {offenders}" + ) diff --git a/nameparser/config/bound_first_names.py b/nameparser/config/bound_first_names.py index b2979f62..7dd29e33 100644 --- a/nameparser/config/bound_first_names.py +++ b/nameparser/config/bound_first_names.py @@ -1,3 +1,5 @@ +from nameparser.config._invariants import assert_normalized + #: Bound Arabic given-name prefixes that attach to the following word to form #: one first name (e.g. "abdul salam" → first name "abdul salam"). They are #: never standalone names. Join logic runs in the given-name region only, @@ -23,8 +25,4 @@ } -# See prefixes.py: entries are looked up in normalized form, so a stray -# capital or trailing space makes one unreachable by direct membership -# test even though the parser's own ingest normalizes it away. -assert all(w == w.strip().lower() for w in BOUND_FIRST_NAMES), \ - "BOUND_FIRST_NAMES entries must be stored lowercase and whitespace-free" +assert_normalized("BOUND_FIRST_NAMES", BOUND_FIRST_NAMES) diff --git a/nameparser/config/capitalization.py b/nameparser/config/capitalization.py index 57c442ab..6973c7d4 100644 --- a/nameparser/config/capitalization.py +++ b/nameparser/config/capitalization.py @@ -1,3 +1,5 @@ +from nameparser.config._invariants import assert_normalized + CAPITALIZATION_EXCEPTIONS = { 'ii': 'II', 'iii': 'III', @@ -10,7 +12,5 @@ """ -# See prefixes.py: keys are looked up in normalized form. Only the keys -- -# the values are the exact-cased replacements, so they are cased on purpose. -assert all(k == k.strip().lower() for k in CAPITALIZATION_EXCEPTIONS), \ - "CAPITALIZATION_EXCEPTIONS keys must be stored lowercase and whitespace-free" +# Keys only -- the values are the exact-cased replacements, cased on purpose. +assert_normalized("CAPITALIZATION_EXCEPTIONS", CAPITALIZATION_EXCEPTIONS) diff --git a/nameparser/config/conjunctions.py b/nameparser/config/conjunctions.py index b6d045ca..cc9a8427 100644 --- a/nameparser/config/conjunctions.py +++ b/nameparser/config/conjunctions.py @@ -1,3 +1,5 @@ +from nameparser.config._invariants import assert_normalized + CONJUNCTIONS = { '&', 'and', @@ -28,8 +30,4 @@ """ -# See prefixes.py: entries are looked up in normalized form, so a stray -# capital or trailing space makes one unreachable by direct membership -# test even though the parser's own ingest normalizes it away. -assert all(w == w.strip().lower() for w in CONJUNCTIONS), \ - "CONJUNCTIONS entries must be stored lowercase and whitespace-free" +assert_normalized("CONJUNCTIONS", CONJUNCTIONS) diff --git a/nameparser/config/maiden_markers.py b/nameparser/config/maiden_markers.py index 8f8719a7..bcb4f6e8 100644 --- a/nameparser/config/maiden_markers.py +++ b/nameparser/config/maiden_markers.py @@ -1,3 +1,5 @@ +from nameparser.config._invariants import assert_normalized + MAIDEN_MARKERS = { 'née', 'né', @@ -37,8 +39,4 @@ """ -# See prefixes.py: entries are looked up in normalized form, so a stray -# capital or trailing space makes one unreachable by direct membership -# test even though the parser's own ingest normalizes it away. -assert all(w == w.strip().lower() for w in MAIDEN_MARKERS), \ - "MAIDEN_MARKERS entries must be stored lowercase and whitespace-free" +assert_normalized("MAIDEN_MARKERS", MAIDEN_MARKERS) diff --git a/nameparser/config/prefixes.py b/nameparser/config/prefixes.py index 7fa9caf0..f7a89b30 100644 --- a/nameparser/config/prefixes.py +++ b/nameparser/config/prefixes.py @@ -1,3 +1,4 @@ +from nameparser.config._invariants import assert_normalized from nameparser.config.bound_first_names import BOUND_FIRST_NAMES #: The sub-set of :py:data:`PREFIXES` that are *never* a standalone first name. @@ -126,8 +127,4 @@ "NON_FIRST_NAME_PREFIXES must stay a subset of PREFIXES" assert not (NON_FIRST_NAME_PREFIXES & BOUND_FIRST_NAMES), \ "NON_FIRST_NAME_PREFIXES must stay disjoint from BOUND_FIRST_NAMES" -# Entries are looked up in normalized form; a stray capital or trailing -# space makes one unreachable by direct membership test even though the -# parser's own ingest normalizes it away. -assert all(w == w.strip().lower() for w in PREFIXES), \ - "PREFIXES entries must be stored lowercase and whitespace-free" +assert_normalized("PREFIXES", PREFIXES) diff --git a/nameparser/config/suffixes.py b/nameparser/config/suffixes.py index 48dd9464..05ae8f78 100644 --- a/nameparser/config/suffixes.py +++ b/nameparser/config/suffixes.py @@ -1,3 +1,5 @@ +from nameparser.config._invariants import assert_normalized + SUFFIX_NOT_ACRONYMS = { # #269: Cyrillic мл/ст (junior/senior, the jr/sr analogs) deferred # pending the within-script collision vetting the issue asks for; @@ -735,8 +737,4 @@ "an ambiguous acronym must not also be a suffix word (the word " \ "branch bypasses its period gate): " \ f"{sorted(SUFFIX_ACRONYMS_AMBIGUOUS & SUFFIX_NOT_ACRONYMS)}" -# Entries are looked up in normalized form, so a stray capital or a -# trailing space makes an entry unreachable by direct membership test even -# though the parser's own ingest would paper over it. -assert all(w == w.strip().lower() for w in SUFFIX_ACRONYMS | SUFFIX_NOT_ACRONYMS), \ - "suffix entries must be stored lowercase and whitespace-free" +assert_normalized("suffix", SUFFIX_ACRONYMS | SUFFIX_NOT_ACRONYMS) diff --git a/nameparser/config/titles.py b/nameparser/config/titles.py index 3b8d3e9c..35371299 100644 --- a/nameparser/config/titles.py +++ b/nameparser/config/titles.py @@ -1,3 +1,5 @@ +from nameparser.config._invariants import assert_normalized + FIRST_NAME_TITLES = { 'aunt', 'auntie', @@ -780,10 +782,5 @@ # caller-supplied vocabulary; `assert` is stripped under `python -O`. assert FIRST_NAME_TITLES <= TITLES, \ "FIRST_NAME_TITLES must stay a subset of TITLES" -# Entries are looked up in normalized form, so a stray capital or a -# trailing space would make an entry unreachable by direct membership -# test -- 'actor ' with a trailing space makes "'actor' in TITLES" -# False -- even though the parser's own ingest normalizes and papers -# over it. TITLES covers FIRST_NAME_TITLES. -assert all(w == w.strip().lower() for w in TITLES), \ - "TITLES entries must be stored lowercase and whitespace-free" +# TITLES covers FIRST_NAME_TITLES, by the subset assert above. +assert_normalized("TITLES", TITLES) From d9a07f74501a42a8c10cd3229a7cdb71701be4f1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 20:49:27 -0700 Subject: [PATCH 269/275] Small cleanups from the review sweep _lexicon.__setstate__: compute the drift set once per field instead of three times, in _VOCAB_FIELDS declaration order rather than alphabetized by formatted string, and drop the intermediate dict that existed only to host a cast. _types.ParsedName.__post_init__: nest the token-subset check under the ambiguities guard rather than giving one local two types to skip work the loop already skips. _config_shim: inline a local used once. tests: drop a bare-string __setstate__ test fully subsumed by the parametrized case above it, and stop parsing the same input twice in the unmatched-close assertion. Co-Authored-By: Claude Opus 4.8 --- nameparser/_types.py | 24 +++++++++++++----------- tests/v2/pipeline/test_extract.py | 7 ++++--- tests/v2/test_facade.py | 13 ------------- 3 files changed, 17 insertions(+), 27 deletions(-) diff --git a/nameparser/_types.py b/nameparser/_types.py index da3433cc..19a1b723 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -380,17 +380,19 @@ def __post_init__(self) -> None: # stray delimiters does), and the linear form made construction # quadratic in their product. Set membership uses the same value # equality the tuple scan did -- Token is frozen and hashable. - known = set(self.tokens) if self.ambiguities else () - for amb in self.ambiguities: - for tok in amb.tokens: - # membership is by Token's value equality, not identity: - # this only guarantees a value-equal token exists in - # self.tokens, not that `tok` IS one of those objects. - if tok not in known: - raise ValueError( - f"Ambiguity token {tok.text!r} is not a subset of " - f"this ParsedName's tokens" - ) + if self.ambiguities: + known = set(self.tokens) + for amb in self.ambiguities: + for tok in amb.tokens: + # membership is by Token's value equality, not + # identity: this only guarantees a value-equal token + # exists in self.tokens, not that `tok` IS one of + # those objects. + if tok not in known: + raise ValueError( + f"Ambiguity token {tok.text!r} is not a " + f"subset of this ParsedName's tokens" + ) def __bool__(self) -> bool: return bool(self.tokens) diff --git a/tests/v2/pipeline/test_extract.py b/tests/v2/pipeline/test_extract.py index 20885d80..b7536070 100644 --- a/tests/v2/pipeline/test_extract.py +++ b/tests/v2/pipeline/test_extract.py @@ -194,9 +194,10 @@ def test_unmatched_close_is_reported(text: str, char: str) -> None: # the scan is opener-driven, so a close with no open to its left was # never in its search space and went unreported -- even though the # AmbiguityKind docstring promises "or closed without opening" - kinds = [a.kind for a in parse(text).ambiguities] - assert kinds == [AmbiguityKind.UNBALANCED_DELIMITER] - assert char in parse(text).ambiguities[0].detail + ambiguities = parse(text).ambiguities + assert [a.kind for a in ambiguities] == [ + AmbiguityKind.UNBALANCED_DELIMITER] + assert char in ambiguities[0].detail def test_word_final_apostrophe_is_not_an_unbalanced_delimiter() -> None: diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index 8de1d1ee..d6658ac1 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -326,19 +326,6 @@ def test_setstate_rejects_malformed_component_lists(bad: object) -> None: loaded.__setstate__(state) -def test_setstate_rejects_a_bare_string_component_list() -> None: - # a genuine v1 pickle stores lists, but __setstate__ reads foreign - # state: a bare str is iterable, so entry.split() over its - # CHARACTERS silently produced a plausible-looking wrong name - # ("John" -> first "J o h n") instead of failing at the load site - n = HumanName("John Smith") - state = n.__getstate__() - state["first_list"] = "John" - loaded = HumanName.__new__(HumanName) - with pytest.raises(TypeError, match="first_list"): - loaded.__setstate__(state) - - def test_setstate_restores_a_foreign_multi_word_suffix_entry() -> None: # the parametrized round-trip tests above go through v2's own # __getstate__, which is self-consistent by construction. The case From 7fbcd9eaf7f044228016aaf1cb868ad885ca732b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 19 Jul 2026 21:49:00 -0700 Subject: [PATCH 270/275] Pin the two-emitter invariant for PARTICLE_OR_GIVEN MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _group and _assign each report one branch of this fork and coordinate only through _group's `j > k + 1` guard, which mirrors _assign's reachability by hand. Nothing checked the mirror: reachability drift in either stage would over- or under-report, and only a case pinning that exact name would notice. Generate the shapes rather than trusting a corpus. The boundary is a suffix straight after the particle ("Dr. Van Jr."), where the chain is a no-op and only _assign should speak -- a shape plausible-name corpora have little reason to hold, and indeed the 486-name differential corpus does not. Swept over every ambiguous particle, so a vocabulary addition is covered without editing the test. Verified to fail in both directions by loosening and tightening the guard. Asserts the count, not which stage spoke, on purpose: whether a fork was decided by a vocabulary merge or by position is NOT recoverable from the finished parse. 'Dr. aan Johnson Jr.' and 'أبو بكر أحمد' end with the same roles and the same tags, and only one is a fork -- so a reconstruction would have to re-implement _group instead of checking it. That is also why the two emitters stay where the decisions are made. Co-Authored-By: Claude Opus 4.8 --- tests/v2/test_properties.py | 97 +++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index 0e4440b0..81895033 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -6,16 +6,30 @@ keeps runs reproducible on shared CI runners -- this layer guards against regressions; exploratory fuzzing happened during review. """ +import json +from pathlib import Path + +import pytest from hypothesis import given, settings from hypothesis import strategies as st from nameparser import Lexicon, Policy, parse from nameparser._pipeline import run from nameparser._pipeline._state import ParseState +from nameparser._types import AmbiguityKind, Role _ALPHABET = st.sampled_from( 'abcdefgh ABC 12 .,،，\'"()«»‏‏\U0001f600éñßЖ-') +# Same one-JSON-name-per-line convention test_locales.py reads; see the +# note there on why tools/ is not imported. +_FORK_CORPUS = [ + json.loads(line) + for line in (Path(__file__).parents[2] / "tools" / "differential" + / "corpus.jsonl").read_text().splitlines() + if line.strip() +] + @given(st.text(alphabet=_ALPHABET, max_size=200)) @settings(max_examples=300, deadline=None, derandomize=True) @@ -76,3 +90,86 @@ def test_every_original_char_is_accounted_for(text: str) -> None: continue raise AssertionError( f"char {ch!r} at {i} in {text!r} is unaccounted for") + + +_NAME_ROLES = (Role.GIVEN, Role.MIDDLE, Role.FAMILY) + + +def _fork_count(state: ParseState) -> int: + return sum(a.kind is AmbiguityKind.PARTICLE_OR_GIVEN + for a in state.ambiguities) + + +def test_a_leading_ambiguous_particle_is_reported_once_and_only_once( +) -> None: + """PARTICLE_OR_GIVEN is the one kind two stages emit: _group takes + the particle branch when a title shifts the particle off index 0 and + the chain claims something ("Dr. Van Johnson"), _assign takes the + given branch when it stays a lone leading piece ("Van Johnson"). + Each reports the side it decides -- see the ParseState docstring -- + but they coordinate only through _group's `j > k + 1` guard, which + mirrors _assign's reachability by hand. Nothing checked the mirror. + + The shape that separates them is the one real-name corpora have + least reason to contain: a suffix straight after the particle, where + the chain is a no-op and only _assign should speak. Generate it, over + every ambiguous particle, so a vocabulary addition is covered too. + + Deliberately asserts the COUNT and not which stage spoke. Whether a + given fork was decided by a vocabulary merge or by position is not + recoverable from the finished parse -- 'Dr. aan Johnson Jr.' and + 'أبو بكر أحمد' end with the same roles and the same tags, and only + one is a fork -- so any reconstruction here would have to + re-implement _group rather than check it. + """ + lex = Lexicon.default() + # bound-given prefixes are excluded, not overlooked: 'abu' is both + # an ambiguous particle and a bound given prefix, so whether it + # forks depends on whether the bound join fired -- a second rule, + # covered by the case corpus rather than by this sweep + particles = sorted(lex.particles_ambiguous - lex.bound_given_names) + assert particles, "no ambiguous particles to exercise" + failures = [] + for particle in particles: + for lead in ("", "Dr. ", "Dr. Ann "): + for body in ("", "Johnson ", "Johnson Smith "): + for tail in ("", "Jr.", "MD", "III"): + text = f"{lead}{particle} {body}{tail}".strip() + state = run(ParseState(original=text, + lexicon=lex, policy=Policy())) + names = [t for t in state.tokens + if t.role in _NAME_ROLES] + # Leading = no name part precedes it. With a given + # name in front ("Dr. Ann van Johnson") the particle + # sits mid-name, where nothing has to choose -- the + # decision-not-a-word rule, so no report. And a lone + # name part is the whole name, not a coin flip. + leads = bool(names) and "vocab:particle-ambiguous" \ + in names[0].tags + want = 1 if leads and len(names) >= 2 else 0 + got = _fork_count(state) + if got != want: + failures.append( + f"{text!r}: {got} report(s), expected {want} " + f"({len(names)} name tokens, " + f"leading={leads})") + assert not failures, ( + f"{len(failures)} shape(s) disagree:\n" + "\n".join(failures[:15])) + + +@pytest.mark.parametrize("text", _FORK_CORPUS) +def test_a_fork_is_never_reported_twice_on_a_real_name(text: str) -> None: + state = run(ParseState(original=text, lexicon=Lexicon.default(), + policy=Policy())) + assert _fork_count(state) <= 1 + + +@given(st.text(alphabet=_ALPHABET, max_size=120)) +@settings(max_examples=400, deadline=None, derandomize=True) +def test_particle_fork_is_never_double_reported(text: str) -> None: + # The half of the invariant that needs no reconstruction: whatever + # the two emitters decide, they must not both fire on one parse. + state = run(ParseState(original=text, lexicon=Lexicon.default(), + policy=Policy())) + assert _fork_count(state) <= 1, ( + f"{text!r} reported the same fork more than once") From 6e53b56293ed819283fcfa5139fd8b4af1d99b53 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 20 Jul 2026 20:18:54 -0700 Subject: [PATCH 271/275] Parse content-free input to an empty name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit parse(".,") returned given='.' and bool() True -- faithful v1 parity (1.4 gives first='.' too), but it defeats the bool(parse(x)) idiom 2.0 documents as "did I get a name?". A caller filtering junk with `if parse(x):` was fooled by all-punctuation input. assemble now empties a parse whose surviving tokens carry no alphanumeric character anywhere. isalnum() is Unicode-aware, so every real name in any script has content -- the guard fires only on pure punctuation/symbols ('.', '- -', the stray '∫≜⩕'). Junk embedded in a name with real content ('John . Smith' keeps its dot) is left alone, since that parse is already truthy and bool() is not misled. Deliberate v1 divergence, both APIs (they share the core). Zero differential impact: the corpus's three content-free names ('', '()', ',') already parsed empty for other reasons. Release-log note added. Co-Authored-By: Claude Opus 4.8 --- docs/release_log.rst | 1 + nameparser/_pipeline/_assemble.py | 10 ++++++++++ tests/v2/pipeline/test_assemble.py | 18 ++++++++++++++++++ tests/v2/test_parser.py | 4 ++++ 4 files changed, 33 insertions(+) diff --git a/docs/release_log.rst b/docs/release_log.rst index 724db8d7..47e66a1b 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -53,6 +53,7 @@ Release Log - Fix the pre-comma piece being routed to ``first`` when everything after the comma is a suffix or title: ``"Andrews, M.D."`` now reads family ``Andrews`` / suffix ``M.D.`` where 1.x read given ``M.D.`` / family ``Andrews``, and ``"Smith, Dr."`` moves ``Smith`` from ``first`` to ``family`` (the title was already correct in 1.x). The piece before a comma is definitionally the family name - Fix a lone recognized trailing suffix with no comma being routed to ``first``/``last``: ``"Johnson PhD"`` and ``"Mr. Johnson PhD"`` now keep the suffix in ``suffix`` - Fix a split ``"Ph. D."`` credential being read as two tokens; it now classifies as one suffix, replacing v1's ``fix_phd`` hook + - Parse an input with no alphanumeric character to an empty name in both APIs. 1.x kept pure punctuation as a name part, so ``"."`` gave ``first="."`` and ``bool()`` was ``True``; 2.0 empties it, keeping ``bool(parse(x))`` an honest "did I get a name?" test. The check is Unicode-aware, so names in any script are unaffected; only inputs that are entirely punctuation or symbols (``"."``, ``"- -"``) change. Junk embedded in a name with real content -- the stray dot in ``"John . Smith"`` -- is still kept, since that parse is already truthy - Fold a leading never-given particle into the family name. Note that ``Lexicon.particles_ambiguous`` is the **complement** of v1's ``non_first_name_prefixes``, not a rename -- it lists the particles that *may* double as a given name, where v1 listed the ones that may not. Copying a v1 customization across without inverting it silently reverses the behavior; see :doc:`migrate` - Add ``ma`` and ``do`` to ``suffix_acronyms_ambiguous``, the set of post-nominals that are also ordinary surnames. An entry there is read as a credential when the name can spare it — written with periods (``"John Smith M.A."``), or when removing it still leaves a given *and* a family name (``"John Smith MA"`` → suffix ``MA``). With only two pieces to go around, the surname reading wins instead: ``"Jack Ma"`` and ``"Anh Do"`` keep their family names. As a side effect, a parenthesized or quoted ``"(MA)"``/``"(DO)"`` now falls through to nickname parsing rather than escaping to ``suffix``, since inside delimiters the nickname reading is the plausible one - Change ``comparison_key()`` and ``matches()`` in both APIs to fold with ``str.casefold()`` where 1.4 used ``str.lower()``, so Unicode case pairs compare equal -- ``"STRASSE"`` matches ``"Straße"``, and a Greek final sigma matches its regular form. This is strictly more permissive: anything 1.4 matched still matches. Vocabulary normalization deliberately still uses ``lower()``, for v1 parity diff --git a/nameparser/_pipeline/_assemble.py b/nameparser/_pipeline/_assemble.py index 42fcef7b..ecd38b84 100644 --- a/nameparser/_pipeline/_assemble.py +++ b/nameparser/_pipeline/_assemble.py @@ -30,6 +30,16 @@ def assemble(state: ParseState) -> ParsedName: continue role = t.role if t.role is not None else Role.GIVEN final[i] = Token(t.text, t.span, role, t.tags) + # No alphanumeric character anywhere means no name: a bare '.' or + # '- -' is not a person. v1 kept such input (parse('.') -> first + # '.'); 2.0 empties it so bool() stays an honest "did I get a name?" + # check. isalnum() is Unicode-aware, so every real name in any + # script has content and only pure punctuation/symbols empty out. + # (Embedded junk in a name with content -- 'John . Smith' -- is left + # alone: that parse is truthy, so bool() is not misled.) + if not any(c.isalnum() for t in final.values() for c in t.text): + return ParsedName(original=state.original, tokens=(), + ambiguities=()) ambiguities = [] for pending in state.ambiguities: materialized = tuple(final[i] for i in pending.indices diff --git a/tests/v2/pipeline/test_assemble.py b/tests/v2/pipeline/test_assemble.py index 793d5db0..8f9289f7 100644 --- a/tests/v2/pipeline/test_assemble.py +++ b/tests/v2/pipeline/test_assemble.py @@ -52,6 +52,24 @@ def test_empty_parse_is_falsy() -> None: assert not _parse(" ") +def test_content_free_input_parses_to_empty() -> None: + # An input with no alphanumeric character anywhere is not a name. + # v1 kept it (parse('.') -> first='.'); 2.0 empties it so bool() + # stays an honest "did I get a name?" check. isalnum is + # Unicode-aware, so this only fires on pure punctuation/symbols. + for junk in [".", ".,", "- -", ". .", "'", "∫≜⩕", "()", "-"]: + pn = _parse(junk) + assert not pn, f"{junk!r} should be falsy" + assert pn.tokens == (), f"{junk!r} should have no tokens" + assert pn.original == junk # the raw input is still remembered + + +def test_a_single_letter_of_content_survives() -> None: + # the guard keys on content, not length: one alnum char is a name + for real in ["a.", ".a", "a", "李", "О"]: + assert _parse(real), f"{real!r} should be truthy" + + def test_assemble_falls_back_to_given_for_unassigned_role() -> None: # This should never happen through the real pipeline -- assign/group # always set a role on every main-stream token before assemble runs. diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 3f3ef1a9..da363ced 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -43,6 +43,10 @@ def test_degenerate_inputs_are_total() -> None: assert not parse("") assert not parse(" ") assert parse("").original == "" + # an input with no alphanumeric character is not a name (v1 kept it) + assert not parse(".,") + assert not parse("- -") + assert parse(".,").original == ".," # but the raw input is kept single = parse("John") assert single.given == "John" family_first = Parser(policy=Policy(name_order=FAMILY_FIRST)) From 7747cef529fc4fe0419f513cb01d556c5c9e26e2 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 20 Jul 2026 20:35:17 -0700 Subject: [PATCH 272/275] docs: reorganize the attaching-words table by word type The table was keyed by destination field, which forced conjunctions into the `family` row as "conjunctions inside a surname" -- misleading, since the very next doctest shows `and` joining two GIVEN names. Keying by word type gives conjunctions one honest row ("both neighbors, whichever field they join") and lets the field column read as a map: the five field-specific attachers cover title/suffix/given/family/ maiden, and the two fields you can't get by attachment (middle, nickname) are exactly the ones absent. Example words for titles and suffixes are drawn from the shipped constants (dr/sir/prof/capt, phd/md/jr/esq). Co-Authored-By: Claude Opus 4.8 --- docs/usage.rst | 58 ++++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index cbf39630..f7f2966d 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -61,35 +61,41 @@ Input shapes tell you where the fields sit. This tells you which words merge into one field instead of standing alone — between them, that is most of what decides a parse. -Most words stand alone. A few pull a neighbor in, and the thing worth -knowing is which field they pull into: +Most words stand alone. A few pull in a neighbor, and each kind pulls +into one particular field — so this table is also a map of which fields +get built by attachment rather than by position. Consecutive titles +chain (``Asst. Vice Chancellor`` is one title), as do consecutive +suffixes: .. list-table:: :header-rows: 1 - :widths: 14 42 44 - - * - Pulls into - - Example - - Words - * - ``family`` - - ``Juan de la Vega`` → ``de la Vega`` - - Particles (``de``, ``van``, ``von``, ``bin``), and conjunctions - inside a surname (``y``) - * - ``given`` - - ``abdul salam ahmed`` → ``abdul salam`` - - Bound given names (``abdul``, ``abu``, ``umm``, ``عبد``) - * - ``title`` - - ``Asst. Vice Chancellor John Smith`` → ``Asst. Vice Chancellor`` - - Consecutive titles chain - * - ``suffix`` - - ``John Smith PhD MD`` → ``PhD, MD`` - - Consecutive suffixes - * - ``maiden`` - - ``Jane Smith née Jones`` → ``Jones`` - - Maiden markers (``née``, ``geb.``) - -A conjunction joins whatever sits on both sides of it, so it can pull -two given names together as easily as two surnames: + :widths: 40 34 26 + + * - Words + - Attach to + - Field + * - Titles — ``dr``, ``sir``, ``prof``, ``capt`` + - adjacent titles + - ``title`` + * - Suffixes — ``phd``, ``md``, ``jr``, ``esq`` + - adjacent suffixes + - ``suffix`` + * - Bound given names — ``abdul``, ``abu``, ``umm``, ``عبد`` + - the following word + - ``given`` + * - Particles — ``de``, ``van``, ``von``, ``bin`` + - the following surname + - ``family`` + * - Maiden markers — ``née``, ``geb.`` + - the following name + - ``maiden`` + * - Conjunctions — ``and``, ``&``, ``y``, ``и`` + - both neighbors, whatever they are + - whichever field they join + +Conjunctions are the exception the last row names: they take the field +of whatever sits on both sides, so the same ``and`` pulls two given +names together as easily as two surnames: .. doctest:: From 8e77693f2119f4ac702f43029e63dc79a3e4f62e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 20 Jul 2026 20:51:27 -0700 Subject: [PATCH 273/275] docs: link the attaching-words rows to their vocabulary sets Each word type now links to its full set on the API reference (the config-module sections render the whole {...}), so the few example words are no longer needed inline -- they move to a restored Example column showing a parse. Drops the Words/Field repetition the inline lists caused. Also tighten the conjunction row (Field "any"; "the words on both sides") and spell out in the lead-in why titles/suffixes say "adjacent" while the others say "the following word": the first attach only to same-kind neighbors, the rest reach forward to the next word of any kind. Co-Authored-By: Claude Opus 4.8 --- docs/usage.rst | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index f7f2966d..4faca1c8 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -63,35 +63,44 @@ most of what decides a parse. Most words stand alone. A few pull in a neighbor, and each kind pulls into one particular field — so this table is also a map of which fields -get built by attachment rather than by position. Consecutive titles -chain (``Asst. Vice Chancellor`` is one title), as do consecutive -suffixes: +get built by attachment rather than by position. Titles and suffixes +attach only to *adjacent words of their own kind* (a run of titles +chains into one, but a title never swallows a plain name); the rest +reach forward to pull in the next word, whatever it is. Follow a row's +name to its full vocabulary set: .. list-table:: :header-rows: 1 - :widths: 40 34 26 + :widths: 22 26 12 40 * - Words - Attach to - Field - * - Titles — ``dr``, ``sir``, ``prof``, ``capt`` + - Example + * - :mod:`Titles ` - adjacent titles - ``title`` - * - Suffixes — ``phd``, ``md``, ``jr``, ``esq`` + - ``Asst. Vice Chancellor John`` → ``Asst. Vice Chancellor`` + * - :mod:`Suffixes ` - adjacent suffixes - ``suffix`` - * - Bound given names — ``abdul``, ``abu``, ``umm``, ``عبد`` + - ``John Smith PhD MD`` → ``PhD, MD`` + * - :mod:`Bound given names ` - the following word - ``given`` - * - Particles — ``de``, ``van``, ``von``, ``bin`` + - ``abdul salam ahmed`` → ``abdul salam`` + * - :mod:`Particles ` - the following surname - ``family`` - * - Maiden markers — ``née``, ``geb.`` + - ``Juan de la Vega`` → ``de la Vega`` + * - :mod:`Maiden markers ` - the following name - ``maiden`` - * - Conjunctions — ``and``, ``&``, ``y``, ``и`` - - both neighbors, whatever they are - - whichever field they join + - ``Jane Smith née Jones`` → ``Jones`` + * - :mod:`Conjunctions ` + - the words on both sides + - any + - ``John and Jane Smith`` → ``John and Jane`` Conjunctions are the exception the last row names: they take the field of whatever sits on both sides, so the same ``and`` pulls two given From 17e101b39bd61e97726209e3c4af663fcb3d3cd0 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 20 Jul 2026 23:34:45 -0700 Subject: [PATCH 274/275] Guard against complexity regressions, not just slow parses Two quadratics shipped this cycle (_extract's mask-overlap scan, ParsedName's token-subset check) and both were caught in review rather than by a test. The existing perf tests cannot catch that class: they bound absolute cost on constant-size input, and their workload is a short name with no delimiters, so a stage quadratic in the number of delimiter pairs stays far inside the one-second bound. Measured: that workload runs in 0.067s against a 1.0s limit. Add a growth test instead -- time a repeated pathological unit at n and 4n, assert the ratio stays near the linear 4. Seven units, one per stage inner loop (matched pairs, the open==close path, both unmatched sweeps, long token streams, comma segments, title chains). Calibrated against the real quadratic rather than guessed, which mattered: the growth is mixed, so the textbook 16x never appears at a testable size. At n=200 the quadratic measures 7.7x and an 8.0 bound passes it -- the first version of this test was vacuous. At n=400 the populations separate (clean 4.1-4.3, quadratic 9.2), so the bound sits at 6.5. Verified both ways: passes clean, and reverting the _extract fix fails exactly the two shapes that build masked spans. Co-Authored-By: Claude Opus 4.8 --- tests/v2/test_benchmark.py | 70 +++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/tests/v2/test_benchmark.py b/tests/v2/test_benchmark.py index c3f77509..314d7af1 100644 --- a/tests/v2/test_benchmark.py +++ b/tests/v2/test_benchmark.py @@ -1,8 +1,21 @@ """Perf smoke (core spec §7 tail): parse cost stays v1-comparable (microseconds per name). Deliberately generous bound -- guards against -order-of-magnitude regressions, does not gate normal variance.""" +order-of-magnitude regressions, does not gate normal variance. + +Two different things are measured here, and the distinction matters. +The thousand-names tests below bound ABSOLUTE cost on constant-size +input; the scaling test bounds GROWTH in input length. Neither +subsumes the other, and the absolute tests are structurally blind to a +complexity regression: their workload is a short name with no +delimiters, so a stage that goes quadratic in the number of delimiter +pairs stays far inside the one-second bound. Two such quadratics +(_extract's mask-overlap scan, ParsedName's token-subset check) shipped +and were caught in review rather than here -- hence the scaling test. +""" import time +import pytest + from nameparser import parse @@ -26,3 +39,58 @@ def test_facade_thousand_names_under_a_second() -> None: HumanName(f"Dr. Juan{i} de la Vega III") elapsed = time.perf_counter() - start assert elapsed < 1.0, f"1000 facade parses took {elapsed:.2f}s" + + +# Pathological shapes: each repeats a unit that drives one stage's inner +# loop. Cheap per unit, so a superlinear stage shows up as growth rather +# than as a slow parse. +_SHAPES = { + "delimiter_pairs": "(a) ", # extract: matched pairs -> masked spans + "quote_pairs": '"a" ', # extract: the open==close path + "unmatched_opens": "( ", # extract: the bulk unmatched-open sweep + "unmatched_closes": ") ", # extract: the unmatched-close sweep + "plain_tokens": "a ", # tokenize/assign: long token stream + "commas": "a, ", # segment: many comma segments + "titles": "Dr. ", # group: one long title chain +} + +_BASE = 400 +_FACTOR = 4 +# Calibrated, not guessed. A quadratic here is MIXED -- linear work +# dominates at small sizes and the quadratic term takes over slowly -- +# so the textbook 16x never appears at a testable size, and picking the +# operating point matters more than the bound. Measured against the +# real _extract mask-overlap quadratic this test exists to catch: +# +# base clean ratio quadratic ratio +# 200 3.9 - 4.3 7.7 <- under an 8.0 bound: useless +# 400 4.1 - 4.3 9.2 +# +# At 400 the two populations separate, so the bound sits between them: +# ~1.5x headroom over the worst clean run (noise on a shared runner) +# and ~1.4x under the quadratic. Lower this only with fresh numbers. +_MAX_RATIO = 6.5 + + +def _best(text: str, repeats: int = 7) -> float: + """Minimum of several runs: on a shared runner the mean carries the + noise of whatever else is running, while the minimum approaches the + true cost.""" + parse(text) # warm the default parser cache + best = float("inf") + for _ in range(repeats): + start = time.perf_counter() + parse(text) + best = min(best, time.perf_counter() - start) + return best + + +@pytest.mark.parametrize("unit", _SHAPES.values(), ids=list(_SHAPES)) +def test_parse_cost_grows_no_worse_than_linearly(unit: str) -> None: + small = _best(unit * _BASE) + large = _best(unit * (_BASE * _FACTOR)) + ratio = large / small + assert ratio < _MAX_RATIO, ( + f"{unit!r} x{_BASE} took {small * 1e3:.2f}ms, " + f"x{_BASE * _FACTOR} took {large * 1e3:.2f}ms -- {ratio:.1f}x for " + f"{_FACTOR}x the input, which is superlinear (linear is ~{_FACTOR})") From 7db3635bb4e9e5eedee780a17ffe65b9a7b62c3d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 20 Jul 2026 23:42:10 -0700 Subject: [PATCH 275/275] Add a second differential corpus harvested from the issue tracker The existing corpus comes from v1's own test suite, which makes it structurally blind to everything 2.0 added: v1's authors had no reason to write a test for a typographic nickname delimiter or a Cyrillic title, so no rule was ever needed to classify those behaviors. Harvest the adversarial half instead -- names users REPORTED as broken, taken from HumanName("...") calls and quoted phrases in issue bodies. 166 of the 198 names were absent from the existing corpus, and the first run paid for the exercise: five intended-but-unclassified 2.0 behaviors (#273 typographic delimiters, #269 non-Latin vocabulary) and one shape no test had considered. That shape is a LEADING "Ph. D.". v1 healed the split 'Ph.'/'D.' pair only when it trailed; leading, it split them into title 'Ph.' and given 'D.' and pushed the real given name to middle. 2.0 keeps the credential whole. Classified as a fix, pinned in cases.py, noted in the release log. The toml's deliberate "leave trailing Ph.D. unclassified so a regression fails" strategy is what surfaced it, and the new rule is anchored to ^ so trailing stays unguarded. Kept as a separate corpus with its own builder: corpus.jsonl is reproducible forever from an immutable git ref, while an issue tracker is mutable, so regenerating this one is an explicit, reviewable act. compare.py now reads every corpus*.jsonl by default -- a corpus you have to ask for by name is one that stops being run -- taking the harness from 486 names to 652, 18 intentional diffs, 0 unexplained. Co-Authored-By: Claude Opus 4.8 --- docs/release_log.rst | 2 +- tests/v2/cases.py | 7 + tools/differential/README.md | 37 ++++ tools/differential/build_issues_corpus.py | 73 ++++++++ tools/differential/compare.py | 22 ++- tools/differential/corpus_issues.jsonl | 198 ++++++++++++++++++++++ tools/differential/expected_changes.toml | 57 ++++++- 7 files changed, 383 insertions(+), 13 deletions(-) create mode 100644 tools/differential/build_issues_corpus.py create mode 100644 tools/differential/corpus_issues.jsonl diff --git a/docs/release_log.rst b/docs/release_log.rst index 47e66a1b..ef2c40da 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -52,7 +52,7 @@ Release Log - Recognize typographic nickname delimiters by default in both APIs (closes #273): smart quotes (``“Jack”``), German and Polish low-high quotes (``„Hansi“``), Swedish right-right quotes (``”Ann”``), guillemets in either direction (``«Petit»``, ``»Hansi«``), CJK corner brackets (``「タロ」``, ``『ハナ』``) and fullwidth parentheses. In 1.x these leaked into ``middle`` as literal text. Curly *single* quotes stay excluded, because U+2019 is the apostrophe in "O’Connor" - Fix the pre-comma piece being routed to ``first`` when everything after the comma is a suffix or title: ``"Andrews, M.D."`` now reads family ``Andrews`` / suffix ``M.D.`` where 1.x read given ``M.D.`` / family ``Andrews``, and ``"Smith, Dr."`` moves ``Smith`` from ``first`` to ``family`` (the title was already correct in 1.x). The piece before a comma is definitionally the family name - Fix a lone recognized trailing suffix with no comma being routed to ``first``/``last``: ``"Johnson PhD"`` and ``"Mr. Johnson PhD"`` now keep the suffix in ``suffix`` - - Fix a split ``"Ph. D."`` credential being read as two tokens; it now classifies as one suffix, replacing v1's ``fix_phd`` hook + - Fix a split ``"Ph. D."`` credential being read as two tokens; it now classifies as one suffix, replacing v1's ``fix_phd`` hook. This now holds wherever the credential sits: 1.x healed the pair only when it trailed, so ``"Ph. D. John Smith"`` parsed as title ``Ph.`` / given ``D.`` with the real given name pushed to ``middle``; it now reads given ``John``, family ``Smith``, suffix ``Ph. D.`` - Parse an input with no alphanumeric character to an empty name in both APIs. 1.x kept pure punctuation as a name part, so ``"."`` gave ``first="."`` and ``bool()`` was ``True``; 2.0 empties it, keeping ``bool(parse(x))`` an honest "did I get a name?" test. The check is Unicode-aware, so names in any script are unaffected; only inputs that are entirely punctuation or symbols (``"."``, ``"- -"``) change. Junk embedded in a name with real content -- the stray dot in ``"John . Smith"`` -- is still kept, since that parse is already truthy - Fold a leading never-given particle into the family name. Note that ``Lexicon.particles_ambiguous`` is the **complement** of v1's ``non_first_name_prefixes``, not a rename -- it lists the particles that *may* double as a given name, where v1 listed the ones that may not. Copying a v1 customization across without inverting it silently reverses the behavior; see :doc:`migrate` - Add ``ma`` and ``do`` to ``suffix_acronyms_ambiguous``, the set of post-nominals that are also ordinary surnames. An entry there is read as a credential when the name can spare it — written with periods (``"John Smith M.A."``), or when removing it still leaves a given *and* a family name (``"John Smith MA"`` → suffix ``MA``). With only two pieces to go around, the surname reading wins instead: ``"Jack Ma"`` and ``"Anh Do"`` keep their family names. As a side effect, a parenthesized or quoted ``"(MA)"``/``"(DO)"`` now falls through to nickname parsing rather than escaping to ``suffix``, since inside delimiters the nickname reading is the plausible one diff --git a/tests/v2/cases.py b/tests/v2/cases.py index ebc43830..cc2d53f6 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -435,6 +435,13 @@ def __post_init__(self) -> None: Case("phd_split_mid_name", "Dr. John Ph. D. Smith", {"title": "Dr.", "given": "John", "family": "Smith", "suffix": "Ph. D."}), + Case("phd_split_leading", "Ph. D. John Smith", + {"given": "John", "family": "Smith", "suffix": "Ph. D."}, + classification="fix", + notes="v1 healed 'Ph.'+'D.' only when trailing; leading it " + "split them (title 'Ph.', given 'D.', real given name " + "pushed to middle). Surfaced by the issue-tracker " + "corpus, which is where this shape existed at all."), Case("leading_never_given_particle", "de la Vega", {"family": "de la Vega"}, notes="v1 handle_non_first_name_prefix: never-given leading " diff --git a/tools/differential/README.md b/tools/differential/README.md index 783f9c58..8f4e0e1d 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -38,6 +38,28 @@ needs widening. The run must exit 0 before a 2.0 release; the classified summary it prints is the source for the "Behavior Changes" section of `docs/release_log.rst`. +## The two corpora + +`compare.py` reads **every** `corpus*.jsonl` beside it by default +(deduped), because a corpus you have to ask for by name is a corpus +that stops being run. Pass `--corpus PATH` (repeatable) to narrow it. + +| File | Source | Blind to | +|---|---|---| +| `corpus.jsonl` | v1's own test suite at a pinned ref | anything 2.0 added — v1's authors had no reason to test a typographic nickname delimiter or a Cyrillic title | +| `corpus_issues.jsonl` | name-like strings harvested from the GitHub issue tracker | anything nobody ever reported | + +They are deliberately separate rather than merged: `corpus.jsonl` is +reproducible forever from an immutable git ref, while the issue +tracker is mutable, so regenerating the second one is an explicit, +reviewable act that can only add names. + +The issue corpus earned its place on the first run — 166 of its 198 +names were not in `corpus.jsonl`, and it immediately surfaced five +intended-but-unclassified 2.0 behaviors (#273 typographic delimiters, +#269 non-Latin vocabulary) plus one shape no test had considered: a +**leading** `"Ph. D."`, which v1 split into title `Ph.` + given `D.`. + ## Corpus provenance `corpus.jsonl` is checked in as a test fixture. It was built by @@ -69,6 +91,21 @@ Regenerate the corpus only if the v1 test banks are revisited again at a still-earlier point in history; otherwise leave the checked-in file alone so the harness stays comparable run to run. +`corpus_issues.jsonl` is built by `build_issues_corpus.py` from the +issue tracker (`gh issue list --state all`), taking `HumanName("...")` +calls and quoted capitalized phrases -- the two ways a reporter writes +the input that broke on them. Regenerate with: + +``` +uv run python tools/differential/build_issues_corpus.py > tools/differential/corpus_issues.jsonl +``` + +Unlike the ref-pinned corpus this is a mutable source, so the +checked-in file is the snapshot under test; re-running only adds names +as new issues arrive. Over-collection is fine in both builders: the +comparator just parses more names, and junk like `Bridge (1.4)` costs +one parse and produces no diff. + ## `expected_changes.toml` Each `[[change]]` entry needs `issue` (a short label, ideally an diff --git a/tools/differential/build_issues_corpus.py b/tools/differential/build_issues_corpus.py new file mode 100644 index 00000000..dceeb40b --- /dev/null +++ b/tools/differential/build_issues_corpus.py @@ -0,0 +1,73 @@ +"""Harvest name-like strings from this project's GitHub issue tracker. + +Companion to build_corpus.py, and deliberately a SEPARATE corpus with a +separate builder. The two have different provenance and different +blind spots: + +- corpus.jsonl comes from v1's own test suite at a pinned git ref. It + is reproducible forever, and it is structurally blind to anything + 2.0 added -- v1's authors had no reason to write a test for a + typographic nickname delimiter or a Cyrillic title. +- corpus_issues.jsonl comes from what USERS REPORTED, which is the + adversarial half: names that broke the parser in the field, in the + reporter's own words. On its first run it surfaced five intended- + but-unclassified 2.0 behaviors and one shape (a LEADING "Ph. D.") + that no test had considered. + +Reproducibility differs, and that is why these are not merged. A git +ref is immutable; an issue tracker is not. Re-running this later can +only ADD names as new issues arrive, so the checked-in file is the +snapshot under test and regeneration is an explicit, reviewable act. + +Over-collection is fine, same as build_corpus.py: the comparator just +parses more names. Junk like 'Bridge (1.4)' costs one parse and +produces no diff. + +Regenerate with (requires `gh` authenticated to the repo): + uv run python tools/differential/build_issues_corpus.py \\ + > tools/differential/corpus_issues.jsonl +""" +import json +import re +import subprocess +import sys + +# Either an explicit HumanName("...") call -- the reporter showing the +# failing input -- or a quoted, capitalized phrase, which is how names +# appear in prose ("parsing 'Hans "Hansi" Müller' gives ..."). +_CANDIDATE = re.compile( + r"""HumanName\(\s*["']([^"']{3,60})["']""" + r"""|["']([A-Z][^"'\n]{4,60})["']""" +) +# Structural characters that mean we caught code or markup, not a name. +_NOT_A_NAME = set('{}<>=/\\|') + + +def _harvest(text: str) -> set[str]: + found = set() + for match in _CANDIDATE.finditer(text): + value = (match.group(1) or match.group(2) or "").strip() + # Require an internal space: a single word is a surname at best + # and carries no structure to disagree about. + if value and " " in value and not (_NOT_A_NAME & set(value)): + found.add(value) + return found + + +def main() -> None: + raw = subprocess.run( + ["gh", "issue", "list", "--state", "all", "--limit", "1000", + "--json", "number,title,body"], + capture_output=True, text=True, check=True).stdout + issues = json.loads(raw) + names: set[str] = set() + for issue in issues: + names |= _harvest( + f"{issue.get('title') or ''}\n{issue.get('body') or ''}") + for name in sorted(names): + print(json.dumps(name, ensure_ascii=False)) + print(f"{len(names)} names from {len(issues)} issues", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/tools/differential/compare.py b/tools/differential/compare.py index f6edef8f..00a89abf 100644 --- a/tools/differential/compare.py +++ b/tools/differential/compare.py @@ -50,13 +50,29 @@ def classify(name: str, diff_fields: set[str], def main() -> int: ap = argparse.ArgumentParser() - ap.add_argument("--corpus", default=str(HERE / "corpus.jsonl")) + # Both corpora by default: they have different blind spots (see + # build_issues_corpus.py), and one that has to be asked for by name + # is one that stops being run. + ap.add_argument("--corpus", action="append", metavar="PATH", + help="corpus file; repeatable. Defaults to every " + "corpus*.jsonl beside this script.") args = ap.parse_args() + paths = ([Path(p) for p in args.corpus] if args.corpus + else sorted(HERE.glob("corpus*.jsonl"))) rules = tomllib.loads( (HERE / "expected_changes.toml").read_text()).get("change", []) validate_rules(rules) - corpus = [json.loads(line) for line in - Path(args.corpus).read_text().splitlines() if line.strip()] + # dedupe across files, keeping first-seen order stable for output + corpus, seen = [], set() + for path in paths: + for line in path.read_text().splitlines(): + if not line.strip(): + continue + name = json.loads(line) + if name not in seen: + seen.add(name) + corpus.append(name) + print(f"corpora: {', '.join(p.name for p in paths)}") proc = subprocess.Popen( ["uv", "run", "--no-project", str(HERE / "worker_v1.py")], diff --git a/tools/differential/corpus_issues.jsonl b/tools/differential/corpus_issues.jsonl new file mode 100644 index 00000000..65884271 --- /dev/null +++ b/tools/differential/corpus_issues.jsonl @@ -0,0 +1,198 @@ +"1 & 2, 3 4 5, Mr." +"Ahmad Jayadi, CHA" +"Ahmad ben Husain" +"Ajay Reddy B" +"Alex Ben Johnson" +"Andrew Ma" +"Ann H" +"Anna Müller geb. Schmidt" +"Attorney General of Minnesota" +"Baker (Johnson), Jenny" +"Barbara Scharf" +"Barbara Scharf, CPPM" +"Beth Edmunds" +"Beyoncé Knowles" +"Bob Jones, author" +"Bob Jones, compositeur" +"Bridge (1.4)" +"Brundridge, Contessa A" +"Charles van der van der Berg" +"Cherice J. (Johnson) Williams" +"Claus SchroederVP of" +"Co-Chairs Miller" +"Co-Chairs Muffy Miller" +"DELA CRUZ" +"David Ben Gurion" +"De Groot" +"Doe, John A." +"Doe, John A. Kenneth" +"Dr John Smith" +"Dr. Jason" +"Dr. Jason Ross Jay" +"Dr. Juan Q. Xavier de la Vega III" +"Dr. King" +"Dr. Mazen Elrouqi" +"Dr. med. univ." +"Dr. med. univ. Margit Popp" +"Dr. med. univ. Margit Popp, MSc" +"Duke Ellington" +"Duke of Edinburgh" +"E Anne D" +"E Anne D,Leonardo" +"E Jones" +"E Maria" +"Ed, Royce" +"Eddie Chang Seng Dee Adecco" +"El Dine" +"Elder Soares" +"Eunsoo Johanna Jeong" +"Flávio Juvenal da Silva Jr" +"Flávio Juvenal da Silva Junior" +"GEORGE SHANAHAN IV" +"GREGORY HOUSE M.D." +"Garcia de Jalon" +"Gary Strawsburg, PMP, CSCP, CLSSBB" +"Gebhart GF" +"General of Minnesota" +"Give me a private config" +"Grand Rounds" +"Hans „Hansi“ Müller" +"Hardie Leannon DVM" +"Hardie Leannon V" +"His Holiness" +"His Holiness the Dalai Lama" +"Hisham Ibrahim, CPA, CFA" +"Howard G. Buffett" +"I don" +"Ivan Ivanov" +"Ivan Tham Jun Hoe" +"Ivanov Ivan" +"Ivanov Ivan Ivanovich" +"J A D" +"J. A. D." +"J. None D." +"J. Smith" +"J. Smith Jr." +"J. Smith Sr." +"J. Smith, MD" +"J. Smith, PhD" +"J.A. K.D." +"JA KD" +"JUAN DELA CRUZ" +"Jack MA" +"Jack Ma" +"Jan Kees de Jager" +"Jane Smith (Jones)" +"Jane Smith née Jones" +"Jean «Petit» Dupont" +"Jill St. John" +"Jo Ann" +"Joao da Silva Do Amaral de Souza" +"Joao da Silva do Amaral de Souza" +"Joe E. Smith" +"Joe Smith" +"John Doe ☁️" +"John Q Doe" +"John An Smith" +"John Doe" +"John Doe, Phd, Jr" +"John F Kennedy" +"John Smith" +"John Smith Jr." +"John Smith MA" +"John Smith Ma" +"John Smith, Dr." +"John Smith, Ph. D." +"John Smith, Ph.D." +"John V" +"John W. Ingram" +"John W. Ingram V" +"John W. Ingram, V" +"John of the Doe" +"John “Jack” Kennedy" +"John, Smith, Dr." +"Jose E Maria Santos" +"Juan de la de la Vega" +"Jun Hoe" +"Kajal Patel Hiring Globally" +"King John V." +"La Shawn K. Ford" +"Lamha Yahya Al Mawali" +"Lastname, Firstname" +"Lisa Schmidt, MBA" +"MD - DO - DDS" +"MD, DO, DDS" +"MD, PHD" +"MSc Dr. med. univ." +"Ma, Andrew" +"Maheen Farooqi, ACA" +"Mary Ann H" +"Mary Ann H Luther" +"Mary Ann H,Luther" +"Matthew Matthew" +"Maura Cellario-Pingor, PHR" +"Md,, Phd" +"Mohamad Ali" +"Mohamad X" +"Mohamad X Surname" +"Mohamad Zein El Dine" +"Mohamad Zeineddine" +"Mr XXX" +"Ms A.J. Smith" +"Mx A.J. Smith" +"Müller geb." +"Nguyễn Thị Minh Khai" +"None John None Doe None (None)" +"Nonez Smith" +"Oren ben Horin" +"Ph. D. John Smith" +"Potential Gotcha" +"QC MP" +"Rafael Sousa dos Anjos" +"Reem Al-Bahrani, CPA" +"Rob Edmunds" +"Royce, Ed" +"Said oglu" +"San San Chiou" +"Sander van" +"Sejal Chaturvedi, CSM" +"Sister Souljah" +"Smith Jr." +"Smith Jr., John" +"Smith née" +"Smith, A.B." +"Smith, J. R." +"Smith, J.R." +"Smith, John" +"Smith, John ," +"Smith, John E, III, Jr" +"St. ___" +"Steven Hardman" +"Steven Hardman, MD - DO - DDS" +"Steven Hardman, MD, DO, DDS" +"Steven Hardman, RN - CRNA" +"Syukri @ Shu Qi Mohd Nor" +"Tham Jun" +"Tham Jun Hoe" +"The Rt Hon" +"The Rt Hon John Jones" +"The Rt Hon Kenneth Clarke QC MP, HMG" +"UNILEVER LTD" +"Unilever Ltd" +"Van Maximilian Carlson" +"Van Nguyen" +"Virginia G. Essandoh, J.D." +"WOOT ALFRED DOUGHERTY III" +"Will De Groot" +"Zein El Dine" +"dr Vincent van Gogh dr" +"e and e" +"joao da silva do amaral de souza" +"scott e. werner" +"scott l. werner" +"van ma van" +"Хосе И Мария Сантос" +"г-н Иван Петров" +"سلمان, محمد" +"سلمان، محمد" +"محمد بن سلمان" diff --git a/tools/differential/expected_changes.toml b/tools/differential/expected_changes.toml index 6feddfe0..83a022c4 100644 --- a/tools/differential/expected_changes.toml +++ b/tools/differential/expected_changes.toml @@ -61,12 +61,51 @@ issue = "feat(#269) Arabic بن prefix chains onto family (non-Latin new-recogni name_regex = "\\bبن\\b" fields = ["middle", "last"] -# Deliberately NOT a [[change]] rule: 'Ph. D.' split-token healing -# ('John Ph. D.', 'John Smith, Ph. D.') is PARITY, not a 2.0 behavior -# change -- v1's fix_phd healing of the adjacent 'Ph.'/'D.' pair into -# one suffix unit is replicated exactly by 2.0's 'joined' vocab tag -# (tests/v2/cases.py classification="parity" on both 'phd_split' and -# 'suffix_comma_split_phd'; verified empirically, zero diff against the -# 1.4 worker). Adding a suppression rule for it would risk masking a -# real regression in this exact shape, so it is intentionally left -# unclassified: if it ever starts diffing, the harness must fail. +[[change]] +issue = "feat(#273) typographic nickname delimiters recognized by default" +# 'John “Jack” Kennedy', 'Hans „Hansi“ Müller', 'Jean «Petit» Dupont': +# v1 knew only straight quotes and parentheses, so a smart-quoted or +# guillemet-wrapped nickname leaked into `middle` as literal text. +# Matches only when such a delimiter is actually present -- the ASCII +# pairs stay unclassified, since a diff there would be a regression. +name_regex = "[“”„«»「」『』（）]" +fields = ["middle", "nickname"] + +[[change]] +issue = "feat(#269) non-Latin titles/conjunctions/particles recognized" +# 'г-н Иван Петров' (Cyrillic title), 'Хосе И Мария Сантос' (Cyrillic +# conjunction И joining two given names). v1 shipped Latin-only +# vocabulary, so these were plain name tokens and everything shifted +# one position. New recognition on non-Latin input -- the point of +# #269 -- not a Latin-corpus regression. Guarded by requiring a +# non-Latin character in the name. +name_regex = "[\\u0400-\\u04FF\\u0370-\\u03FF\\u0590-\\u05FF\\u0600-\\u06FF]" +fields = ["title", "first", "middle", "last", "suffix"] + +[[change]] +issue = "fix(leading-credential) a split 'Ph. D.' before the name stays one unit" +# 'Ph. D. John Smith'. v1 healed the adjacent 'Ph.'/'D.' pair only when +# it TRAILED; leading, it split them -- title 'Ph.', given 'D.', and +# the real given name pushed to middle. 2.0 keeps the credential whole +# and routes it to `suffix`, which is the post-nominal field, so a +# leading credential lands there rather than in `title`. Slightly odd +# positionally, and still strictly better than manufacturing a given +# name out of 'D.'. +# +# Anchored to the START on purpose. Trailing 'Ph. D.' is PARITY and +# stays unclassified below; widening this regex would mask a +# regression in the shape that is the whole reason fix_phd exists. +name_regex = "^Ph\\. ?D\\." +fields = ["title", "first", "middle", "suffix"] + +# Deliberately NOT a [[change]] rule: TRAILING 'Ph. D.' split-token +# healing ('John Ph. D.', 'John Smith, Ph. D.') is PARITY, not a 2.0 +# behavior change -- v1's fix_phd healing of the adjacent 'Ph.'/'D.' +# pair into one suffix unit is replicated exactly by 2.0's 'joined' +# vocab tag (tests/v2/cases.py classification="parity" on both +# 'phd_split' and 'suffix_comma_split_phd'; verified empirically, zero +# diff against the 1.4 worker). Adding a suppression rule for it would +# risk masking a real regression in this exact shape, so it is +# intentionally left unclassified: if it ever starts diffing, the +# harness must fail. (The leading case above is a separate shape, which +# is why it needed its own anchored rule rather than a widened one.)