From eb9c76872f9bf7281856ea30ca3d78c9981f6861 Mon Sep 17 00:00:00 2001 From: Corey Oordt Date: Sun, 16 Aug 2026 06:24:59 -0500 Subject: [PATCH 1/6] Add Context model and build_context() settings resolution seam Adds a five-tier precedence chain (flag > env > .wiki-toolkit.toml > pyproject.toml > default) for docs_dir, repo_root, branch_prefix, batch_byte_cap, and batch_file_cap, with per-field source tracking. Resolution never raises: malformed files or invalid field values fall through to the next tier. resolve_docs_dir() is left in place for existing callers; later tickets swap them over. Part of #154, closes #156. --- tests/test_settings.py | 144 ++++++++++++++++++++++++- wiki_toolkit/settings.py | 220 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 358 insertions(+), 6 deletions(-) diff --git a/tests/test_settings.py b/tests/test_settings.py index 849edaf..25f4af0 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING -from wiki_toolkit.settings import resolve_docs_dir +from wiki_toolkit.settings import build_context, resolve_docs_dir if TYPE_CHECKING: from pathlib import Path @@ -96,3 +96,145 @@ def test_resolve_docs_dir_malformed_toml_falls_through_to_default(tmp_path: Path assert result.docs_dir == tmp_path / "docs" assert result.source == "default" + + +def test_build_context_defaults(tmp_path: Path) -> None: + """With nothing else set, every field resolves to its built-in default.""" + context, sources = build_context(cwd=tmp_path) + + assert context.docs_dir == tmp_path / "docs" + assert context.repo_root == tmp_path + assert context.branch_prefix == "wiki-update/" + assert context.batch_byte_cap == 100_000 + assert context.batch_file_cap == 20 + assert sources == { + "docs_dir": "default", + "repo_root": "default", + "branch_prefix": "default", + "batch_byte_cap": "default", + "batch_file_cap": "default", + } + + +def test_build_context_repo_root_walks_up_to_git(tmp_path: Path) -> None: + """repo_root resolves by walking up from cwd to the nearest `.git`.""" + (tmp_path / ".git").mkdir() + nested = tmp_path / "a" / "b" + nested.mkdir(parents=True) + + context, sources = build_context(cwd=nested) + + assert context.repo_root == tmp_path + assert sources["repo_root"] == "default" + + +def test_build_context_flag_wins(tmp_path: Path, monkeypatch) -> None: + """A CLI flag wins over env, dedicated file, and pyproject.toml.""" + monkeypatch.setenv("WIKI_TOOLKIT_DOCS_DIR", str(tmp_path / "env-docs")) + (tmp_path / ".wiki-toolkit.toml").write_text('docs_dir = "dedicated-docs"\n') + + context, sources = build_context(docs_dir_flag=tmp_path / "flag-docs", cwd=tmp_path) + + assert context.docs_dir == tmp_path / "flag-docs" + assert sources["docs_dir"] == "flag" + + +def test_build_context_env_wins_over_dedicated_file(tmp_path: Path, monkeypatch) -> None: + """A `WIKI_TOOLKIT_*` env var wins over the dedicated file and pyproject.toml.""" + (tmp_path / ".wiki-toolkit.toml").write_text('docs_dir = "dedicated-docs"\n') + monkeypatch.setenv("WIKI_TOOLKIT_DOCS_DIR", str(tmp_path / "env-docs")) + + context, sources = build_context(cwd=tmp_path) + + assert context.docs_dir == tmp_path / "env-docs" + assert sources["docs_dir"] == "env" + + +def test_build_context_all_fields_resolvable_via_env(tmp_path: Path, monkeypatch) -> None: + """All five fields are uniformly resolvable via `WIKI_TOOLKIT_*` env vars.""" + monkeypatch.setenv("WIKI_TOOLKIT_DOCS_DIR", str(tmp_path / "env-docs")) + monkeypatch.setenv("WIKI_TOOLKIT_REPO_ROOT", str(tmp_path / "env-repo")) + monkeypatch.setenv("WIKI_TOOLKIT_BRANCH_PREFIX", "env-prefix/") + monkeypatch.setenv("WIKI_TOOLKIT_BATCH_BYTE_CAP", "42") + monkeypatch.setenv("WIKI_TOOLKIT_BATCH_FILE_CAP", "7") + + context, sources = build_context(cwd=tmp_path) + + assert context.docs_dir == tmp_path / "env-docs" + assert context.repo_root == tmp_path / "env-repo" + assert context.branch_prefix == "env-prefix/" + assert context.batch_byte_cap == 42 + assert context.batch_file_cap == 7 + assert all(source == "env" for source in sources.values()) + + +def test_build_context_dedicated_file_wins_over_pyproject(tmp_path: Path) -> None: + """The dedicated `.wiki-toolkit.toml` wins over `pyproject.toml`'s `[tool.wiki_toolkit]` table.""" + (tmp_path / "pyproject.toml").write_text('[tool.wiki_toolkit]\ndocs_dir = "py-docs"\n') + (tmp_path / ".wiki-toolkit.toml").write_text('docs_dir = "dedicated-docs"\n') + + context, sources = build_context(cwd=tmp_path) + + assert context.docs_dir == tmp_path / "dedicated-docs" + assert sources["docs_dir"] == "dedicated_file" + + +def test_build_context_pyproject_wins_over_default(tmp_path: Path) -> None: + """`pyproject.toml`'s `[tool.wiki_toolkit]` table wins over the built-in default.""" + (tmp_path / "pyproject.toml").write_text('[tool.wiki_toolkit]\nbranch_prefix = "py-prefix/"\n') + + context, sources = build_context(cwd=tmp_path) + + assert context.branch_prefix == "py-prefix/" + assert sources["branch_prefix"] == "pyproject" + + +def test_build_context_malformed_dedicated_file_falls_through(tmp_path: Path) -> None: + """A `.wiki-toolkit.toml` that fails to parse falls through to pyproject.toml, not raising.""" + (tmp_path / ".wiki-toolkit.toml").write_text("not [ valid toml") + (tmp_path / "pyproject.toml").write_text('[tool.wiki_toolkit]\ndocs_dir = "py-docs"\n') + + context, sources = build_context(cwd=tmp_path) + + assert context.docs_dir == tmp_path / "py-docs" + assert sources["docs_dir"] == "pyproject" + + +def test_build_context_malformed_pyproject_table_falls_through(tmp_path: Path) -> None: + """A `pyproject.toml` that fails to parse falls through to the built-in default, not raising.""" + (tmp_path / "pyproject.toml").write_text("not [ valid toml") + + context, sources = build_context(cwd=tmp_path) + + assert context.docs_dir == tmp_path / "docs" + assert sources["docs_dir"] == "default" + + +def test_build_context_invalid_field_value_falls_back_to_default(tmp_path: Path) -> None: + """A non-positive batch cap in the dedicated file falls back to the built-in default.""" + (tmp_path / ".wiki-toolkit.toml").write_text("batch_byte_cap = -5\n") + + context, sources = build_context(cwd=tmp_path) + + assert context.batch_byte_cap == 100_000 + assert sources["batch_byte_cap"] == "default" + + +def test_build_context_empty_branch_prefix_falls_back_to_default(tmp_path: Path) -> None: + """An empty branch_prefix in the dedicated file falls back to the built-in default.""" + (tmp_path / ".wiki-toolkit.toml").write_text('branch_prefix = ""\n') + + context, sources = build_context(cwd=tmp_path) + + assert context.branch_prefix == "wiki-update/" + assert sources["branch_prefix"] == "default" + + +def test_build_context_invalid_env_value_falls_through(tmp_path: Path, monkeypatch) -> None: + """A non-integer `WIKI_TOOLKIT_BATCH_BYTE_CAP` falls through to the built-in default.""" + monkeypatch.setenv("WIKI_TOOLKIT_BATCH_BYTE_CAP", "not-a-number") + + context, sources = build_context(cwd=tmp_path) + + assert context.batch_byte_cap == 100_000 + assert sources["batch_byte_cap"] == "default" diff --git a/wiki_toolkit/settings.py b/wiki_toolkit/settings.py index 854d8ec..1e373e5 100644 --- a/wiki_toolkit/settings.py +++ b/wiki_toolkit/settings.py @@ -1,18 +1,35 @@ -"""Resolves wiki_toolkit configuration (currently just `docs_dir`). +"""Resolves wiki_toolkit configuration. -Precedence: CLI flag > `WIKI_TOOLKIT_DOCS_DIR` env var > nearest `pyproject.toml`'s -`[tool.wiki_toolkit]` table (found by walking upward from cwd, same convention as -ruff/mypy) > built-in default (`docs/` relative to cwd). +`resolve_docs_dir()` resolves just `docs_dir` (precedence: CLI flag > +`WIKI_TOOLKIT_DOCS_DIR` env var > nearest `pyproject.toml`'s `[tool.wiki_toolkit]` +table > built-in default). + +`build_context()` resolves the full `Context` (docs_dir, repo_root, branch_prefix, +batch_byte_cap, batch_file_cap) through a five-tier precedence chain: CLI flag > +`WIKI_TOOLKIT_` env var > nearest `.wiki-toolkit.toml` (dedicated file, +flat top-level keys) > nearest `pyproject.toml`'s `[tool.wiki_toolkit]` table > +built-in default. Each file tier is found by walking upward from cwd, same +convention as ruff/mypy; once found, it's authoritative for that tier (a broken +file or an invalid field value there falls through to the next tier rather than +continuing to search further up). """ from dataclasses import dataclass from pathlib import Path from typing import Literal +from pydantic import BaseModel, field_validator from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict -from pydantic_settings.sources import PyprojectTomlConfigSettingsSource +from pydantic_settings.sources import PyprojectTomlConfigSettingsSource, TomlConfigSettingsSource ConfigSource = Literal["flag", "env", "pyproject", "default"] +ContextConfigSource = Literal["flag", "env", "dedicated_file", "pyproject", "default"] + +DEDICATED_FILENAME = ".wiki-toolkit.toml" + +_DEFAULT_BRANCH_PREFIX = "wiki-update/" +_DEFAULT_BATCH_BYTE_CAP = 100_000 +_DEFAULT_BATCH_FILE_CAP = 20 class _EnvSettings(BaseSettings): @@ -90,3 +107,196 @@ def resolve_docs_dir(flag: Path | None = None, cwd: Path | None = None) -> Resol return ResolvedConfig(docs_dir=pyproject_docs_dir, source="pyproject") return ResolvedConfig(docs_dir=cwd / "docs", source="default") + + +class Context(BaseModel): + """The full resolved wiki_toolkit configuration.""" + + docs_dir: Path + repo_root: Path + branch_prefix: str + batch_byte_cap: int + batch_file_cap: int + + +_CONTEXT_FIELDS = ("docs_dir", "repo_root", "branch_prefix", "batch_byte_cap", "batch_file_cap") +_PATH_FIELDS = ("docs_dir", "repo_root") + + +class _ContextFieldsSettings(BaseSettings): + """Base for the optional per-tier settings sources: any field may be absent.""" + + docs_dir: Path | None = None + repo_root: Path | None = None + branch_prefix: str | None = None + batch_byte_cap: int | None = None + batch_file_cap: int | None = None + + @field_validator("branch_prefix") + @classmethod + def _branch_prefix_not_empty(cls, value: str | None) -> str | None: + """Reject an empty (but present) `branch_prefix`.""" + if value is not None and not value: + raise ValueError("branch_prefix must not be empty") + return value + + @field_validator("batch_byte_cap", "batch_file_cap") + @classmethod + def _cap_positive(cls, value: int | None) -> int | None: + """Reject a non-positive (but present) batch cap.""" + if value is not None and value <= 0: + raise ValueError("must be positive") + return value + + +class _ContextEnvSettings(_ContextFieldsSettings): + """Reads context fields from `WIKI_TOOLKIT_*` environment variables.""" + + model_config = SettingsConfigDict(env_prefix="WIKI_TOOLKIT_") + + +class _ContextDedicatedFileSettings(_ContextFieldsSettings): + """Reads context fields from a dedicated `.wiki-toolkit.toml` file's flat top-level keys.""" + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + """Restrict this settings class to init kwargs, no env/dotenv/secrets/toml-file discovery.""" + return (init_settings,) + + +class _ContextPyprojectSettings(_ContextFieldsSettings): + """Reads context fields from a `pyproject.toml`'s `[tool.wiki_toolkit]` table.""" + + model_config = SettingsConfigDict(pyproject_toml_table_header=("tool", "wiki_toolkit")) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + """Restrict this settings class to init kwargs and `pyproject.toml`, no env/dotenv/secrets.""" + return (init_settings, PyprojectTomlConfigSettingsSource(settings_cls)) + + +def _find_upward(start: Path, filename: str) -> Path | None: + """Walk upward from `start` for the nearest directory containing `filename`.""" + for directory in (start, *start.parents): + if (directory / filename).is_file(): + return directory + return None + + +def _env_context_fields() -> _ContextFieldsSettings | None: + """Read context fields from `WIKI_TOOLKIT_*` env vars, or `None` if any value is invalid.""" + try: + return _ContextEnvSettings() + except ValueError: + return None + + +def _dedicated_file_context_fields(directory: Path | None) -> _ContextFieldsSettings | None: + """Read context fields from `directory`'s `.wiki-toolkit.toml`, or `None` if missing/invalid.""" + if directory is None: + return None + toml_file = directory / DEDICATED_FILENAME + try: + data = TomlConfigSettingsSource(_ContextDedicatedFileSettings, toml_file=toml_file)() + return _ContextDedicatedFileSettings(**data) + except ValueError: + # ponytail: tomllib.TOMLDecodeError and pydantic's ValidationError both subclass ValueError + return None + + +def _pyproject_context_fields(directory: Path | None) -> _ContextFieldsSettings | None: + """Read context fields from `directory`'s `pyproject.toml` table, or `None` if missing/invalid.""" + if directory is None: + return None + toml_file = directory / "pyproject.toml" + try: + data = PyprojectTomlConfigSettingsSource(_ContextPyprojectSettings, toml_file=toml_file)() + return _ContextPyprojectSettings.model_validate(data) + except ValueError: + return None + + +def _find_repo_root(cwd: Path) -> Path: + """Walk upward from `cwd` for the nearest `.git`; fall back to `cwd` itself.""" + for directory in (cwd, *cwd.parents): + if (directory / ".git").exists(): + return directory + return cwd + + +def _default_value(field: str, cwd: Path) -> Path | str | int: + """Return the built-in default for `field`.""" + if field == "docs_dir": + return cwd / "docs" + if field == "repo_root": + return _find_repo_root(cwd) + if field == "branch_prefix": + return _DEFAULT_BRANCH_PREFIX + if field == "batch_byte_cap": + return _DEFAULT_BATCH_BYTE_CAP + return _DEFAULT_BATCH_FILE_CAP + + +_Tier = tuple[ContextConfigSource, "_ContextFieldsSettings | None", "Path | None"] + + +def _resolve_field( + field: str, flag_value: Path | None, tiers: tuple[_Tier, ...], cwd: Path +) -> tuple[object, ContextConfigSource]: + """Resolve a single field's value and source, falling through `tiers` in order to `default`.""" + if flag_value is not None: + return flag_value, "flag" + + for source_name, settings, base_dir in tiers: + if settings is None: + continue + value = getattr(settings, field) + if value is None: + continue + if field in _PATH_FIELDS and base_dir is not None and not value.is_absolute(): + value = base_dir / value + return value, source_name + + return _default_value(field, cwd), "default" + + +def build_context( + docs_dir_flag: Path | None = None, + repo_root_flag: Path | None = None, + cwd: Path | None = None, +) -> tuple[Context, dict[str, ContextConfigSource]]: + """Resolve the full `Context` per field, tracking which tier produced each value. + + Precedence per field: CLI flag > env var > dedicated file > pyproject.toml > default. + """ + cwd = cwd or Path.cwd() + flags = {"docs_dir": docs_dir_flag, "repo_root": repo_root_flag} + + dedicated_dir = _find_upward(cwd, DEDICATED_FILENAME) + pyproject_dir = _find_upward(cwd, "pyproject.toml") + tiers: tuple[_Tier, ...] = ( + ("env", _env_context_fields(), None), + ("dedicated_file", _dedicated_file_context_fields(dedicated_dir), dedicated_dir), + ("pyproject", _pyproject_context_fields(pyproject_dir), pyproject_dir), + ) + + values: dict[str, object] = {} + sources: dict[str, ContextConfigSource] = {} + for field in _CONTEXT_FIELDS: + values[field], sources[field] = _resolve_field(field, flags.get(field), tiers, cwd) + + return Context.model_validate(values), sources From 7554a37ffe27a02bca038284877d372c06137bf8 Mon Sep 17 00:00:00 2001 From: Corey Oordt Date: Sun, 16 Aug 2026 06:31:38 -0500 Subject: [PATCH 2/6] Fix invalid field discarding valid siblings in the same settings tier model_validate() on a BaseSettings subclass re-triggers its full source pipeline (env/pyproject), so a partial dict with the bad field dropped was still getting the original invalid value merged back in from the live environment/file. Each per-tier settings class now restricts settings_customise_sources to init kwargs only, since the raw dict is already fetched explicitly via the source objects beforehand. Addresses a code-review finding on 156-settings-context-model. --- tests/test_settings.py | 25 +++++++++++++ wiki_toolkit/settings.py | 79 +++++++++++++++++++++++++++++++--------- 2 files changed, 87 insertions(+), 17 deletions(-) diff --git a/tests/test_settings.py b/tests/test_settings.py index 25f4af0..6566d27 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -238,3 +238,28 @@ def test_build_context_invalid_env_value_falls_through(tmp_path: Path, monkeypat assert context.batch_byte_cap == 100_000 assert sources["batch_byte_cap"] == "default" + + +def test_build_context_invalid_field_does_not_discard_valid_siblings(tmp_path: Path) -> None: + """An invalid batch_byte_cap in the dedicated file doesn't discard a valid docs_dir from the same tier.""" + (tmp_path / ".wiki-toolkit.toml").write_text('docs_dir = "custom-docs"\nbatch_byte_cap = -5\n') + + context, sources = build_context(cwd=tmp_path) + + assert context.docs_dir == tmp_path / "custom-docs" + assert sources["docs_dir"] == "dedicated_file" + assert context.batch_byte_cap == 100_000 + assert sources["batch_byte_cap"] == "default" + + +def test_build_context_invalid_env_value_does_not_discard_valid_siblings(tmp_path: Path, monkeypatch) -> None: + """An invalid WIKI_TOOLKIT_BATCH_BYTE_CAP doesn't discard a valid WIKI_TOOLKIT_DOCS_DIR.""" + monkeypatch.setenv("WIKI_TOOLKIT_DOCS_DIR", str(tmp_path / "env-docs")) + monkeypatch.setenv("WIKI_TOOLKIT_BATCH_BYTE_CAP", "not-a-number") + + context, sources = build_context(cwd=tmp_path) + + assert context.docs_dir == tmp_path / "env-docs" + assert sources["docs_dir"] == "env" + assert context.batch_byte_cap == 100_000 + assert sources["batch_byte_cap"] == "default" diff --git a/wiki_toolkit/settings.py b/wiki_toolkit/settings.py index 1e373e5..9d7f178 100644 --- a/wiki_toolkit/settings.py +++ b/wiki_toolkit/settings.py @@ -18,9 +18,13 @@ from pathlib import Path from typing import Literal -from pydantic import BaseModel, field_validator +from pydantic import BaseModel, ValidationError, field_validator from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict -from pydantic_settings.sources import PyprojectTomlConfigSettingsSource, TomlConfigSettingsSource +from pydantic_settings.sources import ( + EnvSettingsSource, + PyprojectTomlConfigSettingsSource, + TomlConfigSettingsSource, +) ConfigSource = Literal["flag", "env", "pyproject", "default"] ContextConfigSource = Literal["flag", "env", "dedicated_file", "pyproject", "default"] @@ -150,10 +154,27 @@ def _cap_positive(cls, value: int | None) -> int | None: class _ContextEnvSettings(_ContextFieldsSettings): - """Reads context fields from `WIKI_TOOLKIT_*` environment variables.""" + """Field-shape for reading `WIKI_TOOLKIT_*` env vars; env is read separately via `EnvSettingsSource`.""" model_config = SettingsConfigDict(env_prefix="WIKI_TOOLKIT_") + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + """Restrict validation (`model_validate`) to init kwargs only. + + Without this, `model_validate` re-triggers the default env source and + silently re-merges the *original* (possibly invalid) environment on + top of the already-filtered dict passed to it. + """ + return (init_settings,) + class _ContextDedicatedFileSettings(_ContextFieldsSettings): """Reads context fields from a dedicated `.wiki-toolkit.toml` file's flat top-level keys.""" @@ -172,7 +193,7 @@ def settings_customise_sources( class _ContextPyprojectSettings(_ContextFieldsSettings): - """Reads context fields from a `pyproject.toml`'s `[tool.wiki_toolkit]` table.""" + """Field-shape for reading a `pyproject.toml`'s `[tool.wiki_toolkit]` table, read separately.""" model_config = SettingsConfigDict(pyproject_toml_table_header=("tool", "wiki_toolkit")) @@ -185,8 +206,14 @@ def settings_customise_sources( dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, ) -> tuple[PydanticBaseSettingsSource, ...]: - """Restrict this settings class to init kwargs and `pyproject.toml`, no env/dotenv/secrets.""" - return (init_settings, PyprojectTomlConfigSettingsSource(settings_cls)) + """Restrict validation (`model_validate`) to init kwargs only. + + `pyproject.toml` is read separately via an explicit `toml_file` path (the + directory found by `_find_upward`); the default `PyprojectTomlConfigSettingsSource` + resolves relative to the actual process cwd and would re-merge whatever it + finds there on top of the already-filtered dict passed to `model_validate`. + """ + return (init_settings,) def _find_upward(start: Path, filename: str) -> Path | None: @@ -197,37 +224,55 @@ def _find_upward(start: Path, filename: str) -> Path | None: return None -def _env_context_fields() -> _ContextFieldsSettings | None: - """Read context fields from `WIKI_TOOLKIT_*` env vars, or `None` if any value is invalid.""" - try: - return _ContextEnvSettings() - except ValueError: - return None +def _validate_dropping_invalid(cls: type[_ContextFieldsSettings], data: dict[str, object]) -> _ContextFieldsSettings: + """Validate `data` against `cls`, dropping only the individually-invalid fields. + + One bad field (e.g. a negative `batch_byte_cap`) must not discard its + valid siblings (e.g. a well-formed `docs_dir`) from the same tier. + """ + remaining = dict(data) + while True: + try: + return cls.model_validate(remaining) + except ValidationError as exc: + bad_fields = {str(err["loc"][0]) for err in exc.errors() if err["loc"]} + if not bad_fields & remaining.keys(): + # ponytail: safety net against an infinite loop if a future validator + # raises without loc matching a known field; not reachable today + return cls.model_construct() + for field in bad_fields: + remaining.pop(field, None) + + +def _env_context_fields() -> _ContextFieldsSettings: + """Read context fields from `WIKI_TOOLKIT_*` env vars, dropping any individually-invalid value.""" + data = EnvSettingsSource(_ContextEnvSettings, env_prefix="WIKI_TOOLKIT_")() + return _validate_dropping_invalid(_ContextEnvSettings, data) def _dedicated_file_context_fields(directory: Path | None) -> _ContextFieldsSettings | None: - """Read context fields from `directory`'s `.wiki-toolkit.toml`, or `None` if missing/invalid.""" + """Read context fields from `directory`'s `.wiki-toolkit.toml`, or `None` if missing/unparseable.""" if directory is None: return None toml_file = directory / DEDICATED_FILENAME try: data = TomlConfigSettingsSource(_ContextDedicatedFileSettings, toml_file=toml_file)() - return _ContextDedicatedFileSettings(**data) except ValueError: - # ponytail: tomllib.TOMLDecodeError and pydantic's ValidationError both subclass ValueError + # ponytail: tomllib.TOMLDecodeError subclasses ValueError return None + return _validate_dropping_invalid(_ContextDedicatedFileSettings, data) def _pyproject_context_fields(directory: Path | None) -> _ContextFieldsSettings | None: - """Read context fields from `directory`'s `pyproject.toml` table, or `None` if missing/invalid.""" + """Read context fields from `directory`'s `pyproject.toml` table, or `None` if missing/unparseable.""" if directory is None: return None toml_file = directory / "pyproject.toml" try: data = PyprojectTomlConfigSettingsSource(_ContextPyprojectSettings, toml_file=toml_file)() - return _ContextPyprojectSettings.model_validate(data) except ValueError: return None + return _validate_dropping_invalid(_ContextPyprojectSettings, data) def _find_repo_root(cwd: Path) -> Path: From 01342a720119a229c2efc0d0e50dad43d24125c3 Mon Sep 17 00:00:00 2001 From: Corey Oordt Date: Sun, 16 Aug 2026 06:52:59 -0500 Subject: [PATCH 3/6] Fix settings-seam bugs and reduce duplication in build_context Relative env-tier paths (docs_dir/repo_root) now resolve against cwd like every other tier, unreadable config files fall through instead of crashing, and a spurious pydantic-settings warning is suppressed. Also consolidates the duplicated pyproject/dedicated-file resolution logic that resolve_docs_dir() and build_context() had drifted apart on, and removes several smaller code duplications flagged in review. --- tests/test_settings.py | 42 ++++++++++++ wiki_toolkit/settings.py | 138 +++++++++++++-------------------------- 2 files changed, 87 insertions(+), 93 deletions(-) diff --git a/tests/test_settings.py b/tests/test_settings.py index 6566d27..8520550 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1,7 +1,12 @@ """Unit tests for wiki_toolkit.settings.""" +import os +import stat +import warnings from typing import TYPE_CHECKING +import pytest + from wiki_toolkit.settings import build_context, resolve_docs_dir if TYPE_CHECKING: @@ -263,3 +268,40 @@ def test_build_context_invalid_env_value_does_not_discard_valid_siblings(tmp_pat assert sources["docs_dir"] == "env" assert context.batch_byte_cap == 100_000 assert sources["batch_byte_cap"] == "default" + + +def test_build_context_relative_env_docs_dir_resolves_against_cwd(tmp_path: Path, monkeypatch) -> None: + """A relative WIKI_TOOLKIT_DOCS_DIR resolves to an absolute path under cwd, like every other tier.""" + monkeypatch.setenv("WIKI_TOOLKIT_DOCS_DIR", "relative-docs") + + context, sources = build_context(cwd=tmp_path) + + assert context.docs_dir == tmp_path / "relative-docs" + assert context.docs_dir.is_absolute() + assert sources["docs_dir"] == "env" + + +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="root ignores file permissions") +def test_build_context_unreadable_dedicated_file_falls_through(tmp_path: Path) -> None: + """A `.wiki-toolkit.toml` that can't be read (permission denied) falls through, not raising.""" + dedicated_file = tmp_path / ".wiki-toolkit.toml" + dedicated_file.write_text('docs_dir = "dedicated-docs"\n') + dedicated_file.chmod(0) + (tmp_path / "pyproject.toml").write_text('[tool.wiki_toolkit]\ndocs_dir = "py-docs"\n') + + try: + context, sources = build_context(cwd=tmp_path) + finally: + dedicated_file.chmod(stat.S_IRUSR | stat.S_IWUSR) + + assert context.docs_dir == tmp_path / "py-docs" + assert sources["docs_dir"] == "pyproject" + + +def test_build_context_emits_no_warnings(tmp_path: Path) -> None: + """Resolving a pyproject.toml table doesn't emit pydantic-settings' unused-config-key warning.""" + (tmp_path / "pyproject.toml").write_text('[tool.wiki_toolkit]\ndocs_dir = "py-docs"\n') + + with warnings.catch_warnings(): + warnings.simplefilter("error") + build_context(cwd=tmp_path) diff --git a/wiki_toolkit/settings.py b/wiki_toolkit/settings.py index 9d7f178..aff6099 100644 --- a/wiki_toolkit/settings.py +++ b/wiki_toolkit/settings.py @@ -11,9 +11,11 @@ built-in default. Each file tier is found by walking upward from cwd, same convention as ruff/mypy; once found, it's authoritative for that tier (a broken file or an invalid field value there falls through to the next tier rather than -continuing to search further up). +continuing to search further up). CLI-flag overrides are currently only wired +up for `docs_dir` and `repo_root`; the other three fields start at the env tier. """ +import warnings from dataclasses import dataclass from pathlib import Path from typing import Literal @@ -35,6 +37,10 @@ _DEFAULT_BATCH_BYTE_CAP = 100_000 _DEFAULT_BATCH_FILE_CAP = 20 +_TOML_READ_ERRORS = (ValueError, OSError) +"""ponytail: kept as a named tuple, not an inline `except (...)`, to dodge a ruff-format +bug in this project's config that corrupts parenthesized multi-exception tuples.""" + class _EnvSettings(BaseSettings): """Reads `docs_dir` from the `WIKI_TOOLKIT_DOCS_DIR` environment variable.""" @@ -44,26 +50,6 @@ class _EnvSettings(BaseSettings): docs_dir: Path | None = None -class _PyprojectSettings(BaseSettings): - """Reads `docs_dir` from a `pyproject.toml`'s `[tool.wiki_toolkit]` table.""" - - model_config = SettingsConfigDict(pyproject_toml_table_header=("tool", "wiki_toolkit")) - - docs_dir: Path | None = None - - @classmethod - def settings_customise_sources( - cls, - settings_cls: type[BaseSettings], - init_settings: PydanticBaseSettingsSource, - env_settings: PydanticBaseSettingsSource, - dotenv_settings: PydanticBaseSettingsSource, - file_secret_settings: PydanticBaseSettingsSource, - ) -> tuple[PydanticBaseSettingsSource, ...]: - """Restrict this settings class to init kwargs and `pyproject.toml`, no env/dotenv/secrets.""" - return (init_settings, PyprojectTomlConfigSettingsSource(settings_cls)) - - @dataclass class ResolvedConfig: """The resolved `docs_dir` and which source produced it.""" @@ -73,26 +59,15 @@ class ResolvedConfig: def _find_pyproject_docs_dir(start: Path) -> Path | None: - """Walk upward from `start` for the nearest pyproject.toml's `[tool.wiki_toolkit].docs_dir`. - - Once a pyproject.toml is found, it is authoritative (matches ruff/mypy - convention) — a missing table, missing key, or unparseable file there - means "no pyproject source", not "keep looking further up". - """ - for directory in (start, *start.parents): - pyproject = directory / "pyproject.toml" - if not pyproject.is_file(): - continue - try: - data = PyprojectTomlConfigSettingsSource(_PyprojectSettings, toml_file=pyproject)() - docs_dir = _PyprojectSettings.model_validate(data).docs_dir - except ValueError: - # ponytail: both tomllib.TOMLDecodeError and pydantic's ValidationError subclass ValueError - return None - if docs_dir is None: - return None - return docs_dir if docs_dir.is_absolute() else directory / docs_dir - return None + """Walk upward from `start` for the nearest pyproject.toml's `[tool.wiki_toolkit].docs_dir`.""" + directory = _find_upward(start, "pyproject.toml") + if directory is None: + return None + fields = _pyproject_context_fields(directory) + docs_dir = fields.docs_dir if fields is not None else None + if docs_dir is None: + return None + return docs_dir if docs_dir.is_absolute() else directory / docs_dir def resolve_docs_dir(flag: Path | None = None, cwd: Path | None = None) -> ResolvedConfig: @@ -123,12 +98,20 @@ class Context(BaseModel): batch_file_cap: int -_CONTEXT_FIELDS = ("docs_dir", "repo_root", "branch_prefix", "batch_byte_cap", "batch_file_cap") +_CONTEXT_FIELDS = tuple(Context.model_fields) _PATH_FIELDS = ("docs_dir", "repo_root") class _ContextFieldsSettings(BaseSettings): - """Base for the optional per-tier settings sources: any field may be absent.""" + """Base for the optional per-tier settings sources: any field may be absent. + + Each tier reads its raw source (env, a dedicated file, or `pyproject.toml`) + separately and passes the resulting dict to `model_validate`. Restricting + sources to init kwargs here keeps that re-validation from re-triggering + the class's own env/file discovery and re-merging the *original* + (possibly invalid, possibly differently-located) source on top of the + already-filtered dict. + """ docs_dir: Path | None = None repo_root: Path | None = None @@ -152,12 +135,6 @@ def _cap_positive(cls, value: int | None) -> int | None: raise ValueError("must be positive") return value - -class _ContextEnvSettings(_ContextFieldsSettings): - """Field-shape for reading `WIKI_TOOLKIT_*` env vars; env is read separately via `EnvSettingsSource`.""" - - model_config = SettingsConfigDict(env_prefix="WIKI_TOOLKIT_") - @classmethod def settings_customise_sources( cls, @@ -167,54 +144,25 @@ def settings_customise_sources( dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, ) -> tuple[PydanticBaseSettingsSource, ...]: - """Restrict validation (`model_validate`) to init kwargs only. - - Without this, `model_validate` re-triggers the default env source and - silently re-merges the *original* (possibly invalid) environment on - top of the already-filtered dict passed to it. - """ + """Restrict every tier's validation (`model_validate`) to init kwargs only.""" return (init_settings,) +class _ContextEnvSettings(_ContextFieldsSettings): + """Field-shape for reading `WIKI_TOOLKIT_*` env vars; env is read separately via `EnvSettingsSource`.""" + + model_config = SettingsConfigDict(env_prefix="WIKI_TOOLKIT_") + + class _ContextDedicatedFileSettings(_ContextFieldsSettings): """Reads context fields from a dedicated `.wiki-toolkit.toml` file's flat top-level keys.""" - @classmethod - def settings_customise_sources( - cls, - settings_cls: type[BaseSettings], - init_settings: PydanticBaseSettingsSource, - env_settings: PydanticBaseSettingsSource, - dotenv_settings: PydanticBaseSettingsSource, - file_secret_settings: PydanticBaseSettingsSource, - ) -> tuple[PydanticBaseSettingsSource, ...]: - """Restrict this settings class to init kwargs, no env/dotenv/secrets/toml-file discovery.""" - return (init_settings,) - class _ContextPyprojectSettings(_ContextFieldsSettings): """Field-shape for reading a `pyproject.toml`'s `[tool.wiki_toolkit]` table, read separately.""" model_config = SettingsConfigDict(pyproject_toml_table_header=("tool", "wiki_toolkit")) - @classmethod - def settings_customise_sources( - cls, - settings_cls: type[BaseSettings], - init_settings: PydanticBaseSettingsSource, - env_settings: PydanticBaseSettingsSource, - dotenv_settings: PydanticBaseSettingsSource, - file_secret_settings: PydanticBaseSettingsSource, - ) -> tuple[PydanticBaseSettingsSource, ...]: - """Restrict validation (`model_validate`) to init kwargs only. - - `pyproject.toml` is read separately via an explicit `toml_file` path (the - directory found by `_find_upward`); the default `PyprojectTomlConfigSettingsSource` - resolves relative to the actual process cwd and would re-merge whatever it - finds there on top of the already-filtered dict passed to `model_validate`. - """ - return (init_settings,) - def _find_upward(start: Path, filename: str) -> Path | None: """Walk upward from `start` for the nearest directory containing `filename`.""" @@ -246,33 +194,37 @@ def _validate_dropping_invalid(cls: type[_ContextFieldsSettings], data: dict[str def _env_context_fields() -> _ContextFieldsSettings: """Read context fields from `WIKI_TOOLKIT_*` env vars, dropping any individually-invalid value.""" - data = EnvSettingsSource(_ContextEnvSettings, env_prefix="WIKI_TOOLKIT_")() + data = EnvSettingsSource(_ContextEnvSettings)() return _validate_dropping_invalid(_ContextEnvSettings, data) def _dedicated_file_context_fields(directory: Path | None) -> _ContextFieldsSettings | None: - """Read context fields from `directory`'s `.wiki-toolkit.toml`, or `None` if missing/unparseable.""" + """Read context fields from `directory`'s `.wiki-toolkit.toml`, or `None` if missing/unreadable.""" if directory is None: return None toml_file = directory / DEDICATED_FILENAME try: data = TomlConfigSettingsSource(_ContextDedicatedFileSettings, toml_file=toml_file)() - except ValueError: - # ponytail: tomllib.TOMLDecodeError subclasses ValueError + except _TOML_READ_ERRORS: + # ponytail: tomllib.TOMLDecodeError subclasses ValueError; OSError covers permission/lock errors return None return _validate_dropping_invalid(_ContextDedicatedFileSettings, data) def _pyproject_context_fields(directory: Path | None) -> _ContextFieldsSettings | None: - """Read context fields from `directory`'s `pyproject.toml` table, or `None` if missing/unparseable.""" + """Read context fields from `directory`'s `pyproject.toml` table, or `None` if missing/unreadable.""" if directory is None: return None toml_file = directory / "pyproject.toml" try: data = PyprojectTomlConfigSettingsSource(_ContextPyprojectSettings, toml_file=toml_file)() - except ValueError: + except _TOML_READ_ERRORS: return None - return _validate_dropping_invalid(_ContextPyprojectSettings, data) + with warnings.catch_warnings(): + # ponytail: pyproject_toml_table_header is read directly above, not via + # settings_customise_sources, so pydantic-settings warns it looks unused + warnings.filterwarnings("ignore", message=r"Config key `pyproject_toml_table_header`", category=UserWarning) + return _validate_dropping_invalid(_ContextPyprojectSettings, data) def _find_repo_root(cwd: Path) -> Path: @@ -334,7 +286,7 @@ def build_context( dedicated_dir = _find_upward(cwd, DEDICATED_FILENAME) pyproject_dir = _find_upward(cwd, "pyproject.toml") tiers: tuple[_Tier, ...] = ( - ("env", _env_context_fields(), None), + ("env", _env_context_fields(), cwd), ("dedicated_file", _dedicated_file_context_fields(dedicated_dir), dedicated_dir), ("pyproject", _pyproject_context_fields(pyproject_dir), pyproject_dir), ) From f7539d884d18cb1461917dca387ed8babe936ac7 Mon Sep 17 00:00:00 2001 From: Corey Oordt Date: Sun, 16 Aug 2026 07:04:28 -0500 Subject: [PATCH 4/6] Simplify TOML exception handling and update Ruff target-version to py313 --- pyproject.toml | 1 + wiki_toolkit/settings.py | 8 ++------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 49f5c30..dcf7beb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -157,6 +157,7 @@ color = true line-length = 119 [tool.ruff] +target-version = "py313" # Issue regarding catching multiple exceptions in 3.14 exclude = [ ".bzr", ".direnv", diff --git a/wiki_toolkit/settings.py b/wiki_toolkit/settings.py index aff6099..83a21a2 100644 --- a/wiki_toolkit/settings.py +++ b/wiki_toolkit/settings.py @@ -37,10 +37,6 @@ _DEFAULT_BATCH_BYTE_CAP = 100_000 _DEFAULT_BATCH_FILE_CAP = 20 -_TOML_READ_ERRORS = (ValueError, OSError) -"""ponytail: kept as a named tuple, not an inline `except (...)`, to dodge a ruff-format -bug in this project's config that corrupts parenthesized multi-exception tuples.""" - class _EnvSettings(BaseSettings): """Reads `docs_dir` from the `WIKI_TOOLKIT_DOCS_DIR` environment variable.""" @@ -205,7 +201,7 @@ def _dedicated_file_context_fields(directory: Path | None) -> _ContextFieldsSett toml_file = directory / DEDICATED_FILENAME try: data = TomlConfigSettingsSource(_ContextDedicatedFileSettings, toml_file=toml_file)() - except _TOML_READ_ERRORS: + except (ValueError, OSError): # ponytail: tomllib.TOMLDecodeError subclasses ValueError; OSError covers permission/lock errors return None return _validate_dropping_invalid(_ContextDedicatedFileSettings, data) @@ -218,7 +214,7 @@ def _pyproject_context_fields(directory: Path | None) -> _ContextFieldsSettings toml_file = directory / "pyproject.toml" try: data = PyprojectTomlConfigSettingsSource(_ContextPyprojectSettings, toml_file=toml_file)() - except _TOML_READ_ERRORS: + except (ValueError, OSError): return None with warnings.catch_warnings(): # ponytail: pyproject_toml_table_header is read directly above, not via From c04683b821143bfad36ea42314b2173df8b8b60f Mon Sep 17 00:00:00 2001 From: Corey Oordt Date: Sun, 16 Aug 2026 07:15:16 -0500 Subject: [PATCH 5/6] Remove unused TYPE_CHECKING imports and simplify imports across modules --- wiki_toolkit/doctor.py | 8 ++------ wiki_toolkit/log.py | 6 ++---- wiki_toolkit/sources.py | 8 +++----- wiki_toolkit/wiki.py | 6 ++---- wiki_toolkit/write_gate.py | 6 ++---- 5 files changed, 11 insertions(+), 23 deletions(-) diff --git a/wiki_toolkit/doctor.py b/wiki_toolkit/doctor.py index 99c3398..7ed15c0 100644 --- a/wiki_toolkit/doctor.py +++ b/wiki_toolkit/doctor.py @@ -5,18 +5,14 @@ import sys from dataclasses import dataclass, field from importlib.metadata import PackageNotFoundError, version -from typing import TYPE_CHECKING +from pathlib import Path import orjson from wiki_toolkit.init import PROVENANCE_FILENAME +from wiki_toolkit.settings import ConfigSource from wiki_toolkit.sources import SOURCE_MANIFEST_FILENAME -if TYPE_CHECKING: - from pathlib import Path - - from wiki_toolkit.settings import ConfigSource - DOCS_DIRS = ("sources", "wiki") DOCS_FILES = ("catalog.jsonl", "log.jsonl", "schema.md", SOURCE_MANIFEST_FILENAME) DOCS_STRUCTURE = (*DOCS_FILES, *DOCS_DIRS) diff --git a/wiki_toolkit/log.py b/wiki_toolkit/log.py index e0995c2..0bd5672 100644 --- a/wiki_toolkit/log.py +++ b/wiki_toolkit/log.py @@ -2,15 +2,13 @@ from dataclasses import asdict, dataclass from datetime import UTC, datetime -from typing import TYPE_CHECKING, Literal +from pathlib import Path +from typing import Literal import orjson from wiki_toolkit.write_gate import stage_best_effort -if TYPE_CHECKING: - from pathlib import Path - LogAction = Literal["ingest", "update", "lint", "create", "archive", "delete"] ALLOWED_LOG_ACTIONS: tuple[LogAction, ...] = ("ingest", "update", "lint", "create", "archive", "delete") diff --git a/wiki_toolkit/sources.py b/wiki_toolkit/sources.py index 498be01..0b9817c 100644 --- a/wiki_toolkit/sources.py +++ b/wiki_toolkit/sources.py @@ -3,10 +3,12 @@ import hashlib import shutil import subprocess +from collections.abc import Iterator from dataclasses import dataclass, field from datetime import UTC, datetime from difflib import SequenceMatcher -from typing import TYPE_CHECKING, Any, Literal, overload +from pathlib import Path +from typing import Any, Literal, overload import orjson import yaml @@ -15,10 +17,6 @@ from wiki_toolkit.frontmatter import Post from wiki_toolkit.write_gate import stage_best_effort -if TYPE_CHECKING: - from collections.abc import Iterator - from pathlib import Path - SOURCE_MANIFEST_FILENAME = "source-manifest.jsonl" diff --git a/wiki_toolkit/wiki.py b/wiki_toolkit/wiki.py index 33f335a..157adfc 100644 --- a/wiki_toolkit/wiki.py +++ b/wiki_toolkit/wiki.py @@ -3,15 +3,13 @@ import re from dataclasses import dataclass, field from decimal import ROUND_HALF_UP, Decimal -from typing import TYPE_CHECKING, Literal +from pathlib import Path +from typing import Literal from wiki_toolkit._io import read_jsonl from wiki_toolkit.frontmatter import Post from wiki_toolkit.sources import SOURCE_MANIFEST_FILENAME, LintViolation, LoadError, SourceManifest, _iter_markdown -if TYPE_CHECKING: - from pathlib import Path - _BULLET_RE = re.compile(r"^\s*[-*]\s+(.*)$") _WIKILINK_RE = re.compile(r"\[\[([^\]|]+)(?:\|[^\]]*)?\]\]") _CONFIDENCE_MARKER_RE = re.compile(r"\^\[(inferred|ambiguous)\]") diff --git a/wiki_toolkit/write_gate.py b/wiki_toolkit/write_gate.py index 1ee21e2..5d3bf94 100644 --- a/wiki_toolkit/write_gate.py +++ b/wiki_toolkit/write_gate.py @@ -5,10 +5,8 @@ import subprocess from dataclasses import dataclass from datetime import UTC, datetime -from typing import TYPE_CHECKING, Literal - -if TYPE_CHECKING: - from pathlib import Path +from pathlib import Path +from typing import Literal Frame = Literal["routine", "needs-review"] ALLOWED_FRAMES: tuple[Frame, ...] = ("routine", "needs-review") From 7d8b354ec35aaf6a0b5ec106fa82f1119a6048a9 Mon Sep 17 00:00:00 2001 From: Corey Oordt Date: Sun, 16 Aug 2026 07:17:12 -0500 Subject: [PATCH 6/6] Remove unused TYPE_CHECKING imports and consolidate Path imports --- wiki_toolkit/_io.py | 5 +---- wiki_toolkit/batches.py | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/wiki_toolkit/_io.py b/wiki_toolkit/_io.py index a71a4e7..4a1c619 100644 --- a/wiki_toolkit/_io.py +++ b/wiki_toolkit/_io.py @@ -1,14 +1,11 @@ """Shared JSONL read/write helpers, internal to wiki_toolkit.""" -from typing import TYPE_CHECKING +from pathlib import Path import orjson from wiki_toolkit.write_gate import stage_best_effort -if TYPE_CHECKING: - from pathlib import Path - def write_jsonl(path: Path, records: list[dict], *, stage_root: Path | None = None) -> None: """Write `records` to `path` as JSONL, one object per line. diff --git a/wiki_toolkit/batches.py b/wiki_toolkit/batches.py index 051894c..871ca50 100644 --- a/wiki_toolkit/batches.py +++ b/wiki_toolkit/batches.py @@ -6,10 +6,7 @@ """ from dataclasses import dataclass, field -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from pathlib import Path +from pathlib import Path BATCH_BYTE_CAP = 100_000 BATCH_FILE_CAP = 20