From dbace4c27dd475f62573ed0f5ca389fc994f2e04 Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Fri, 21 Aug 2026 19:22:28 +0800 Subject: [PATCH 1/7] refactor: replace normalize_heading_text with normalize_heading_label and normalize_match_text - Updated multiple files to replace instances of normalize_heading_text with the new function normalize_heading_label for improved heading normalization. - Introduced normalize_match_text for consistent text matching, ensuring CJK-aware spacing and case-insensitivity. - Adjusted related functions and tests to reflect these changes, enhancing overall text processing accuracy. --- .../document_agent/calibration/phase1.py | 4 +- .../document_agent/calibration/procedure.py | 8 +- .../app/services/document_agent/pdf_text.py | 8 +- .../structure/anchoring_primitives.py | 6 +- .../structure/hierarchy_locator.py | 31 +- .../document_agent/structure/outline_check.py | 11 +- .../document_agent/structure/toc_anchoring.py | 4 +- .../tools/find_toc_anchor_pages.py | 26 +- .../document_agent/tools/grep_text.py | 69 +- .../structure/body_boundary.py | 46 +- .../structure/toc_hierarchy.py | 6 +- .../document_parser/structure/toc_parser.py | 7 +- .../document_parser/support/text_helpers.py | 8 - .../services/page_memory/fine_hierarchy.py | 3 +- .../page_memory/tmp_probe_null_page_leaves.py | 833 ++++++++++++++++++ .../contract/test_body_boundary_contract.py | 8 + .../test_doc_profile_anatomy_contract.py | 2 +- .../test_structure_anchoring_contract.py | 19 +- .../test_tool_registry_smoke_contract.py | 28 + 19 files changed, 1042 insertions(+), 85 deletions(-) create mode 100644 apps/worker/scripts/page_memory/tmp_probe_null_page_leaves.py diff --git a/apps/worker/app/services/document_agent/calibration/phase1.py b/apps/worker/app/services/document_agent/calibration/phase1.py index 87ae08bdf..caff9f871 100644 --- a/apps/worker/app/services/document_agent/calibration/phase1.py +++ b/apps/worker/app/services/document_agent/calibration/phase1.py @@ -64,7 +64,7 @@ def _regime_probes( ) -> dict[str, list[_Probe]]: """Keep leaf probes from the first distinct printed pages of each kind.""" from app.services.document_parser.structure.body_boundary import ( - normalize_heading_text, + normalize_heading_label, ) probes: dict[str, list[_Probe]] = {} @@ -82,7 +82,7 @@ def _regime_probes( continue if printed in seen_printed.get(kind, set()): continue - title = normalize_heading_text(str(entry.get("heading") or "")) + title = normalize_heading_label(str(entry.get("heading") or "")) if not title: continue probes.setdefault(kind, []).append(_Probe(title=title, printed=printed)) diff --git a/apps/worker/app/services/document_agent/calibration/procedure.py b/apps/worker/app/services/document_agent/calibration/procedure.py index 90dbc78ac..ff22d105e 100644 --- a/apps/worker/app/services/document_agent/calibration/procedure.py +++ b/apps/worker/app/services/document_agent/calibration/procedure.py @@ -142,7 +142,7 @@ def _entry_titles_for_regime( ) -> set[str] | None: """Titles belonging to this regime; None means fall back to page_kind match.""" from app.services.document_parser.structure.body_boundary import ( - normalize_heading_text, + normalize_heading_label, ) kind = normalize_kind(regime.kind) @@ -160,7 +160,7 @@ def _entry_titles_for_regime( if idx < 0 or idx >= len(entries): continue heading = entries[idx].get("heading") - title = normalize_heading_text(str(heading or "")) + title = normalize_heading_label(str(heading or "")) if title: titles.add(title) return titles or None @@ -445,10 +445,10 @@ def _annotate_regimes_from_anchor( continue heading = str(entries[idx].get("heading") or "") from app.services.document_parser.structure.body_boundary import ( - normalize_heading_text, + normalize_heading_label, ) - title = normalize_heading_text(heading) + title = normalize_heading_label(heading) path = path_by_title.get(title) if path is not None and path in (anchor.match_overrides or {}): ok_indices.append(idx) diff --git a/apps/worker/app/services/document_agent/pdf_text.py b/apps/worker/app/services/document_agent/pdf_text.py index 7a6da6522..f3851058f 100644 --- a/apps/worker/app/services/document_agent/pdf_text.py +++ b/apps/worker/app/services/document_agent/pdf_text.py @@ -9,10 +9,7 @@ run_in_child_process, worker, ) - - -def normalize_spaces(text: str) -> str: - return " ".join((text or "").split()) +from app.services.document_parser.structure.body_boundary import normalize_heading_label @worker @@ -54,7 +51,8 @@ def read_page_texts( def meaningful_lines(text: str) -> list[str]: - return [normalize_spaces(line) for line in text.splitlines() if normalize_spaces(line)] + lines = [normalize_heading_label(line) for line in text.splitlines()] + return [line for line in lines if line] def top_lines(text: str, *, max_lines: int = 20) -> list[str]: diff --git a/apps/worker/app/services/document_agent/structure/anchoring_primitives.py b/apps/worker/app/services/document_agent/structure/anchoring_primitives.py index 91a11b7b4..f7d514f2a 100644 --- a/apps/worker/app/services/document_agent/structure/anchoring_primitives.py +++ b/apps/worker/app/services/document_agent/structure/anchoring_primitives.py @@ -15,7 +15,7 @@ first_leaf_start_under, iter_leaf_title_nodes, last_leaf_start_under, - locate_title_compact_strict, + locate_title_normalized_strict, ) from app.services.document_agent.structure.section_page_verify import ( verify_section_page_choice, @@ -334,7 +334,7 @@ def _resolve_null_parent_with_sibling_window( report: list[dict[str, Any]], ) -> None: scope_pages = [page for page in body_pages if left <= page <= right] - match = locate_title_compact_strict( + match = locate_title_normalized_strict( title, scope_pages=scope_pages, page_texts=page_texts, @@ -383,7 +383,7 @@ def _resolve_null_parent_first_sibling( ) scope_pages = [page for page in body_pages if left <= page <= right] - match = locate_title_compact_strict( + match = locate_title_normalized_strict( title, scope_pages=scope_pages, page_texts=page_texts, diff --git a/apps/worker/app/services/document_agent/structure/hierarchy_locator.py b/apps/worker/app/services/document_agent/structure/hierarchy_locator.py index bf94692b8..694415205 100644 --- a/apps/worker/app/services/document_agent/structure/hierarchy_locator.py +++ b/apps/worker/app/services/document_agent/structure/hierarchy_locator.py @@ -2,7 +2,7 @@ Deterministic range assembly from PROFILE ``match_overrides``. Leaf starts come only from those overrides. Null-page parents are located upstream via -compact-strict (cross-line) + optional VLM, then resolved here including +normalized-strict text matching + optional VLM, then resolved here including parent self-only spans for interstitial pages. Parents without an override may still inherit start from the earliest located descendant leaf. """ @@ -15,7 +15,8 @@ from app.services.document_parser.structure.body_boundary import ( clean_toc_title, - normalize_heading_text, + normalize_heading_label, + normalize_match_text, ) TitleMatchSource = Literal[ @@ -133,26 +134,26 @@ class ResolvedHierarchyRange: evidence: dict[str, Any] = field(default_factory=dict) -def locate_title_compact_strict( +def locate_title_normalized_strict( title: str, *, scope_pages: list[int], page_texts: dict[int, str], ) -> TitleMatch | None: - """Locate *title* after cross-line compact cleanup; accept only a unique page. + """Locate *title* after unified text normalization; accept one unique page. - Pipeline: compact(page text) → contiguous strict match of compact(title) → - accept iff exactly one page in ``scope_pages`` hits. Handles PyMuPDF line - splits; does not use token/normalized weak matching. + Query and page text both preserve one space between non-CJK words while + removing whitespace adjacent to CJK. Accept iff exactly one page in + ``scope_pages`` hits. """ - needle = _compact_match_text(clean_toc_title(title) or title) + needle = normalize_match_text(clean_toc_title(title) or title) if not needle or not scope_pages: return None hit_pages: list[int] = [] matched_preview = "" for page in scope_pages: - haystack = _compact_match_text(page_texts.get(page, "")) + haystack = normalize_match_text(page_texts.get(page, "")) if not haystack or needle not in haystack: continue hit_pages.append(page) @@ -169,7 +170,7 @@ def locate_title_compact_strict( source="anchored", matched_line=matched_preview, candidates=[page], - evidence={"accept": "compact_strict_unique"}, + evidence={"accept": "normalized_strict_unique"}, ) @@ -403,7 +404,7 @@ def _locate_match_for_node( if match is not None: return match if node.children: - # Parent active locate is upstream (compact-strict / visual). + # Parent active locate is upstream (normalized-strict / visual). return _infer_start_from_descendant_overrides( node, parent_titles=path_titles[:-1], match_overrides=match_overrides, scope_pages=scope_pages, @@ -550,10 +551,6 @@ def _allowed_pages_between(start: int, end: int, allowed_pages: set[int]) -> lis return [page for page in range(start, end + 1) if page in allowed_pages] -def _compact_match_text(text: str) -> str: - return re.sub(r"\s+", "", normalize_heading_text(text)).casefold() - - def _extract_flat_entries(payload: Any) -> list[dict[str, Any]]: if isinstance(payload, list): return [ @@ -617,8 +614,8 @@ def _entries_to_tree(entries: list[dict[str, Any]]) -> list[TitleNode]: for entry in entries: # Keep original TOC heading (incl. numbering). Prefix stripping belongs - # only in text/compact match helpers used for null-page parents. - title = normalize_heading_text(str(entry.get("heading") or "")) + # only in normalized text-match helpers used for null-page parents. + title = normalize_heading_label(str(entry.get("heading") or "")) level = _safe_int(entry.get("level")) or 1 if not title or len(title) < 2: continue diff --git a/apps/worker/app/services/document_agent/structure/outline_check.py b/apps/worker/app/services/document_agent/structure/outline_check.py index d3975af8e..1cbaf2d06 100644 --- a/apps/worker/app/services/document_agent/structure/outline_check.py +++ b/apps/worker/app/services/document_agent/structure/outline_check.py @@ -2,12 +2,11 @@ from __future__ import annotations -import re from typing import Any from app.services.document_parser.structure.body_boundary import ( clean_toc_title, - normalize_heading_text, + normalize_match_text, ) _DEFAULT_DIGEST_TITLE_CHARS = 80 @@ -94,12 +93,8 @@ def build_tree_digest_from_entries( def _title_on_page(title: str, page: int, page_texts: dict[int, str]) -> bool: - needle = _compact(clean_toc_title(title) or title) + needle = normalize_match_text(clean_toc_title(title) or title) if not needle: return False - haystack = _compact(page_texts.get(page, "")) + haystack = normalize_match_text(page_texts.get(page, "")) return bool(haystack) and needle in haystack - - -def _compact(text: str) -> str: - return re.sub(r"\s+", "", normalize_heading_text(text)).casefold() diff --git a/apps/worker/app/services/document_agent/structure/toc_anchoring.py b/apps/worker/app/services/document_agent/structure/toc_anchoring.py index 74eff2f09..e52bef364 100644 --- a/apps/worker/app/services/document_agent/structure/toc_anchoring.py +++ b/apps/worker/app/services/document_agent/structure/toc_anchoring.py @@ -25,8 +25,8 @@ collapse_intermediate_single_child_chains, extract_toc_nodes, iter_leaf_title_nodes, - normalize_heading_text, ) +from app.services.document_parser.structure.body_boundary import normalize_heading_label _LOG_PREFIX = "[profile.toc_anchoring]" PENDING_TOC_CALIBRATION_CONCURRENCY = 10 @@ -253,7 +253,7 @@ def outline_physical_overrides( for entry in entries: if not isinstance(entry, dict): continue - title = normalize_heading_text(str(entry.get("heading") or "")) + title = normalize_heading_label(str(entry.get("heading") or "")) if not title or len(title) < 2: continue try: diff --git a/apps/worker/app/services/document_agent/tools/find_toc_anchor_pages.py b/apps/worker/app/services/document_agent/tools/find_toc_anchor_pages.py index cd161a5c7..716ab6f87 100644 --- a/apps/worker/app/services/document_agent/tools/find_toc_anchor_pages.py +++ b/apps/worker/app/services/document_agent/tools/find_toc_anchor_pages.py @@ -10,6 +10,7 @@ from app.services.document_agent.manifest import TocAnchorPage, ToolContext, ToolResult from app.services.document_agent.registry import has_page_full_text, has_page_labels, register_tool +from app.services.document_parser.structure.body_boundary import normalize_match_text from app.services.document_parser.formats.pdf.pymupdf_subprocess import ( run_in_child_process, worker, @@ -17,7 +18,7 @@ from loguru import logger # CJK and English TOC keywords used for first-pass anchor detection. -TOC_KEYWORDS = frozenset({"目录", "目次", "contents", "tableofcontents"}) +TOC_KEYWORDS = frozenset({"目录", "目次", "contents", "table of contents"}) # If a TOC keyword fingerprint appears on more than this fraction of total # pages, it is treated as a recurring navigation element (header/footer link) @@ -33,9 +34,19 @@ _MAX_KEYWORD_SPLIT_LINES = max(len(keyword) for keyword in TOC_KEYWORDS) -def _normalize_for_toc(text: str) -> str: - """Collapse whitespace for keyword matching.""" - return text.replace(" ", "").replace("\u3000", "").lower() +def _match_toc_keyword_parts(parts: list[str]) -> str | None: + candidates = {normalize_match_text(parts[0])} if parts else set() + for part in parts[1:]: + next_candidates: set[str] = set() + for prefix in candidates: + for separator in ("", " "): + candidate = normalize_match_text(f"{prefix}{separator}{part}") + if any(keyword.startswith(candidate) for keyword in TOC_KEYWORDS): + next_candidates.add(candidate) + candidates = next_candidates + if not candidates: + return None + return next((candidate for candidate in candidates if candidate in TOC_KEYWORDS), None) def _meaningful_text_lines(text: str) -> list[str]: @@ -57,9 +68,10 @@ def _merge_keyword_split_lines( upper = min(_MAX_KEYWORD_SPLIT_LINES, len(lines) - index) for part_count in range(upper, 1, -1): parts = [lines[index + offset].strip() for offset in range(part_count)] - if _normalize_for_toc("".join(parts)) not in TOC_KEYWORDS: + keyword = _match_toc_keyword_parts(parts) + if keyword is None: continue - joined = ("".join(parts), index, index + part_count - 1) + joined = (keyword, index, index + part_count - 1) break if joined is not None: merged.append(joined) @@ -74,7 +86,7 @@ def _find_toc_text_matches(lines: list[str]) -> list[dict[str, Any]]: """Match TOC keywords only as whole lines after keyword-split repair.""" matches: list[dict[str, Any]] = [] for raw_line, start_idx, end_idx in _merge_keyword_split_lines(lines): - keyword = _normalize_for_toc(raw_line) + keyword = normalize_match_text(raw_line) if keyword not in TOC_KEYWORDS: continue matches.append( diff --git a/apps/worker/app/services/document_agent/tools/grep_text.py b/apps/worker/app/services/document_agent/tools/grep_text.py index 3663c8268..ebf6ef74e 100644 --- a/apps/worker/app/services/document_agent/tools/grep_text.py +++ b/apps/worker/app/services/document_agent/tools/grep_text.py @@ -13,19 +13,24 @@ not_is_scanned, register_tool, ) +from app.services.document_parser.structure.body_boundary import normalize_match_text @register_tool( name="grep.text", - description="Search full PDF text for a substring or regex. Available only for native PDFs.", + description=( + "Search normalized PDF text for a substring or regex. Whitespace is " + "collapsed with CJK-aware spacing and matching is case-insensitive." + ), parameters={ "type": "object", "properties": { "query": {"type": "string"}, "regex": {"type": "boolean", "default": False}, - "case_sensitive": {"type": "boolean", "default": False}, "max_results": {"type": "integer", "default": 30}, "context_chars": {"type": "integer", "default": 80}, + "start_page": {"type": "integer"}, + "end_page": {"type": "integer"}, }, "required": ["query"], }, @@ -41,34 +46,68 @@ def grep_text(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: latency_ms=int((time.monotonic() - start) * 1000), ) use_regex = bool(args.get("regex", False)) - case_sensitive = bool(args.get("case_sensitive", False)) max_results = max(1, min(int(args.get("max_results") or 30), 100)) context_chars = max(20, min(int(args.get("context_chars") or 80), 300)) - flags = 0 if case_sensitive else re.IGNORECASE - pattern = re.compile(query if use_regex else re.escape(query), flags) + start_page = max(1, int(args.get("start_page") or 1)) + end_page = int(args.get("end_page") or ctx.blackboard.page_count or 0) + normalized_query = normalize_match_text(query) + if not normalized_query: + return ToolResult( + status="error", + error="grep.text normalized query is empty", + latency_ms=int((time.monotonic() - start) * 1000), + ) + pattern = re.compile( + normalized_query if use_regex else re.escape(normalized_query) + ) results: list[dict[str, Any]] = [] + hit_count = 0 + hit_pages: list[int] = [] for page, text in sorted(ctx.blackboard.page_full_text_cache.items()): - for match in pattern.finditer(text): + if page < start_page or (end_page and page > end_page): + continue + normalized_text = normalize_match_text(text) + page_hit = False + for match in pattern.finditer(normalized_text): + hit_count += 1 + page_hit = True + if len(results) >= max_results: + continue start_idx = max(match.start() - context_chars, 0) - end_idx = min(match.end() + context_chars, len(text)) + end_idx = min(match.end() + context_chars, len(normalized_text)) results.append( { "page": page, "char_offset": match.start(), - "snippet": text[start_idx:end_idx].replace("\n", " "), + "snippet": normalized_text[start_idx:end_idx], } ) - if len(results) >= max_results: - break - if len(results) >= max_results: - break - summary = {"query": query, "hit_count": len(results), "results": results} + if page_hit: + hit_pages.append(page) + summary = { + "query": query, + "normalized_query": normalized_query, + "hit_count": hit_count, + "hit_page_count": len(hit_pages), + "hit_pages": hit_pages, + "results": results, + } ctx.blackboard.global_signals.setdefault("grep_history", []).append( - {"query": query, "hit_count": len(results), "sample_pages": [item["page"] for item in results[:10]]} + { + "query": query, + "normalized_query": normalized_query, + "hit_count": hit_count, + "hit_page_count": len(hit_pages), + "sample_pages": hit_pages[:10], + } ) return ToolResult( status="ok", payload=summary, latency_ms=int((time.monotonic() - start) * 1000), - output_summary={"query": query, "hit_count": len(results)}, + output_summary={ + "query": query, + "hit_count": hit_count, + "hit_page_count": len(hit_pages), + }, ) diff --git a/apps/worker/app/services/document_parser/structure/body_boundary.py b/apps/worker/app/services/document_parser/structure/body_boundary.py index f0ce15e42..b3d4a6122 100644 --- a/apps/worker/app/services/document_parser/structure/body_boundary.py +++ b/apps/worker/app/services/document_parser/structure/body_boundary.py @@ -26,13 +26,49 @@ _PAGE_SUFFIX_RE = re.compile(r"[\s\.\-·…]+\d+\s*$") -def normalize_heading_text(text: str) -> str: - """Normalize text for fuzzy heading matching.""" +def normalize_heading_label(text: str) -> str: + """Normalize heading text while preserving display casing.""" text = unicodedata.normalize("NFKC", text or "") text = re.sub(r"\s+", " ", text).strip() return text +def _is_cjk_char(char: str) -> bool: + codepoint = ord(char) + return ( + 0x3000 <= codepoint <= 0x303F + or 0x3040 <= codepoint <= 0x30FF + or 0x31F0 <= codepoint <= 0x31FF + or 0x3400 <= codepoint <= 0x4DBF + or 0x4E00 <= codepoint <= 0x9FFF + or 0xF900 <= codepoint <= 0xFAFF + or 0x1100 <= codepoint <= 0x11FF + or 0x3130 <= codepoint <= 0x318F + or 0xAC00 <= codepoint <= 0xD7AF + or 0x20000 <= codepoint <= 0x2FA1F + ) + + +def normalize_match_text(text: str) -> str: + """Normalize query and corpus text for every deterministic text match. + + Whitespace becomes one space between non-CJK text, and disappears whenever + either adjacent character is CJK. Matching is case-insensitive. + """ + normalized = unicodedata.normalize("NFKC", text or "").casefold().strip() + parts = re.split(r"\s+", normalized) + if not parts or not parts[0]: + return "" + + output = parts[0] + for part in parts[1:]: + if not part: + continue + separator = "" if _is_cjk_char(output[-1]) or _is_cjk_char(part[0]) else " " + output = f"{output}{separator}{part}" + return output + + def clean_toc_title(title: str) -> str: """Remove leading numbering/hashes and trailing page numbers from a TOC title.""" cleaned = _PAGE_SUFFIX_RE.sub("", title or "").strip() @@ -78,15 +114,15 @@ def find_first_body_boundary( ) -> int | None: """Return the first line index matching a TOC level-1 title, if any.""" normalized_titles = [ - normalize_heading_text(title) + normalize_match_text(title) for title in level1_titles - if normalize_heading_text(title) + if normalize_match_text(title) ] if not normalized_titles: return None for index, line in enumerate(lines): - normalized_line = normalize_heading_text(line.lstrip("#").strip()) + normalized_line = normalize_match_text(line.lstrip("#").strip()) if any(title in normalized_line for title in normalized_titles): return index return None diff --git a/apps/worker/app/services/document_parser/structure/toc_hierarchy.py b/apps/worker/app/services/document_parser/structure/toc_hierarchy.py index 439223b38..6f921477c 100644 --- a/apps/worker/app/services/document_parser/structure/toc_hierarchy.py +++ b/apps/worker/app/services/document_parser/structure/toc_hierarchy.py @@ -1,10 +1,10 @@ from __future__ import annotations import pandas as pd +from app.services.document_parser.structure.body_boundary import normalize_match_text from app.services.document_parser.structure.layout_parser import hiearchy_llm from app.services.document_parser.support.stage_profiler import stage_timer from app.services.document_parser.tables.table_text_parser import df2md -from app.services.document_parser.support.text_helpers import normalize_md from loguru import logger from pandas import Index @@ -109,14 +109,14 @@ def build_toc_hierarchy_payload( def eval_toc_levels( toc_lines: list[str], model_name: str | None = None, max_depth: int = 6 ) -> tuple[str, dict]: - toc_title_keywords = {"目录", "目次", "tableofcontents", "contents"} + toc_title_keywords = {"目录", "目次", "table of contents", "contents"} valid_data = [] for index, line in enumerate(toc_lines): heading = line.strip() if not heading: continue - if normalize_md(heading) in toc_title_keywords: + if normalize_match_text(heading.lstrip("#").strip()) in toc_title_keywords: logger.debug( f"eval_toc_levels: skipping TOC keyword title line id={index}: {heading[:60]}" ) diff --git a/apps/worker/app/services/document_parser/structure/toc_parser.py b/apps/worker/app/services/document_parser/structure/toc_parser.py index 884437421..d6a28d39a 100644 --- a/apps/worker/app/services/document_parser/structure/toc_parser.py +++ b/apps/worker/app/services/document_parser/structure/toc_parser.py @@ -12,8 +12,9 @@ import gevent import pandas as pd +from app.services.document_parser.structure.body_boundary import normalize_match_text from app.services.document_parser.structure.toc_hierarchy import eval_toc_levels -from app.services.document_parser.support.text_helpers import normalize_md, truncate_text_by_tokens +from app.services.document_parser.support.text_helpers import truncate_text_by_tokens from app.services.document_parser.support.stage_profiler import stage_timer from app.services.document_parser.tables.table_text_parser import df2md from gevent.pool import Pool as GeventPool @@ -111,12 +112,12 @@ def detect_toc_candidates(md_lines: list, limit_: int = 100) -> tuple: - area_ranges: List[(start_idx, end_idx)] - full raw md_lines ranges for later filtering """ - toc_keywords = {"目录", "目次", "tableofcontents", "contents"} + toc_keywords = {"目录", "目次", "table of contents", "contents"} # Step 1: find all TOC keywords start_indices = [] for i, line in enumerate(md_lines): - if normalize_md(line) in toc_keywords: + if normalize_match_text(line.lstrip("#").strip()) in toc_keywords: start_indices.append(i) # Step 2: if no TOC keywords found, use the first line as the candidate area diff --git a/apps/worker/app/services/document_parser/support/text_helpers.py b/apps/worker/app/services/document_parser/support/text_helpers.py index b11fdc772..b87af9af5 100644 --- a/apps/worker/app/services/document_parser/support/text_helpers.py +++ b/apps/worker/app/services/document_parser/support/text_helpers.py @@ -8,14 +8,6 @@ EN_START_LIMIT = 15 CN_RATIO_THRESHOLD = 0.3 - -def normalize_md(text: str) -> str: - """Normalize markdown string for comparison.""" - text = re.sub(r"^\s*#+\s*", "", text) - text = re.sub(r"\s+", "", text) - return text.lower() - - def truncate_text(text: str, start_limit: int, end_limit: int) -> str: """Truncate text by raw character count, keeping start and end parts.""" text = str(text) diff --git a/apps/worker/app/services/page_memory/fine_hierarchy.py b/apps/worker/app/services/page_memory/fine_hierarchy.py index 96fd18de4..57db6362b 100644 --- a/apps/worker/app/services/page_memory/fine_hierarchy.py +++ b/apps/worker/app/services/page_memory/fine_hierarchy.py @@ -21,6 +21,7 @@ from loguru import logger +from app.services.document_parser.structure.body_boundary import normalize_match_text from app.services.page_memory.page_tagger import PageTagResult from app.services.page_memory.skeleton_extractor import SectionSkeleton from app.services.page_memory._utils import page_scope_info, sort_skeletons @@ -461,7 +462,7 @@ def _exclusive_end(skeletons: list[SectionSkeleton], index: int) -> int: def _title_key(title: str | None) -> str: - normalized = re.sub(r"\s+", "", str(title or "")).casefold() + normalized = normalize_match_text(str(title or "")) normalized = re.sub(r"[^\w\u4e00-\u9fff]+", "", normalized) return normalized diff --git a/apps/worker/scripts/page_memory/tmp_probe_null_page_leaves.py b/apps/worker/scripts/page_memory/tmp_probe_null_page_leaves.py new file mode 100644 index 000000000..228ceb8fe --- /dev/null +++ b/apps/worker/scripts/page_memory/tmp_probe_null_page_leaves.py @@ -0,0 +1,833 @@ +#!/usr/bin/env python3 +# ruff: noqa: E402 +"""TEMP: locate null-page TOC nodes with normalized grep ReAct + VLM. + +Experiment only — patches production symbols for one run, then restores them. + +Policy under test: + - Prune only printed-page leaves that failed anchoring; keep null-page nodes. + - Null-page parents and leaves use a bounded mini-ReAct search planner. + - Loop budget and hit/visual budget both equal PROFILE TOC + ``BOUNDARY_STEP_PAGES`` (currently 5). + - ReAct grep uses the registered ``grep.text`` normalized-text tool. + - hit_count > budget → too many; reflect and change query (no VLM). + - hit_count in 1..budget → confirm one page at a time until accepted or budget used. + - Under one parent, siblings are serial: left cursor advances on success; + on first failure, remaining null siblings are skipped (no window reset). + - Search scope ends at the next located sibling or the enclosing parent scope; + there is no fixed 22-page cap and no peer-TOC homepage clip. + +Usage: + cd apps/worker + uv run python scripts/page_memory/tmp_probe_null_page_leaves.py \\ + --file "/path/to/EN_Sydney Streets Code.pdf" +""" + +from __future__ import annotations + +import json +import sys +import time +from dataclasses import replace +from pathlib import Path as _Path +from typing import Any, cast + +sys.path.insert(0, str(_Path(__file__).resolve().parent)) + +from loguru import logger + +from _debug_pm_shared import ( + _build_debug_coordinator, + base_argparser, + load_anatomy_cache, + load_stage0_into_coordinator, + page_text_cache_path, + require_file, + resolve_anatomy_cache_path, + resolve_paths, + stage0_state_path, + write_debug_json, +) + +# Same constant as PROFILE TOC boundary / confirm batch size. +from app.services.document_agent.tools.extract_toc_with_boundaries import ( + BOUNDARY_STEP_PAGES, +) + +REACT_BUDGET = int(BOUNDARY_STEP_PAGES) +_HISTORY_SAMPLE_PAGES = 3 + +_REACT_INSTRUCTIONS = """\ +You are the search planner in a small ReAct loop. Propose one plain-text grep +query that may appear on the physical START page of a section. Grep collapses +whitespace/newlines to one space between non-CJK words, removes whitespace +adjacent to CJK, and matches case-insensitively. A separate visual check +confirms candidates one page at a time. + +Return one strict json object: +{"action":"grep","query":"...","reason":"..."} +or, only when no useful untried query remains: +{"action":"give_up","query":"","reason":"..."} + +General query tactics: +- Do not guess page numbers. +- Account for differences between TOC labels and body headings. +- Try removing numbering, lettering, punctuation, or decorative prefixes. +- When supported by the title or parent path, try a structural prefix such as + chapter, part, section, annex, or appendix. +- Try a distinctive leading, middle, or trailing title phrase when the full + title is unlikely to be printed verbatim. +- Prefer queries specific enough to avoid running headers and passing mentions. + +Reflection rules (mandatory): +- Read previous_attempts. Reflect on hit_count and observation before answering. +- If the last observation is no_normalized_hits, too_many_hits, visual_rejected, + or duplicate_normalized_query, you MUST change the query. Emitting the same + query again (same text after whitespace/case normalization) is invalid. +- too_many_hits means hit_count exceeded the visual budget; narrow the query. +- no_normalized_hits means broaden, rephrase, add/drop a structural prefix, or + try another title fragment. +- visual_rejected means the pages were not the section beginning; change the + query rather than repeating it. + +Generic cases: +- A TOC label like "B Safety requirements" under an appendices parent may be + printed as "Appendix B", "Safety requirements", or both together. +- A TOC label like "4.2 Access control — Technical requirements" may be printed + with the number removed or with only one distinctive title phrase. +""" + + +def _react_history_item(item: dict[str, Any]) -> dict[str, Any]: + hit_pages = [int(page) for page in (item.get("hit_pages") or [])] + return { + "query": item.get("query"), + "normalized_query": item.get("normalized_query"), + "hit_count": int(item.get("hit_count") or len(hit_pages)), + "sample_pages": hit_pages[:_HISTORY_SAMPLE_PAGES], + "observation": item.get("observation"), + "visual_selected_page": item.get("visual_selected_page"), + "visual_reason": item.get("visual_reason"), + "visual_pages_checked": item.get("visual_pages_checked"), + } + + +def prune_unanchored_keep_null_pages( + nodes: list[Any], + *, + match_overrides: dict[tuple[str, ...], Any], +) -> tuple[list[Any], int]: + """Drop only printed-page leaves that never got a physical override.""" + from app.services.document_agent.structure.hierarchy_locator import TitleNode + + removed = 0 + + def _prune(node: TitleNode, parent_titles: tuple[str, ...]) -> TitleNode | None: + nonlocal removed + path = (*parent_titles, node.title) + if node.children: + children: list[TitleNode] = [] + for child in node.children: + kept = _prune(child, path) + if kept is not None: + children.append(kept) + if children: + return replace(node, children=children) + if path in match_overrides or node.printed_page is None: + return replace(node, children=[]) + removed += 1 + return None + if path in match_overrides or node.printed_page is None: + return node + removed += 1 + return None + + out: list[TitleNode] = [] + for node in nodes: + kept = _prune(node, ()) + if kept is not None: + out.append(kept) + if removed: + logger.info( + "[tmp.null_leaf] pruned {} printed-page unanchored leaves " + "(null-page nodes kept)", + removed, + ) + return out, removed + + +def _next_located_bound( + *, + sibling_nodes: list[Any], + index: int, + parent_titles: tuple[str, ...], + overrides: dict[tuple[str, ...], Any], +) -> int | None: + from app.services.document_agent.structure.hierarchy_locator import ( + first_leaf_start_under, + ) + + for later in sibling_nodes[index + 1 :]: + path = (*parent_titles, later.title) + if path in overrides: + return int(overrides[path].page) + bound = first_leaf_start_under(later, parent_titles, overrides) + if bound is not None: + return int(bound) + return None + + +def _infer_offset( + nodes: list[Any], + match_overrides: dict[tuple[str, ...], Any], +) -> int: + from collections import Counter + + from app.services.document_agent.structure.hierarchy_locator import ( + iter_leaf_title_nodes, + ) + + diffs: list[int] = [] + for path, node in iter_leaf_title_nodes(nodes): + if node.printed_page is None or path not in match_overrides: + continue + diffs.append(int(match_overrides[path].page) - int(node.printed_page)) + if not diffs: + return 0 + return int(Counter(diffs).most_common(1)[0][0]) + + +def _normalized_grep( + *, + ctx: Any, + query: str, + left: int, + right: int, +) -> tuple[str, list[int], int]: + from app.services.document_agent.tools.grep_text import grep_text + + result = grep_text( + ctx, + { + "query": query, + "start_page": left, + "end_page": right, + }, + ) + if result.status != "ok": + return "", [], 0 + payload = result.payload or {} + return ( + str(payload.get("normalized_query") or ""), + [int(page) for page in (payload.get("hit_pages") or [])], + int(payload.get("hit_count") or 0), + ) + + +def _propose_react_query( + *, + title: str, + parent_titles: tuple[str, ...], + left: int, + right: int, + attempts: list[dict[str, Any]], + budget: int, +) -> tuple[dict[str, Any] | None, dict[str, Any]]: + state = { + "toc_title": title, + "parent_path": list(parent_titles), + "physical_search_scope": [left, right], + "react_budget": budget, + "visual_budget": budget, + "previous_attempts": [_react_history_item(item) for item in attempts], + } + prompt = ( + f"{_REACT_INSTRUCTIONS}\n\nCurrent state:\n" + f"{json.dumps(state, ensure_ascii=False)}" + ) + + try: + from shared.services.ai.llm_overrides import get_text_client + + client, model = get_text_client() + raw, usage = client.chat_completion_with_usage( + messages=cast(Any, [{"role": "user", "content": prompt}]), + model=model, + temperature=0.0, + max_tokens=300, + response_format={"type": "json_object"}, + usage_task="document_agent.null_page_title_react", + ) + payload = json.loads(raw) if raw else {} + except Exception as exc: + logger.warning("[tmp.null_react] planner failed for {!r}: {}", title, exc) + return None, {"error": f"planner failed: {exc}"} + + action = str(payload.get("action") or "").strip().lower() + query = str(payload.get("query") or "").strip() + if action not in {"grep", "give_up"}: + return None, { + "error": f"unknown planner action: {action!r}", + "usage": usage, + } + if action == "grep" and not query: + return None, {"error": "planner returned empty grep query", "usage": usage} + return ( + { + "action": action, + "query": query, + "reason": str(payload.get("reason") or ""), + }, + {"usage": usage}, + ) + + +def _verify_section_beginning_page( + *, + ctx: Any, + title: str, + page: int, + query: str, +) -> tuple[bool, str, int]: + """Confirm one physical page as the section beginning. Returns (ok, reason, tokens).""" + from app.services.document_agent.calibration.prompts import ( + coerce_found, + coerce_found_page, + ) + from app.services.document_agent.tools.inspect_pages import inspect_pages + + question = ( + f"Does this page mark the physical BEGINNING of the document section " + f"corresponding to the TOC entry {title!r}? A cover page, section " + "title page, or first body-heading page can be the beginning. Allow " + "equivalent wording and added or omitted numbering, lettering, or " + "structural prefixes. Do not accept a table-of-contents line, running " + "header or footer, passing mention, or continuation page. " + f"The normalized text query that nominated this page was {query!r}. " + "Report the physical page number printed in the page label above the image." + ) + verify_result = inspect_pages( + ctx, + { + "pages": [page], + "page_cap": 1, + "question": question, + "answer_keys": { + "found": ( + "boolean, true only when this page is the physical beginning " + "of the requested section" + ), + "found_page": ( + "number|null, the physical page number where the section begins" + ), + }, + "folder_name": "null_page_react_verify", + "prefix": "verify", + "usage_task": "document_agent.null_page_react_verify", + }, + ) + tokens = int(verify_result.tokens_used or 0) + if verify_result.status != "ok": + return False, str(verify_result.error or "inspect.pages failed"), tokens + fields = (verify_result.payload or {}).get("fields") or {} + found_page = coerce_found_page(fields.get("found_page"), pages=[page]) + ok = coerce_found(fields.get("found")) and found_page == page + reason = str((verify_result.payload or {}).get("answer") or "") + return ok, reason, tokens + + +def _locate_with_react( + *, + path_titles: tuple[str, ...], + title: str, + left: int, + right: int, + ctx: Any, +) -> tuple[Any | None, list[dict[str, Any]], int, str]: + from app.services.document_agent.structure.hierarchy_locator import TitleMatch + + budget = REACT_BUDGET + attempts: list[dict[str, Any]] = [] + attempted_needles: set[str] = set() + visual_calls = 0 + visual_remaining = budget + + for loop_index in range(1, budget + 1): + proposal, planner_meta = _propose_react_query( + title=title, + parent_titles=path_titles[:-1], + left=left, + right=right, + attempts=attempts, + budget=budget, + ) + if proposal is None: + attempts.append( + { + "loop": loop_index, + "action": "planner_error", + "hit_count": 0, + "hit_pages": [], + **planner_meta, + } + ) + continue + + action = proposal["action"] + if action == "give_up": + attempts.append( + { + "loop": loop_index, + **proposal, + "hit_count": 0, + "hit_pages": [], + **planner_meta, + } + ) + return None, attempts, visual_calls, "react_give_up" + + query = proposal["query"] + needle, hit_pages, match_count = _normalized_grep( + ctx=ctx, + query=query, + left=left, + right=right, + ) + attempt: dict[str, Any] = { + "loop": loop_index, + **proposal, + "normalized_query": needle, + "hit_count": len(hit_pages), + "hit_pages": hit_pages, + "match_count": match_count, + "visual_budget_remaining_before": visual_remaining, + **planner_meta, + } + if not needle or needle in attempted_needles: + attempt["observation"] = "duplicate_normalized_query" + attempts.append(attempt) + continue + attempted_needles.add(needle) + + if not hit_pages: + attempt["observation"] = "no_normalized_hits" + attempts.append(attempt) + continue + + if len(hit_pages) > budget: + attempt["observation"] = "too_many_hits" + attempts.append(attempt) + continue + + if visual_remaining <= 0: + attempt["observation"] = "visual_budget_exhausted" + attempts.append(attempt) + continue + + checked: list[dict[str, Any]] = [] + selected: int | None = None + last_reason = "" + for page in hit_pages: + if visual_remaining <= 0: + break + visual_remaining -= 1 + visual_calls += 1 + ok, reason, tokens = _verify_section_beginning_page( + ctx=ctx, + title=title, + page=page, + query=query, + ) + checked.append( + { + "page": page, + "confirmed": ok, + "reason": reason, + "tokens_used": tokens, + } + ) + last_reason = reason + if ok: + selected = page + break + + attempt["visual_pages_checked"] = checked + attempt["visual_selected_page"] = selected + attempt["visual_reason"] = last_reason + attempt["visual_budget_remaining_after"] = visual_remaining + if selected is not None: + attempt["observation"] = "section_start_confirmed" + attempts.append(attempt) + return ( + TitleMatch( + page=int(selected), + source="react_normalized_grep_vlm", + matched_line=query, + candidates=hit_pages, + evidence={ + "accept": "react_normalized_grep_vlm", + "null_page_react": True, + "loop": loop_index, + "normalized_query": needle, + "visual_reason": last_reason, + "visual_pages_checked": [item["page"] for item in checked], + }, + ), + attempts, + visual_calls, + "react_normalized_grep_vlm", + ) + + if visual_remaining <= 0 and len(checked) < len(hit_pages): + attempt["observation"] = "visual_budget_exhausted" + else: + attempt["observation"] = "visual_rejected" + attempts.append(attempt) + + return None, attempts, visual_calls, "react_loop_limit" + + +def locate_null_page_nodes_unified( + *, + nodes: list[Any], + match_overrides: dict[tuple[str, ...], Any], + page_texts: dict[int, str], + body_pages: list[int], + ctx: Any, + offset: int | None = None, +) -> tuple[dict[tuple[str, ...], Any], list[dict[str, Any]]]: + """Locate null-page nodes serially with normalized grep ReAct + VLM.""" + from app.services.document_agent.structure.hierarchy_locator import ( + TitleMatch, + last_leaf_start_under, + ) + + if not nodes or not body_pages: + return dict(match_overrides), [] + + out = dict(match_overrides) + body_set = set(body_pages) + report: list[dict[str, Any]] = [] + primary_offset = ( + int(offset) if offset is not None else _infer_offset(nodes, out) + ) + + def _seed_printed( + sibling_nodes: list[Any], parent_titles: tuple[str, ...] + ) -> None: + for node in sibling_nodes: + path = (*parent_titles, node.title) + if node.printed_page is not None and path not in out: + page = int(node.printed_page) + primary_offset + if page in body_set: + out[path] = TitleMatch( + page=page, + source="offset_seed", + matched_line="", + candidates=[page], + evidence={ + "accept": "printed_plus_offset_seed", + "tmp_null_leaf_probe": True, + }, + ) + if node.children: + _seed_printed(node.children, path) + + def _skip_rest( + sibling_nodes: list[Any], + start_index: int, + parent_titles: tuple[str, ...], + failed_title: str, + ) -> None: + for later in sibling_nodes[start_index:]: + path = (*parent_titles, later.title) + if later.printed_page is not None or path in out: + continue + report.append( + { + "path_titles": list(path), + "title": later.title, + "kind": "leaf" if not later.children else "parent", + "printed_page": None, + "search_scope": None, + "result": "skipped_after_sibling_failure", + "page": None, + "accept": None, + "visual_verify_calls": 0, + "react_attempts": [], + "failed_sibling": failed_title, + } + ) + + def _probe_succeeded(entry: dict[str, Any]) -> bool: + return entry.get("page") is not None + + _seed_printed(nodes, ()) + + def walk( + sibling_nodes: list[Any], + parent_titles: tuple[str, ...], + scope_start: int, + scope_end: int, + ) -> None: + cursor = int(scope_start) + for index, node in enumerate(sibling_nodes): + path_titles = (*parent_titles, node.title) + next_bound = _next_located_bound( + sibling_nodes=sibling_nodes, + index=index, + parent_titles=parent_titles, + overrides=out, + ) + node_scope_end = ( + min(int(next_bound), scope_end) + if next_bound is not None + else int(scope_end) + ) + + if path_titles in out: + cursor = max(cursor, int(out[path_titles].page)) + + needs_probe = node.printed_page is None and path_titles not in out + if needs_probe: + is_leaf = not node.children + entry: dict[str, Any] = { + "path_titles": list(path_titles), + "title": node.title, + "kind": "leaf" if is_leaf else "parent", + "printed_page": None, + "search_scope": None, + "result": "unresolved", + "page": None, + "accept": None, + "visual_verify_calls": 0, + "react_attempts": [], + } + + left = int(cursor) + right = int(node_scope_end) + if right < left: + entry["result"] = "skipped_bad_window" + entry["search_scope"] = [left, right] + report.append(entry) + _skip_rest( + sibling_nodes, index + 1, parent_titles, node.title + ) + return + + entry["search_scope"] = [left, right] + match, attempts, visual_calls, result = _locate_with_react( + path_titles=path_titles, + title=node.title, + left=left, + right=right, + ctx=ctx, + ) + entry["react_attempts"] = attempts + entry["visual_verify_calls"] = visual_calls + entry["result"] = result + if match is not None: + out[path_titles] = match + entry["page"] = int(match.page) + entry["accept"] = match.evidence.get("accept") + report.append(entry) + + if not _probe_succeeded(entry): + _skip_rest( + sibling_nodes, index + 1, parent_titles, node.title + ) + return + + cursor = int(entry["page"]) + + if node.children: + child_scope_start = ( + int(out[path_titles].page) + if path_titles in out + else cursor + ) + walk( + node.children, + path_titles, + child_scope_start, + node_scope_end, + ) + last_under = last_leaf_start_under(node, parent_titles, out) + if last_under is not None: + cursor = max(cursor, int(last_under)) + elif path_titles in out: + cursor = max(cursor, int(out[path_titles].page)) + + walk(nodes, (), body_pages[0], body_pages[-1]) + located = sum(1 for row in report if row.get("page") is not None) + skipped = sum( + 1 + for row in report + if row.get("result") == "skipped_after_sibling_failure" + ) + logger.info( + "[tmp.null_leaf] serial null-page ReAct: attempted={} located={} " + "unresolved={} skipped_after_fail={} budget={}", + len(report), + located, + sum( + 1 + for row in report + if row.get("result") + in {"react_give_up", "react_loop_limit", "planner_error"} + ), + skipped, + REACT_BUDGET, + ) + return out, report + + +def main() -> int: + parser = base_argparser( + "TEMP: null-page normalized grep ReAct + VLM (stop siblings on fail)" + ) + args = parser.parse_args() + + from app.services.document_agent.calibration import procedure as procedure_mod + from app.services.document_agent.structure import anchoring_primitives as ap + from app.services.document_agent.structure.toc_anchoring import run_toc_anchoring + from app.services.document_agent.validators import single_shard_plan + from shared.core.config import settings + + pdf_path, filename, out_dir = resolve_paths(args) + anatomy_cache = resolve_anatomy_cache_path(out_dir) + require_file(stage0_state_path(out_dir), hint="Run Stage 0 first") + require_file(page_text_cache_path(out_dir), hint="Re-run Stage 0") + require_file(anatomy_cache, hint="Run Stage 1 first") + + anatomy = load_anatomy_cache(anatomy_cache, pdf_path, filename) + page_count = int(anatomy.page_count or 0) + hierarchies = list(getattr(anatomy, "toc_hierarchies", None) or []) + + original_prune = ( + procedure_mod.prune_unanchored_toc_leaves, + ap.prune_unanchored_toc_leaves, + ) + original_locate = ( + procedure_mod.locate_null_page_parent_overrides, + ap.locate_null_page_parent_overrides, + ) + + def _patched_locate(*, nodes, match_overrides, page_texts, body_pages, ctx): + return locate_null_page_nodes_unified( + nodes=nodes, + match_overrides=match_overrides, + page_texts=page_texts, + body_pages=body_pages, + ctx=ctx, + offset=None, + ) + + procedure_mod.prune_unanchored_toc_leaves = prune_unanchored_keep_null_pages + ap.prune_unanchored_toc_leaves = prune_unanchored_keep_null_pages + procedure_mod.locate_null_page_parent_overrides = _patched_locate + ap.locate_null_page_parent_overrides = _patched_locate + + logger.info("█" * 70) + logger.info(" TEMP null-page normalized grep ReAct — {}", filename) + logger.info(" OUTPUT: {}", out_dir) + logger.info("█" * 70) + + t0 = time.time() + previous_image_model = settings.IMAGE_MODEL + try: + coordinator = _build_debug_coordinator( + pdf_path=pdf_path, + job_id=filename, + out_dir=out_dir, + model=None if args.no_vlm else args.model, + settings_extra={"skip_toc_anchoring": False}, + ) + load_stage0_into_coordinator(coordinator, out_dir) + bb = coordinator.blackboard + bb.toc_result = anatomy.toc_result + bb.toc_hierarchies = hierarchies + bb.shard_plan = anatomy.shard_plan or single_shard_plan(page_count) + bb.skeleton_anchor = None + bb.skeleton_nodes = None + bb.pending_skeleton_anchors = [] + + run_toc_anchoring(coordinator.ctx) + + anchor = bb.skeleton_anchor or {} + report = list(anchor.get("null_page_report") or []) + overrides = dict(anchor.get("match_overrides") or {}) + react_hits = [] + for path, match in overrides.items(): + source = ( + match.get("source") + if isinstance(match, dict) + else getattr(match, "source", None) + ) + if source != "react_normalized_grep_vlm": + continue + titles = path if isinstance(path, (list, tuple)) else (path,) + page = ( + match.get("page") + if isinstance(match, dict) + else getattr(match, "page", None) + ) + react_hits.append({"path": list(titles), "page": page}) + + payload = { + "policy": { + "prune": "keep_null_page_nodes; drop printed-page unanchored leaves only", + "probe": ( + "normalized-grep ReAct; loop/hit/visual budget=" + f"{REACT_BUDGET} (=BOUNDARY_STEP_PAGES); " + "hit_count>budget → too_many_hits + reflect; " + "else confirm one page at a time; " + "search next located sibling or enclosing parent scope; " + "serial under parent; stop siblings after first failure" + ), + "react_budget": REACT_BUDGET, + "boundary_step_pages": BOUNDARY_STEP_PAGES, + }, + "offset": anchor.get("offset"), + "pruned_count": anchor.get("pruned_count"), + "bulk_count": anchor.get("bulk_count"), + "override_count": len(overrides), + "null_page_report": report, + "react_override_hits": react_hits, + "elapsed_s": round(time.time() - t0, 2), + } + + out_path = out_dir / "_doc_agent" / "tmp_null_page_leaf_probe.json" + write_debug_json(out_path, payload) + logger.info("wrote {}", out_path) + logger.info( + "null_page rows={} react_override_hits={}", + len(report), + len(react_hits), + ) + for row in report: + logger.info( + " [{}] {} search_scope={} result={} page={} loops={} failed_sibling={}", + row.get("kind"), + row.get("path_titles"), + row.get("search_scope"), + row.get("result"), + row.get("page"), + len(row.get("react_attempts") or []), + row.get("failed_sibling"), + ) + for hit in react_hits: + logger.info(" OVERRIDE {} -> p{}", hit["path"], hit["page"]) + finally: + procedure_mod.prune_unanchored_toc_leaves = original_prune[0] + ap.prune_unanchored_toc_leaves = original_prune[1] + procedure_mod.locate_null_page_parent_overrides = original_locate[0] + ap.locate_null_page_parent_overrides = original_locate[1] + if args.model: + settings.IMAGE_MODEL = previous_image_model + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/apps/worker/tests/contract/test_body_boundary_contract.py b/apps/worker/tests/contract/test_body_boundary_contract.py index 3512c9f89..56b9906d3 100644 --- a/apps/worker/tests/contract/test_body_boundary_contract.py +++ b/apps/worker/tests/contract/test_body_boundary_contract.py @@ -14,6 +14,7 @@ from app.services.document_parser.structure.body_boundary import ( extract_level1_titles, find_first_body_boundary, + normalize_match_text, ) from app.services.document_parser.structure.layout_parser import ( _supports_multi_toc_zones, @@ -40,6 +41,13 @@ def test_extract_level1_titles_reads_toc_with_level_not_toc_tree() -> None: assert titles == ["Overview", "Requirements"] +def test_normalize_match_text_uses_cjk_aware_spacing_and_lowercase() -> None: + assert normalize_match_text("附录 A OVERVIEW") == "附录a overview" + assert normalize_match_text("Public\n Domain\tManual") == "public domain manual" + assert normalize_match_text("目\n录") == "目录" + assert normalize_match_text("Chapter 1 概述") == "chapter 1概述" + + def test_extract_level1_titles_ignores_empty_or_non_list_payloads() -> None: assert extract_level1_titles([]) == [] assert extract_level1_titles([{"toc_with_level": None}]) == [] diff --git a/apps/worker/tests/contract/test_doc_profile_anatomy_contract.py b/apps/worker/tests/contract/test_doc_profile_anatomy_contract.py index a8c27f2dc..31ba78f1c 100644 --- a/apps/worker/tests/contract/test_doc_profile_anatomy_contract.py +++ b/apps/worker/tests/contract/test_doc_profile_anatomy_contract.py @@ -100,7 +100,7 @@ def test_toc_anchor_text_scan_whole_line_keyword_and_split_repair() -> None: assert late_matches[0]["line_index"] == 60 assert late_matches[0]["match_kind"] == "keyword:目录" - assert split_matches[0]["match_kind"] == "keyword:tableofcontents" + assert split_matches[0]["match_kind"] == "keyword:table of contents" assert split_matches[0]["line_index"] == 0 assert split_matches[0]["line_end_index"] == 2 assert false_matches == [] diff --git a/apps/worker/tests/contract/test_structure_anchoring_contract.py b/apps/worker/tests/contract/test_structure_anchoring_contract.py index e87104b4a..ac6ad7e09 100644 --- a/apps/worker/tests/contract/test_structure_anchoring_contract.py +++ b/apps/worker/tests/contract/test_structure_anchoring_contract.py @@ -93,7 +93,7 @@ def test_null_page_parent_skipped_without_right_anchor() -> None: assert report[0]["result"] == "skipped_no_right" -def test_null_page_parent_located_via_compact_text() -> None: +def test_null_page_parent_located_via_normalized_text() -> None: child = TitleNode(title="1.1 Detail", level=2, printed_page=5, children=[]) parent = TitleNode( title="1 Overview", @@ -124,6 +124,23 @@ def test_null_page_parent_located_via_compact_text() -> None: assert report[0]["window"] == [1, 5] +def test_normalized_title_match_preserves_english_word_boundary() -> None: + from app.services.document_agent.structure.hierarchy_locator import ( + locate_title_normalized_strict, + ) + + match = locate_title_normalized_strict( + "附录 A OVERVIEW", + scope_pages=[7], + page_texts={7: "附录\nA OVERVIEW"}, + ) + + assert match is not None + assert match.page == 7 + assert match.matched_line == "附录a overview" + assert match.evidence["accept"] == "normalized_strict_unique" + + def test_first_sibling_null_parent_uses_scan_forward_not_wide_rtl() -> None: """No left sibling: miss text → ``scan_title_forward`` within 2+4+6+10 budget.""" child = TitleNode(title="22.1 Intro", level=2, printed_page=278, children=[]) diff --git a/apps/worker/tests/contract/test_tool_registry_smoke_contract.py b/apps/worker/tests/contract/test_tool_registry_smoke_contract.py index 031266c02..a9930f69e 100644 --- a/apps/worker/tests/contract/test_tool_registry_smoke_contract.py +++ b/apps/worker/tests/contract/test_tool_registry_smoke_contract.py @@ -12,7 +12,10 @@ os.environ.setdefault("S3_TEMP_PATH", "/tmp") import app.services.document_agent.tools as _tools # noqa: F401 +from app.services.document_agent.manifest import ToolContext from app.services.document_agent.registry import REGISTRY +from app.services.document_agent.state import ProfileBlackboard +from app.services.document_agent.tools.grep_text import grep_text from app.services.document_agent.tools.inspect_pages import inspect_pages @@ -34,5 +37,30 @@ def test_inspect_pages_handler_is_same_callable() -> None: assert spec.handler is inspect_pages +def test_grep_text_normalizes_query_and_corpus_by_default() -> None: + blackboard = ProfileBlackboard(page_count=2) + blackboard.page_full_text_cache = { + 1: "Public\nDomain Manual", + 2: "附录\nA OVERVIEW", + } + ctx = ToolContext( + pdf_path="/tmp/doc.pdf", + job_id="grep-normalization", + blackboard=blackboard, + trace=None, + settings={}, + ) + + result = grep_text( + ctx, + {"query": "附录 A OVERVIEW", "start_page": 2, "end_page": 2}, + ) + + assert result.status == "ok" + assert result.payload["normalized_query"] == "附录a overview" + assert result.payload["hit_page_count"] == 1 + assert result.payload["hit_pages"] == [2] + + def test_openai_specs_removed() -> None: assert not hasattr(REGISTRY, "openai_specs") From 791bcac078e288914f0e2f78e6c268aa43539cfe Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Fri, 21 Aug 2026 22:23:04 +0800 Subject: [PATCH 2/7] refactor: update document agent to use PageTextBands for text extraction - Replaced the read_page_texts function with read_page_text_bands to extract page content, header, and footer as structured PageTextBands. - Updated ProfileCoordinator and related services to accommodate the new text extraction method, ensuring compatibility with existing workflows. - Adjusted the blackboard state to include page_text_search_view and modified tests to validate the new structure and functionality. - Enhanced overall text processing by introducing a more granular approach to handling page content. --- .../document_agent/calibration/procedure.py | 32 +- .../document_agent/calibration/service.py | 3 +- .../services/document_agent/coordinator.py | 28 +- .../app/services/document_agent/pdf_text.py | 216 ++++- .../app/services/document_agent/state.py | 5 +- .../structure/anchoring_primitives.py | 397 ++------- .../structure/hierarchy_locator.py | 11 +- .../structure/null_page_react.py | 766 ++++++++++++++++++ .../document_agent/structure/toc_anchoring.py | 3 +- .../services/document_agent/tools/__init__.py | 1 + .../tools/find_toc_anchor_pages.py | 3 +- .../document_agent/tools/grep_text.py | 14 +- .../document_agent/tools/judge_toc_source.py | 3 +- .../document_agent/tools/ocr_pages.py | 10 +- .../tools/text_strip_margins.py | 132 +++ .../document_parser/profiling/doc_profiler.py | 5 +- .../scripts/page_memory/_debug_pm_shared.py | 9 +- .../debug_pm_stage2_calibration.py | 4 +- .../page_memory/tmp_probe_null_page_leaves.py | 699 +--------------- .../test_doc_profile_anatomy_contract.py | 42 +- .../tests/contract/test_ocr_pages_contract.py | 5 +- .../test_outline_short_circuit_contract.py | 4 +- .../test_structure_anchoring_contract.py | 253 +++--- .../test_tool_registry_smoke_contract.py | 41 +- 24 files changed, 1458 insertions(+), 1228 deletions(-) create mode 100644 apps/worker/app/services/document_agent/structure/null_page_react.py create mode 100644 apps/worker/app/services/document_agent/tools/text_strip_margins.py diff --git a/apps/worker/app/services/document_agent/calibration/procedure.py b/apps/worker/app/services/document_agent/calibration/procedure.py index ff22d105e..7216ebd00 100644 --- a/apps/worker/app/services/document_agent/calibration/procedure.py +++ b/apps/worker/app/services/document_agent/calibration/procedure.py @@ -4,7 +4,7 @@ 1. Builds TitleNodes the same way production does 2. Runs Phase-2 **per regime** (prune → bulk/bisect → recalibrate) 3. Merges physical-page ``match_overrides`` across regimes -4. Runs null-page parent locate once on the combined tree +4. Runs null-page ReAct locate once on the combined tree, then final prune Returns production ``SkeletonAnchor`` plus regime diagnostics for debug payloads. """ @@ -34,10 +34,12 @@ from app.services.document_agent.structure.anchoring_primitives import ( SkeletonAnchor, backfill_parent_offset_matches, - locate_null_page_parent_overrides, prune_unanchored_toc_leaves, serialize_skeleton_anchor, ) +from app.services.document_agent.structure.null_page_react import ( + locate_null_page_node_overrides, +) from app.services.document_agent.structure import anchoring_primitives as _anchoring # Re-export under prior names so existing imports keep working. @@ -350,9 +352,11 @@ def anchor_hierarchy_from_regimes( len(regime_seed), ) - # Failed suffix / never-confirmed printed leaves → drop from TOC tree. + # Failed printed-page leaves → drop; keep null-page nodes for ReAct. working, unanchored_removed = prune_unanchored_toc_leaves( - working, match_overrides=merged + working, + match_overrides=merged, + keep_null_page_nodes=True, ) total_pruned += unanchored_removed if working: @@ -379,7 +383,7 @@ def anchor_hierarchy_from_regimes( len(parent_matches), ) - match_overrides, null_page_report = locate_null_page_parent_overrides( + match_overrides, null_page_report = locate_null_page_node_overrides( nodes=working, match_overrides=merged, page_texts=page_texts, @@ -387,6 +391,24 @@ def anchor_hierarchy_from_regimes( ctx=ctx, ) + # Drop still-unanchored null-page nodes (avoid sticky inherited ranges). + working, failed_null_removed = prune_unanchored_toc_leaves( + working, + match_overrides=match_overrides, + keep_null_page_nodes=False, + ) + total_pruned += failed_null_removed + if working: + surviving_paths = { + path + for path, _node in _iter_all_title_nodes(working) + } + match_overrides = { + path: match + for path, match in match_overrides.items() + if path in surviving_paths + } + primary = pick_primary_offset(result) if primary is None and usable_regimes: primary = int(usable_regimes[0].offset) # type: ignore[arg-type] diff --git a/apps/worker/app/services/document_agent/calibration/service.py b/apps/worker/app/services/document_agent/calibration/service.py index 39a609d77..7c3bae11f 100644 --- a/apps/worker/app/services/document_agent/calibration/service.py +++ b/apps/worker/app/services/document_agent/calibration/service.py @@ -16,6 +16,7 @@ CalibrationResult, ) from app.services.document_agent.manifest import ToolContext +from app.services.document_agent.pdf_text import page_bands_map def calibrate_offset( @@ -44,7 +45,7 @@ def calibrate_offset( if page_count and not ctx.blackboard.page_count: ctx.blackboard.page_count = int(page_count) if page_texts and not ctx.blackboard.page_full_text_cache: - ctx.blackboard.page_full_text_cache = dict(page_texts) + ctx.blackboard.page_full_text_cache = page_bands_map(page_texts) try: phase1 = run_calibration_phase1( diff --git a/apps/worker/app/services/document_agent/coordinator.py b/apps/worker/app/services/document_agent/coordinator.py index f9a43b15d..7cb899279 100644 --- a/apps/worker/app/services/document_agent/coordinator.py +++ b/apps/worker/app/services/document_agent/coordinator.py @@ -20,7 +20,7 @@ ToolContext, ToolResult, ) -from app.services.document_agent.pdf_text import read_page_texts +from app.services.document_agent.pdf_text import read_page_text_bands from app.services.document_agent.persist import build_anatomy_map, persist_anatomy_map from app.services.document_agent.coarse_profile import CoarseProfiler from app.services.document_agent.registry import REGISTRY @@ -297,6 +297,7 @@ def _run_text_scan(self) -> None: pages = list(range(1, page_count + 1)) if not pages: self.blackboard.page_full_text_cache = {} + self.blackboard.page_text_search_view = None return if profile.is_scanned: result = REGISTRY.dispatch("ocr.pages", self.ctx, {"pages": pages}) @@ -312,19 +313,30 @@ def _run_text_scan(self) -> None: raise RuntimeError(result.error or "ocr.pages failed") self.round_index += 1 return - texts = read_page_texts(self.ctx.pdf_path, pages, timeout=300) - self.blackboard.page_full_text_cache = texts + bands = read_page_text_bands( + self.ctx.pdf_path, + pages, + header_y=profile.header_y, + footer_y=profile.footer_y, + timeout=300, + ) + self.blackboard.page_full_text_cache = bands + self.blackboard.page_text_search_view = None self.trace.record_step( round_index=self.round_index, - actor="scan:read_page_texts", + actor="scan:read_page_text_bands", action_type="scan", result=ToolResult( status="ok", - payload={"page_count": len(texts)}, - output_summary={"page_count": len(texts)}, + payload={"page_count": len(bands)}, + output_summary={"page_count": len(bands)}, ), - tool_name="read_page_texts", - tool_args={"pages": pages}, + tool_name="read_page_text_bands", + tool_args={ + "pages": pages, + "header_y": profile.header_y, + "footer_y": profile.footer_y, + }, ) self.round_index += 1 diff --git a/apps/worker/app/services/document_agent/pdf_text.py b/apps/worker/app/services/document_agent/pdf_text.py index f3851058f..202e2417f 100644 --- a/apps/worker/app/services/document_agent/pdf_text.py +++ b/apps/worker/app/services/document_agent/pdf_text.py @@ -3,6 +3,7 @@ from __future__ import annotations import gc +from dataclasses import dataclass from typing import Any from app.services.document_parser.formats.pdf.pymupdf_subprocess import ( @@ -12,6 +13,151 @@ from app.services.document_parser.structure.body_boundary import normalize_heading_label +@dataclass(frozen=True) +class PageTextBands: + """Per-page text from one span pass: full content plus edge-band extracts. + + ``content`` is the full page text (includes header/footer span text). + ``header`` / ``footer`` are the same spans whose vertical centers fall in the + coarse ``header_y`` / ``footer_y`` bands. Strip tools remove those extracts + from a search view; they do not rewrite the stored ``content``. + """ + + content: str + header: str = "" + footer: str = "" + + def to_dict(self) -> dict[str, str]: + return { + "content": self.content, + "header": self.header, + "footer": self.footer, + } + + @classmethod + def from_any(cls, value: Any) -> "PageTextBands": + if isinstance(value, PageTextBands): + return value + if isinstance(value, str): + return cls(content=value) + if isinstance(value, dict): + content = value.get("content") + if content is None and "text" in value: + content = value.get("text") + return cls( + content=str(content or ""), + header=str(value.get("header") or ""), + footer=str(value.get("footer") or ""), + ) + return cls(content=str(value or "")) + + +def page_content(value: Any) -> str: + return PageTextBands.from_any(value).content + + +def page_content_map(raw: Any) -> dict[int, str]: + """Map page -> content string (legacy-compatible plain cache shape).""" + if not isinstance(raw, dict): + return {} + out: dict[int, str] = {} + for page, value in raw.items(): + out[int(page)] = page_content(value) + return out + + +def page_bands_map(raw: Any) -> dict[int, PageTextBands]: + if not isinstance(raw, dict): + return {} + return {int(page): PageTextBands.from_any(value) for page, value in raw.items()} + + +def strip_margin_text(content: str, margin: str) -> str: + """Remove one margin extract from full content (homologous span join). + + Tries the full margin blob first, then each non-empty line once, so + non-contiguous edge lines still drop when they appear as content lines. + """ + if not content or not margin: + return content + if margin in content: + return content.replace(margin, "", 1) + out = content + for frag in margin.split("\n"): + if frag and frag in out: + out = out.replace(frag, "", 1) + return out + + +def _band_for_center( + cy: float, + *, + page_h: float, + header_y: float | None, + footer_y: float | None, +) -> str: + if page_h <= 0: + return "content" + if header_y is not None and cy < float(header_y) * page_h: + return "header" + if footer_y is not None and cy > float(footer_y) * page_h: + return "footer" + return "content" + + +def _extract_page_bands_from_pymupdf_page( + page: Any, + *, + header_y: float | None, + footer_y: float | None, +) -> PageTextBands: + """Build content/header/footer from the same span walk (line-join with ``\\n``).""" + page_h = float(getattr(page.rect, "height", 0) or 0) + data = page.get_text("dict") or {} + content_lines: list[str] = [] + header_chunks: list[str] = [] + footer_chunks: list[str] = [] + + for block in data.get("blocks") or []: + if int(block.get("type") or 0) != 0: + continue + for line in block.get("lines") or []: + line_content_parts: list[str] = [] + line_header_parts: list[str] = [] + line_footer_parts: list[str] = [] + for span in line.get("spans") or []: + text = str(span.get("text") or "") + if not text: + continue + bbox = span.get("bbox") or (0, 0, 0, 0) + try: + y0 = float(bbox[1]) + y1 = float(bbox[3]) + except (TypeError, ValueError, IndexError): + y0, y1 = 0.0, 0.0 + cy = (y0 + y1) / 2.0 + band = _band_for_center( + cy, page_h=page_h, header_y=header_y, footer_y=footer_y + ) + line_content_parts.append(text) + if band == "header": + line_header_parts.append(text) + elif band == "footer": + line_footer_parts.append(text) + if line_content_parts: + content_lines.append("".join(line_content_parts)) + if line_header_parts: + header_chunks.append("".join(line_header_parts)) + if line_footer_parts: + footer_chunks.append("".join(line_footer_parts)) + + return PageTextBands( + content="\n".join(content_lines), + header="\n".join(header_chunks), + footer="\n".join(footer_chunks), + ) + + @worker def _read_page_texts_worker(queue, pdf_path: str, pages: list[int]) -> None: import pymupdf # type: ignore[import] @@ -32,10 +178,40 @@ def _read_page_texts_worker(queue, pdf_path: str, pages: list[int]) -> None: queue.put({"ok": True, "texts": texts}) +@worker +def _read_page_text_bands_worker( + queue, + pdf_path: str, + pages: list[int], + header_y: float | None, + footer_y: float | None, +) -> None: + import pymupdf # type: ignore[import] + + bands: dict[int, dict[str, str]] = {} + try: + doc = pymupdf.open(pdf_path) + for page in pages: + idx = page - 1 + if 0 <= idx < doc.page_count: + record = _extract_page_bands_from_pymupdf_page( + doc[idx], + header_y=header_y, + footer_y=footer_y, + ) + bands[page] = record.to_dict() + finally: + try: + doc.close() + except Exception: + pass + gc.collect() + queue.put({"ok": True, "bands": bands}) + + def coerce_page_text_cache(raw: Any) -> dict[int, str]: - if not isinstance(raw, dict): - return {} - return {int(page): str(text) for page, text in raw.items()} + """Legacy helper: page -> content string only.""" + return page_content_map(raw) def read_page_texts( @@ -44,12 +220,44 @@ def read_page_texts( *, timeout: int = 180, ) -> dict[int, str]: + """Plain ``get_text()`` map for call sites that need a one-shot full dump. + + PROFILE text-scan uses :func:`read_page_text_bands` instead. + """ if not pages: return {} - result = run_in_child_process(_read_page_texts_worker, pdf_path, pages, timeout=timeout) + result = run_in_child_process( + _read_page_texts_worker, pdf_path, pages, timeout=timeout + ) return {int(k): str(v) for k, v in (result.get("texts") or {}).items()} +def read_page_text_bands( + pdf_path: str, + pages: list[int], + *, + header_y: float | None = None, + footer_y: float | None = None, + timeout: int = 180, +) -> dict[int, PageTextBands]: + """Span-homogeneous content/header/footer for PROFILE text scan.""" + if not pages: + return {} + result = run_in_child_process( + _read_page_text_bands_worker, + pdf_path, + pages, + header_y, + footer_y, + timeout=timeout, + ) + raw_bands = result.get("bands") or {} + return { + int(page): PageTextBands.from_any(value) + for page, value in raw_bands.items() + } + + def meaningful_lines(text: str) -> list[str]: lines = [normalize_heading_label(line) for line in text.splitlines()] return [line for line in lines if line] diff --git a/apps/worker/app/services/document_agent/state.py b/apps/worker/app/services/document_agent/state.py index 67b434022..8f1e703b6 100644 --- a/apps/worker/app/services/document_agent/state.py +++ b/apps/worker/app/services/document_agent/state.py @@ -42,5 +42,8 @@ class ProfileBlackboard: shard_plan: ShardPlan | None = None validation_report: dict[str, Any] | None = None verdict: ProfileVerdict | None = None - page_full_text_cache: dict[int, str] = field(default_factory=dict) + # Values are PageTextBands (or legacy plain str / {"content","header","footer"}). + page_full_text_cache: dict[int, Any] = field(default_factory=dict) + # Optional temporary grep view after text.strip_*; None means use each page's content. + page_text_search_view: dict[int, str] | None = None global_signals: dict[str, Any] = field(default_factory=dict) diff --git a/apps/worker/app/services/document_agent/structure/anchoring_primitives.py b/apps/worker/app/services/document_agent/structure/anchoring_primitives.py index f7d514f2a..abb6948b6 100644 --- a/apps/worker/app/services/document_agent/structure/anchoring_primitives.py +++ b/apps/worker/app/services/document_agent/structure/anchoring_primitives.py @@ -12,10 +12,10 @@ from app.services.document_agent.structure.hierarchy_locator import ( TitleMatch, TitleNode, - first_leaf_start_under, iter_leaf_title_nodes, - last_leaf_start_under, - locate_title_normalized_strict, +) +from app.services.document_agent.structure.null_page_react import ( + locate_null_page_node_overrides, ) from app.services.document_agent.structure.section_page_verify import ( verify_section_page_choice, @@ -23,17 +23,6 @@ from loguru import logger -def _first_sibling_null_parent_scan_start(right: int) -> int: - """Left edge for first-at-level null parents: at most one 2+4+6+10 budget. - - Does not inherit a wider parent/body scope. Floors at document page 1. - """ - from app.services.document_agent.calibration.scan import DEFAULT_WINDOW_SCHEDULE - - budget = sum(DEFAULT_WINDOW_SCHEDULE) - return max(1, int(right) - budget + 1) - - def prune_out_of_scope_nodes( nodes: list[TitleNode], *, @@ -88,16 +77,26 @@ def prune_unanchored_toc_leaves( nodes: list[TitleNode], *, match_overrides: dict[tuple[str, ...], TitleMatch], + keep_null_page_nodes: bool = False, ) -> tuple[list[TitleNode], int]: - """Remove TOC leaves that have no physical ``match_overrides`` entry. + """Remove TOC nodes that have no physical ``match_overrides`` entry. Implements Phase-2 ``suffix = no TOC``: after bulk/bisect/recalibrate, any leaf that was not successfully anchored is dropped from the coarse tree instead of sticky ``inherited_unlocated`` ranges. Childless parents are removed unless they themselves have an override. + + When ``keep_null_page_nodes`` is True (pre null-page ReAct), nodes with + ``printed_page is None`` are retained so they can be probed. Call again with + the default after locate to drop still-unanchored null-page nodes. """ removed = 0 + def _keep(path: tuple[str, ...], node: TitleNode) -> bool: + if path in match_overrides: + return True + return keep_null_page_nodes and node.printed_page is None + def _prune( node: TitleNode, parent_titles: tuple[str, ...] ) -> TitleNode | None: @@ -111,11 +110,11 @@ def _prune( children.append(kept) if children: return replace(node, children=children) - if path in match_overrides: + if _keep(path, node): return replace(node, children=[]) removed += 1 return None - if path in match_overrides: + if _keep(path, node): return node removed += 1 return None @@ -129,8 +128,9 @@ def _prune( if removed: logger.info( "[structure_anchoring] pruned {} unanchored TOC nodes " - "(suffix / no match_overrides → no TOC)", + "(keep_null_page_nodes={} → no TOC)", removed, + keep_null_page_nodes, ) return out, removed @@ -155,320 +155,8 @@ def toc_range_end(hierarchy: dict[str, Any]) -> int | None: return None -# ── Null-page parent locate (sibling window / first-sibling scan) ─────────── - - -def locate_null_page_parent_overrides( - *, - nodes: list[TitleNode], - match_overrides: dict[tuple[str, ...], TitleMatch], - page_texts: dict[int, str], - body_pages: list[int], - ctx: ToolContext | None, -) -> tuple[dict[tuple[str, ...], TitleMatch], list[dict[str, Any]]]: - """Locate TOC parents with ``printed_page=None`` into ``match_overrides``. - - Window for parent P with a previous same-level sibling: ``[last leaf under - that sibling, first leaf under P]``; text then RTL visual verify. - - First-at-level parents (no left sibling) do **not** inherit a wider parent - or body scope. Left edge is one Phase-1 ``2+4+6+10`` budget before the first - child (floor page 1). Text runs in that window; on miss, reuse - ``scan_title_forward`` (same schedule, early exit). Miss → unresolved. - - Returns ``(overrides, report)`` where *report* lists every null-page parent - attempt (for debug / LLM-call accounting). - """ - if not nodes or not body_pages: - return dict(match_overrides), [] - - out = dict(match_overrides) - body_set = set(body_pages) - parent_scope_start = body_pages[0] - report: list[dict[str, Any]] = [] - - def walk( - sibling_nodes: list[TitleNode], - parent_titles: tuple[str, ...], - scope_start: int, - ) -> None: - for index, node in enumerate(sibling_nodes): - path_titles = (*parent_titles, node.title) - if ( - node.children - and node.printed_page is None - and path_titles not in out - ): - right = first_leaf_start_under(node, parent_titles, out) - entry: dict[str, Any] = { - "path_titles": list(path_titles), - "title": node.title, - "printed_page": None, - "window": None, - "result": "skipped_no_right", - "page": None, - "accept": None, - "visual_verify_calls": 0, - } - if right is None: - report.append(entry) - logger.info( - "[structure_anchoring] null-page parent skipped: " - "title={!r} reason=no_located_first_child", - node.title, - ) - elif index > 0: - left = last_leaf_start_under( - sibling_nodes[index - 1], parent_titles, out - ) - if left is None: - left = scope_start - if right < left: - report.append(entry) - logger.info( - "[structure_anchoring] null-page parent skipped: " - "title={!r} reason=no_located_first_child left={}", - node.title, - left, - ) - else: - _resolve_null_parent_with_sibling_window( - path_titles=path_titles, - title=node.title, - left=left, - right=right, - body_pages=body_pages, - body_set=body_set, - page_texts=page_texts, - ctx=ctx, - out=out, - entry=entry, - report=report, - ) - else: - left = _first_sibling_null_parent_scan_start(right) - _resolve_null_parent_first_sibling( - path_titles=path_titles, - title=node.title, - left=left, - right=right, - body_pages=body_pages, - body_set=body_set, - page_texts=page_texts, - ctx=ctx, - out=out, - entry=entry, - report=report, - ) - if node.children: - child_scope_start = ( - out[path_titles].page if path_titles in out else scope_start - ) - walk(node.children, path_titles, child_scope_start) - - walk(nodes, (), parent_scope_start) - logger.info( - "[structure_anchoring] null-page parent locate summary: " - "attempted={} located={} unresolved={} visual_verify_calls={}", - len(report), - sum(1 for row in report if row.get("page") is not None), - sum(1 for row in report if row.get("result") == "unresolved"), - sum(int(row.get("visual_verify_calls") or 0) for row in report), - ) - return out, report - - -def _record_null_parent_outcome( - *, - path_titles: tuple[str, ...], - title: str, - left: int, - right: int, - match: TitleMatch | None, - visual_calls: int, - body_set: set[int], - out: dict[tuple[str, ...], TitleMatch], - entry: dict[str, Any], - report: list[dict[str, Any]], -) -> None: - entry["window"] = [left, right] - entry["visual_verify_calls"] = visual_calls - if match is not None and match.page in body_set: - out[path_titles] = match - entry["result"] = str(match.evidence.get("accept") or match.source) - entry["page"] = match.page - entry["accept"] = match.evidence.get("accept") - logger.info( - "[structure_anchoring] null-page parent located: " - "title={!r} page={} window={} accept={} visual_calls={}", - title, - match.page, - [left, right], - match.evidence.get("accept"), - visual_calls, - ) - else: - entry["result"] = "unresolved" - logger.info( - "[structure_anchoring] null-page parent unresolved: " - "title={!r} window={} visual_calls={}", - title, - [left, right], - visual_calls, - ) - report.append(entry) - - -def _resolve_null_parent_with_sibling_window( - *, - path_titles: tuple[str, ...], - title: str, - left: int, - right: int, - body_pages: list[int], - body_set: set[int], - page_texts: dict[int, str], - ctx: ToolContext | None, - out: dict[tuple[str, ...], TitleMatch], - entry: dict[str, Any], - report: list[dict[str, Any]], -) -> None: - scope_pages = [page for page in body_pages if left <= page <= right] - match = locate_title_normalized_strict( - title, - scope_pages=scope_pages, - page_texts=page_texts, - ) - visual_calls = 0 - if match is None and ctx is not None: - match, visual_calls = _visual_rtl_locate_parent( - title=title, - left=left, - right=right, - body_set=body_set, - ctx=ctx, - ) - _record_null_parent_outcome( - path_titles=path_titles, - title=title, - left=left, - right=right, - match=match, - visual_calls=visual_calls, - body_set=body_set, - out=out, - entry=entry, - report=report, - ) - - -def _resolve_null_parent_first_sibling( - *, - path_titles: tuple[str, ...], - title: str, - left: int, - right: int, - body_pages: list[int], - body_set: set[int], - page_texts: dict[int, str], - ctx: ToolContext | None, - out: dict[tuple[str, ...], TitleMatch], - entry: dict[str, Any], - report: list[dict[str, Any]], -) -> None: - """First-at-level null parent: capped text window, then ``scan_title_forward``.""" - from app.services.document_agent.calibration.scan import ( - DEFAULT_WINDOW_SCHEDULE, - scan_title_forward, - ) - - scope_pages = [page for page in body_pages if left <= page <= right] - match = locate_title_normalized_strict( - title, - scope_pages=scope_pages, - page_texts=page_texts, - ) - visual_calls = 0 - if match is None and ctx is not None: - scan = scan_title_forward( - ctx=ctx, - title=title, - start_page=left, - page_count=right, - window_schedule=DEFAULT_WINDOW_SCHEDULE, - ) - visual_calls = len(scan.scanned_pages) - if scan.found and scan.found_page is not None: - match = TitleMatch( - page=int(scan.found_page), - source="inspect_vlm", - matched_line="", - candidates=[int(scan.found_page)], - evidence={ - "accept": "scan_forward", - "null_page_parent_probe": True, - "scanned_pages": list(scan.scanned_pages), - }, - ) - _record_null_parent_outcome( - path_titles=path_titles, - title=title, - left=left, - right=right, - match=match, - visual_calls=visual_calls, - body_set=body_set, - out=out, - entry=entry, - report=report, - ) - - -def _visual_rtl_locate_parent( - *, - title: str, - left: int, - right: int, - body_set: set[int], - ctx: ToolContext, -) -> tuple[TitleMatch | None, int]: - """Confirm parent title from right boundary toward left via VLM verify.""" - visual_calls = 0 - for page in range(right, left - 1, -1): - if page not in body_set: - continue - candidate = TitleMatch( - page=page, - source="inspect_vlm", - matched_line="", - candidates=[page], - evidence={"null_page_parent_probe": True}, - ) - visual_calls += 1 - result = verify_section_page_choice( - ctx=ctx, - title=title, - candidate_matches=[candidate], - candidate_page_cap=1, - ) - selected = result.get("selected_page") - if selected != page: - continue - return ( - TitleMatch( - page=page, - source="inspect_vlm", - matched_line="", - candidates=[page], - evidence={ - "accept": "visual_rtl", - "reason": result.get("reason", ""), - "visual_verify_calls": visual_calls, - }, - ), - visual_calls, - ) - return None, visual_calls +# Null-page locate lives in ``null_page_react.locate_null_page_node_overrides``. +# Imported above for ``anchor_hierarchy_from_offset``. # ── Offset-guided bulk anchoring with recursive recalibrate (Phase-2) ─────── @@ -980,6 +668,34 @@ def deserialize_skeleton_anchor(data: dict[str, Any]) -> SkeletonAnchor: ) +def _iter_all_title_nodes( + nodes: list[TitleNode], + *, + parent_titles: tuple[str, ...] = (), +) -> list[tuple[tuple[str, ...], TitleNode]]: + rows: list[tuple[tuple[str, ...], TitleNode]] = [] + for node in nodes: + path = (*parent_titles, node.title) + rows.append((path, node)) + if node.children: + rows.extend(_iter_all_title_nodes(node.children, parent_titles=path)) + return rows + + +def _filter_overrides_to_tree( + nodes: list[TitleNode], + match_overrides: dict[tuple[str, ...], TitleMatch], +) -> dict[tuple[str, ...], TitleMatch]: + if not nodes: + return {} + surviving = {path for path, _node in _iter_all_title_nodes(nodes)} + return { + path: match + for path, match in match_overrides.items() + if path in surviving + } + + def anchor_hierarchy_from_offset( *, nodes: list[TitleNode], @@ -990,7 +706,7 @@ def anchor_hierarchy_from_offset( page_count: int, ctx: ToolContext | None, ) -> tuple[list[TitleNode], SkeletonAnchor]: - """Production prune → bulk → null-page given a precomputed offset. + """Production prune → bulk → null-page ReAct → final prune. Phase-2 entry after ``calibrate_offset`` (Phase-1). """ @@ -1022,11 +738,14 @@ def anchor_hierarchy_from_offset( bulk_count = 0 working, unanchored_removed = prune_unanchored_toc_leaves( - working, match_overrides=match_overrides + working, + match_overrides=match_overrides, + keep_null_page_nodes=True, ) pruned_count += unanchored_removed + match_overrides = _filter_overrides_to_tree(working, match_overrides) - match_overrides, null_page_report = locate_null_page_parent_overrides( + match_overrides, null_page_report = locate_null_page_node_overrides( nodes=working, match_overrides=match_overrides, page_texts=page_texts, @@ -1034,6 +753,14 @@ def anchor_hierarchy_from_offset( ctx=ctx, ) + working, failed_null_removed = prune_unanchored_toc_leaves( + working, + match_overrides=match_overrides, + keep_null_page_nodes=False, + ) + pruned_count += failed_null_removed + match_overrides = _filter_overrides_to_tree(working, match_overrides) + if offset_hint is None: offset_status = "failed" if ctx is not None else "skipped" else: diff --git a/apps/worker/app/services/document_agent/structure/hierarchy_locator.py b/apps/worker/app/services/document_agent/structure/hierarchy_locator.py index 694415205..ab7fef614 100644 --- a/apps/worker/app/services/document_agent/structure/hierarchy_locator.py +++ b/apps/worker/app/services/document_agent/structure/hierarchy_locator.py @@ -1,10 +1,10 @@ """Locate hierarchy titles on PDF pages and resolve page ranges. Deterministic range assembly from PROFILE ``match_overrides``. Leaf starts -come only from those overrides. Null-page parents are located upstream via -normalized-strict text matching + optional VLM, then resolved here including -parent self-only spans for interstitial pages. Parents without an override -may still inherit start from the earliest located descendant leaf. +come only from those overrides. Null-page parents and leaves are located +upstream via bounded grep ReAct + VLM (``null_page_react``), then resolved +here including parent self-only spans for interstitial pages. Parents without +an override may still inherit start from the earliest located descendant leaf. """ from __future__ import annotations @@ -25,6 +25,7 @@ "inspect_vlm", "inferred_descendant", "pdf_outline", + "react_normalized_grep_vlm", ] @@ -404,7 +405,7 @@ def _locate_match_for_node( if match is not None: return match if node.children: - # Parent active locate is upstream (normalized-strict / visual). + # Parent active locate is upstream (null_page_react / backfill). return _infer_start_from_descendant_overrides( node, parent_titles=path_titles[:-1], match_overrides=match_overrides, scope_pages=scope_pages, diff --git a/apps/worker/app/services/document_agent/structure/null_page_react.py b/apps/worker/app/services/document_agent/structure/null_page_react.py new file mode 100644 index 000000000..53b95dea2 --- /dev/null +++ b/apps/worker/app/services/document_agent/structure/null_page_react.py @@ -0,0 +1,766 @@ +"""Bounded ReAct locate for TOC nodes with ``printed_page=None``. + +After Phase-2 printed-page bulk/bisect, null-page parents and leaves share one +serial probe: text LLM plans a ``grep.text`` query inside a sibling window, +then ``inspect.pages`` confirms the physical section start one hit page at a +time. Loop / hit / visual budgets equal ``BOUNDARY_STEP_PAGES``. + +No offset seed, no fixed page-count cap, and no fallback to normalized-strict +unique hit / RTL / ``scan_title_forward``. +""" + +from __future__ import annotations + +import json +from typing import Any, cast + +from loguru import logger + +from app.services.document_agent.manifest import ToolContext +from app.services.document_agent.structure.hierarchy_locator import ( + TitleMatch, + TitleNode, + first_leaf_start_under, + last_leaf_start_under, +) + +_HISTORY_SAMPLE_PAGES = 3 + + +def react_budget() -> int: + """Loop / hit / visual budget; same constant as TOC ``BOUNDARY_STEP_PAGES``.""" + from app.services.document_agent.tools.extract_toc_with_boundaries import ( + BOUNDARY_STEP_PAGES, + ) + + return int(BOUNDARY_STEP_PAGES) + +_REACT_INSTRUCTIONS = """\ +You are the search planner in a small ReAct loop. Propose the next action to +find the physical START page of a section. Grep collapses whitespace/newlines +to one space between non-CJK words, removes whitespace adjacent to CJK, and +matches case-insensitively. A separate visual check confirms candidates one +page at a time. + +The system already grepped the full TOC title once before this loop (see +previous_attempts). Do not repeat that exact full-title query. + +Return one strict json object with action one of (no other keys): +{"action":"grep","query":"..."} +{"action":"strip_header","query":""} +{"action":"strip_footer","query":""} +{"action":"give_up","query":""} + +Do not include a reason field. Use give_up only when no useful untried query +or strip remains. + +Ordered query strategy after the automatic full-title grep (follow this order; +skip a step only if already tried or not applicable to the TOC title / parent +path). Pattern-level only — do not invent document-specific titles: +1. Remove the leading number / letter / punctuation prefix from the TOC title + and grep the remaining title body. +2. When the parent path indicates appendices/annexes (or the TOC label is a + lettered appendix-style entry): grep the structural form + "Appendix " using the letter taken from the TOC label. Prefer this + before inventing other phrases. +3. Only after the above: try other variants such as "Appendix " plus + the title body, a shorter distinctive fragment of the title, or another + structural prefix (chapter / part / section / annex) when supported by the + title or parent path. +4. Prefer queries specific enough to avoid running headers and passing mentions. + Do not guess page numbers. + +Reflection rules (mandatory): +- Read previous_attempts. Reflect on hit_count and observation before answering. +- If the last observation is no_normalized_hits, too_many_hits, visual_rejected, + or duplicate_normalized_query, you MUST change the query when choosing grep. + Emitting the same grep query again (same text after whitespace/case + normalization) is invalid for planner-chosen greps. +- too_many_hits means hit_count exceeded the visual budget. Prefer + strip_header or strip_footer when hits look scattered by running + headers/footers. Each strip automatically re-greps the last query once + (same action; do not spend a planner turn to repeat that query). Otherwise + go to the next step in the ordered strategy (narrower / different query). +- strip_header / strip_footer only update a temporary search view; they do not + change stored page text. They do NOT consume react_budget. Call each at most + once per locate. +- Planner greps consume react_budget (grep_loops_remaining). Strip auto-retries + never consume it. +- no_normalized_hits / visual_rejected: advance to the next ordered strategy + step rather than repeating the same query. +""" + + +def _react_history_item(item: dict[str, Any]) -> dict[str, Any]: + hit_pages = [int(page) for page in (item.get("hit_pages") or [])] + out = { + "query": item.get("query"), + "normalized_query": item.get("normalized_query"), + "hit_count": int(item.get("hit_count") or len(hit_pages)), + "sample_pages": hit_pages[:_HISTORY_SAMPLE_PAGES], + "observation": item.get("observation"), + "visual_selected_page": item.get("visual_selected_page"), + "visual_reason": item.get("visual_reason"), + "visual_pages_checked": item.get("visual_pages_checked"), + } + if item.get("seed_full_title"): + out["seed_full_title"] = True + if item.get("post_strip"): + out["post_strip"] = item.get("post_strip") + return out + + +def _next_located_bound( + *, + sibling_nodes: list[TitleNode], + index: int, + parent_titles: tuple[str, ...], + overrides: dict[tuple[str, ...], TitleMatch], +) -> int | None: + for later in sibling_nodes[index + 1 :]: + path = (*parent_titles, later.title) + if path in overrides: + return int(overrides[path].page) + bound = first_leaf_start_under(later, parent_titles, overrides) + if bound is not None: + return int(bound) + return None + + +def _normalized_grep( + *, + ctx: ToolContext, + query: str, + left: int, + right: int, +) -> tuple[str, list[int], int]: + from app.services.document_agent.tools.grep_text import grep_text + + result = grep_text( + ctx, + { + "query": query, + "start_page": left, + "end_page": right, + }, + ) + if result.status != "ok": + return "", [], 0 + payload = result.payload or {} + return ( + str(payload.get("normalized_query") or ""), + [int(page) for page in (payload.get("hit_pages") or [])], + int(payload.get("hit_count") or 0), + ) + + +def _propose_react_query( + *, + title: str, + parent_titles: tuple[str, ...], + left: int, + right: int, + attempts: list[dict[str, Any]], + budget: int, + grep_loops_used: int, +) -> tuple[dict[str, Any] | None, dict[str, Any]]: + state = { + "toc_title": title, + "parent_path": list(parent_titles), + "physical_search_scope": [left, right], + "react_budget": budget, + "grep_loops_used": grep_loops_used, + "grep_loops_remaining": max(0, budget - grep_loops_used), + "visual_budget": budget, + "previous_attempts": [_react_history_item(item) for item in attempts], + "note": ( + "Full TOC title was already grepped automatically before this loop. " + "Planner greps consume react_budget. strip_header/strip_footer are " + "free and each auto-retries the last grep query once." + ), + } + prompt = ( + f"{_REACT_INSTRUCTIONS}\n\nCurrent state:\n" + f"{json.dumps(state, ensure_ascii=False)}" + ) + + try: + from shared.services.ai.llm_overrides import get_text_client + + client, model = get_text_client() + raw, usage = client.chat_completion_with_usage( + messages=cast(Any, [{"role": "user", "content": prompt}]), + model=model, + temperature=0.0, + max_tokens=120, + response_format={"type": "json_object"}, + usage_task="document_agent.null_page_title_react", + ) + payload = json.loads(raw) if raw else {} + except Exception as exc: + logger.warning("[null_page_react] planner failed for {!r}: {}", title, exc) + return None, {"error": f"planner failed: {exc}"} + + action = str(payload.get("action") or "").strip().lower() + query = str(payload.get("query") or "").strip() + if action not in {"grep", "give_up", "strip_header", "strip_footer"}: + return None, { + "error": f"unknown planner action: {action!r}", + "usage": usage, + } + if action == "grep" and not query: + return None, {"error": "planner returned empty grep query", "usage": usage} + return ( + { + "action": action, + "query": query, + }, + {"usage": usage}, + ) + + +def _verify_section_beginning_page( + *, + ctx: ToolContext, + title: str, + page: int, + query: str, +) -> tuple[bool, str, int]: + """Confirm one physical page as the section beginning.""" + from app.services.document_agent.calibration.prompts import ( + coerce_found, + coerce_found_page, + ) + from app.services.document_agent.tools.inspect_pages import inspect_pages + + question = ( + f"Does this page mark the physical BEGINNING of the document section " + f"corresponding to the TOC entry {title!r}? A cover page, section " + "title page, or first body-heading page can be the beginning. Allow " + "equivalent wording and added or omitted numbering, lettering, or " + "structural prefixes. Do not accept a table-of-contents line, running " + "header or footer, passing mention, or continuation page. " + f"The normalized text query that nominated this page was {query!r}. " + "Report the physical page number printed in the page label above the image." + ) + verify_result = inspect_pages( + ctx, + { + "pages": [page], + "page_cap": 1, + "question": question, + "answer_keys": { + "found": ( + "boolean, true only when this page is the physical beginning " + "of the requested section" + ), + "found_page": ( + "number|null, the physical page number where the section begins" + ), + }, + "folder_name": "null_page_react_verify", + "prefix": "verify", + "usage_task": "document_agent.null_page_react_verify", + }, + ) + tokens = int(verify_result.tokens_used or 0) + if verify_result.status != "ok": + return False, str(verify_result.error or "inspect.pages failed"), tokens + fields = (verify_result.payload or {}).get("fields") or {} + found_page = coerce_found_page(fields.get("found_page"), pages=[page]) + ok = coerce_found(fields.get("found")) and found_page == page + reason = str((verify_result.payload or {}).get("answer") or "") + return ok, reason, tokens + + +def _locate_with_react( + *, + path_titles: tuple[str, ...], + title: str, + left: int, + right: int, + ctx: ToolContext, +) -> tuple[TitleMatch | None, list[dict[str, Any]], int, str]: + budget = react_budget() + attempts: list[dict[str, Any]] = [] + attempted_needles: set[str] = set() + visual_calls = 0 + visual_remaining = budget + stripped: set[str] = set() + last_grep_query: str | None = None + # Each locate starts from stored content (no cross-node strip leakage). + ctx.blackboard.page_text_search_view = None + # Automatic full-title grep is free (no planner). Planner greps consume + # budget. strip_* (+ auto same-query re-grep) is free. + grep_loops_used = 0 + planner_turn = 0 + max_planner_turns = budget + 2 + budget + + def _visual_confirm( + *, + query: str, + hit_pages: list[int], + attempt: dict[str, Any], + ) -> TitleMatch | None: + nonlocal visual_calls, visual_remaining + if visual_remaining <= 0: + attempt["observation"] = "visual_budget_exhausted" + attempts.append(attempt) + return None + + checked: list[dict[str, Any]] = [] + selected: int | None = None + last_reason = "" + for page in hit_pages: + if visual_remaining <= 0: + break + visual_remaining -= 1 + visual_calls += 1 + ok, reason, tokens = _verify_section_beginning_page( + ctx=ctx, + title=title, + page=page, + query=query, + ) + checked.append( + { + "page": page, + "confirmed": ok, + "reason": reason, + "tokens_used": tokens, + } + ) + last_reason = reason + if ok: + selected = page + break + + attempt["visual_pages_checked"] = checked + attempt["visual_selected_page"] = selected + attempt["visual_reason"] = last_reason + attempt["visual_budget_remaining_after"] = visual_remaining + if selected is not None: + attempt["observation"] = "section_start_confirmed" + attempts.append(attempt) + return TitleMatch( + page=int(selected), + source="react_normalized_grep_vlm", + matched_line=query, + candidates=hit_pages, + evidence={ + "accept": "react_normalized_grep_vlm", + "null_page_react": True, + "loop": grep_loops_used, + "normalized_query": attempt.get("normalized_query"), + "visual_reason": last_reason, + "visual_pages_checked": [item["page"] for item in checked], + "post_strip": attempt.get("post_strip"), + "seed_full_title": attempt.get("seed_full_title"), + }, + ) + + if visual_remaining <= 0 and len(checked) < len(hit_pages): + attempt["observation"] = "visual_budget_exhausted" + else: + attempt["observation"] = "visual_rejected" + attempts.append(attempt) + return None + + def _apply_grep_result( + *, + query: str, + planner_turn_index: int, + consume_budget: bool, + allow_duplicate: bool, + post_strip: str | None, + planner_meta: dict[str, Any], + seed_full_title: bool = False, + ) -> TitleMatch | None: + """Grep + classify. Appends to attempts; returns match on visual confirm.""" + nonlocal grep_loops_used, last_grep_query + + needle, hit_pages, match_count = _normalized_grep( + ctx=ctx, + query=query, + left=left, + right=right, + ) + if consume_budget and needle and needle not in attempted_needles: + grep_loops_used += 1 + attempt: dict[str, Any] = { + "loop": planner_turn_index, + "grep_loop": grep_loops_used, + "action": "grep", + "query": query, + "normalized_query": needle, + "hit_count": len(hit_pages), + "hit_pages": hit_pages, + "match_count": match_count, + "visual_budget_remaining_before": visual_remaining, + **planner_meta, + } + if post_strip is not None: + attempt["post_strip"] = post_strip + if seed_full_title: + attempt["seed_full_title"] = True + + if not needle or (needle in attempted_needles and not allow_duplicate): + attempt["observation"] = "duplicate_normalized_query" + attempts.append(attempt) + return None + + last_grep_query = query + attempted_needles.add(needle) + + if not hit_pages: + attempt["observation"] = ( + "post_strip_no_normalized_hits" if post_strip else "no_normalized_hits" + ) + attempts.append(attempt) + return None + + if len(hit_pages) > budget: + attempt["observation"] = ( + "post_strip_too_many_hits" if post_strip else "too_many_hits" + ) + attempts.append(attempt) + return None + + return _visual_confirm(query=query, hit_pages=hit_pages, attempt=attempt) + + # Free automatic probe: full TOC title (no planner, no react_budget). + seed_match = _apply_grep_result( + query=title, + planner_turn_index=0, + consume_budget=False, + allow_duplicate=False, + post_strip=None, + planner_meta={}, + seed_full_title=True, + ) + if seed_match is not None: + return seed_match, attempts, visual_calls, "react_normalized_grep_vlm" + + while grep_loops_used < budget and planner_turn < max_planner_turns: + planner_turn += 1 + proposal, planner_meta = _propose_react_query( + title=title, + parent_titles=path_titles[:-1], + left=left, + right=right, + attempts=attempts, + budget=budget, + grep_loops_used=grep_loops_used, + ) + if proposal is None: + attempts.append( + { + "loop": planner_turn, + "grep_loop": grep_loops_used, + "action": "planner_error", + "hit_count": 0, + "hit_pages": [], + **planner_meta, + } + ) + continue + + action = proposal["action"] + if action == "give_up": + attempts.append( + { + "loop": planner_turn, + "grep_loop": grep_loops_used, + **proposal, + "hit_count": 0, + "hit_pages": [], + **planner_meta, + } + ) + return None, attempts, visual_calls, "react_give_up" + + if action in {"strip_header", "strip_footer"}: + from app.services.document_agent.tools.text_strip_margins import ( + strip_footer, + strip_header, + ) + + which = "header" if action == "strip_header" else "footer" + if which in stripped: + attempts.append( + { + "loop": planner_turn, + "grep_loop": grep_loops_used, + **proposal, + "hit_count": 0, + "hit_pages": [], + "observation": f"duplicate_strip_{which}", + **planner_meta, + } + ) + continue + strip_fn = strip_header if which == "header" else strip_footer + strip_result = strip_fn( + ctx, + {"start_page": left, "end_page": right}, + ) + stripped.add(which) + payload = strip_result.payload or {} + strip_ok = strip_result.status == "ok" + attempts.append( + { + "loop": planner_turn, + "grep_loop": grep_loops_used, + **proposal, + "hit_count": 0, + "hit_pages": [], + "observation": ( + f"stripped_{which}" if strip_ok else f"strip_{which}_failed" + ), + "pages_updated": int(payload.get("pages_updated") or 0), + "strip_error": strip_result.error, + **planner_meta, + } + ) + if not strip_ok or not last_grep_query: + continue + + # Same action: auto re-grep last query on stripped view (no budget). + from app.services.document_parser.structure.body_boundary import ( + normalize_match_text, + ) + + prior_needle = normalize_match_text(last_grep_query) + if prior_needle: + attempted_needles.discard(prior_needle) + match = _apply_grep_result( + query=last_grep_query, + planner_turn_index=planner_turn, + consume_budget=False, + allow_duplicate=True, + post_strip=which, + planner_meta={}, + ) + if match is not None: + return match, attempts, visual_calls, "react_normalized_grep_vlm" + continue + + query = proposal["query"] + match = _apply_grep_result( + query=query, + planner_turn_index=planner_turn, + consume_budget=True, + allow_duplicate=False, + post_strip=None, + planner_meta=planner_meta, + ) + if match is not None: + return match, attempts, visual_calls, "react_normalized_grep_vlm" + + return None, attempts, visual_calls, "react_loop_limit" + + +def locate_null_page_node_overrides( + *, + nodes: list[TitleNode], + match_overrides: dict[tuple[str, ...], TitleMatch], + page_texts: dict[int, str], + body_pages: list[int], + ctx: ToolContext | None, +) -> tuple[dict[tuple[str, ...], TitleMatch], list[dict[str, Any]]]: + """Locate null-page parents and leaves with normalized grep ReAct + VLM. + + ``page_texts`` is accepted for call-site stability; grep reads + ``ctx.blackboard.page_full_text_cache`` instead. When ``ctx`` is None, every + null-page node is recorded as unresolved (no text-unique fallback). + """ + del page_texts # grep uses blackboard cache via ctx + + if not nodes or not body_pages: + return dict(match_overrides), [] + + out = dict(match_overrides) + report: list[dict[str, Any]] = [] + + def _skip_rest( + sibling_nodes: list[TitleNode], + start_index: int, + parent_titles: tuple[str, ...], + failed_title: str, + ) -> None: + for later in sibling_nodes[start_index:]: + path = (*parent_titles, later.title) + if later.printed_page is not None or path in out: + continue + report.append( + { + "path_titles": list(path), + "title": later.title, + "kind": "leaf" if not later.children else "parent", + "printed_page": None, + "search_scope": None, + "result": "skipped_after_sibling_failure", + "page": None, + "accept": None, + "visual_verify_calls": 0, + "react_attempts": [], + "failed_sibling": failed_title, + } + ) + + def _record_unresolved_no_ctx( + sibling_nodes: list[TitleNode], + parent_titles: tuple[str, ...], + ) -> None: + for node in sibling_nodes: + path = (*parent_titles, node.title) + if node.printed_page is None and path not in out: + report.append( + { + "path_titles": list(path), + "title": node.title, + "kind": "leaf" if not node.children else "parent", + "printed_page": None, + "search_scope": None, + "result": "unresolved_no_ctx", + "page": None, + "accept": None, + "visual_verify_calls": 0, + "react_attempts": [], + } + ) + if node.children: + _record_unresolved_no_ctx(node.children, path) + + if ctx is None: + _record_unresolved_no_ctx(nodes, ()) + logger.info( + "[null_page_react] ctx is None: {} null-page node(s) unresolved " + "(no LLM/VLM probe)", + len(report), + ) + return out, report + + def walk( + sibling_nodes: list[TitleNode], + parent_titles: tuple[str, ...], + scope_start: int, + scope_end: int, + ) -> None: + cursor = int(scope_start) + for index, node in enumerate(sibling_nodes): + path_titles = (*parent_titles, node.title) + next_bound = _next_located_bound( + sibling_nodes=sibling_nodes, + index=index, + parent_titles=parent_titles, + overrides=out, + ) + node_scope_end = ( + min(int(next_bound), scope_end) + if next_bound is not None + else int(scope_end) + ) + + if path_titles in out: + cursor = max(cursor, int(out[path_titles].page)) + + needs_probe = node.printed_page is None and path_titles not in out + if needs_probe: + is_leaf = not node.children + entry: dict[str, Any] = { + "path_titles": list(path_titles), + "title": node.title, + "kind": "leaf" if is_leaf else "parent", + "printed_page": None, + "search_scope": None, + "result": "unresolved", + "page": None, + "accept": None, + "visual_verify_calls": 0, + "react_attempts": [], + } + + left = int(cursor) + right = int(node_scope_end) + if right < left: + entry["result"] = "skipped_bad_window" + entry["search_scope"] = [left, right] + report.append(entry) + _skip_rest( + sibling_nodes, index + 1, parent_titles, node.title + ) + return + + entry["search_scope"] = [left, right] + match, attempts, visual_calls, result = _locate_with_react( + path_titles=path_titles, + title=node.title, + left=left, + right=right, + ctx=ctx, + ) + entry["react_attempts"] = attempts + entry["visual_verify_calls"] = visual_calls + entry["result"] = result + if match is not None: + out[path_titles] = match + entry["page"] = int(match.page) + entry["accept"] = match.evidence.get("accept") + report.append(entry) + + if entry.get("page") is None: + _skip_rest( + sibling_nodes, index + 1, parent_titles, node.title + ) + return + + cursor = int(entry["page"]) + + if node.children: + child_scope_start = ( + int(out[path_titles].page) + if path_titles in out + else cursor + ) + walk( + node.children, + path_titles, + child_scope_start, + node_scope_end, + ) + last_under = last_leaf_start_under(node, parent_titles, out) + if last_under is not None: + cursor = max(cursor, int(last_under)) + elif path_titles in out: + cursor = max(cursor, int(out[path_titles].page)) + + walk(nodes, (), body_pages[0], body_pages[-1]) + located = sum(1 for row in report if row.get("page") is not None) + skipped = sum( + 1 + for row in report + if row.get("result") == "skipped_after_sibling_failure" + ) + budget = react_budget() + logger.info( + "[null_page_react] serial null-page ReAct: attempted={} located={} " + "unresolved={} skipped_after_fail={} budget={}", + len(report), + located, + sum( + 1 + for row in report + if row.get("result") + in { + "react_give_up", + "react_loop_limit", + "planner_error", + "unresolved", + "skipped_bad_window", + } + ), + skipped, + budget, + ) + return out, report diff --git a/apps/worker/app/services/document_agent/structure/toc_anchoring.py b/apps/worker/app/services/document_agent/structure/toc_anchoring.py index e52bef364..3053ee096 100644 --- a/apps/worker/app/services/document_agent/structure/toc_anchoring.py +++ b/apps/worker/app/services/document_agent/structure/toc_anchoring.py @@ -9,6 +9,7 @@ from loguru import logger from app.services.document_agent.manifest import ToolContext +from app.services.document_agent.pdf_text import page_content_map from app.services.document_agent.structure.anchoring_primitives import ( SkeletonAnchor, anchor_hierarchy_from_offset, @@ -48,7 +49,7 @@ def run_toc_anchoring(ctx: ToolContext) -> None: if page_count <= 0: return - page_texts = dict(ctx.blackboard.page_full_text_cache) + page_texts = page_content_map(ctx.blackboard.page_full_text_cache) if not page_texts: raise ValueError( "page_full_text_cache missing; run text scan before TOC anchoring" diff --git a/apps/worker/app/services/document_agent/tools/__init__.py b/apps/worker/app/services/document_agent/tools/__init__.py index bc94a8fda..72eee350b 100644 --- a/apps/worker/app/services/document_agent/tools/__init__.py +++ b/apps/worker/app/services/document_agent/tools/__init__.py @@ -11,6 +11,7 @@ from . import probe_links as probe_links # noqa: F401 from . import probe_outline as probe_outline # noqa: F401 from . import propose_shard_plan as propose_shard_plan # noqa: F401 +from . import text_strip_margins as text_strip_margins # noqa: F401 from . import validate_anatomy_map as validate_anatomy_map # noqa: F401 from . import verdict as verdict # noqa: F401 diff --git a/apps/worker/app/services/document_agent/tools/find_toc_anchor_pages.py b/apps/worker/app/services/document_agent/tools/find_toc_anchor_pages.py index 716ab6f87..dc272cbc8 100644 --- a/apps/worker/app/services/document_agent/tools/find_toc_anchor_pages.py +++ b/apps/worker/app/services/document_agent/tools/find_toc_anchor_pages.py @@ -9,6 +9,7 @@ from typing import Any from app.services.document_agent.manifest import TocAnchorPage, ToolContext, ToolResult +from app.services.document_agent.pdf_text import page_content_map from app.services.document_agent.registry import has_page_full_text, has_page_labels, register_tool from app.services.document_parser.structure.body_boundary import normalize_match_text from app.services.document_parser.formats.pdf.pymupdf_subprocess import ( @@ -214,7 +215,7 @@ def find_toc_anchor_pages(ctx: ToolContext, _args: dict[str, Any]) -> ToolResult total_pages = ctx.blackboard.page_count keyword_matches = _scan_toc_from_page_texts( - ctx.blackboard.page_full_text_cache, + page_content_map(ctx.blackboard.page_full_text_cache), page_count=total_pages, ) raw_hit_pages = {int(match["page"]) for match in keyword_matches} diff --git a/apps/worker/app/services/document_agent/tools/grep_text.py b/apps/worker/app/services/document_agent/tools/grep_text.py index ebf6ef74e..bb318e236 100644 --- a/apps/worker/app/services/document_agent/tools/grep_text.py +++ b/apps/worker/app/services/document_agent/tools/grep_text.py @@ -7,6 +7,7 @@ from typing import Any from app.services.document_agent.manifest import ToolContext, ToolResult +from app.services.document_agent.pdf_text import page_content_map from app.services.document_agent.registry import ( has_page_features, has_page_full_text, @@ -20,7 +21,9 @@ name="grep.text", description=( "Search normalized PDF text for a substring or regex. Whitespace is " - "collapsed with CJK-aware spacing and matching is case-insensitive." + "collapsed with CJK-aware spacing and matching is case-insensitive. " + "Uses page_text_search_view when set (after text.strip_*), else each " + "page's stored content field." ), parameters={ "type": "object", @@ -60,13 +63,18 @@ def grep_text(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: pattern = re.compile( normalized_query if use_regex else re.escape(normalized_query) ) + view = ctx.blackboard.page_text_search_view + if view is None: + texts = page_content_map(ctx.blackboard.page_full_text_cache) + else: + texts = {int(page): str(text) for page, text in view.items()} results: list[dict[str, Any]] = [] hit_count = 0 hit_pages: list[int] = [] - for page, text in sorted(ctx.blackboard.page_full_text_cache.items()): + for page, text in sorted(texts.items()): if page < start_page or (end_page and page > end_page): continue - normalized_text = normalize_match_text(text) + normalized_text = normalize_match_text(str(text or "")) page_hit = False for match in pattern.finditer(normalized_text): hit_count += 1 diff --git a/apps/worker/app/services/document_agent/tools/judge_toc_source.py b/apps/worker/app/services/document_agent/tools/judge_toc_source.py index 8f053ac27..a535846aa 100644 --- a/apps/worker/app/services/document_agent/tools/judge_toc_source.py +++ b/apps/worker/app/services/document_agent/tools/judge_toc_source.py @@ -14,6 +14,7 @@ from loguru import logger from app.services.document_agent.manifest import ToolContext, ToolResult +from app.services.document_agent.pdf_text import page_content_map from app.services.document_agent.registry import has_page_full_text, register_tool OUTLINE_CHOICE = "outline" @@ -118,7 +119,7 @@ def judge_toc_source(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: latency_ms=int((time.monotonic() - start) * 1000), ) - cache = dict(ctx.blackboard.page_full_text_cache) + cache = page_content_map(ctx.blackboard.page_full_text_cache) printed_toc = merge_printed_toc_texts([cache.get(page, "") for page in pages]) if not printed_toc.strip(): return ToolResult( diff --git a/apps/worker/app/services/document_agent/tools/ocr_pages.py b/apps/worker/app/services/document_agent/tools/ocr_pages.py index 737a8c377..01ff59d14 100644 --- a/apps/worker/app/services/document_agent/tools/ocr_pages.py +++ b/apps/worker/app/services/document_agent/tools/ocr_pages.py @@ -6,6 +6,7 @@ from typing import Any from app.services.document_agent.manifest import ToolContext, ToolResult +from app.services.document_agent.pdf_text import PageTextBands from app.services.document_agent.registry import has_page_features, register_tool from app.services.document_agent.visual import render_pages @@ -73,6 +74,7 @@ def ocr_pages(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: engine = RapidOCR() page_texts: dict[int, str] = {} + page_bands: dict[int, PageTextBands] = {} page_lines: dict[int, list[dict[str, Any]]] = {} for page in pages: image_path = png_by_page.get(page) @@ -89,11 +91,15 @@ def ocr_pages(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: } ) page_lines[page] = lines - page_texts[page] = "\n".join(line["text"] for line in lines if line["text"]) + content = "\n".join(line["text"] for line in lines if line["text"]) + page_texts[page] = content + # OCR has no reliable Y bands; content only, empty header/footer. + page_bands[page] = PageTextBands(content=content) cache = dict(ctx.blackboard.page_full_text_cache) - cache.update(page_texts) + cache.update(page_bands) ctx.blackboard.page_full_text_cache = cache + ctx.blackboard.page_text_search_view = None return ToolResult( status="ok", payload={"page_texts": page_texts, "page_lines": page_lines}, diff --git a/apps/worker/app/services/document_agent/tools/text_strip_margins.py b/apps/worker/app/services/document_agent/tools/text_strip_margins.py new file mode 100644 index 000000000..44ac2a98d --- /dev/null +++ b/apps/worker/app/services/document_agent/tools/text_strip_margins.py @@ -0,0 +1,132 @@ +"""Temporary search-view strip of stored header/footer page bands.""" + +from __future__ import annotations + +import time +from typing import Any, Literal + +from app.services.document_agent.manifest import ToolContext, ToolResult +from app.services.document_agent.pdf_text import ( + page_bands_map, + page_content_map, + strip_margin_text, +) +from app.services.document_agent.registry import ( + has_page_features, + has_page_full_text, + not_is_scanned, + register_tool, +) + + +def _resolve_page_range( + ctx: ToolContext, args: dict[str, Any] +) -> tuple[int, int] | ToolResult: + start_page = max(1, int(args.get("start_page") or 1)) + end_page = int(args.get("end_page") or ctx.blackboard.page_count or 0) + if end_page < start_page: + return ToolResult( + status="error", + error="end_page must be >= start_page", + latency_ms=0, + ) + return start_page, end_page + + +def _apply_strip( + ctx: ToolContext, + *, + which: Literal["header", "footer"], + start_page: int, + end_page: int, +) -> dict[str, Any]: + bands = page_bands_map(ctx.blackboard.page_full_text_cache) + view = ctx.blackboard.page_text_search_view + if view is None: + view = page_content_map(bands) + else: + view = dict(view) + + pages_updated = 0 + for page in range(start_page, end_page + 1): + record = bands.get(page) + if record is None: + continue + margin = record.header if which == "header" else record.footer + before = view.get(page, record.content) + after = strip_margin_text(before, margin) + view[page] = after + if after != before: + pages_updated += 1 + + ctx.blackboard.page_text_search_view = view + return { + "strip": which, + "start_page": start_page, + "end_page": end_page, + "pages_updated": pages_updated, + "view_active": True, + } + + +def _strip_tool( + ctx: ToolContext, + args: dict[str, Any], + *, + which: Literal["header", "footer"], +) -> ToolResult: + start = time.monotonic() + resolved = _resolve_page_range(ctx, args) + if isinstance(resolved, ToolResult): + resolved.latency_ms = int((time.monotonic() - start) * 1000) + return resolved + start_page, end_page = resolved + payload = _apply_strip( + ctx, which=which, start_page=start_page, end_page=end_page + ) + return ToolResult( + status="ok", + payload=payload, + latency_ms=int((time.monotonic() - start) * 1000), + output_summary={ + "strip": which, + "pages_updated": payload["pages_updated"], + }, + ) + + +_STRIP_PARAMS = { + "type": "object", + "properties": { + "start_page": {"type": "integer"}, + "end_page": {"type": "integer"}, + }, +} + + +@register_tool( + name="text.strip_header", + description=( + "Temporarily remove stored header-band text from the grep search view " + "for the given page range. Does not mutate page_full_text_cache. " + "Subsequent grep.text uses the stripped view until cleared." + ), + parameters=_STRIP_PARAMS, + preconditions=(has_page_features, has_page_full_text, not_is_scanned), +) +def strip_header(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: + return _strip_tool(ctx, args, which="header") + + +@register_tool( + name="text.strip_footer", + description=( + "Temporarily remove stored footer-band text from the grep search view " + "for the given page range. Does not mutate page_full_text_cache. " + "Subsequent grep.text uses the stripped view until cleared." + ), + parameters=_STRIP_PARAMS, + preconditions=(has_page_features, has_page_full_text, not_is_scanned), +) +def strip_footer(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: + return _strip_tool(ctx, args, which="footer") diff --git a/apps/worker/app/services/document_parser/profiling/doc_profiler.py b/apps/worker/app/services/document_parser/profiling/doc_profiler.py index 612d73fbe..03bbd4c44 100644 --- a/apps/worker/app/services/document_parser/profiling/doc_profiler.py +++ b/apps/worker/app/services/document_parser/profiling/doc_profiler.py @@ -8,6 +8,7 @@ from loguru import logger from app.services.document_agent.coordinator import ProfileCoordinator +from app.services.document_agent.pdf_text import page_content_map from app.services.document_agent.visual import purge_debug_visual_dirs, visual_debug_enabled from app.services.document_parser.orchestration.oversized_pdf_policy import ( build_oversized_pdf_profile_failed_exception, @@ -159,7 +160,9 @@ def _profile_pdf_with_db( {}, ), }, - page_full_text_cache=dict(coordinator.blackboard.page_full_text_cache), + page_full_text_cache=page_content_map( + coordinator.blackboard.page_full_text_cache + ), ) if profile.page_count > settings.MAX_PDF_PAGE_LIMIT: if oversized_policy != "page_memory": diff --git a/apps/worker/scripts/page_memory/_debug_pm_shared.py b/apps/worker/scripts/page_memory/_debug_pm_shared.py index 13270c720..be93e6030 100644 --- a/apps/worker/scripts/page_memory/_debug_pm_shared.py +++ b/apps/worker/scripts/page_memory/_debug_pm_shared.py @@ -462,8 +462,10 @@ def persist_stage0_state(out_dir: Path, coordinator) -> Path: doc_agent_dir = out_dir / "_doc_agent" doc_agent_dir.mkdir(parents=True, exist_ok=True) + from app.services.document_agent.pdf_text import PageTextBands + texts = { - str(page): text + str(page): PageTextBands.from_any(text).to_dict() for page, text in dict(coordinator.blackboard.page_full_text_cache or {}).items() } write_debug_json(page_text_cache_path(out_dir), texts) @@ -505,7 +507,9 @@ def load_stage0_into_coordinator(coordinator, out_dir: Path) -> None: hint="Stage-0 page_full_text_cache.json missing; re-run Stage 0", ) raw_texts = json.loads(text_path.read_text(encoding="utf-8")) - page_texts = {int(page): str(text) for page, text in dict(raw_texts or {}).items()} + from app.services.document_agent.pdf_text import page_bands_map + + page_texts = page_bands_map(raw_texts) bb = coordinator.blackboard bb.page_count = int(state.get("page_count") or 0) @@ -515,6 +519,7 @@ def load_stage0_into_coordinator(coordinator, out_dir: Path) -> None: bb.doc_stats = dict(state.get("doc_stats") or {}) bb.global_signals = dict(state.get("global_signals") or {}) bb.page_full_text_cache = page_texts + bb.page_text_search_view = None # Stage-1 owns TOC + assets from here. bb.toc_result = None bb.toc_hierarchies = None diff --git a/apps/worker/scripts/page_memory/debug_pm_stage2_calibration.py b/apps/worker/scripts/page_memory/debug_pm_stage2_calibration.py index 5034d429d..775e25b9c 100644 --- a/apps/worker/scripts/page_memory/debug_pm_stage2_calibration.py +++ b/apps/worker/scripts/page_memory/debug_pm_stage2_calibration.py @@ -148,7 +148,9 @@ def main() -> int: except FileNotFoundError: pass - page_texts = dict(bb.page_full_text_cache or {}) + from app.services.document_agent.pdf_text import page_content_map + + page_texts = page_content_map(bb.page_full_text_cache or {}) skeletons = extract_section_skeletons( anatomy=anchored, filename=filename, diff --git a/apps/worker/scripts/page_memory/tmp_probe_null_page_leaves.py b/apps/worker/scripts/page_memory/tmp_probe_null_page_leaves.py index 228ceb8fe..91815d3cf 100644 --- a/apps/worker/scripts/page_memory/tmp_probe_null_page_leaves.py +++ b/apps/worker/scripts/page_memory/tmp_probe_null_page_leaves.py @@ -1,36 +1,20 @@ #!/usr/bin/env python3 # ruff: noqa: E402 -"""TEMP: locate null-page TOC nodes with normalized grep ReAct + VLM. +"""Debug: run production Stage-2 TOC anchoring and dump null_page_report. -Experiment only — patches production symbols for one run, then restores them. - -Policy under test: - - Prune only printed-page leaves that failed anchoring; keep null-page nodes. - - Null-page parents and leaves use a bounded mini-ReAct search planner. - - Loop budget and hit/visual budget both equal PROFILE TOC - ``BOUNDARY_STEP_PAGES`` (currently 5). - - ReAct grep uses the registered ``grep.text`` normalized-text tool. - - hit_count > budget → too many; reflect and change query (no VLM). - - hit_count in 1..budget → confirm one page at a time until accepted or budget used. - - Under one parent, siblings are serial: left cursor advances on success; - on first failure, remaining null siblings are skipped (no window reset). - - Search scope ends at the next located sibling or the enclosing parent scope; - there is no fixed 22-page cap and no peer-TOC homepage clip. +Uses the live prune + ``locate_null_page_node_overrides`` path (no patches). Usage: cd apps/worker uv run python scripts/page_memory/tmp_probe_null_page_leaves.py \\ - --file "/path/to/EN_Sydney Streets Code.pdf" + --file "/path/to/doc.pdf" """ from __future__ import annotations -import json import sys import time -from dataclasses import replace from pathlib import Path as _Path -from typing import Any, cast sys.path.insert(0, str(_Path(__file__).resolve().parent)) @@ -49,647 +33,18 @@ write_debug_json, ) -# Same constant as PROFILE TOC boundary / confirm batch size. +from app.services.document_agent.structure.null_page_react import react_budget from app.services.document_agent.tools.extract_toc_with_boundaries import ( BOUNDARY_STEP_PAGES, ) -REACT_BUDGET = int(BOUNDARY_STEP_PAGES) -_HISTORY_SAMPLE_PAGES = 3 - -_REACT_INSTRUCTIONS = """\ -You are the search planner in a small ReAct loop. Propose one plain-text grep -query that may appear on the physical START page of a section. Grep collapses -whitespace/newlines to one space between non-CJK words, removes whitespace -adjacent to CJK, and matches case-insensitively. A separate visual check -confirms candidates one page at a time. - -Return one strict json object: -{"action":"grep","query":"...","reason":"..."} -or, only when no useful untried query remains: -{"action":"give_up","query":"","reason":"..."} - -General query tactics: -- Do not guess page numbers. -- Account for differences between TOC labels and body headings. -- Try removing numbering, lettering, punctuation, or decorative prefixes. -- When supported by the title or parent path, try a structural prefix such as - chapter, part, section, annex, or appendix. -- Try a distinctive leading, middle, or trailing title phrase when the full - title is unlikely to be printed verbatim. -- Prefer queries specific enough to avoid running headers and passing mentions. - -Reflection rules (mandatory): -- Read previous_attempts. Reflect on hit_count and observation before answering. -- If the last observation is no_normalized_hits, too_many_hits, visual_rejected, - or duplicate_normalized_query, you MUST change the query. Emitting the same - query again (same text after whitespace/case normalization) is invalid. -- too_many_hits means hit_count exceeded the visual budget; narrow the query. -- no_normalized_hits means broaden, rephrase, add/drop a structural prefix, or - try another title fragment. -- visual_rejected means the pages were not the section beginning; change the - query rather than repeating it. - -Generic cases: -- A TOC label like "B Safety requirements" under an appendices parent may be - printed as "Appendix B", "Safety requirements", or both together. -- A TOC label like "4.2 Access control — Technical requirements" may be printed - with the number removed or with only one distinctive title phrase. -""" - - -def _react_history_item(item: dict[str, Any]) -> dict[str, Any]: - hit_pages = [int(page) for page in (item.get("hit_pages") or [])] - return { - "query": item.get("query"), - "normalized_query": item.get("normalized_query"), - "hit_count": int(item.get("hit_count") or len(hit_pages)), - "sample_pages": hit_pages[:_HISTORY_SAMPLE_PAGES], - "observation": item.get("observation"), - "visual_selected_page": item.get("visual_selected_page"), - "visual_reason": item.get("visual_reason"), - "visual_pages_checked": item.get("visual_pages_checked"), - } - - -def prune_unanchored_keep_null_pages( - nodes: list[Any], - *, - match_overrides: dict[tuple[str, ...], Any], -) -> tuple[list[Any], int]: - """Drop only printed-page leaves that never got a physical override.""" - from app.services.document_agent.structure.hierarchy_locator import TitleNode - - removed = 0 - - def _prune(node: TitleNode, parent_titles: tuple[str, ...]) -> TitleNode | None: - nonlocal removed - path = (*parent_titles, node.title) - if node.children: - children: list[TitleNode] = [] - for child in node.children: - kept = _prune(child, path) - if kept is not None: - children.append(kept) - if children: - return replace(node, children=children) - if path in match_overrides or node.printed_page is None: - return replace(node, children=[]) - removed += 1 - return None - if path in match_overrides or node.printed_page is None: - return node - removed += 1 - return None - - out: list[TitleNode] = [] - for node in nodes: - kept = _prune(node, ()) - if kept is not None: - out.append(kept) - if removed: - logger.info( - "[tmp.null_leaf] pruned {} printed-page unanchored leaves " - "(null-page nodes kept)", - removed, - ) - return out, removed - - -def _next_located_bound( - *, - sibling_nodes: list[Any], - index: int, - parent_titles: tuple[str, ...], - overrides: dict[tuple[str, ...], Any], -) -> int | None: - from app.services.document_agent.structure.hierarchy_locator import ( - first_leaf_start_under, - ) - - for later in sibling_nodes[index + 1 :]: - path = (*parent_titles, later.title) - if path in overrides: - return int(overrides[path].page) - bound = first_leaf_start_under(later, parent_titles, overrides) - if bound is not None: - return int(bound) - return None - - -def _infer_offset( - nodes: list[Any], - match_overrides: dict[tuple[str, ...], Any], -) -> int: - from collections import Counter - - from app.services.document_agent.structure.hierarchy_locator import ( - iter_leaf_title_nodes, - ) - - diffs: list[int] = [] - for path, node in iter_leaf_title_nodes(nodes): - if node.printed_page is None or path not in match_overrides: - continue - diffs.append(int(match_overrides[path].page) - int(node.printed_page)) - if not diffs: - return 0 - return int(Counter(diffs).most_common(1)[0][0]) - - -def _normalized_grep( - *, - ctx: Any, - query: str, - left: int, - right: int, -) -> tuple[str, list[int], int]: - from app.services.document_agent.tools.grep_text import grep_text - - result = grep_text( - ctx, - { - "query": query, - "start_page": left, - "end_page": right, - }, - ) - if result.status != "ok": - return "", [], 0 - payload = result.payload or {} - return ( - str(payload.get("normalized_query") or ""), - [int(page) for page in (payload.get("hit_pages") or [])], - int(payload.get("hit_count") or 0), - ) - - -def _propose_react_query( - *, - title: str, - parent_titles: tuple[str, ...], - left: int, - right: int, - attempts: list[dict[str, Any]], - budget: int, -) -> tuple[dict[str, Any] | None, dict[str, Any]]: - state = { - "toc_title": title, - "parent_path": list(parent_titles), - "physical_search_scope": [left, right], - "react_budget": budget, - "visual_budget": budget, - "previous_attempts": [_react_history_item(item) for item in attempts], - } - prompt = ( - f"{_REACT_INSTRUCTIONS}\n\nCurrent state:\n" - f"{json.dumps(state, ensure_ascii=False)}" - ) - - try: - from shared.services.ai.llm_overrides import get_text_client - - client, model = get_text_client() - raw, usage = client.chat_completion_with_usage( - messages=cast(Any, [{"role": "user", "content": prompt}]), - model=model, - temperature=0.0, - max_tokens=300, - response_format={"type": "json_object"}, - usage_task="document_agent.null_page_title_react", - ) - payload = json.loads(raw) if raw else {} - except Exception as exc: - logger.warning("[tmp.null_react] planner failed for {!r}: {}", title, exc) - return None, {"error": f"planner failed: {exc}"} - - action = str(payload.get("action") or "").strip().lower() - query = str(payload.get("query") or "").strip() - if action not in {"grep", "give_up"}: - return None, { - "error": f"unknown planner action: {action!r}", - "usage": usage, - } - if action == "grep" and not query: - return None, {"error": "planner returned empty grep query", "usage": usage} - return ( - { - "action": action, - "query": query, - "reason": str(payload.get("reason") or ""), - }, - {"usage": usage}, - ) - - -def _verify_section_beginning_page( - *, - ctx: Any, - title: str, - page: int, - query: str, -) -> tuple[bool, str, int]: - """Confirm one physical page as the section beginning. Returns (ok, reason, tokens).""" - from app.services.document_agent.calibration.prompts import ( - coerce_found, - coerce_found_page, - ) - from app.services.document_agent.tools.inspect_pages import inspect_pages - - question = ( - f"Does this page mark the physical BEGINNING of the document section " - f"corresponding to the TOC entry {title!r}? A cover page, section " - "title page, or first body-heading page can be the beginning. Allow " - "equivalent wording and added or omitted numbering, lettering, or " - "structural prefixes. Do not accept a table-of-contents line, running " - "header or footer, passing mention, or continuation page. " - f"The normalized text query that nominated this page was {query!r}. " - "Report the physical page number printed in the page label above the image." - ) - verify_result = inspect_pages( - ctx, - { - "pages": [page], - "page_cap": 1, - "question": question, - "answer_keys": { - "found": ( - "boolean, true only when this page is the physical beginning " - "of the requested section" - ), - "found_page": ( - "number|null, the physical page number where the section begins" - ), - }, - "folder_name": "null_page_react_verify", - "prefix": "verify", - "usage_task": "document_agent.null_page_react_verify", - }, - ) - tokens = int(verify_result.tokens_used or 0) - if verify_result.status != "ok": - return False, str(verify_result.error or "inspect.pages failed"), tokens - fields = (verify_result.payload or {}).get("fields") or {} - found_page = coerce_found_page(fields.get("found_page"), pages=[page]) - ok = coerce_found(fields.get("found")) and found_page == page - reason = str((verify_result.payload or {}).get("answer") or "") - return ok, reason, tokens - - -def _locate_with_react( - *, - path_titles: tuple[str, ...], - title: str, - left: int, - right: int, - ctx: Any, -) -> tuple[Any | None, list[dict[str, Any]], int, str]: - from app.services.document_agent.structure.hierarchy_locator import TitleMatch - - budget = REACT_BUDGET - attempts: list[dict[str, Any]] = [] - attempted_needles: set[str] = set() - visual_calls = 0 - visual_remaining = budget - - for loop_index in range(1, budget + 1): - proposal, planner_meta = _propose_react_query( - title=title, - parent_titles=path_titles[:-1], - left=left, - right=right, - attempts=attempts, - budget=budget, - ) - if proposal is None: - attempts.append( - { - "loop": loop_index, - "action": "planner_error", - "hit_count": 0, - "hit_pages": [], - **planner_meta, - } - ) - continue - - action = proposal["action"] - if action == "give_up": - attempts.append( - { - "loop": loop_index, - **proposal, - "hit_count": 0, - "hit_pages": [], - **planner_meta, - } - ) - return None, attempts, visual_calls, "react_give_up" - - query = proposal["query"] - needle, hit_pages, match_count = _normalized_grep( - ctx=ctx, - query=query, - left=left, - right=right, - ) - attempt: dict[str, Any] = { - "loop": loop_index, - **proposal, - "normalized_query": needle, - "hit_count": len(hit_pages), - "hit_pages": hit_pages, - "match_count": match_count, - "visual_budget_remaining_before": visual_remaining, - **planner_meta, - } - if not needle or needle in attempted_needles: - attempt["observation"] = "duplicate_normalized_query" - attempts.append(attempt) - continue - attempted_needles.add(needle) - - if not hit_pages: - attempt["observation"] = "no_normalized_hits" - attempts.append(attempt) - continue - - if len(hit_pages) > budget: - attempt["observation"] = "too_many_hits" - attempts.append(attempt) - continue - - if visual_remaining <= 0: - attempt["observation"] = "visual_budget_exhausted" - attempts.append(attempt) - continue - - checked: list[dict[str, Any]] = [] - selected: int | None = None - last_reason = "" - for page in hit_pages: - if visual_remaining <= 0: - break - visual_remaining -= 1 - visual_calls += 1 - ok, reason, tokens = _verify_section_beginning_page( - ctx=ctx, - title=title, - page=page, - query=query, - ) - checked.append( - { - "page": page, - "confirmed": ok, - "reason": reason, - "tokens_used": tokens, - } - ) - last_reason = reason - if ok: - selected = page - break - - attempt["visual_pages_checked"] = checked - attempt["visual_selected_page"] = selected - attempt["visual_reason"] = last_reason - attempt["visual_budget_remaining_after"] = visual_remaining - if selected is not None: - attempt["observation"] = "section_start_confirmed" - attempts.append(attempt) - return ( - TitleMatch( - page=int(selected), - source="react_normalized_grep_vlm", - matched_line=query, - candidates=hit_pages, - evidence={ - "accept": "react_normalized_grep_vlm", - "null_page_react": True, - "loop": loop_index, - "normalized_query": needle, - "visual_reason": last_reason, - "visual_pages_checked": [item["page"] for item in checked], - }, - ), - attempts, - visual_calls, - "react_normalized_grep_vlm", - ) - - if visual_remaining <= 0 and len(checked) < len(hit_pages): - attempt["observation"] = "visual_budget_exhausted" - else: - attempt["observation"] = "visual_rejected" - attempts.append(attempt) - - return None, attempts, visual_calls, "react_loop_limit" - - -def locate_null_page_nodes_unified( - *, - nodes: list[Any], - match_overrides: dict[tuple[str, ...], Any], - page_texts: dict[int, str], - body_pages: list[int], - ctx: Any, - offset: int | None = None, -) -> tuple[dict[tuple[str, ...], Any], list[dict[str, Any]]]: - """Locate null-page nodes serially with normalized grep ReAct + VLM.""" - from app.services.document_agent.structure.hierarchy_locator import ( - TitleMatch, - last_leaf_start_under, - ) - - if not nodes or not body_pages: - return dict(match_overrides), [] - - out = dict(match_overrides) - body_set = set(body_pages) - report: list[dict[str, Any]] = [] - primary_offset = ( - int(offset) if offset is not None else _infer_offset(nodes, out) - ) - - def _seed_printed( - sibling_nodes: list[Any], parent_titles: tuple[str, ...] - ) -> None: - for node in sibling_nodes: - path = (*parent_titles, node.title) - if node.printed_page is not None and path not in out: - page = int(node.printed_page) + primary_offset - if page in body_set: - out[path] = TitleMatch( - page=page, - source="offset_seed", - matched_line="", - candidates=[page], - evidence={ - "accept": "printed_plus_offset_seed", - "tmp_null_leaf_probe": True, - }, - ) - if node.children: - _seed_printed(node.children, path) - - def _skip_rest( - sibling_nodes: list[Any], - start_index: int, - parent_titles: tuple[str, ...], - failed_title: str, - ) -> None: - for later in sibling_nodes[start_index:]: - path = (*parent_titles, later.title) - if later.printed_page is not None or path in out: - continue - report.append( - { - "path_titles": list(path), - "title": later.title, - "kind": "leaf" if not later.children else "parent", - "printed_page": None, - "search_scope": None, - "result": "skipped_after_sibling_failure", - "page": None, - "accept": None, - "visual_verify_calls": 0, - "react_attempts": [], - "failed_sibling": failed_title, - } - ) - - def _probe_succeeded(entry: dict[str, Any]) -> bool: - return entry.get("page") is not None - - _seed_printed(nodes, ()) - - def walk( - sibling_nodes: list[Any], - parent_titles: tuple[str, ...], - scope_start: int, - scope_end: int, - ) -> None: - cursor = int(scope_start) - for index, node in enumerate(sibling_nodes): - path_titles = (*parent_titles, node.title) - next_bound = _next_located_bound( - sibling_nodes=sibling_nodes, - index=index, - parent_titles=parent_titles, - overrides=out, - ) - node_scope_end = ( - min(int(next_bound), scope_end) - if next_bound is not None - else int(scope_end) - ) - - if path_titles in out: - cursor = max(cursor, int(out[path_titles].page)) - - needs_probe = node.printed_page is None and path_titles not in out - if needs_probe: - is_leaf = not node.children - entry: dict[str, Any] = { - "path_titles": list(path_titles), - "title": node.title, - "kind": "leaf" if is_leaf else "parent", - "printed_page": None, - "search_scope": None, - "result": "unresolved", - "page": None, - "accept": None, - "visual_verify_calls": 0, - "react_attempts": [], - } - - left = int(cursor) - right = int(node_scope_end) - if right < left: - entry["result"] = "skipped_bad_window" - entry["search_scope"] = [left, right] - report.append(entry) - _skip_rest( - sibling_nodes, index + 1, parent_titles, node.title - ) - return - - entry["search_scope"] = [left, right] - match, attempts, visual_calls, result = _locate_with_react( - path_titles=path_titles, - title=node.title, - left=left, - right=right, - ctx=ctx, - ) - entry["react_attempts"] = attempts - entry["visual_verify_calls"] = visual_calls - entry["result"] = result - if match is not None: - out[path_titles] = match - entry["page"] = int(match.page) - entry["accept"] = match.evidence.get("accept") - report.append(entry) - - if not _probe_succeeded(entry): - _skip_rest( - sibling_nodes, index + 1, parent_titles, node.title - ) - return - - cursor = int(entry["page"]) - - if node.children: - child_scope_start = ( - int(out[path_titles].page) - if path_titles in out - else cursor - ) - walk( - node.children, - path_titles, - child_scope_start, - node_scope_end, - ) - last_under = last_leaf_start_under(node, parent_titles, out) - if last_under is not None: - cursor = max(cursor, int(last_under)) - elif path_titles in out: - cursor = max(cursor, int(out[path_titles].page)) - - walk(nodes, (), body_pages[0], body_pages[-1]) - located = sum(1 for row in report if row.get("page") is not None) - skipped = sum( - 1 - for row in report - if row.get("result") == "skipped_after_sibling_failure" - ) - logger.info( - "[tmp.null_leaf] serial null-page ReAct: attempted={} located={} " - "unresolved={} skipped_after_fail={} budget={}", - len(report), - located, - sum( - 1 - for row in report - if row.get("result") - in {"react_give_up", "react_loop_limit", "planner_error"} - ), - skipped, - REACT_BUDGET, - ) - return out, report - def main() -> int: parser = base_argparser( - "TEMP: null-page normalized grep ReAct + VLM (stop siblings on fail)" + "Debug: production null-page ReAct via run_toc_anchoring (Stage 2)" ) args = parser.parse_args() - from app.services.document_agent.calibration import procedure as procedure_mod - from app.services.document_agent.structure import anchoring_primitives as ap from app.services.document_agent.structure.toc_anchoring import run_toc_anchoring from app.services.document_agent.validators import single_shard_plan from shared.core.config import settings @@ -704,32 +59,8 @@ def main() -> int: page_count = int(anatomy.page_count or 0) hierarchies = list(getattr(anatomy, "toc_hierarchies", None) or []) - original_prune = ( - procedure_mod.prune_unanchored_toc_leaves, - ap.prune_unanchored_toc_leaves, - ) - original_locate = ( - procedure_mod.locate_null_page_parent_overrides, - ap.locate_null_page_parent_overrides, - ) - - def _patched_locate(*, nodes, match_overrides, page_texts, body_pages, ctx): - return locate_null_page_nodes_unified( - nodes=nodes, - match_overrides=match_overrides, - page_texts=page_texts, - body_pages=body_pages, - ctx=ctx, - offset=None, - ) - - procedure_mod.prune_unanchored_toc_leaves = prune_unanchored_keep_null_pages - ap.prune_unanchored_toc_leaves = prune_unanchored_keep_null_pages - procedure_mod.locate_null_page_parent_overrides = _patched_locate - ap.locate_null_page_parent_overrides = _patched_locate - logger.info("█" * 70) - logger.info(" TEMP null-page normalized grep ReAct — {}", filename) + logger.info(" Production null-page ReAct dump — {}", filename) logger.info(" OUTPUT: {}", out_dir) logger.info("█" * 70) @@ -776,16 +107,10 @@ def _patched_locate(*, nodes, match_overrides, page_texts, body_pages, ctx): payload = { "policy": { - "prune": "keep_null_page_nodes; drop printed-page unanchored leaves only", - "probe": ( - "normalized-grep ReAct; loop/hit/visual budget=" - f"{REACT_BUDGET} (=BOUNDARY_STEP_PAGES); " - "hit_count>budget → too_many_hits + reflect; " - "else confirm one page at a time; " - "search next located sibling or enclosing parent scope; " - "serial under parent; stop siblings after first failure" - ), - "react_budget": REACT_BUDGET, + "prune_pre": "keep_null_page_nodes=True", + "probe": "null_page_react.locate_null_page_node_overrides", + "prune_post": "keep_null_page_nodes=False (drop unresolved)", + "react_budget": react_budget(), "boundary_step_pages": BOUNDARY_STEP_PAGES, }, "offset": anchor.get("offset"), @@ -819,10 +144,6 @@ def _patched_locate(*, nodes, match_overrides, page_texts, body_pages, ctx): for hit in react_hits: logger.info(" OVERRIDE {} -> p{}", hit["path"], hit["page"]) finally: - procedure_mod.prune_unanchored_toc_leaves = original_prune[0] - ap.prune_unanchored_toc_leaves = original_prune[1] - procedure_mod.locate_null_page_parent_overrides = original_locate[0] - ap.locate_null_page_parent_overrides = original_locate[1] if args.model: settings.IMAGE_MODEL = previous_image_model diff --git a/apps/worker/tests/contract/test_doc_profile_anatomy_contract.py b/apps/worker/tests/contract/test_doc_profile_anatomy_contract.py index 31ba78f1c..b73f1aa26 100644 --- a/apps/worker/tests/contract/test_doc_profile_anatomy_contract.py +++ b/apps/worker/tests/contract/test_doc_profile_anatomy_contract.py @@ -618,10 +618,12 @@ def fake_finalize() -> None: assert anatomy.toc_result.toc_pages == [17] -def test_run_text_scan_native_uses_read_page_texts( +def test_run_text_scan_native_uses_read_page_text_bands( monkeypatch, tmp_path: Path, ) -> None: + from app.services.document_agent.pdf_text import PageTextBands + coordinator = ProfileCoordinator( pdf_path=str(tmp_path / "doc.pdf"), job_id="job-scan-native", @@ -632,21 +634,41 @@ def test_run_text_scan_native_uses_read_page_texts( is_scanned=False, category="Report", routing_category=PdfRoutingCategory.GENERIC.value, + header_y=0.06, + footer_y=0.95, ) - def fake_read(_pdf_path: str, pages: list[int], timeout: int = 300) -> dict[int, str]: + def fake_read( + _pdf_path: str, + pages: list[int], + *, + header_y: float | None = None, + footer_y: float | None = None, + timeout: int = 300, + ) -> dict[int, PageTextBands]: assert pages == [1, 2] - return {1: "a", 2: "b"} + assert header_y == 0.06 + assert footer_y == 0.95 + return { + 1: PageTextBands(content="a", header="h1", footer="f1"), + 2: PageTextBands(content="b", header="", footer="f2"), + } - monkeypatch.setattr(coordinator_module, "read_page_texts", fake_read) + monkeypatch.setattr(coordinator_module, "read_page_text_bands", fake_read) coordinator._run_text_scan() - assert coordinator.blackboard.page_full_text_cache == {1: "a", 2: "b"} + assert coordinator.blackboard.page_full_text_cache == { + 1: PageTextBands(content="a", header="h1", footer="f1"), + 2: PageTextBands(content="b", header="", footer="f2"), + } + assert coordinator.blackboard.page_text_search_view is None def test_run_text_scan_scanned_dispatches_ocr_pages( monkeypatch, tmp_path: Path, ) -> None: + from app.services.document_agent.pdf_text import PageTextBands + coordinator = ProfileCoordinator( pdf_path=str(tmp_path / "doc.pdf"), job_id="job-scan-ocr", @@ -662,12 +684,18 @@ def test_run_text_scan_scanned_dispatches_ocr_pages( def fake_dispatch(name: str, ctx, args): assert name == "ocr.pages" assert args == {"pages": [1, 2]} - ctx.blackboard.page_full_text_cache = {1: "ocr-1", 2: "ocr-2"} + ctx.blackboard.page_full_text_cache = { + 1: PageTextBands(content="ocr-1"), + 2: PageTextBands(content="ocr-2"), + } return ToolResult(status="ok", payload={}) monkeypatch.setattr(coordinator_module.REGISTRY, "dispatch", fake_dispatch) coordinator._run_text_scan() - assert coordinator.blackboard.page_full_text_cache == {1: "ocr-1", 2: "ocr-2"} + assert coordinator.blackboard.page_full_text_cache == { + 1: PageTextBands(content="ocr-1"), + 2: PageTextBands(content="ocr-2"), + } def test_anchor_confirmation_failure_requires_one_strict_retry(tmp_path: Path) -> None: diff --git a/apps/worker/tests/contract/test_ocr_pages_contract.py b/apps/worker/tests/contract/test_ocr_pages_contract.py index d55254046..efb28a8ce 100644 --- a/apps/worker/tests/contract/test_ocr_pages_contract.py +++ b/apps/worker/tests/contract/test_ocr_pages_contract.py @@ -72,5 +72,8 @@ def fake_render(*_args, **_kwargs): result = ocr_pages(ctx, {"pages": [1]}) assert result.status == "ok" - assert ctx.blackboard.page_full_text_cache[1] == "Hello" + bands = ctx.blackboard.page_full_text_cache[1] + assert getattr(bands, "content", None) == "Hello" + assert getattr(bands, "header", None) == "" + assert getattr(bands, "footer", None) == "" assert result.payload["page_lines"][1][0]["text"] == "Hello" diff --git a/apps/worker/tests/contract/test_outline_short_circuit_contract.py b/apps/worker/tests/contract/test_outline_short_circuit_contract.py index cb038742f..bdbbc882b 100644 --- a/apps/worker/tests/contract/test_outline_short_circuit_contract.py +++ b/apps/worker/tests/contract/test_outline_short_circuit_contract.py @@ -117,7 +117,7 @@ def fake_calibrate(*args: Any, **kwargs: Any) -> Any: ), patch( "app.services.document_agent.structure.anchoring_primitives." - "locate_null_page_parent_overrides", + "locate_null_page_node_overrides", side_effect=_no_null_parent_locate, ), ): @@ -153,7 +153,7 @@ def fake_judge(tool_ctx: ToolContext, args: dict[str, Any]) -> ToolResult: ), patch( "app.services.document_agent.structure.anchoring_primitives." - "locate_null_page_parent_overrides", + "locate_null_page_node_overrides", side_effect=_no_null_parent_locate, ), ): diff --git a/apps/worker/tests/contract/test_structure_anchoring_contract.py b/apps/worker/tests/contract/test_structure_anchoring_contract.py index ac6ad7e09..6eaff8127 100644 --- a/apps/worker/tests/contract/test_structure_anchoring_contract.py +++ b/apps/worker/tests/contract/test_structure_anchoring_contract.py @@ -74,14 +74,19 @@ def test_prune_out_of_scope_nodes_removes_overflow_leaves() -> None: assert [n.title for n in pruned] == ["A"] -def test_null_page_parent_skipped_without_right_anchor() -> None: +def test_null_page_nodes_unresolved_without_ctx() -> None: + """No ctx → no LLM/VLM and no text-unique fallback.""" + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_node_overrides, + ) + parent = TitleNode( title="Chapter", level=1, printed_page=None, children=[TitleNode(title="Orphan", level=2, printed_page=None, children=[])], ) - overrides, report = anchoring.locate_null_page_parent_overrides( + overrides, report = locate_null_page_node_overrides( nodes=[parent], match_overrides={}, page_texts={1: "Chapter\nHello"}, @@ -89,183 +94,117 @@ def test_null_page_parent_skipped_without_right_anchor() -> None: ctx=None, ) assert overrides == {} - assert len(report) == 1 - assert report[0]["result"] == "skipped_no_right" + assert len(report) == 2 + assert {row["result"] for row in report} == {"unresolved_no_ctx"} -def test_null_page_parent_located_via_normalized_text() -> None: - child = TitleNode(title="1.1 Detail", level=2, printed_page=5, children=[]) - parent = TitleNode( - title="1 Overview", - level=1, - printed_page=None, - children=[child], - ) - leaf_match = anchoring.bulk_offset_matches( - [(("1 Overview", "1.1 Detail"), child)], +def test_null_page_leaf_kept_by_pre_react_prune() -> None: + """Printed unanchored leaves drop; null-page leaves survive until ReAct.""" + nodes = [ + _leaf("PrintedOk", 1), + _leaf("PrintedMiss", 10), + TitleNode(title="NullLeaf", level=1, printed_page=None, children=[]), + ] + overrides = anchoring.bulk_offset_matches( + [(("PrintedOk",), nodes[0])], offset=0, ) - page_texts = { - 4: "noise", - 5: "1 Overview\n1.1 Detail\nbody", - 6: "more", - } - overrides, report = anchoring.locate_null_page_parent_overrides( - nodes=[parent], - match_overrides=leaf_match, - page_texts=page_texts, - body_pages=[4, 5, 6], - ctx=None, + kept, removed = anchoring.prune_unanchored_toc_leaves( + nodes, + match_overrides=overrides, + keep_null_page_nodes=True, ) - assert ("1 Overview",) in overrides - assert overrides[("1 Overview",)].page == 5 - assert report[0]["result"] != "unresolved" - assert report[0]["page"] == 5 - assert report[0]["window"] == [1, 5] + assert removed == 1 + assert [n.title for n in kept] == ["PrintedOk", "NullLeaf"] -def test_normalized_title_match_preserves_english_word_boundary() -> None: - from app.services.document_agent.structure.hierarchy_locator import ( - locate_title_normalized_strict, +def test_null_page_react_skips_later_siblings_after_failure() -> None: + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_node_overrides, ) - match = locate_title_normalized_strict( - "附录 A OVERVIEW", - scope_pages=[7], - page_texts={7: "附录\nA OVERVIEW"}, - ) - - assert match is not None - assert match.page == 7 - assert match.matched_line == "附录a overview" - assert match.evidence["accept"] == "normalized_strict_unique" + nodes = [ + TitleNode(title="A", level=1, printed_page=None, children=[]), + TitleNode(title="B", level=1, printed_page=None, children=[]), + ] + ctx = _ctx() + def fail_a(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, str]: + assert kwargs["title"] == "A" + return None, [{"loop": 1, "observation": "react_give_up"}], 0, "react_give_up" -def test_first_sibling_null_parent_uses_scan_forward_not_wide_rtl() -> None: - """No left sibling: miss text → ``scan_title_forward`` within 2+4+6+10 budget.""" - child = TitleNode(title="22.1 Intro", level=2, printed_page=278, children=[]) - parent = TitleNode( - title="Chapter 22", - level=1, - printed_page=None, - children=[child], - ) - leaf_match = { - ("Chapter 22", "22.1 Intro"): TitleMatch( - page=278, - source="test", - matched_line="", - candidates=[278], - evidence={}, + with patch.object(npr, "_locate_with_react", side_effect=fail_a): + overrides, report = locate_null_page_node_overrides( + nodes=nodes, + match_overrides={}, + page_texts={}, + body_pages=[1, 2, 3, 4, 5], + ctx=ctx, ) - } - body_pages = list(range(1, 301)) - page_texts = {page: "noise" for page in body_pages} - ctx = _ctx() - scanned_starts: list[int] = [] + assert overrides == {} + assert report[0]["result"] == "react_give_up" + assert report[1]["result"] == "skipped_after_sibling_failure" + + +def test_null_page_react_hit_writes_override() -> None: + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_node_overrides, + ) - def fake_scan(**kwargs: Any) -> Any: - from app.services.document_agent.calibration.scan import TitleScanResult + leaf = TitleNode(title="Appendix B", level=1, printed_page=None, children=[]) + ctx = _ctx() + match = TitleMatch( + page=12, + source="react_normalized_grep_vlm", + matched_line="Appendix B", + candidates=[12], + evidence={ + "accept": "react_normalized_grep_vlm", + "null_page_react": True, + "normalized_query": "appendix b", + }, + ) - scanned_starts.append(int(kwargs["start_page"])) - assert int(kwargs["page_count"]) == 278 - assert int(kwargs["start_page"]) == anchoring._first_sibling_null_parent_scan_start( - 278 + def fake_react(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, str]: + assert kwargs["left"] == 1 + assert kwargs["right"] == 20 + return match, [{"loop": 1, "observation": "section_start_confirmed"}], 1, ( + "react_normalized_grep_vlm" ) - return TitleScanResult( - title=str(kwargs["title"]), - found=True, - found_page=270, - scanned_pages=list(range(int(kwargs["start_page"]), 271)), - next_start=271, + + with patch.object(npr, "_locate_with_react", side_effect=fake_react): + overrides, report = locate_null_page_node_overrides( + nodes=[leaf], + match_overrides={}, + page_texts={}, + body_pages=list(range(1, 21)), + ctx=ctx, ) - with patch( - "app.services.document_agent.calibration.scan.scan_title_forward", - side_effect=fake_scan, - ): - with patch.object(anchoring, "_visual_rtl_locate_parent") as rtl: - overrides, report = anchoring.locate_null_page_parent_overrides( - nodes=[parent], - match_overrides=leaf_match, - page_texts=page_texts, - body_pages=body_pages, - ctx=ctx, - ) - rtl.assert_not_called() - - assert scanned_starts == [anchoring._first_sibling_null_parent_scan_start(278)] - assert overrides[("Chapter 22",)].page == 270 - assert report[0]["accept"] == "scan_forward" - assert report[0]["window"] == [ - anchoring._first_sibling_null_parent_scan_start(278), - 278, - ] + assert overrides[("Appendix B",)].page == 12 + assert overrides[("Appendix B",)].source == "react_normalized_grep_vlm" + assert report[0]["page"] == 12 + assert report[0]["result"] == "react_normalized_grep_vlm" -def test_null_page_parent_with_left_sibling_still_uses_rtl() -> None: - left_child = TitleNode(title="A.1", level=2, printed_page=10, children=[]) - left = TitleNode(title="A", level=1, printed_page=10, children=[left_child]) - right_child = TitleNode(title="B.1", level=2, printed_page=50, children=[]) - right = TitleNode(title="B", level=1, printed_page=None, children=[right_child]) - overrides_in = { - ("A",): TitleMatch( - page=10, - source="test", - matched_line="", - candidates=[10], - evidence={}, - ), - ("A", "A.1"): TitleMatch( - page=10, - source="test", - matched_line="", - candidates=[10], - evidence={}, - ), - ("B", "B.1"): TitleMatch( - page=50, - source="test", - matched_line="", - candidates=[50], - evidence={}, - ), - } - page_texts = {p: "noise" for p in range(1, 61)} - ctx = _ctx() - - def fake_rtl(**kwargs: Any) -> tuple[TitleMatch, int]: - assert kwargs["left"] == 10 - assert kwargs["right"] == 50 - return ( - TitleMatch( - page=40, - source="inspect_vlm", - matched_line="", - candidates=[40], - evidence={"accept": "visual_rtl"}, - ), - 3, - ) +def test_normalized_title_match_preserves_english_word_boundary() -> None: + from app.services.document_agent.structure.hierarchy_locator import ( + locate_title_normalized_strict, + ) - with patch( - "app.services.document_agent.calibration.scan.scan_title_forward" - ) as scan: - with patch.object( - anchoring, "_visual_rtl_locate_parent", side_effect=fake_rtl - ): - overrides, report = anchoring.locate_null_page_parent_overrides( - nodes=[left, right], - match_overrides=overrides_in, - page_texts=page_texts, - body_pages=list(range(1, 61)), - ctx=ctx, - ) - scan.assert_not_called() + match = locate_title_normalized_strict( + "附录 A OVERVIEW", + scope_pages=[7], + page_texts={7: "附录\nA OVERVIEW"}, + ) - assert overrides[("B",)].page == 40 - assert report[0]["accept"] == "visual_rtl" + assert match is not None + assert match.page == 7 + assert match.matched_line == "附录a overview" + assert match.evidence["accept"] == "normalized_strict_unique" def test_phase2_bulk_via_mocked_offset() -> None: diff --git a/apps/worker/tests/contract/test_tool_registry_smoke_contract.py b/apps/worker/tests/contract/test_tool_registry_smoke_contract.py index a9930f69e..78d431138 100644 --- a/apps/worker/tests/contract/test_tool_registry_smoke_contract.py +++ b/apps/worker/tests/contract/test_tool_registry_smoke_contract.py @@ -27,6 +27,8 @@ def test_probe_and_inspect_registered() -> None: "inspect.pages", "ocr.pages", "grep.text", + "text.strip_header", + "text.strip_footer", ): assert REGISTRY.get(name) is not None, name @@ -62,5 +64,42 @@ def test_grep_text_normalizes_query_and_corpus_by_default() -> None: assert result.payload["hit_pages"] == [2] +def test_strip_footer_updates_search_view_for_grep() -> None: + from app.services.document_agent.pdf_text import PageTextBands + from app.services.document_agent.tools.text_strip_margins import strip_footer + + blackboard = ProfileBlackboard(page_count=1) + blackboard.page_full_text_cache = { + 1: PageTextBands( + content="Section Start\nPublic Domain Manual", + header="", + footer="Public Domain Manual", + ) + } + ctx = ToolContext( + pdf_path="/tmp/doc.pdf", + job_id="strip-footer", + blackboard=blackboard, + trace=None, + settings={}, + ) + + before = grep_text(ctx, {"query": "Public Domain Manual", "start_page": 1, "end_page": 1}) + assert before.status == "ok" + assert before.payload["hit_count"] == 1 + + strip = strip_footer(ctx, {"start_page": 1, "end_page": 1}) + assert strip.status == "ok" + assert strip.payload["pages_updated"] == 1 + assert ctx.blackboard.page_text_search_view[1] == "Section Start\n" + + after = grep_text(ctx, {"query": "Public Domain Manual", "start_page": 1, "end_page": 1}) + assert after.status == "ok" + assert after.payload["hit_count"] == 0 + + # Stored cache must remain untouched. + assert blackboard.page_full_text_cache[1].content == "Section Start\nPublic Domain Manual" + + def test_openai_specs_removed() -> None: - assert not hasattr(REGISTRY, "openai_specs") + assert not hasattr(REGISTRY, "openai_specs") \ No newline at end of file From 655c2b82cd557389318352b76427ab1afb4d93fa Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Fri, 21 Aug 2026 22:52:04 +0800 Subject: [PATCH 3/7] refactor: enhance null-page processing and introduce parent locating functionality - Updated the null-page locating logic to differentiate between leaves and parents, improving the handling of null-page nodes. - Introduced a new function, locate_null_page_parent_overrides, to locate null-page parents using sibling windows, enhancing the accuracy of parent-child relationships. - Adjusted existing functions to accommodate the new parent locating logic, ensuring compatibility with the overall document structure. - Refined test cases to validate the new functionality and ensure robustness in null-page handling. --- .../document_agent/calibration/procedure.py | 12 +- .../structure/anchoring_primitives.py | 339 +++++++++++++++++- .../structure/hierarchy_locator.py | 9 +- .../structure/null_page_react.py | 127 +++++-- ..._leaves.py => debug_pm_null_page_react.py} | 15 +- .../test_structure_anchoring_contract.py | 277 +++++++++++++- 6 files changed, 723 insertions(+), 56 deletions(-) rename apps/worker/scripts/page_memory/{tmp_probe_null_page_leaves.py => debug_pm_null_page_react.py} (88%) diff --git a/apps/worker/app/services/document_agent/calibration/procedure.py b/apps/worker/app/services/document_agent/calibration/procedure.py index 7216ebd00..6b4775b54 100644 --- a/apps/worker/app/services/document_agent/calibration/procedure.py +++ b/apps/worker/app/services/document_agent/calibration/procedure.py @@ -4,7 +4,7 @@ 1. Builds TitleNodes the same way production does 2. Runs Phase-2 **per regime** (prune → bulk/bisect → recalibrate) 3. Merges physical-page ``match_overrides`` across regimes -4. Runs null-page ReAct locate once on the combined tree, then final prune +4. Locates null-page leaves, then null-page parents, then final prune Returns production ``SkeletonAnchor`` plus regime diagnostics for debug payloads. """ @@ -34,6 +34,7 @@ from app.services.document_agent.structure.anchoring_primitives import ( SkeletonAnchor, backfill_parent_offset_matches, + locate_null_page_parent_overrides, prune_unanchored_toc_leaves, serialize_skeleton_anchor, ) @@ -383,13 +384,20 @@ def anchor_hierarchy_from_regimes( len(parent_matches), ) - match_overrides, null_page_report = locate_null_page_node_overrides( + match_overrides, leaf_report = locate_null_page_node_overrides( nodes=working, match_overrides=merged, + body_pages=body_pages, + ctx=ctx, + ) + match_overrides, parent_report = locate_null_page_parent_overrides( + nodes=working, + match_overrides=match_overrides, page_texts=page_texts, body_pages=body_pages, ctx=ctx, ) + null_page_report = [*leaf_report, *parent_report] # Drop still-unanchored null-page nodes (avoid sticky inherited ranges). working, failed_null_removed = prune_unanchored_toc_leaves( diff --git a/apps/worker/app/services/document_agent/structure/anchoring_primitives.py b/apps/worker/app/services/document_agent/structure/anchoring_primitives.py index abb6948b6..0cb974b6f 100644 --- a/apps/worker/app/services/document_agent/structure/anchoring_primitives.py +++ b/apps/worker/app/services/document_agent/structure/anchoring_primitives.py @@ -12,7 +12,10 @@ from app.services.document_agent.structure.hierarchy_locator import ( TitleMatch, TitleNode, + first_leaf_start_under, iter_leaf_title_nodes, + last_leaf_start_under, + locate_title_normalized_strict, ) from app.services.document_agent.structure.null_page_react import ( locate_null_page_node_overrides, @@ -23,6 +26,17 @@ from loguru import logger +def _first_sibling_null_parent_scan_start(right: int) -> int: + """Left edge for first-at-level null parents: at most one 2+4+6+10 budget. + + Does not inherit a wider parent/body scope. Floors at document page 1. + """ + from app.services.document_agent.calibration.scan import DEFAULT_WINDOW_SCHEDULE + + budget = sum(DEFAULT_WINDOW_SCHEDULE) + return max(1, int(right) - budget + 1) + + def prune_out_of_scope_nodes( nodes: list[TitleNode], *, @@ -155,8 +169,320 @@ def toc_range_end(hierarchy: dict[str, Any]) -> int | None: return None -# Null-page locate lives in ``null_page_react.locate_null_page_node_overrides``. -# Imported above for ``anchor_hierarchy_from_offset``. +# ── Null-page parent locate (sibling window / first-sibling scan) ─────────── + + +def locate_null_page_parent_overrides( + *, + nodes: list[TitleNode], + match_overrides: dict[tuple[str, ...], TitleMatch], + page_texts: dict[int, str], + body_pages: list[int], + ctx: ToolContext | None, +) -> tuple[dict[tuple[str, ...], TitleMatch], list[dict[str, Any]]]: + """Locate TOC parents with ``printed_page=None`` into ``match_overrides``. + + Window for parent P with a previous same-level sibling: ``[last leaf under + that sibling, first leaf under P]``; text then RTL visual verify. + + First-at-level parents (no left sibling) do **not** inherit a wider parent + or body scope. Left edge is one Phase-1 ``2+4+6+10`` budget before the first + child (floor page 1). Text runs in that window; on miss, reuse + ``scan_title_forward`` (same schedule, early exit). Miss → unresolved. + + Returns ``(overrides, report)`` where *report* lists every null-page parent + attempt (for debug / LLM-call accounting). + """ + if not nodes or not body_pages: + return dict(match_overrides), [] + + out = dict(match_overrides) + body_set = set(body_pages) + parent_scope_start = body_pages[0] + report: list[dict[str, Any]] = [] + + def walk( + sibling_nodes: list[TitleNode], + parent_titles: tuple[str, ...], + scope_start: int, + ) -> None: + for index, node in enumerate(sibling_nodes): + path_titles = (*parent_titles, node.title) + if ( + node.children + and node.printed_page is None + and path_titles not in out + ): + right = first_leaf_start_under(node, parent_titles, out) + entry: dict[str, Any] = { + "path_titles": list(path_titles), + "title": node.title, + "printed_page": None, + "window": None, + "result": "skipped_no_right", + "page": None, + "accept": None, + "visual_verify_calls": 0, + } + if right is None: + report.append(entry) + logger.info( + "[structure_anchoring] null-page parent skipped: " + "title={!r} reason=no_located_first_child", + node.title, + ) + elif index > 0: + left = last_leaf_start_under( + sibling_nodes[index - 1], parent_titles, out + ) + if left is None: + left = scope_start + if right < left: + report.append(entry) + logger.info( + "[structure_anchoring] null-page parent skipped: " + "title={!r} reason=no_located_first_child left={}", + node.title, + left, + ) + else: + _resolve_null_parent_with_sibling_window( + path_titles=path_titles, + title=node.title, + left=left, + right=right, + body_pages=body_pages, + body_set=body_set, + page_texts=page_texts, + ctx=ctx, + out=out, + entry=entry, + report=report, + ) + else: + left = _first_sibling_null_parent_scan_start(right) + _resolve_null_parent_first_sibling( + path_titles=path_titles, + title=node.title, + left=left, + right=right, + body_pages=body_pages, + body_set=body_set, + page_texts=page_texts, + ctx=ctx, + out=out, + entry=entry, + report=report, + ) + if node.children: + child_scope_start = ( + out[path_titles].page if path_titles in out else scope_start + ) + walk(node.children, path_titles, child_scope_start) + + walk(nodes, (), parent_scope_start) + logger.info( + "[structure_anchoring] null-page parent locate summary: " + "attempted={} located={} unresolved={} visual_verify_calls={}", + len(report), + sum(1 for row in report if row.get("page") is not None), + sum(1 for row in report if row.get("result") == "unresolved"), + sum(int(row.get("visual_verify_calls") or 0) for row in report), + ) + return out, report + + +def _record_null_parent_outcome( + *, + path_titles: tuple[str, ...], + title: str, + left: int, + right: int, + match: TitleMatch | None, + visual_calls: int, + body_set: set[int], + out: dict[tuple[str, ...], TitleMatch], + entry: dict[str, Any], + report: list[dict[str, Any]], +) -> None: + entry["window"] = [left, right] + entry["visual_verify_calls"] = visual_calls + if match is not None and match.page in body_set: + out[path_titles] = match + entry["result"] = str(match.evidence.get("accept") or match.source) + entry["page"] = match.page + entry["accept"] = match.evidence.get("accept") + logger.info( + "[structure_anchoring] null-page parent located: " + "title={!r} page={} window={} accept={} visual_calls={}", + title, + match.page, + [left, right], + match.evidence.get("accept"), + visual_calls, + ) + else: + entry["result"] = "unresolved" + logger.info( + "[structure_anchoring] null-page parent unresolved: " + "title={!r} window={} visual_calls={}", + title, + [left, right], + visual_calls, + ) + report.append(entry) + + +def _resolve_null_parent_with_sibling_window( + *, + path_titles: tuple[str, ...], + title: str, + left: int, + right: int, + body_pages: list[int], + body_set: set[int], + page_texts: dict[int, str], + ctx: ToolContext | None, + out: dict[tuple[str, ...], TitleMatch], + entry: dict[str, Any], + report: list[dict[str, Any]], +) -> None: + scope_pages = [page for page in body_pages if left <= page <= right] + match = locate_title_normalized_strict( + title, + scope_pages=scope_pages, + page_texts=page_texts, + ) + visual_calls = 0 + if match is None and ctx is not None: + match, visual_calls = _visual_rtl_locate_parent( + title=title, + left=left, + right=right, + body_set=body_set, + ctx=ctx, + ) + _record_null_parent_outcome( + path_titles=path_titles, + title=title, + left=left, + right=right, + match=match, + visual_calls=visual_calls, + body_set=body_set, + out=out, + entry=entry, + report=report, + ) + + +def _resolve_null_parent_first_sibling( + *, + path_titles: tuple[str, ...], + title: str, + left: int, + right: int, + body_pages: list[int], + body_set: set[int], + page_texts: dict[int, str], + ctx: ToolContext | None, + out: dict[tuple[str, ...], TitleMatch], + entry: dict[str, Any], + report: list[dict[str, Any]], +) -> None: + """First-at-level null parent: capped text window, then ``scan_title_forward``.""" + from app.services.document_agent.calibration.scan import ( + DEFAULT_WINDOW_SCHEDULE, + scan_title_forward, + ) + + scope_pages = [page for page in body_pages if left <= page <= right] + match = locate_title_normalized_strict( + title, + scope_pages=scope_pages, + page_texts=page_texts, + ) + visual_calls = 0 + if match is None and ctx is not None: + scan = scan_title_forward( + ctx=ctx, + title=title, + start_page=left, + page_count=right, + window_schedule=DEFAULT_WINDOW_SCHEDULE, + ) + visual_calls = len(scan.scanned_pages) + if scan.found and scan.found_page is not None: + match = TitleMatch( + page=int(scan.found_page), + source="inspect_vlm", + matched_line="", + candidates=[int(scan.found_page)], + evidence={ + "accept": "scan_forward", + "null_page_parent_probe": True, + "scanned_pages": list(scan.scanned_pages), + }, + ) + _record_null_parent_outcome( + path_titles=path_titles, + title=title, + left=left, + right=right, + match=match, + visual_calls=visual_calls, + body_set=body_set, + out=out, + entry=entry, + report=report, + ) + + +def _visual_rtl_locate_parent( + *, + title: str, + left: int, + right: int, + body_set: set[int], + ctx: ToolContext, +) -> tuple[TitleMatch | None, int]: + """Confirm parent title from right boundary toward left via VLM verify.""" + visual_calls = 0 + for page in range(right, left - 1, -1): + if page not in body_set: + continue + candidate = TitleMatch( + page=page, + source="inspect_vlm", + matched_line="", + candidates=[page], + evidence={"null_page_parent_probe": True}, + ) + visual_calls += 1 + result = verify_section_page_choice( + ctx=ctx, + title=title, + candidate_matches=[candidate], + candidate_page_cap=1, + ) + selected = result.get("selected_page") + if selected != page: + continue + return ( + TitleMatch( + page=page, + source="inspect_vlm", + matched_line="", + candidates=[page], + evidence={ + "accept": "visual_rtl", + "reason": result.get("reason", ""), + "visual_verify_calls": visual_calls, + }, + ), + visual_calls, + ) + return None, visual_calls # ── Offset-guided bulk anchoring with recursive recalibrate (Phase-2) ─────── @@ -745,13 +1071,20 @@ def anchor_hierarchy_from_offset( pruned_count += unanchored_removed match_overrides = _filter_overrides_to_tree(working, match_overrides) - match_overrides, null_page_report = locate_null_page_node_overrides( + match_overrides, leaf_report = locate_null_page_node_overrides( + nodes=working, + match_overrides=match_overrides, + body_pages=body_pages, + ctx=ctx, + ) + match_overrides, parent_report = locate_null_page_parent_overrides( nodes=working, match_overrides=match_overrides, page_texts=page_texts, body_pages=body_pages, ctx=ctx, ) + null_page_report = [*leaf_report, *parent_report] working, failed_null_removed = prune_unanchored_toc_leaves( working, diff --git a/apps/worker/app/services/document_agent/structure/hierarchy_locator.py b/apps/worker/app/services/document_agent/structure/hierarchy_locator.py index ab7fef614..4ed61933a 100644 --- a/apps/worker/app/services/document_agent/structure/hierarchy_locator.py +++ b/apps/worker/app/services/document_agent/structure/hierarchy_locator.py @@ -1,10 +1,11 @@ """Locate hierarchy titles on PDF pages and resolve page ranges. Deterministic range assembly from PROFILE ``match_overrides``. Leaf starts -come only from those overrides. Null-page parents and leaves are located -upstream via bounded grep ReAct + VLM (``null_page_react``), then resolved -here including parent self-only spans for interstitial pages. Parents without -an override may still inherit start from the earliest located descendant leaf. +come only from those overrides. Null-page leaves are located upstream via +bounded grep ReAct + VLM; null-page parents use their sibling/first-child +window. They are then resolved here, including parent self-only spans for +interstitial pages. Parents without an override may still inherit start from +the earliest located descendant leaf. """ from __future__ import annotations diff --git a/apps/worker/app/services/document_agent/structure/null_page_react.py b/apps/worker/app/services/document_agent/structure/null_page_react.py index 53b95dea2..4e824efae 100644 --- a/apps/worker/app/services/document_agent/structure/null_page_react.py +++ b/apps/worker/app/services/document_agent/structure/null_page_react.py @@ -1,9 +1,9 @@ -"""Bounded ReAct locate for TOC nodes with ``printed_page=None``. +"""Bounded ReAct locate for TOC leaves with ``printed_page=None``. -After Phase-2 printed-page bulk/bisect, null-page parents and leaves share one -serial probe: text LLM plans a ``grep.text`` query inside a sibling window, -then ``inspect.pages`` confirms the physical section start one hit page at a -time. Loop / hit / visual budgets equal ``BOUNDARY_STEP_PAGES``. +After Phase-2 printed-page bulk/bisect, null-page leaves use a serial probe: +text LLM plans a ``grep.text`` query inside a sibling window, then +``inspect.pages`` confirms the physical section start one hit page at a time. +Loop / hit / visual budgets equal ``BOUNDARY_STEP_PAGES``. No offset seed, no fixed page-count cap, and no fallback to normalized-strict unique hit / RTL / ``scan_title_forward``. @@ -564,24 +564,73 @@ def locate_null_page_node_overrides( *, nodes: list[TitleNode], match_overrides: dict[tuple[str, ...], TitleMatch], - page_texts: dict[int, str], body_pages: list[int], ctx: ToolContext | None, ) -> tuple[dict[tuple[str, ...], TitleMatch], list[dict[str, Any]]]: - """Locate null-page parents and leaves with normalized grep ReAct + VLM. + """Locate null-page leaves with normalized grep ReAct + VLM. - ``page_texts`` is accepted for call-site stability; grep reads - ``ctx.blackboard.page_full_text_cache`` instead. When ``ctx`` is None, every - null-page node is recorded as unresolved (no text-unique fallback). + Grep reads ``ctx.blackboard.page_full_text_cache``. When ``ctx`` is None, + every null-page leaf is recorded as unresolved (no text-unique fallback). """ - del page_texts # grep uses blackboard cache via ctx - if not nodes or not body_pages: return dict(match_overrides), [] out = dict(match_overrides) report: list[dict[str, Any]] = [] + def _skip_entry( + *, + node: TitleNode, + path: tuple[str, ...], + result: str, + failed_sibling: str | None = None, + ) -> dict[str, Any]: + entry: dict[str, Any] = { + "path_titles": list(path), + "title": node.title, + "kind": "leaf", + "printed_page": None, + "search_scope": None, + "result": result, + "page": None, + "accept": None, + "visual_verify_calls": 0, + "react_attempts": [], + } + if failed_sibling is not None: + entry["failed_sibling"] = failed_sibling + return entry + + def _record_skipped_null_leaves( + node: TitleNode, + parent_titles: tuple[str, ...], + *, + result: str, + failed_sibling: str | None = None, + ) -> None: + for child in node.children: + path = (*parent_titles, child.title) + if ( + not child.children + and child.printed_page is None + and path not in out + ): + report.append( + _skip_entry( + node=child, + path=path, + result=result, + failed_sibling=failed_sibling, + ) + ) + if child.children: + _record_skipped_null_leaves( + child, + path, + result=result, + failed_sibling=failed_sibling, + ) + def _skip_rest( sibling_nodes: list[TitleNode], start_index: int, @@ -590,22 +639,24 @@ def _skip_rest( ) -> None: for later in sibling_nodes[start_index:]: path = (*parent_titles, later.title) - if later.printed_page is not None or path in out: - continue - report.append( - { - "path_titles": list(path), - "title": later.title, - "kind": "leaf" if not later.children else "parent", - "printed_page": None, - "search_scope": None, - "result": "skipped_after_sibling_failure", - "page": None, - "accept": None, - "visual_verify_calls": 0, - "react_attempts": [], - "failed_sibling": failed_title, - } + if ( + not later.children + and later.printed_page is None + and path not in out + ): + report.append( + _skip_entry( + node=later, + path=path, + result="skipped_after_sibling_failure", + failed_sibling=failed_title, + ) + ) + _record_skipped_null_leaves( + later, + path, + result="skipped_after_sibling_failure", + failed_sibling=failed_title, ) def _record_unresolved_no_ctx( @@ -614,12 +665,16 @@ def _record_unresolved_no_ctx( ) -> None: for node in sibling_nodes: path = (*parent_titles, node.title) - if node.printed_page is None and path not in out: + if ( + not node.children + and node.printed_page is None + and path not in out + ): report.append( { "path_titles": list(path), "title": node.title, - "kind": "leaf" if not node.children else "parent", + "kind": "leaf", "printed_page": None, "search_scope": None, "result": "unresolved_no_ctx", @@ -635,7 +690,7 @@ def _record_unresolved_no_ctx( if ctx is None: _record_unresolved_no_ctx(nodes, ()) logger.info( - "[null_page_react] ctx is None: {} null-page node(s) unresolved " + "[null_page_react] ctx is None: {} null-page leaf/leaves unresolved " "(no LLM/VLM probe)", len(report), ) @@ -665,13 +720,16 @@ def walk( if path_titles in out: cursor = max(cursor, int(out[path_titles].page)) - needs_probe = node.printed_page is None and path_titles not in out + needs_probe = ( + not node.children + and node.printed_page is None + and path_titles not in out + ) if needs_probe: - is_leaf = not node.children entry: dict[str, Any] = { "path_titles": list(path_titles), "title": node.title, - "kind": "leaf" if is_leaf else "parent", + "kind": "leaf", "printed_page": None, "search_scope": None, "result": "unresolved", @@ -757,6 +815,7 @@ def walk( "react_loop_limit", "planner_error", "unresolved", + "unresolved_no_ctx", "skipped_bad_window", } ), diff --git a/apps/worker/scripts/page_memory/tmp_probe_null_page_leaves.py b/apps/worker/scripts/page_memory/debug_pm_null_page_react.py similarity index 88% rename from apps/worker/scripts/page_memory/tmp_probe_null_page_leaves.py rename to apps/worker/scripts/page_memory/debug_pm_null_page_react.py index 91815d3cf..bfb28bd45 100644 --- a/apps/worker/scripts/page_memory/tmp_probe_null_page_leaves.py +++ b/apps/worker/scripts/page_memory/debug_pm_null_page_react.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 # ruff: noqa: E402 -"""Debug: run production Stage-2 TOC anchoring and dump null_page_report. +"""Debug: dump production null-page locate report via Stage-2 anchoring. -Uses the live prune + ``locate_null_page_node_overrides`` path (no patches). +Uses the live leaf ReAct + parent window locator path (no patches). Usage: cd apps/worker - uv run python scripts/page_memory/tmp_probe_null_page_leaves.py \\ + uv run python scripts/page_memory/debug_pm_null_page_react.py \\ --file "/path/to/doc.pdf" """ @@ -41,7 +41,7 @@ def main() -> int: parser = base_argparser( - "Debug: production null-page ReAct via run_toc_anchoring (Stage 2)" + "Debug: production null-page locate via run_toc_anchoring (Stage 2)" ) args = parser.parse_args() @@ -60,7 +60,7 @@ def main() -> int: hierarchies = list(getattr(anatomy, "toc_hierarchies", None) or []) logger.info("█" * 70) - logger.info(" Production null-page ReAct dump — {}", filename) + logger.info(" Production null-page locate dump — {}", filename) logger.info(" OUTPUT: {}", out_dir) logger.info("█" * 70) @@ -108,7 +108,8 @@ def main() -> int: payload = { "policy": { "prune_pre": "keep_null_page_nodes=True", - "probe": "null_page_react.locate_null_page_node_overrides", + "leaf_probe": "null_page_react.locate_null_page_node_overrides", + "parent_probe": "anchoring_primitives.locate_null_page_parent_overrides", "prune_post": "keep_null_page_nodes=False (drop unresolved)", "react_budget": react_budget(), "boundary_step_pages": BOUNDARY_STEP_PAGES, @@ -122,7 +123,7 @@ def main() -> int: "elapsed_s": round(time.time() - t0, 2), } - out_path = out_dir / "_doc_agent" / "tmp_null_page_leaf_probe.json" + out_path = out_dir / "_doc_agent" / "null_page_react_report.json" write_debug_json(out_path, payload) logger.info("wrote {}", out_path) logger.info( diff --git a/apps/worker/tests/contract/test_structure_anchoring_contract.py b/apps/worker/tests/contract/test_structure_anchoring_contract.py index 6eaff8127..5f3097e59 100644 --- a/apps/worker/tests/contract/test_structure_anchoring_contract.py +++ b/apps/worker/tests/contract/test_structure_anchoring_contract.py @@ -74,8 +74,8 @@ def test_prune_out_of_scope_nodes_removes_overflow_leaves() -> None: assert [n.title for n in pruned] == ["A"] -def test_null_page_nodes_unresolved_without_ctx() -> None: - """No ctx → no LLM/VLM and no text-unique fallback.""" +def test_null_page_leaf_unresolved_without_ctx() -> None: + """No ctx → no leaf ReAct; null-page parent is not handled here.""" from app.services.document_agent.structure.null_page_react import ( locate_null_page_node_overrides, ) @@ -89,12 +89,12 @@ def test_null_page_nodes_unresolved_without_ctx() -> None: overrides, report = locate_null_page_node_overrides( nodes=[parent], match_overrides={}, - page_texts={1: "Chapter\nHello"}, body_pages=[1, 2, 3], ctx=None, ) assert overrides == {} - assert len(report) == 2 + assert len(report) == 1 + assert report[0]["path_titles"] == ["Chapter", "Orphan"] assert {row["result"] for row in report} == {"unresolved_no_ctx"} @@ -138,7 +138,6 @@ def fail_a(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, overrides, report = locate_null_page_node_overrides( nodes=nodes, match_overrides={}, - page_texts={}, body_pages=[1, 2, 3, 4, 5], ctx=ctx, ) @@ -148,6 +147,39 @@ def fail_a(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, assert report[1]["result"] == "skipped_after_sibling_failure" +def test_null_page_react_does_not_probe_parent() -> None: + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_node_overrides, + ) + + parent = TitleNode( + title="Parent", + level=1, + printed_page=None, + children=[ + TitleNode(title="Child", level=2, printed_page=None, children=[]), + ], + ) + ctx = _ctx() + + def fail_child(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, str]: + assert kwargs["title"] == "Child" + return None, [], 0, "react_give_up" + + with patch.object(npr, "_locate_with_react", side_effect=fail_child): + overrides, report = locate_null_page_node_overrides( + nodes=[parent], + match_overrides={}, + body_pages=[1, 2, 3, 4, 5], + ctx=ctx, + ) + + assert overrides == {} + assert [row["path_titles"] for row in report] == [["Parent", "Child"]] + assert report[0]["result"] == "react_give_up" + + def test_null_page_react_hit_writes_override() -> None: from app.services.document_agent.structure import null_page_react as npr from app.services.document_agent.structure.null_page_react import ( @@ -179,7 +211,6 @@ def fake_react(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], overrides, report = locate_null_page_node_overrides( nodes=[leaf], match_overrides={}, - page_texts={}, body_pages=list(range(1, 21)), ctx=ctx, ) @@ -190,6 +221,240 @@ def fake_react(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], assert report[0]["result"] == "react_normalized_grep_vlm" +def test_null_page_parent_skipped_without_right_anchor() -> None: + parent = TitleNode( + title="Chapter", + level=1, + printed_page=None, + children=[TitleNode(title="Orphan", level=2, printed_page=None, children=[])], + ) + overrides, report = anchoring.locate_null_page_parent_overrides( + nodes=[parent], + match_overrides={}, + page_texts={1: "Chapter\nHello"}, + body_pages=[1, 2, 3], + ctx=None, + ) + assert overrides == {} + assert len(report) == 1 + assert report[0]["result"] == "skipped_no_right" + + +def test_null_page_parent_located_via_normalized_text() -> None: + child = TitleNode(title="1.1 Detail", level=2, printed_page=5, children=[]) + parent = TitleNode( + title="1 Overview", + level=1, + printed_page=None, + children=[child], + ) + leaf_match = anchoring.bulk_offset_matches( + [(("1 Overview", "1.1 Detail"), child)], + offset=0, + ) + page_texts = { + 4: "noise", + 5: "1 Overview\n1.1 Detail\nbody", + 6: "more", + } + overrides, report = anchoring.locate_null_page_parent_overrides( + nodes=[parent], + match_overrides=leaf_match, + page_texts=page_texts, + body_pages=[4, 5, 6], + ctx=None, + ) + assert ("1 Overview",) in overrides + assert overrides[("1 Overview",)].page == 5 + assert report[0]["result"] != "unresolved" + assert report[0]["page"] == 5 + assert report[0]["window"] == [1, 5] + + +def test_first_sibling_null_parent_uses_scan_forward_not_wide_rtl() -> None: + """No left sibling: miss text → ``scan_title_forward`` within 2+4+6+10 budget.""" + child = TitleNode(title="22.1 Intro", level=2, printed_page=278, children=[]) + parent = TitleNode( + title="Chapter 22", + level=1, + printed_page=None, + children=[child], + ) + leaf_match = { + ("Chapter 22", "22.1 Intro"): TitleMatch( + page=278, + source="anchored", + matched_line="", + candidates=[278], + evidence={}, + ) + } + body_pages = list(range(1, 301)) + page_texts = {page: "noise" for page in body_pages} + ctx = _ctx() + + scanned_starts: list[int] = [] + + def fake_scan(**kwargs: Any) -> Any: + from app.services.document_agent.calibration.scan import TitleScanResult + + scanned_starts.append(int(kwargs["start_page"])) + assert int(kwargs["page_count"]) == 278 + assert int(kwargs["start_page"]) == anchoring._first_sibling_null_parent_scan_start( + 278 + ) + return TitleScanResult( + title=str(kwargs["title"]), + found=True, + found_page=270, + scanned_pages=list(range(int(kwargs["start_page"]), 271)), + next_start=271, + ) + + with patch( + "app.services.document_agent.calibration.scan.scan_title_forward", + side_effect=fake_scan, + ): + with patch.object(anchoring, "_visual_rtl_locate_parent") as rtl: + overrides, report = anchoring.locate_null_page_parent_overrides( + nodes=[parent], + match_overrides=leaf_match, + page_texts=page_texts, + body_pages=body_pages, + ctx=ctx, + ) + rtl.assert_not_called() + + assert scanned_starts == [anchoring._first_sibling_null_parent_scan_start(278)] + assert overrides[("Chapter 22",)].page == 270 + assert report[0]["accept"] == "scan_forward" + assert report[0]["window"] == [ + anchoring._first_sibling_null_parent_scan_start(278), + 278, + ] + + +def test_null_page_parent_with_left_sibling_still_uses_rtl() -> None: + left_child = TitleNode(title="A.1", level=2, printed_page=10, children=[]) + left = TitleNode(title="A", level=1, printed_page=10, children=[left_child]) + right_child = TitleNode(title="B.1", level=2, printed_page=50, children=[]) + right = TitleNode(title="B", level=1, printed_page=None, children=[right_child]) + overrides_in = { + ("A",): TitleMatch( + page=10, + source="anchored", + matched_line="", + candidates=[10], + evidence={}, + ), + ("A", "A.1"): TitleMatch( + page=10, + source="anchored", + matched_line="", + candidates=[10], + evidence={}, + ), + ("B", "B.1"): TitleMatch( + page=50, + source="anchored", + matched_line="", + candidates=[50], + evidence={}, + ), + } + page_texts = {p: "noise" for p in range(1, 61)} + ctx = _ctx() + + def fake_rtl(**kwargs: Any) -> tuple[TitleMatch, int]: + assert kwargs["left"] == 10 + assert kwargs["right"] == 50 + return ( + TitleMatch( + page=40, + source="inspect_vlm", + matched_line="", + candidates=[40], + evidence={"accept": "visual_rtl"}, + ), + 3, + ) + + with patch( + "app.services.document_agent.calibration.scan.scan_title_forward" + ) as scan: + with patch.object( + anchoring, "_visual_rtl_locate_parent", side_effect=fake_rtl + ): + overrides, report = anchoring.locate_null_page_parent_overrides( + nodes=[left, right], + match_overrides=overrides_in, + page_texts=page_texts, + body_pages=list(range(1, 61)), + ctx=ctx, + ) + scan.assert_not_called() + + assert overrides[("B",)].page == 40 + assert report[0]["accept"] == "visual_rtl" + + +def test_null_page_leaf_runs_before_parent_in_production_flow() -> None: + child = TitleNode(title="Child", level=2, printed_page=None, children=[]) + parent = TitleNode( + title="Parent", + level=1, + printed_page=None, + children=[child], + ) + calls: list[str] = [] + + def fake_leaf(**kwargs: Any) -> tuple[dict[tuple[str, ...], TitleMatch], list[dict[str, Any]]]: + calls.append("leaf") + overrides = dict(kwargs["match_overrides"]) + overrides[("Parent", "Child")] = TitleMatch( + page=5, + source="anchored", + matched_line="Child", + candidates=[5], + evidence={}, + ) + return overrides, [{"kind": "leaf", "page": 5}] + + def fake_parent( + **kwargs: Any, + ) -> tuple[dict[tuple[str, ...], TitleMatch], list[dict[str, Any]]]: + calls.append("parent") + overrides = dict(kwargs["match_overrides"]) + assert ("Parent", "Child") in overrides + overrides[("Parent",)] = TitleMatch( + page=4, + source="anchored", + matched_line="Parent", + candidates=[4], + evidence={}, + ) + return overrides, [{"kind": "parent", "page": 4}] + + with ( + patch.object(anchoring, "locate_null_page_node_overrides", side_effect=fake_leaf), + patch.object(anchoring, "locate_null_page_parent_overrides", side_effect=fake_parent), + ): + resolved, anchor = anchoring.anchor_hierarchy_from_offset( + nodes=[parent], + offset_hint=None, + calibration_overrides={}, + page_texts={page: "noise" for page in range(1, 11)}, + body_pages=list(range(1, 11)), + page_count=10, + ctx=None, + ) + + assert calls == ["leaf", "parent"] + assert [node.title for node in resolved] == ["Parent"] + assert set(anchor.match_overrides) == {("Parent",), ("Parent", "Child")} + assert [row["kind"] for row in anchor.null_page_report] == ["leaf", "parent"] + + def test_normalized_title_match_preserves_english_word_boundary() -> None: from app.services.document_agent.structure.hierarchy_locator import ( locate_title_normalized_strict, From ea0edb9a6e4c35f455ea181082582ed6c58f3406 Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Sat, 22 Aug 2026 12:22:14 +0800 Subject: [PATCH 4/7] refactor: enhance PDF text processing and null-page handling - Updated the strip_margin_text function to support edge-specific margin stripping for headers and footers, improving content extraction accuracy. - Introduced a new utility function, apply_null_page_locates_and_prune, to streamline null-page processing and enhance the handling of parent-child relationships in document structures. - Refactored related functions and tests to ensure compatibility with the new logic, improving overall robustness in null-page handling and text processing. - Enhanced test coverage for edge cases in margin stripping and null-page processing to validate the new functionality. --- .../document_agent/calibration/procedure.py | 54 +- .../app/services/document_agent/pdf_text.py | 47 +- .../structure/anchoring_primitives.py | 94 ++- .../structure/null_page_react.py | 552 +++++++++--------- .../document_agent/structure/toc_anchoring.py | 32 +- .../document_agent/tools/grep_text.py | 9 - .../tools/text_strip_margins.py | 2 +- .../page_memory/skeleton_extractor.py | 54 +- .../page_memory/debug_pm_null_page_react.py | 11 +- ...profile_skeleton_anchor_wiring_contract.py | 43 +- .../test_structure_anchoring_contract.py | 529 +++++++++++++++++ .../test_tool_registry_smoke_contract.py | 16 + 12 files changed, 1057 insertions(+), 386 deletions(-) diff --git a/apps/worker/app/services/document_agent/calibration/procedure.py b/apps/worker/app/services/document_agent/calibration/procedure.py index 6b4775b54..7ff1fdc59 100644 --- a/apps/worker/app/services/document_agent/calibration/procedure.py +++ b/apps/worker/app/services/document_agent/calibration/procedure.py @@ -33,14 +33,11 @@ ) from app.services.document_agent.structure.anchoring_primitives import ( SkeletonAnchor, + apply_null_page_locates_and_prune, backfill_parent_offset_matches, - locate_null_page_parent_overrides, prune_unanchored_toc_leaves, serialize_skeleton_anchor, ) -from app.services.document_agent.structure.null_page_react import ( - locate_null_page_node_overrides, -) from app.services.document_agent.structure import anchoring_primitives as _anchoring # Re-export under prior names so existing imports keep working. @@ -353,6 +350,13 @@ def anchor_hierarchy_from_regimes( len(regime_seed), ) + # Capture parents before prune: empty shells after keep_null prune must + # not enter leaf ReAct (H1). + structural_parent_paths = { + path + for path, node in _iter_all_title_nodes(working) + if node.children + } # Failed printed-page leaves → drop; keep null-page nodes for ReAct. working, unanchored_removed = prune_unanchored_toc_leaves( working, @@ -384,38 +388,21 @@ def anchor_hierarchy_from_regimes( len(parent_matches), ) - match_overrides, leaf_report = locate_null_page_node_overrides( - nodes=working, - match_overrides=merged, - body_pages=body_pages, - ctx=ctx, - ) - match_overrides, parent_report = locate_null_page_parent_overrides( - nodes=working, - match_overrides=match_overrides, - page_texts=page_texts, - body_pages=body_pages, - ctx=ctx, - ) - null_page_report = [*leaf_report, *parent_report] + # H4: freeze bulk before null-page leaf/parent ReAct (same as offset path: + # offset_guided → len(overrides then); else 0). Never recount after ReAct. + bulk_count = len(merged) if regime_bulk > 0 else 0 - # Drop still-unanchored null-page nodes (avoid sticky inherited ranges). - working, failed_null_removed = prune_unanchored_toc_leaves( - working, - match_overrides=match_overrides, - keep_null_page_nodes=False, + working, match_overrides, null_page_report, failed_null_removed = ( + apply_null_page_locates_and_prune( + nodes=working, + match_overrides=merged, + body_pages=body_pages, + page_texts=page_texts, + ctx=ctx, + structural_parent_paths=structural_parent_paths, + ) ) total_pruned += failed_null_removed - if working: - surviving_paths = { - path - for path, _node in _iter_all_title_nodes(working) - } - match_overrides = { - path: match - for path, match in match_overrides.items() - if path in surviving_paths - } primary = pick_primary_offset(result) if primary is None and usable_regimes: @@ -431,7 +418,6 @@ def anchor_hierarchy_from_regimes( if match_overrides and (regime_bulk > 0 or seed) else "offset_only" ) - bulk_count = len(match_overrides) return working, SkeletonAnchor( offset=primary, diff --git a/apps/worker/app/services/document_agent/pdf_text.py b/apps/worker/app/services/document_agent/pdf_text.py index 202e2417f..12994405b 100644 --- a/apps/worker/app/services/document_agent/pdf_text.py +++ b/apps/worker/app/services/document_agent/pdf_text.py @@ -4,7 +4,7 @@ import gc from dataclasses import dataclass -from typing import Any +from typing import Any, Literal from app.services.document_parser.formats.pdf.pymupdf_subprocess import ( run_in_child_process, @@ -72,20 +72,47 @@ def page_bands_map(raw: Any) -> dict[int, PageTextBands]: return {int(page): PageTextBands.from_any(value) for page, value in raw.items()} -def strip_margin_text(content: str, margin: str) -> str: - """Remove one margin extract from full content (homologous span join). +def strip_margin_text( + content: str, + margin: str, + *, + edge: Literal["header", "footer"], +) -> str: + """Remove a header/footer band extract from page content for search view. - Tries the full margin blob first, then each non-empty line once, so - non-contiguous edge lines still drop when they appear as content lines. + Header is removed only from the content prefix; footer only from the + content suffix. Never uses a whole-string first-occurrence replace (that + can delete a same-looking body span). Falls back to edge-aligned line + fragments when the full margin blob is not contiguous at that edge. """ if not content or not margin: return content - if margin in content: - return content.replace(margin, "", 1) + + if edge == "header": + if content.startswith(margin): + return content[len(margin) :] + out = content + for frag in margin.split("\n"): + if not frag: + continue + if out.startswith(frag + "\n"): + out = out[len(frag) + 1 :] + elif out.startswith(frag): + out = out[len(frag) :] + else: + break + return out + + if content.endswith(margin): + return content[: -len(margin)] out = content - for frag in margin.split("\n"): - if frag and frag in out: - out = out.replace(frag, "", 1) + for frag in reversed([part for part in margin.split("\n") if part]): + if out.endswith("\n" + frag): + out = out[: -(len(frag) + 1)] + elif out.endswith(frag): + out = out[: -len(frag)] + else: + break return out diff --git a/apps/worker/app/services/document_agent/structure/anchoring_primitives.py b/apps/worker/app/services/document_agent/structure/anchoring_primitives.py index 0cb974b6f..fb0884c4c 100644 --- a/apps/worker/app/services/document_agent/structure/anchoring_primitives.py +++ b/apps/worker/app/services/document_agent/structure/anchoring_primitives.py @@ -97,8 +97,12 @@ def prune_unanchored_toc_leaves( Implements Phase-2 ``suffix = no TOC``: after bulk/bisect/recalibrate, any leaf that was not successfully anchored is dropped from the coarse tree - instead of sticky ``inherited_unlocated`` ranges. Childless parents are - removed unless they themselves have an override. + instead of sticky ``inherited_unlocated`` ranges. + + Parents that still have at least one surviving child after recursive prune + are kept even without their own override, so later descendant-based infer + can use the subtree. Childless parents are removed unless they themselves + have an override. When ``keep_null_page_nodes`` is True (pre null-page ReAct), nodes with ``printed_page is None`` are retained so they can be probed. Call again with @@ -217,6 +221,7 @@ def walk( entry: dict[str, Any] = { "path_titles": list(path_titles), "title": node.title, + "kind": "parent", "printed_page": None, "window": None, "result": "skipped_no_right", @@ -1022,6 +1027,54 @@ def _filter_overrides_to_tree( } +def apply_null_page_locates_and_prune( + *, + nodes: list[TitleNode], + match_overrides: dict[tuple[str, ...], TitleMatch], + body_pages: list[int], + page_texts: dict[int, str], + ctx: ToolContext | None, + structural_parent_paths: set[tuple[str, ...]], +) -> tuple[ + list[TitleNode], + dict[tuple[str, ...], TitleMatch], + list[dict[str, Any]], + int, +]: + """Shared Phase-2 tail: leaf ReAct → null-page parent locate → final prune. + + Callers collect ``structural_parent_paths`` and run ``prune(..., keep_null=True)`` + before this helper. Regimes also runs printed-page parent backfill before + calling; the offset path intentionally does not (known asymmetry). + """ + working = nodes + overrides = dict(match_overrides) + + overrides, leaf_report = locate_null_page_node_overrides( + nodes=working, + match_overrides=overrides, + body_pages=body_pages, + ctx=ctx, + structural_parent_paths=structural_parent_paths, + ) + overrides, parent_report = locate_null_page_parent_overrides( + nodes=working, + match_overrides=overrides, + page_texts=page_texts, + body_pages=body_pages, + ctx=ctx, + ) + null_page_report = [*leaf_report, *parent_report] + + working, failed_null_removed = prune_unanchored_toc_leaves( + working, + match_overrides=overrides, + keep_null_page_nodes=False, + ) + overrides = _filter_overrides_to_tree(working, overrides) + return working, overrides, null_page_report, failed_null_removed + + def anchor_hierarchy_from_offset( *, nodes: list[TitleNode], @@ -1063,6 +1116,13 @@ def anchor_hierarchy_from_offset( locate_method = "offset_only" bulk_count = 0 + # Capture parents before prune: empty shells after keep_null prune must + # not enter leaf ReAct (H1). + structural_parent_paths = { + path + for path, node in _iter_all_title_nodes(working) + if node.children + } working, unanchored_removed = prune_unanchored_toc_leaves( working, match_overrides=match_overrides, @@ -1071,28 +1131,18 @@ def anchor_hierarchy_from_offset( pruned_count += unanchored_removed match_overrides = _filter_overrides_to_tree(working, match_overrides) - match_overrides, leaf_report = locate_null_page_node_overrides( - nodes=working, - match_overrides=match_overrides, - body_pages=body_pages, - ctx=ctx, - ) - match_overrides, parent_report = locate_null_page_parent_overrides( - nodes=working, - match_overrides=match_overrides, - page_texts=page_texts, - body_pages=body_pages, - ctx=ctx, - ) - null_page_report = [*leaf_report, *parent_report] - - working, failed_null_removed = prune_unanchored_toc_leaves( - working, - match_overrides=match_overrides, - keep_null_page_nodes=False, + # Offset path does not run printed-page parent backfill (regimes does). + working, match_overrides, null_page_report, failed_null_removed = ( + apply_null_page_locates_and_prune( + nodes=working, + match_overrides=match_overrides, + body_pages=body_pages, + page_texts=page_texts, + ctx=ctx, + structural_parent_paths=structural_parent_paths, + ) ) pruned_count += failed_null_removed - match_overrides = _filter_overrides_to_tree(working, match_overrides) if offset_hint is None: offset_status = "failed" if ctx is not None else "skipped" diff --git a/apps/worker/app/services/document_agent/structure/null_page_react.py b/apps/worker/app/services/document_agent/structure/null_page_react.py index 4e824efae..1ef30cb11 100644 --- a/apps/worker/app/services/document_agent/structure/null_page_react.py +++ b/apps/worker/app/services/document_agent/structure/null_page_react.py @@ -2,8 +2,15 @@ After Phase-2 printed-page bulk/bisect, null-page leaves use a serial probe: text LLM plans a ``grep.text`` query inside a sibling window, then -``inspect.pages`` confirms the physical section start one hit page at a time. -Loop / hit / visual budgets equal ``BOUNDARY_STEP_PAGES``. +``inspect.pages`` confirms only the first hit page (binary section-start check). +Later hits from the same grep are discarded; a reject advances to the next +planner turn. + +Budget (``REACT_PLANNER_GREP_BUDGET``): +- Seed full-title grep is free (does not consume the budget). +- Planner-chosen greps: at most ``REACT_PLANNER_GREP_BUDGET`` times. +- strip_header / strip_footer each auto re-grep the last query once for free. +- Planner turn cap = budget + 2 (room for the two strip actions). No offset seed, no fixed page-count cap, and no fallback to normalized-strict unique hit / RTL / ``scan_title_forward``. @@ -24,23 +31,26 @@ last_leaf_start_under, ) -_HISTORY_SAMPLE_PAGES = 3 +# Planner grep rounds after the free seed full-title probe. Not a page count. +REACT_PLANNER_GREP_BUDGET = 5 def react_budget() -> int: - """Loop / hit / visual budget; same constant as TOC ``BOUNDARY_STEP_PAGES``.""" - from app.services.document_agent.tools.extract_toc_with_boundaries import ( - BOUNDARY_STEP_PAGES, - ) + """Planner grep-loop budget (and shared visual-confirm cap). - return int(BOUNDARY_STEP_PAGES) + Seed full-title grep does not consume this. Strip auto re-greps do not + consume it. Value is ``REACT_PLANNER_GREP_BUDGET``, independent of TOC + page-window constants. + """ + return int(REACT_PLANNER_GREP_BUDGET) _REACT_INSTRUCTIONS = """\ You are the search planner in a small ReAct loop. Propose the next action to find the physical START page of a section. Grep collapses whitespace/newlines to one space between non-CJK words, removes whitespace adjacent to CJK, and -matches case-insensitively. A separate visual check confirms candidates one -page at a time. +matches case-insensitively. After each grep, a visual check confirms only the +first hit page (binary section-start). Later hits are discarded; a reject +means try a different query next. The system already grepped the full TOC title once before this loop (see previous_attempts). Do not repeat that exact full-title query. @@ -72,20 +82,21 @@ def react_budget() -> int: Reflection rules (mandatory): - Read previous_attempts. Reflect on hit_count and observation before answering. -- If the last observation is no_normalized_hits, too_many_hits, visual_rejected, - or duplicate_normalized_query, you MUST change the query when choosing grep. - Emitting the same grep query again (same text after whitespace/case - normalization) is invalid for planner-chosen greps. -- too_many_hits means hit_count exceeded the visual budget. Prefer - strip_header or strip_footer when hits look scattered by running - headers/footers. Each strip automatically re-greps the last query once - (same action; do not spend a planner turn to repeat that query). Otherwise - go to the next step in the ordered strategy (narrower / different query). +- If the last observation is no_normalized_hits, visual_rejected, + empty_normalized_query, grep_tool_error, or duplicate_normalized_query, you + MUST change the query when choosing grep. Emitting the same grep query again + (same text after whitespace/case normalization) is invalid for planner-chosen + greps. +- Prefer strip_header or strip_footer when hits look scattered by running + headers/footers. Each strip automatically re-greps the last query once for + free (same action; do not spend a planner turn to repeat that query). + Otherwise go to the next step in the ordered strategy (narrower / different + query). - strip_header / strip_footer only update a temporary search view; they do not change stored page text. They do NOT consume react_budget. Call each at most once per locate. -- Planner greps consume react_budget (grep_loops_remaining). Strip auto-retries - never consume it. +- Planner greps consume react_budget (grep_loops_remaining). The automatic + full-title seed grep is free and does not consume it. - no_normalized_hits / visual_rejected: advance to the next ordered strategy step rather than repeating the same query. """ @@ -97,7 +108,6 @@ def _react_history_item(item: dict[str, Any]) -> dict[str, Any]: "query": item.get("query"), "normalized_query": item.get("normalized_query"), "hit_count": int(item.get("hit_count") or len(hit_pages)), - "sample_pages": hit_pages[:_HISTORY_SAMPLE_PAGES], "observation": item.get("observation"), "visual_selected_page": item.get("visual_selected_page"), "visual_reason": item.get("visual_reason"), @@ -133,23 +143,65 @@ def _normalized_grep( query: str, left: int, right: int, -) -> tuple[str, list[int], int]: + body_pages: list[int], +) -> tuple[str, str, list[int], int]: + """Grep only ``body_pages ∩ [left, right]``. + + Returns ``(status, needle_or_error, hit_pages, match_count)``. + ``status`` is ``"ok"`` or ``"error"``; on error the second field is a short + error summary (not a normalized needle). + """ + from app.services.document_agent.pdf_text import page_content_map from app.services.document_agent.tools.grep_text import grep_text - result = grep_text( - ctx, - { - "query": query, - "start_page": left, - "end_page": right, - }, - ) + scope_pages = [page for page in body_pages if left <= page <= right] + if not scope_pages: + return "ok", "", [], 0 + + body_set = set(scope_pages) + view = ctx.blackboard.page_text_search_view + if view is not None: + texts = { + int(page): str(view[page]) + for page in scope_pages + if page in view + } + else: + full = page_content_map(ctx.blackboard.page_full_text_cache) + texts = { + int(page): str(full[page]) + for page in scope_pages + if page in full + } + if not texts: + return "ok", "", [], 0 + + prev_view = ctx.blackboard.page_text_search_view + ctx.blackboard.page_text_search_view = texts + try: + result = grep_text( + ctx, + { + "query": query, + "start_page": min(texts), + "end_page": max(texts), + }, + ) + finally: + ctx.blackboard.page_text_search_view = prev_view + if result.status != "ok": - return "", [], 0 + return "error", str(result.error or "grep.text failed"), [], 0 payload = result.payload or {} + hit_pages = [ + int(page) + for page in (payload.get("hit_pages") or []) + if int(page) in body_set + ] return ( + "ok", str(payload.get("normalized_query") or ""), - [int(page) for page in (payload.get("hit_pages") or [])], + hit_pages, int(payload.get("hit_count") or 0), ) @@ -174,9 +226,10 @@ def _propose_react_query( "visual_budget": budget, "previous_attempts": [_react_history_item(item) for item in attempts], "note": ( - "Full TOC title was already grepped automatically before this loop. " - "Planner greps consume react_budget. strip_header/strip_footer are " - "free and each auto-retries the last grep query once." + "Full TOC title was already grepped automatically before this loop " + "(free; does not consume react_budget). Planner greps consume " + "react_budget. strip_header/strip_footer are free and each " + "auto-retries the last grep query once for free." ), } prompt = ( @@ -279,6 +332,7 @@ def _locate_with_react( title: str, left: int, right: int, + body_pages: list[int], ctx: ToolContext, ) -> tuple[TitleMatch | None, list[dict[str, Any]], int, str]: budget = react_budget() @@ -290,11 +344,12 @@ def _locate_with_react( last_grep_query: str | None = None # Each locate starts from stored content (no cross-node strip leakage). ctx.blackboard.page_text_search_view = None - # Automatic full-title grep is free (no planner). Planner greps consume - # budget. strip_* (+ auto same-query re-grep) is free. + # Seed full-title grep is free. Planner greps consume budget (≤ budget). + # strip_* each auto re-greps last query once for free. + # +2 planner turns: room for strip_header and strip_footer actions. grep_loops_used = 0 planner_turn = 0 - max_planner_turns = budget + 2 + budget + max_planner_turns = budget + 2 def _visual_confirm( *, @@ -302,67 +357,55 @@ def _visual_confirm( hit_pages: list[int], attempt: dict[str, Any], ) -> TitleMatch | None: + """Confirm only the first hit page; discard the rest of this grep.""" nonlocal visual_calls, visual_remaining if visual_remaining <= 0: attempt["observation"] = "visual_budget_exhausted" attempts.append(attempt) return None - checked: list[dict[str, Any]] = [] - selected: int | None = None - last_reason = "" - for page in hit_pages: - if visual_remaining <= 0: - break - visual_remaining -= 1 - visual_calls += 1 - ok, reason, tokens = _verify_section_beginning_page( - ctx=ctx, - title=title, - page=page, - query=query, - ) - checked.append( - { - "page": page, - "confirmed": ok, - "reason": reason, - "tokens_used": tokens, - } - ) - last_reason = reason - if ok: - selected = page - break - + page = int(hit_pages[0]) + visual_remaining -= 1 + visual_calls += 1 + ok, reason, tokens = _verify_section_beginning_page( + ctx=ctx, + title=title, + page=page, + query=query, + ) + checked = [ + { + "page": page, + "confirmed": ok, + "reason": reason, + "tokens_used": tokens, + } + ] attempt["visual_pages_checked"] = checked - attempt["visual_selected_page"] = selected - attempt["visual_reason"] = last_reason + attempt["visual_selected_page"] = page if ok else None + attempt["visual_reason"] = reason attempt["visual_budget_remaining_after"] = visual_remaining - if selected is not None: + if ok: attempt["observation"] = "section_start_confirmed" attempts.append(attempt) return TitleMatch( - page=int(selected), + page=page, source="react_normalized_grep_vlm", matched_line=query, - candidates=hit_pages, + candidates=[page], evidence={ "accept": "react_normalized_grep_vlm", "null_page_react": True, "loop": grep_loops_used, "normalized_query": attempt.get("normalized_query"), - "visual_reason": last_reason, - "visual_pages_checked": [item["page"] for item in checked], + "visual_reason": reason, + "visual_pages_checked": [page], "post_strip": attempt.get("post_strip"), "seed_full_title": attempt.get("seed_full_title"), }, ) - if visual_remaining <= 0 and len(checked) < len(hit_pages): - attempt["observation"] = "visual_budget_exhausted" - else: - attempt["observation"] = "visual_rejected" + attempt["observation"] = "visual_rejected" attempts.append(attempt) return None @@ -379,13 +422,15 @@ def _apply_grep_result( """Grep + classify. Appends to attempts; returns match on visual confirm.""" nonlocal grep_loops_used, last_grep_query - needle, hit_pages, match_count = _normalized_grep( + status, needle_or_error, hit_pages, match_count = _normalized_grep( ctx=ctx, query=query, left=left, right=right, + body_pages=body_pages, ) - if consume_budget and needle and needle not in attempted_needles: + needle = needle_or_error if status == "ok" else "" + if consume_budget and status == "ok" and needle and needle not in attempted_needles: grep_loops_used += 1 attempt: dict[str, Any] = { "loop": planner_turn_index, @@ -404,7 +449,18 @@ def _apply_grep_result( if seed_full_title: attempt["seed_full_title"] = True - if not needle or (needle in attempted_needles and not allow_duplicate): + if status != "ok": + attempt["observation"] = "grep_tool_error" + attempt["error"] = needle_or_error + attempts.append(attempt) + return None + + if not needle: + attempt["observation"] = "empty_normalized_query" + attempts.append(attempt) + return None + + if needle in attempted_needles and not allow_duplicate: attempt["observation"] = "duplicate_normalized_query" attempts.append(attempt) return None @@ -419,74 +475,48 @@ def _apply_grep_result( attempts.append(attempt) return None - if len(hit_pages) > budget: - attempt["observation"] = ( - "post_strip_too_many_hits" if post_strip else "too_many_hits" - ) - attempts.append(attempt) - return None - return _visual_confirm(query=query, hit_pages=hit_pages, attempt=attempt) - # Free automatic probe: full TOC title (no planner, no react_budget). - seed_match = _apply_grep_result( - query=title, - planner_turn_index=0, - consume_budget=False, - allow_duplicate=False, - post_strip=None, - planner_meta={}, - seed_full_title=True, - ) - if seed_match is not None: - return seed_match, attempts, visual_calls, "react_normalized_grep_vlm" - - while grep_loops_used < budget and planner_turn < max_planner_turns: - planner_turn += 1 - proposal, planner_meta = _propose_react_query( - title=title, - parent_titles=path_titles[:-1], - left=left, - right=right, - attempts=attempts, - budget=budget, - grep_loops_used=grep_loops_used, + try: + # Free automatic probe: full TOC title (no planner, no react_budget). + seed_match = _apply_grep_result( + query=title, + planner_turn_index=0, + consume_budget=False, + allow_duplicate=False, + post_strip=None, + planner_meta={}, + seed_full_title=True, ) - if proposal is None: - attempts.append( - { - "loop": planner_turn, - "grep_loop": grep_loops_used, - "action": "planner_error", - "hit_count": 0, - "hit_pages": [], - **planner_meta, - } - ) - continue - - action = proposal["action"] - if action == "give_up": - attempts.append( - { - "loop": planner_turn, - "grep_loop": grep_loops_used, - **proposal, - "hit_count": 0, - "hit_pages": [], - **planner_meta, - } - ) - return None, attempts, visual_calls, "react_give_up" + if seed_match is not None: + return seed_match, attempts, visual_calls, "react_normalized_grep_vlm" - if action in {"strip_header", "strip_footer"}: - from app.services.document_agent.tools.text_strip_margins import ( - strip_footer, - strip_header, + while grep_loops_used < budget and planner_turn < max_planner_turns: + planner_turn += 1 + proposal, planner_meta = _propose_react_query( + title=title, + parent_titles=path_titles[:-1], + left=left, + right=right, + attempts=attempts, + budget=budget, + grep_loops_used=grep_loops_used, ) + if proposal is None: + attempts.append( + { + "loop": planner_turn, + "grep_loop": grep_loops_used, + "action": "planner_error", + "hit_count": 0, + "hit_pages": [], + **planner_meta, + } + ) + continue - which = "header" if action == "strip_header" else "footer" - if which in stripped: + action = proposal["action"] + if action == "give_up": attempts.append( { "loop": planner_turn, @@ -494,70 +524,95 @@ def _apply_grep_result( **proposal, "hit_count": 0, "hit_pages": [], - "observation": f"duplicate_strip_{which}", **planner_meta, } ) - continue - strip_fn = strip_header if which == "header" else strip_footer - strip_result = strip_fn( - ctx, - {"start_page": left, "end_page": right}, - ) - stripped.add(which) - payload = strip_result.payload or {} - strip_ok = strip_result.status == "ok" - attempts.append( - { - "loop": planner_turn, - "grep_loop": grep_loops_used, - **proposal, - "hit_count": 0, - "hit_pages": [], - "observation": ( - f"stripped_{which}" if strip_ok else f"strip_{which}_failed" - ), - "pages_updated": int(payload.get("pages_updated") or 0), - "strip_error": strip_result.error, - **planner_meta, - } - ) - if not strip_ok or not last_grep_query: - continue + return None, attempts, visual_calls, "react_give_up" - # Same action: auto re-grep last query on stripped view (no budget). - from app.services.document_parser.structure.body_boundary import ( - normalize_match_text, - ) + if action in {"strip_header", "strip_footer"}: + from app.services.document_agent.tools.text_strip_margins import ( + strip_footer, + strip_header, + ) + + which = "header" if action == "strip_header" else "footer" + if which in stripped: + attempts.append( + { + "loop": planner_turn, + "grep_loop": grep_loops_used, + **proposal, + "hit_count": 0, + "hit_pages": [], + "observation": f"duplicate_strip_{which}", + **planner_meta, + } + ) + continue + strip_fn = strip_header if which == "header" else strip_footer + strip_result = strip_fn( + ctx, + {"start_page": left, "end_page": right}, + ) + stripped.add(which) + payload = strip_result.payload or {} + strip_ok = strip_result.status == "ok" + attempts.append( + { + "loop": planner_turn, + "grep_loop": grep_loops_used, + **proposal, + "hit_count": 0, + "hit_pages": [], + "observation": ( + f"stripped_{which}" + if strip_ok + else f"strip_{which}_failed" + ), + "pages_updated": int(payload.get("pages_updated") or 0), + "strip_error": strip_result.error, + **planner_meta, + } + ) + if not strip_ok or not last_grep_query: + continue + + # Same action: auto re-grep last query on stripped view (no budget). + from app.services.document_parser.structure.body_boundary import ( + normalize_match_text, + ) + + prior_needle = normalize_match_text(last_grep_query) + if prior_needle: + attempted_needles.discard(prior_needle) + match = _apply_grep_result( + query=last_grep_query, + planner_turn_index=planner_turn, + consume_budget=False, + allow_duplicate=True, + post_strip=which, + planner_meta={}, + ) + if match is not None: + return match, attempts, visual_calls, "react_normalized_grep_vlm" + continue - prior_needle = normalize_match_text(last_grep_query) - if prior_needle: - attempted_needles.discard(prior_needle) + query = proposal["query"] match = _apply_grep_result( - query=last_grep_query, + query=query, planner_turn_index=planner_turn, - consume_budget=False, - allow_duplicate=True, - post_strip=which, - planner_meta={}, + consume_budget=True, + allow_duplicate=False, + post_strip=None, + planner_meta=planner_meta, ) if match is not None: return match, attempts, visual_calls, "react_normalized_grep_vlm" - continue - - query = proposal["query"] - match = _apply_grep_result( - query=query, - planner_turn_index=planner_turn, - consume_budget=True, - allow_duplicate=False, - post_strip=None, - planner_meta=planner_meta, - ) - if match is not None: - return match, attempts, visual_calls, "react_normalized_grep_vlm" - return None, attempts, visual_calls, "react_loop_limit" + return None, attempts, visual_calls, "react_loop_limit" + finally: + # Do not leak strip view across nodes / locate paths. + ctx.blackboard.page_text_search_view = None def locate_null_page_node_overrides( @@ -566,15 +621,21 @@ def locate_null_page_node_overrides( match_overrides: dict[tuple[str, ...], TitleMatch], body_pages: list[int], ctx: ToolContext | None, + structural_parent_paths: set[tuple[str, ...]] | None = None, ) -> tuple[dict[tuple[str, ...], TitleMatch], list[dict[str, Any]]]: """Locate null-page leaves with normalized grep ReAct + VLM. Grep reads ``ctx.blackboard.page_full_text_cache``. When ``ctx`` is None, every null-page leaf is recorded as unresolved (no text-unique fallback). + + ``structural_parent_paths`` are paths that had children *before* + ``prune(keep_null=True)``. Empty-shell parents after prune must not be + leaf-probed; they go to parent locate instead. """ if not nodes or not body_pages: return dict(match_overrides), [] + parent_paths = structural_parent_paths or set() out = dict(match_overrides) report: list[dict[str, Any]] = [] @@ -601,64 +662,6 @@ def _skip_entry( entry["failed_sibling"] = failed_sibling return entry - def _record_skipped_null_leaves( - node: TitleNode, - parent_titles: tuple[str, ...], - *, - result: str, - failed_sibling: str | None = None, - ) -> None: - for child in node.children: - path = (*parent_titles, child.title) - if ( - not child.children - and child.printed_page is None - and path not in out - ): - report.append( - _skip_entry( - node=child, - path=path, - result=result, - failed_sibling=failed_sibling, - ) - ) - if child.children: - _record_skipped_null_leaves( - child, - path, - result=result, - failed_sibling=failed_sibling, - ) - - def _skip_rest( - sibling_nodes: list[TitleNode], - start_index: int, - parent_titles: tuple[str, ...], - failed_title: str, - ) -> None: - for later in sibling_nodes[start_index:]: - path = (*parent_titles, later.title) - if ( - not later.children - and later.printed_page is None - and path not in out - ): - report.append( - _skip_entry( - node=later, - path=path, - result="skipped_after_sibling_failure", - failed_sibling=failed_title, - ) - ) - _record_skipped_null_leaves( - later, - path, - result="skipped_after_sibling_failure", - failed_sibling=failed_title, - ) - def _record_unresolved_no_ctx( sibling_nodes: list[TitleNode], parent_titles: tuple[str, ...], @@ -667,6 +670,7 @@ def _record_unresolved_no_ctx( path = (*parent_titles, node.title) if ( not node.children + and path not in parent_paths and node.printed_page is None and path not in out ): @@ -703,6 +707,9 @@ def walk( scope_end: int, ) -> None: cursor = int(scope_start) + # A failed leaf only invalidates the serial cursor for later leaves at + # this level. Later parents keep their own walk (fresh flag). + failed_sibling: str | None = None for index, node in enumerate(sibling_nodes): path_titles = (*parent_titles, node.title) next_bound = _next_located_bound( @@ -722,10 +729,22 @@ def walk( needs_probe = ( not node.children + and path_titles not in parent_paths and node.printed_page is None and path_titles not in out ) if needs_probe: + if failed_sibling is not None: + report.append( + _skip_entry( + node=node, + path=path_titles, + result="skipped_after_sibling_failure", + failed_sibling=failed_sibling, + ) + ) + continue + entry: dict[str, Any] = { "path_titles": list(path_titles), "title": node.title, @@ -741,21 +760,28 @@ def walk( left = int(cursor) right = int(node_scope_end) + entry["search_scope"] = [left, right] + scope_pages = [ + page for page in body_pages if left <= page <= right + ] if right < left: entry["result"] = "skipped_bad_window" - entry["search_scope"] = [left, right] report.append(entry) - _skip_rest( - sibling_nodes, index + 1, parent_titles, node.title - ) - return + failed_sibling = node.title + continue + + if not scope_pages: + entry["result"] = "no_scope_pages" + report.append(entry) + failed_sibling = node.title + continue - entry["search_scope"] = [left, right] match, attempts, visual_calls, result = _locate_with_react( path_titles=path_titles, title=node.title, left=left, right=right, + body_pages=body_pages, ctx=ctx, ) entry["react_attempts"] = attempts @@ -768,12 +794,11 @@ def walk( report.append(entry) if entry.get("page") is None: - _skip_rest( - sibling_nodes, index + 1, parent_titles, node.title - ) - return + failed_sibling = node.title + continue cursor = int(entry["page"]) + continue if node.children: child_scope_start = ( @@ -817,6 +842,7 @@ def walk( "unresolved", "unresolved_no_ctx", "skipped_bad_window", + "no_scope_pages", } ), skipped, diff --git a/apps/worker/app/services/document_agent/structure/toc_anchoring.py b/apps/worker/app/services/document_agent/structure/toc_anchoring.py index 3053ee096..1e9ccae0b 100644 --- a/apps/worker/app/services/document_agent/structure/toc_anchoring.py +++ b/apps/worker/app/services/document_agent/structure/toc_anchoring.py @@ -25,7 +25,6 @@ TitleNode, collapse_intermediate_single_child_chains, extract_toc_nodes, - iter_leaf_title_nodes, ) from app.services.document_parser.structure.body_boundary import normalize_heading_label @@ -33,6 +32,22 @@ PENDING_TOC_CALIBRATION_CONCURRENCY = 10 +def fork_ctx_for_pending(ctx: ToolContext) -> ToolContext: + """Shallow-fork ctx for one pending TOC job. + + Shares ``page_full_text_cache`` and other blackboard fields by reference; + isolates ``page_text_search_view`` so concurrent strip/grep cannot race. + """ + return ToolContext( + pdf_path=ctx.pdf_path, + job_id=ctx.job_id, + blackboard=replace(ctx.blackboard, page_text_search_view=None), + trace=ctx.trace, + output_dir=ctx.output_dir, + settings=ctx.settings, + ) + + def run_toc_anchoring(ctx: ToolContext) -> None: """Anchor TOC structure onto the profile blackboard. @@ -510,19 +525,6 @@ def _calibrate_one_pending_toc( pending_toc.get("toc_range"), ) return None - if not any( - node.printed_page is not None - for _path, node in iter_leaf_title_nodes(nodes) - ): - logger.info( - "{} pending TOC toc_range={}: no printed pages, unresolvable", - _LOG_PREFIX, - pending_toc.get("toc_range"), - ) - return { - "toc": pending_toc, - "relationship": "unresolvable", - } resolve_nodes, skeleton_anchor, _finalized = finalize_calibration_result( result=phase1, entries=list(pending_toc.get("toc_with_level") or []), @@ -577,7 +579,7 @@ def _calibrate_pending_tocs( index=i, pending_toc=pending_toc, pending_tocs=pending_tocs, - ctx=ctx, + ctx=fork_ctx_for_pending(ctx), page_texts=page_texts, page_count=page_count, body_pages=body_pages, diff --git a/apps/worker/app/services/document_agent/tools/grep_text.py b/apps/worker/app/services/document_agent/tools/grep_text.py index bb318e236..1c5bf173c 100644 --- a/apps/worker/app/services/document_agent/tools/grep_text.py +++ b/apps/worker/app/services/document_agent/tools/grep_text.py @@ -100,15 +100,6 @@ def grep_text(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: "hit_pages": hit_pages, "results": results, } - ctx.blackboard.global_signals.setdefault("grep_history", []).append( - { - "query": query, - "normalized_query": normalized_query, - "hit_count": hit_count, - "hit_page_count": len(hit_pages), - "sample_pages": hit_pages[:10], - } - ) return ToolResult( status="ok", payload=summary, diff --git a/apps/worker/app/services/document_agent/tools/text_strip_margins.py b/apps/worker/app/services/document_agent/tools/text_strip_margins.py index 44ac2a98d..cb5f3b9f3 100644 --- a/apps/worker/app/services/document_agent/tools/text_strip_margins.py +++ b/apps/worker/app/services/document_agent/tools/text_strip_margins.py @@ -54,7 +54,7 @@ def _apply_strip( continue margin = record.header if which == "header" else record.footer before = view.get(page, record.content) - after = strip_margin_text(before, margin) + after = strip_margin_text(before, margin, edge=which) view[page] = after if after != before: pages_updated += 1 diff --git a/apps/worker/app/services/page_memory/skeleton_extractor.py b/apps/worker/app/services/page_memory/skeleton_extractor.py index f7faa8420..07b366277 100644 --- a/apps/worker/app/services/page_memory/skeleton_extractor.py +++ b/apps/worker/app/services/page_memory/skeleton_extractor.py @@ -34,6 +34,24 @@ ) +def _null_page_locate_bucket( + null_page_report: list[dict[str, Any]], + *, + kind: str, +) -> dict[str, Any]: + """Summarize leaf or parent null-page locate rows (M2: no mixed bucket).""" + rows = [row for row in null_page_report if row.get("kind") == kind] + return { + "attempted": len(rows), + "located": sum(1 for row in rows if row.get("page") is not None), + "unresolved": sum(1 for row in rows if row.get("result") == "unresolved"), + "visual_verify_calls": sum( + int(row.get("visual_verify_calls") or 0) for row in rows + ), + "entries": rows, + } + + @dataclass(frozen=True) class SectionSkeleton: section_path: str @@ -143,17 +161,12 @@ def extract_section_skeletons( "reason": "offset_guided_anchoring_skipped_or_empty", "pruned_out_of_scope": skeleton_anchor.pruned_count, } - locate_summary["null_page_parent_locate"] = { - "attempted": len(null_page_report), - "located": sum(1 for row in null_page_report if row.get("page") is not None), - "unresolved": sum( - 1 for row in null_page_report if row.get("result") == "unresolved" - ), - "visual_verify_calls": sum( - int(row.get("visual_verify_calls") or 0) for row in null_page_report - ), - "entries": null_page_report, - } + locate_summary["null_page_leaf_locate"] = _null_page_locate_bucket( + null_page_report, kind="leaf" + ) + locate_summary["null_page_parent_locate"] = _null_page_locate_bucket( + null_page_report, kind="parent" + ) ranges = resolve_hierarchy_page_ranges( resolve_nodes, @@ -332,19 +345,12 @@ def _resolve_pending_tocs( "pruned_out_of_scope": skeleton_anchor.pruned_count, "toc_relationship": relationship, } - locate_summary["null_page_parent_locate"] = { - "attempted": len(null_page_report), - "located": sum( - 1 for row in null_page_report if row.get("page") is not None - ), - "unresolved": sum( - 1 for row in null_page_report if row.get("result") == "unresolved" - ), - "visual_verify_calls": sum( - int(row.get("visual_verify_calls") or 0) for row in null_page_report - ), - "entries": null_page_report, - } + locate_summary["null_page_leaf_locate"] = _null_page_locate_bucket( + null_page_report, kind="leaf" + ) + locate_summary["null_page_parent_locate"] = _null_page_locate_bucket( + null_page_report, kind="parent" + ) ranges = resolve_hierarchy_page_ranges( resolve_nodes, diff --git a/apps/worker/scripts/page_memory/debug_pm_null_page_react.py b/apps/worker/scripts/page_memory/debug_pm_null_page_react.py index bfb28bd45..2e64c3561 100644 --- a/apps/worker/scripts/page_memory/debug_pm_null_page_react.py +++ b/apps/worker/scripts/page_memory/debug_pm_null_page_react.py @@ -33,9 +33,9 @@ write_debug_json, ) -from app.services.document_agent.structure.null_page_react import react_budget -from app.services.document_agent.tools.extract_toc_with_boundaries import ( - BOUNDARY_STEP_PAGES, +from app.services.document_agent.structure.null_page_react import ( + REACT_PLANNER_GREP_BUDGET, + react_budget, ) @@ -112,7 +112,10 @@ def main() -> int: "parent_probe": "anchoring_primitives.locate_null_page_parent_overrides", "prune_post": "keep_null_page_nodes=False (drop unresolved)", "react_budget": react_budget(), - "boundary_step_pages": BOUNDARY_STEP_PAGES, + "react_planner_grep_budget": REACT_PLANNER_GREP_BUDGET, + "seed_full_title_grep": "free", + "strip_auto_regrep": "free", + "max_planner_turns": REACT_PLANNER_GREP_BUDGET + 2, }, "offset": anchor.get("offset"), "pruned_count": anchor.get("pruned_count"), diff --git a/apps/worker/tests/contract/test_profile_skeleton_anchor_wiring_contract.py b/apps/worker/tests/contract/test_profile_skeleton_anchor_wiring_contract.py index fc10dcaa8..19bc37727 100644 --- a/apps/worker/tests/contract/test_profile_skeleton_anchor_wiring_contract.py +++ b/apps/worker/tests/contract/test_profile_skeleton_anchor_wiring_contract.py @@ -307,16 +307,26 @@ def fake_finalize(**kwargs): assert records[0]["nodes"][0]["title"] == "App" -def test_profile_skips_finalize_for_unresolvable_pending_toc() -> None: +def test_profile_pending_toc_without_printed_pages_still_finalizes() -> None: + """H7a: all-null printed leaves still run finalize; unresolvable only if no span.""" ctx = _ctx(page_count=30) hierarchies = _pending_tocs() hierarchies[1]["toc_with_level"] = [{"heading": "App", "level": 1}] ctx.blackboard.toc_hierarchies = hierarchies ctx.blackboard.toc_result = TocResult(method="vlm_batch", toc_pages=[1, 20, 21]) ctx.blackboard.page_full_text_cache = {page: "body" for page in range(1, 31)} + finalize_calls: list[object] = [] - def _boom(*_args, **_kwargs): - raise AssertionError("unresolvable pending TOC must not finalize") + def fake_finalize(**kwargs): + finalize_calls.append(kwargs) + empty = SkeletonAnchor( + offset=0, + offset_status="ok", + match_overrides={}, + null_page_report=[], + bulk_count=0, + ) + return [], empty, True with ( patch( @@ -333,11 +343,12 @@ def _boom(*_args, **_kwargs): ), patch( "app.services.document_agent.calibration.procedure.finalize_calibration_result", - side_effect=_boom, + side_effect=fake_finalize, ), ): run_toc_anchoring(ctx) + assert finalize_calls, "H7a: finalize must run even when leaves lack printed pages" records = ctx.blackboard.pending_skeleton_anchors assert len(records) == 1 assert records[0]["relationship"] == "unresolvable" @@ -345,6 +356,30 @@ def _boom(*_args, **_kwargs): assert "skeleton_anchor" not in records[0] +def test_fork_ctx_for_pending_isolates_search_view() -> None: + """H7b: each pending job gets its own page_text_search_view.""" + from app.services.document_agent.structure.toc_anchoring import fork_ctx_for_pending + + ctx = _ctx(page_count=10) + ctx.blackboard.page_full_text_cache = {1: "shared"} + ctx.blackboard.page_text_search_view = {1: "parent-stale"} + + job_a = fork_ctx_for_pending(ctx) + job_b = fork_ctx_for_pending(ctx) + + assert job_a.blackboard.page_text_search_view is None + assert job_b.blackboard.page_text_search_view is None + # Shared cache by reference; search views independent. + assert job_a.blackboard.page_full_text_cache is ctx.blackboard.page_full_text_cache + assert job_b.blackboard.page_full_text_cache is ctx.blackboard.page_full_text_cache + + job_a.blackboard.page_text_search_view = {1: "a-strip"} + job_b.blackboard.page_text_search_view = {1: "b-strip"} + assert job_a.blackboard.page_text_search_view == {1: "a-strip"} + assert job_b.blackboard.page_text_search_view == {1: "b-strip"} + assert ctx.blackboard.page_text_search_view == {1: "parent-stale"} + + def test_c4_uses_persisted_pending_relationship_and_does_not_classify() -> None: pending_toc = _pending_tocs()[1] anatomy = _anatomy(with_anchor=True) diff --git a/apps/worker/tests/contract/test_structure_anchoring_contract.py b/apps/worker/tests/contract/test_structure_anchoring_contract.py index 5f3097e59..ccd7d2336 100644 --- a/apps/worker/tests/contract/test_structure_anchoring_contract.py +++ b/apps/worker/tests/contract/test_structure_anchoring_contract.py @@ -147,6 +147,69 @@ def fail_a(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, assert report[1]["result"] == "skipped_after_sibling_failure" +def test_null_page_react_still_walks_later_parent_subtree_after_failure() -> None: + """H5: a failed leaf must not block later parents' subtrees.""" + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_node_overrides, + ) + + nodes = [ + TitleNode(title="A", level=1, printed_page=None, children=[]), + TitleNode( + title="B", + level=1, + printed_page=None, + children=[ + TitleNode(title="B1", level=2, printed_page=None, children=[]), + TitleNode(title="B2", level=2, printed_page=None, children=[]), + ], + ), + TitleNode(title="C", level=1, printed_page=None, children=[]), + ] + ctx = _ctx() + probed: list[str] = [] + + def probe(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, str]: + title = str(kwargs["title"]) + probed.append(title) + if title == "A": + return None, [], 0, "react_give_up" + page = 10 if title == "B1" else 11 + return ( + TitleMatch( + page=page, + source="react_normalized_grep_vlm", + matched_line=title, + candidates=[page], + evidence={"accept": "react_normalized_grep_vlm"}, + ), + [], + 1, + "react_normalized_grep_vlm", + ) + + with patch.object(npr, "_locate_with_react", side_effect=probe): + overrides, report = locate_null_page_node_overrides( + nodes=nodes, + match_overrides={}, + body_pages=list(range(1, 21)), + ctx=ctx, + ) + + # A failed; B1/B2 under the later parent are still probed and located. + assert probed == ["A", "B1", "B2"] + assert overrides[("B", "B1")].page == 10 + assert overrides[("B", "B2")].page == 11 + by_path = {tuple(row["path_titles"]): row for row in report} + assert by_path[("A",)]["result"] == "react_give_up" + assert by_path[("B", "B1")]["page"] == 10 + assert by_path[("B", "B2")]["page"] == 11 + # C is a later leaf at A's level: still skipped (serial cursor invalid). + assert by_path[("C",)]["result"] == "skipped_after_sibling_failure" + assert by_path[("C",)]["failed_sibling"] == "A" + + def test_null_page_react_does_not_probe_parent() -> None: from app.services.document_agent.structure import null_page_react as npr from app.services.document_agent.structure.null_page_react import ( @@ -180,6 +243,321 @@ def fail_child(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], assert report[0]["result"] == "react_give_up" +def test_null_page_react_skips_empty_shell_structural_parent() -> None: + """H1: pre-prune parents that become childless must not leaf-probe.""" + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_node_overrides, + ) + + # After prune(keep_null=True): former parent survives as empty shell. + shell = TitleNode(title="Appendix", level=1, printed_page=None, children=[]) + leaf = TitleNode(title="RealLeaf", level=1, printed_page=None, children=[]) + ctx = _ctx() + probed: list[str] = [] + + def track(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, str]: + probed.append(str(kwargs["title"])) + return None, [], 0, "react_give_up" + + with patch.object(npr, "_locate_with_react", side_effect=track): + overrides, report = locate_null_page_node_overrides( + nodes=[shell, leaf], + match_overrides={}, + body_pages=[1, 2, 3, 4, 5], + ctx=ctx, + structural_parent_paths={("Appendix",)}, + ) + + assert overrides == {} + assert probed == ["RealLeaf"] + assert [row["path_titles"] for row in report] == [["RealLeaf"]] + + +def test_anchor_offset_collects_structural_parent_paths_before_prune() -> None: + """H1 production wiring: paths collected before keep_null prune.""" + parent = TitleNode( + title="P", + level=1, + printed_page=None, + children=[ + TitleNode(title="PrintedMiss", level=2, printed_page=10, children=[]), + ], + ) + captured: dict[str, Any] = {} + + def fake_leaf(**kwargs: Any) -> tuple[dict, list]: + captured["structural_parent_paths"] = set( + kwargs.get("structural_parent_paths") or set() + ) + captured["nodes"] = kwargs["nodes"] + return dict(kwargs["match_overrides"]), [] + + def fake_parent(**kwargs: Any) -> tuple[dict, list]: + return dict(kwargs["match_overrides"]), [] + + with ( + patch.object(anchoring, "locate_null_page_node_overrides", side_effect=fake_leaf), + patch.object( + anchoring, "locate_null_page_parent_overrides", side_effect=fake_parent + ), + ): + _working, _anchor = anchoring.anchor_hierarchy_from_offset( + nodes=[parent], + offset_hint=None, + page_texts={1: "x"}, + body_pages=[1, 2, 3], + page_count=3, + ctx=_ctx(), + ) + + assert ("P",) in captured["structural_parent_paths"] + # PrintedMiss dropped by keep_null prune; empty-shell P still passed to leaf. + assert len(captured["nodes"]) == 1 + assert captured["nodes"][0].title == "P" + assert captured["nodes"][0].children == [] + +def test_null_page_normalized_grep_excludes_pages_outside_body() -> None: + """H2: grep page map is body_pages ∩ [left, right] (TOC pages dropped).""" + from app.services.document_agent.pdf_text import PageTextBands + from app.services.document_agent.structure.null_page_react import _normalized_grep + + ctx = _ctx() + ctx.blackboard.page_count = 10 + # Page 1 is TOC-excluded; title only there would inflate hits without filter. + ctx.blackboard.page_full_text_cache = { + 1: PageTextBands(content="Appendix F Overview\nAppendix F Overview"), + 2: PageTextBands(content="noise"), + 5: PageTextBands(content="Appendix F Overview"), + } + status, needle, hit_pages, _count = _normalized_grep( + ctx=ctx, + query="Appendix F Overview", + left=1, + right=10, + body_pages=[2, 3, 4, 5, 6], + ) + assert status == "ok" + assert needle + assert hit_pages == [5] + + +def test_null_page_react_clears_page_text_search_view() -> None: + """H3: locate always clears strip view in finally (no cross-node leak).""" + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import _locate_with_react + + ctx = _ctx() + ctx.blackboard.page_count = 5 + ctx.blackboard.page_full_text_cache = {page: "X" for page in range(1, 6)} + ctx.blackboard.page_text_search_view = {1: "stale-before"} + + def dirty_grep(**kwargs: Any) -> tuple[str, str, list[int], int]: + ctx.blackboard.page_text_search_view = {1: "dirty-during"} + return "ok", "x", [], 0 + + with ( + patch.object(npr, "_normalized_grep", side_effect=dirty_grep), + patch.object( + npr, + "_propose_react_query", + return_value=({"action": "give_up", "query": ""}, {}), + ), + ): + match, _attempts, _visual, result = _locate_with_react( + path_titles=("T",), + title="T", + left=1, + right=5, + body_pages=[1, 2, 3, 4, 5], + ctx=ctx, + ) + + assert match is None + assert result == "react_give_up" + assert ctx.blackboard.page_text_search_view is None + + +def test_null_page_react_confirms_only_first_hit_page() -> None: + """H6: each grep visually confirms only the first hit; rest are discarded.""" + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import _locate_with_react + + ctx = _ctx() + verified_pages: list[int] = [] + + def fake_grep(**kwargs: Any) -> tuple[str, str, list[int], int]: + return "ok", "appendix f", [10, 11, 12, 13, 14], 5 + + def fake_verify(**kwargs: Any) -> tuple[bool, str, int]: + page = int(kwargs["page"]) + verified_pages.append(page) + return False, "not start", 0 + + with ( + patch.object(npr, "_normalized_grep", side_effect=fake_grep), + patch.object(npr, "_verify_section_beginning_page", side_effect=fake_verify), + patch.object( + npr, + "_propose_react_query", + return_value=({"action": "give_up", "query": ""}, {}), + ), + ): + match, attempts, visual_calls, result = _locate_with_react( + path_titles=("Appendix F",), + title="Appendix F", + left=1, + right=20, + body_pages=list(range(1, 21)), + ctx=ctx, + ) + + assert match is None + assert result == "react_give_up" + # Seed rejected first hit only; did not walk 11..14. + assert verified_pages == [10] + assert visual_calls == 1 + assert attempts[0]["observation"] == "visual_rejected" + assert attempts[0]["visual_pages_checked"] == [ + {"page": 10, "confirmed": False, "reason": "not start", "tokens_used": 0} + ] + assert "too_many_hits" not in {a.get("observation") for a in attempts} + + +def test_react_planner_grep_budget_is_not_page_constant() -> None: + """M5: dedicated grep budget; seed free; max turns = budget + 2 (strips).""" + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + REACT_PLANNER_GREP_BUDGET, + _locate_with_react, + react_budget, + ) + + assert REACT_PLANNER_GREP_BUDGET == 5 + assert react_budget() == REACT_PLANNER_GREP_BUDGET + + ctx = _ctx() + planner_turns = {"n": 0} + + def fake_grep(**kwargs: Any) -> tuple[str, str, list[int], int]: + # Seed + any strip re-grep: empty hits, never consume via consume_budget=False path + return "ok", "needle", [], 0 + + def fake_propose(**kwargs: Any) -> tuple[dict[str, Any] | None, dict[str, Any]]: + planner_turns["n"] += 1 + return None, {"error": "planner failed"} + + with ( + patch.object(npr, "react_budget", return_value=1), + patch.object(npr, "_normalized_grep", side_effect=fake_grep), + patch.object(npr, "_propose_react_query", side_effect=fake_propose), + ): + _match, attempts, _visual, result = _locate_with_react( + path_titles=("T",), + title="T", + left=1, + right=5, + body_pages=[1, 2, 3, 4, 5], + ctx=ctx, + ) + + # max_planner_turns = budget + 2 = 3 (not budget + 2 + budget). + assert planner_turns["n"] == 3 + assert result == "react_loop_limit" + assert all(a.get("action") == "planner_error" or a.get("action") == "grep" for a in attempts) + +def test_null_page_grep_observations_distinguish_error_empty_duplicate() -> None: + """M1: tool error / empty needle / true duplicate are distinct labels.""" + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import _locate_with_react + + ctx = _ctx() + calls = {"n": 0} + + def fake_grep(**kwargs: Any) -> tuple[str, str, list[int], int]: + calls["n"] += 1 + if calls["n"] == 1: + return "error", "grep.text failed", [], 0 + if calls["n"] == 2: + return "ok", "", [], 0 + if calls["n"] == 3: + return "ok", "same needle", [5], 1 + return "ok", "same needle", [5], 1 + + proposals = iter( + [ + ({"action": "grep", "query": "q2"}, {}), + ({"action": "grep", "query": "q3"}, {}), + ({"action": "grep", "query": "q3 again"}, {}), + ({"action": "give_up", "query": ""}, {}), + ] + ) + + def fake_propose(**kwargs: Any) -> tuple[dict[str, Any] | None, dict[str, Any]]: + return next(proposals) + + def fake_verify(**kwargs: Any) -> tuple[bool, str, int]: + return False, "reject", 0 + + with ( + patch.object(npr, "_normalized_grep", side_effect=fake_grep), + patch.object(npr, "_propose_react_query", side_effect=fake_propose), + patch.object(npr, "_verify_section_beginning_page", side_effect=fake_verify), + patch.object(npr, "react_budget", return_value=5), + ): + _match, attempts, _visual, _result = _locate_with_react( + path_titles=("T",), + title="T", + left=1, + right=10, + body_pages=list(range(1, 11)), + ctx=ctx, + ) + + observations = [a.get("observation") for a in attempts if a.get("action") == "grep"] + assert observations[0] == "grep_tool_error" + assert observations[1] == "empty_normalized_query" + assert observations[2] == "visual_rejected" + assert observations[3] == "duplicate_normalized_query" + + +def test_null_page_locate_summary_splits_leaf_and_parent() -> None: + """M2: C4 locate_summary keeps leaf and parent buckets separate.""" + from app.services.page_memory.skeleton_extractor import _null_page_locate_bucket + + report = [ + { + "kind": "leaf", + "page": 5, + "result": "react_normalized_grep_vlm", + "visual_verify_calls": 1, + }, + { + "kind": "leaf", + "page": None, + "result": "unresolved", + "visual_verify_calls": 0, + }, + { + "kind": "parent", + "page": 4, + "result": "visual_rtl", + "visual_verify_calls": 2, + }, + ] + leaf = _null_page_locate_bucket(report, kind="leaf") + parent = _null_page_locate_bucket(report, kind="parent") + assert leaf["attempted"] == 2 + assert leaf["located"] == 1 + assert leaf["unresolved"] == 1 + assert leaf["visual_verify_calls"] == 1 + assert parent["attempted"] == 1 + assert parent["located"] == 1 + assert parent["visual_verify_calls"] == 2 + assert [e["kind"] for e in leaf["entries"]] == ["leaf", "leaf"] + assert [e["kind"] for e in parent["entries"]] == ["parent"] + + def test_null_page_react_hit_writes_override() -> None: from app.services.document_agent.structure import null_page_react as npr from app.services.document_agent.structure.null_page_react import ( @@ -620,6 +998,96 @@ def fake_verify(**kwargs: Any) -> dict[str, Any]: assert bp == -1 +def test_regimes_bulk_count_excludes_null_page_react_hits() -> None: + """H4: bulk_count frozen before leaf/parent ReAct; ReAct hits stay in report only.""" + from app.services.document_agent.calibration import procedure as proc + from app.services.document_agent.calibration.procedure import ( + anchor_hierarchy_from_regimes, + ) + from app.services.document_agent.calibration.types import ( + CalibrationRegime, + CalibrationResult, + CalibrationSample, + ) + + nodes = [ + _leaf("Ch1", 1), + TitleNode(title="NullLeaf", level=1, printed_page=None, children=[]), + ] + phase1 = CalibrationResult( + status="ok", + offset=10, + regimes=[ + CalibrationRegime( + kind="decimal", + offset=10, + offset_status="ok", + entry_indices=[0, 1], + samples=[CalibrationSample(title="Ch1", physical=11)], + ) + ], + ) + bulk_match = TitleMatch( + page=11, + source="bulk_offset", + matched_line="Ch1", + candidates=[11], + evidence={}, + ) + react_match = TitleMatch( + page=20, + source="react_normalized_grep_vlm", + matched_line="NullLeaf", + candidates=[20], + evidence={"accept": "react_normalized_grep_vlm"}, + ) + + def fake_offset(**kwargs: Any) -> dict[tuple[str, ...], TitleMatch]: + return {("Ch1",): bulk_match} + + def fake_apply(**kwargs: Any) -> tuple[list, dict, list, int]: + out = dict(kwargs["match_overrides"]) + out[("NullLeaf",)] = react_match + return ( + list(kwargs["nodes"]), + out, + [ + { + "path_titles": ["NullLeaf"], + "kind": "leaf", + "page": 20, + "result": "react_normalized_grep_vlm", + } + ], + 0, + ) + + with ( + patch.object(proc, "offset_guided_anchoring", side_effect=fake_offset), + patch.object(proc, "apply_null_page_locates_and_prune", side_effect=fake_apply), + ): + _working, anchor = anchor_hierarchy_from_regimes( + nodes=nodes, + result=phase1, + entries=[ + {"heading": "Ch1", "level": 1, "page_number": 1}, + {"heading": "NullLeaf", "level": 1, "page_number": None}, + ], + page_texts={11: "Ch1", 20: "NullLeaf"}, + body_pages=list(range(1, 30)), + page_count=30, + ctx=_ctx(), + ) + + assert ("Ch1",) in anchor.match_overrides + assert ("NullLeaf",) in anchor.match_overrides + assert len(anchor.match_overrides) == 2 + assert anchor.bulk_count == 1 + assert any( + row.get("path_titles") == ["NullLeaf"] for row in anchor.null_page_report + ) + + def test_phase2_all_bisect_fail_does_not_invent_first_leaf() -> None: """When every Phase-2 probe fails, do not bulk-anchor the first TOC leaf.""" from app.services.document_agent.calibration.procedure import ( @@ -895,6 +1363,67 @@ def test_parent_backfill_uses_descendant_regime_offset() -> None: assert parents[("Section A",)].evidence["parent_backfill"] is True +def test_parent_backfill_ignores_react_located_children_for_offset() -> None: + """ReAct leaves have no printed page, so they carry no offset for the parent.""" + from app.services.document_agent.structure import anchoring_primitives as primitives + + react_child = TitleNode(title="Intro", level=2, printed_page=None, children=[]) + printed_child = TitleNode(title="Body", level=2, printed_page=12, children=[]) + section = TitleNode( + title="Section A", + level=1, + printed_page=10, + children=[react_child, printed_child], + ) + matches = { + # ReAct hit comes first in document order and must not set the offset. + ("Section A", "Intro"): TitleMatch( + page=99, + source="react_normalized_grep_vlm", + matched_line="Intro", + candidates=[99], + evidence={"accept": "react_normalized_grep_vlm", "null_page_react": True}, + ), + **primitives.bulk_offset_matches( + [(("Section A", "Body"), printed_child)], 5 + ), + } + + parents = primitives.backfill_parent_offset_matches( + nodes=[section], + matches=matches, + page_count=200, + ) + + assert parents[("Section A",)].page == 15 + assert parents[("Section A",)].evidence["offset"] == 5 + + +def test_parent_backfill_unresolved_when_only_react_children() -> None: + """No printed-page descendant → no offset → parent left to its own locate.""" + from app.services.document_agent.structure import anchoring_primitives as primitives + + react_child = TitleNode(title="Intro", level=2, printed_page=None, children=[]) + section = _printed_page_parent("Section A", 10, react_child) + matches = { + ("Section A", "Intro"): TitleMatch( + page=99, + source="react_normalized_grep_vlm", + matched_line="Intro", + candidates=[99], + evidence={"accept": "react_normalized_grep_vlm", "null_page_react": True}, + ) + } + + parents = primitives.backfill_parent_offset_matches( + nodes=[section], + matches=matches, + page_count=200, + ) + + assert parents == {} + + def test_parent_backfill_skips_unanchored_and_out_of_range() -> None: from app.services.document_agent.structure import anchoring_primitives as primitives diff --git a/apps/worker/tests/contract/test_tool_registry_smoke_contract.py b/apps/worker/tests/contract/test_tool_registry_smoke_contract.py index 78d431138..61785a191 100644 --- a/apps/worker/tests/contract/test_tool_registry_smoke_contract.py +++ b/apps/worker/tests/contract/test_tool_registry_smoke_contract.py @@ -101,5 +101,21 @@ def test_strip_footer_updates_search_view_for_grep() -> None: assert blackboard.page_full_text_cache[1].content == "Section Start\nPublic Domain Manual" +def test_strip_margin_text_edge_aligned_not_first_occurrence() -> None: + """M4: footer/header strip only the matching edge, not body duplicates.""" + from app.services.document_agent.pdf_text import strip_margin_text + + body_and_footer = "Public Domain Manual\nSection body\nPublic Domain Manual" + assert ( + strip_margin_text(body_and_footer, "Public Domain Manual", edge="footer") + == "Public Domain Manual\nSection body\n" + ) + body_and_header = "Running Header\nSection body\nRunning Header" + assert ( + strip_margin_text(body_and_header, "Running Header", edge="header") + == "\nSection body\nRunning Header" + ) + + def test_openai_specs_removed() -> None: assert not hasattr(REGISTRY, "openai_specs") \ No newline at end of file From 335627da25ebb636d56f0c9a242c9c23aeb10574 Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Sat, 22 Aug 2026 15:49:10 +0800 Subject: [PATCH 5/7] refactor: improve null-page handling and introduce progressive page windows - Refactored the anchor_hierarchy_from_regimes function to enhance clarity and maintainability, specifically in the handling of structural parent paths. - Introduced a new function, progressive_page_windows, to generate non-overlapping page windows for scanning, improving the efficiency of page inspections. - Updated scan_title_forward to utilize the new progressive page windows function, ensuring better management of page scanning logic. - Adjusted related tests to validate the new functionality and ensure robustness in null-page processing and page window generation. --- .../document_agent/calibration/procedure.py | 6 +- .../document_agent/calibration/scan.py | 41 +- .../structure/anchoring_primitives.py | 350 +--------- .../structure/hierarchy_locator.py | 57 +- .../structure/null_page_react.py | 645 ++++++++---------- .../document_agent/tools/grep_text.py | 58 +- .../page_memory/debug_pm_null_page_react.py | 13 +- .../test_calibration_scan_contract.py | 9 + .../test_outline_short_circuit_contract.py | 4 +- .../test_structure_anchoring_contract.py | 574 +++++++++------- .../test_tool_registry_smoke_contract.py | 33 + 11 files changed, 749 insertions(+), 1041 deletions(-) diff --git a/apps/worker/app/services/document_agent/calibration/procedure.py b/apps/worker/app/services/document_agent/calibration/procedure.py index 7ff1fdc59..b59684674 100644 --- a/apps/worker/app/services/document_agent/calibration/procedure.py +++ b/apps/worker/app/services/document_agent/calibration/procedure.py @@ -350,8 +350,7 @@ def anchor_hierarchy_from_regimes( len(regime_seed), ) - # Capture parents before prune: empty shells after keep_null prune must - # not enter leaf ReAct (H1). + # Capture parent identity before prune so empty shells retain ``kind=parent``. structural_parent_paths = { path for path, node in _iter_all_title_nodes(working) @@ -388,7 +387,7 @@ def anchor_hierarchy_from_regimes( len(parent_matches), ) - # H4: freeze bulk before null-page leaf/parent ReAct (same as offset path: + # Freeze bulk before unified null-page ReAct (same as offset path: # offset_guided → len(overrides then); else 0). Never recount after ReAct. bulk_count = len(merged) if regime_bulk > 0 else 0 @@ -397,7 +396,6 @@ def anchor_hierarchy_from_regimes( nodes=working, match_overrides=merged, body_pages=body_pages, - page_texts=page_texts, ctx=ctx, structural_parent_paths=structural_parent_paths, ) diff --git a/apps/worker/app/services/document_agent/calibration/scan.py b/apps/worker/app/services/document_agent/calibration/scan.py index 95b468e75..5cf0399e1 100644 --- a/apps/worker/app/services/document_agent/calibration/scan.py +++ b/apps/worker/app/services/document_agent/calibration/scan.py @@ -25,6 +25,27 @@ DEFAULT_WINDOW_SCHEDULE: tuple[int, ...] = (2, 4, 6, 10) +def progressive_page_windows( + *, + start_page: int, + end_page: int, + window_schedule: tuple[int, ...] = DEFAULT_WINDOW_SCHEDULE, +) -> list[list[int]]: + """Return non-overlapping physical-page windows clipped to ``end_page``.""" + cursor = max(int(start_page), 1) + last_page = int(end_page) + windows: list[list[int]] = [] + for size in window_schedule: + if cursor > last_page: + break + pages = list(range(cursor, min(cursor + int(size), last_page + 1))) + if not pages: + break + windows.append(pages) + cursor = pages[-1] + 1 + return windows + + @dataclass class ScanRound: pages: list[int] @@ -76,17 +97,15 @@ def scan_title_forward( Each round opens ``window_schedule[i]`` consecutive pages starting at the cursor left by the previous round, so no page is inspected twice. """ - cursor = max(int(start_page), 1) scanned: list[int] = [] rounds: list[ScanRound] = [] + next_start = max(int(start_page), 1) - for size in window_schedule: - if cursor > page_count: - break - pages = [page for page in range(cursor, cursor + size) if page <= page_count] - if not pages: - break - + for pages in progressive_page_windows( + start_page=start_page, + end_page=page_count, + window_schedule=window_schedule, + ): result = inspect_pages( ctx, { @@ -99,7 +118,7 @@ def scan_title_forward( "usage_task": "calibration.scan_title_forward", }, ) - cursor = pages[-1] + 1 + next_start = pages[-1] + 1 scanned.extend(pages) if result.status != "ok": @@ -128,7 +147,7 @@ def scan_title_forward( found=True, found_page=found_page, scanned_pages=scanned, - next_start=cursor if cursor <= page_count else None, + next_start=next_start if next_start <= page_count else None, rounds=rounds, ) @@ -142,6 +161,6 @@ def scan_title_forward( found=False, found_page=None, scanned_pages=scanned, - next_start=cursor if cursor <= page_count else None, + next_start=next_start if next_start <= page_count else None, rounds=rounds, ) diff --git a/apps/worker/app/services/document_agent/structure/anchoring_primitives.py b/apps/worker/app/services/document_agent/structure/anchoring_primitives.py index fb0884c4c..ab580f1d0 100644 --- a/apps/worker/app/services/document_agent/structure/anchoring_primitives.py +++ b/apps/worker/app/services/document_agent/structure/anchoring_primitives.py @@ -12,13 +12,10 @@ from app.services.document_agent.structure.hierarchy_locator import ( TitleMatch, TitleNode, - first_leaf_start_under, iter_leaf_title_nodes, - last_leaf_start_under, - locate_title_normalized_strict, ) from app.services.document_agent.structure.null_page_react import ( - locate_null_page_node_overrides, + locate_null_page_overrides, ) from app.services.document_agent.structure.section_page_verify import ( verify_section_page_choice, @@ -26,17 +23,6 @@ from loguru import logger -def _first_sibling_null_parent_scan_start(right: int) -> int: - """Left edge for first-at-level null parents: at most one 2+4+6+10 budget. - - Does not inherit a wider parent/body scope. Floors at document page 1. - """ - from app.services.document_agent.calibration.scan import DEFAULT_WINDOW_SCHEDULE - - budget = sum(DEFAULT_WINDOW_SCHEDULE) - return max(1, int(right) - budget + 1) - - def prune_out_of_scope_nodes( nodes: list[TitleNode], *, @@ -173,323 +159,6 @@ def toc_range_end(hierarchy: dict[str, Any]) -> int | None: return None -# ── Null-page parent locate (sibling window / first-sibling scan) ─────────── - - -def locate_null_page_parent_overrides( - *, - nodes: list[TitleNode], - match_overrides: dict[tuple[str, ...], TitleMatch], - page_texts: dict[int, str], - body_pages: list[int], - ctx: ToolContext | None, -) -> tuple[dict[tuple[str, ...], TitleMatch], list[dict[str, Any]]]: - """Locate TOC parents with ``printed_page=None`` into ``match_overrides``. - - Window for parent P with a previous same-level sibling: ``[last leaf under - that sibling, first leaf under P]``; text then RTL visual verify. - - First-at-level parents (no left sibling) do **not** inherit a wider parent - or body scope. Left edge is one Phase-1 ``2+4+6+10`` budget before the first - child (floor page 1). Text runs in that window; on miss, reuse - ``scan_title_forward`` (same schedule, early exit). Miss → unresolved. - - Returns ``(overrides, report)`` where *report* lists every null-page parent - attempt (for debug / LLM-call accounting). - """ - if not nodes or not body_pages: - return dict(match_overrides), [] - - out = dict(match_overrides) - body_set = set(body_pages) - parent_scope_start = body_pages[0] - report: list[dict[str, Any]] = [] - - def walk( - sibling_nodes: list[TitleNode], - parent_titles: tuple[str, ...], - scope_start: int, - ) -> None: - for index, node in enumerate(sibling_nodes): - path_titles = (*parent_titles, node.title) - if ( - node.children - and node.printed_page is None - and path_titles not in out - ): - right = first_leaf_start_under(node, parent_titles, out) - entry: dict[str, Any] = { - "path_titles": list(path_titles), - "title": node.title, - "kind": "parent", - "printed_page": None, - "window": None, - "result": "skipped_no_right", - "page": None, - "accept": None, - "visual_verify_calls": 0, - } - if right is None: - report.append(entry) - logger.info( - "[structure_anchoring] null-page parent skipped: " - "title={!r} reason=no_located_first_child", - node.title, - ) - elif index > 0: - left = last_leaf_start_under( - sibling_nodes[index - 1], parent_titles, out - ) - if left is None: - left = scope_start - if right < left: - report.append(entry) - logger.info( - "[structure_anchoring] null-page parent skipped: " - "title={!r} reason=no_located_first_child left={}", - node.title, - left, - ) - else: - _resolve_null_parent_with_sibling_window( - path_titles=path_titles, - title=node.title, - left=left, - right=right, - body_pages=body_pages, - body_set=body_set, - page_texts=page_texts, - ctx=ctx, - out=out, - entry=entry, - report=report, - ) - else: - left = _first_sibling_null_parent_scan_start(right) - _resolve_null_parent_first_sibling( - path_titles=path_titles, - title=node.title, - left=left, - right=right, - body_pages=body_pages, - body_set=body_set, - page_texts=page_texts, - ctx=ctx, - out=out, - entry=entry, - report=report, - ) - if node.children: - child_scope_start = ( - out[path_titles].page if path_titles in out else scope_start - ) - walk(node.children, path_titles, child_scope_start) - - walk(nodes, (), parent_scope_start) - logger.info( - "[structure_anchoring] null-page parent locate summary: " - "attempted={} located={} unresolved={} visual_verify_calls={}", - len(report), - sum(1 for row in report if row.get("page") is not None), - sum(1 for row in report if row.get("result") == "unresolved"), - sum(int(row.get("visual_verify_calls") or 0) for row in report), - ) - return out, report - - -def _record_null_parent_outcome( - *, - path_titles: tuple[str, ...], - title: str, - left: int, - right: int, - match: TitleMatch | None, - visual_calls: int, - body_set: set[int], - out: dict[tuple[str, ...], TitleMatch], - entry: dict[str, Any], - report: list[dict[str, Any]], -) -> None: - entry["window"] = [left, right] - entry["visual_verify_calls"] = visual_calls - if match is not None and match.page in body_set: - out[path_titles] = match - entry["result"] = str(match.evidence.get("accept") or match.source) - entry["page"] = match.page - entry["accept"] = match.evidence.get("accept") - logger.info( - "[structure_anchoring] null-page parent located: " - "title={!r} page={} window={} accept={} visual_calls={}", - title, - match.page, - [left, right], - match.evidence.get("accept"), - visual_calls, - ) - else: - entry["result"] = "unresolved" - logger.info( - "[structure_anchoring] null-page parent unresolved: " - "title={!r} window={} visual_calls={}", - title, - [left, right], - visual_calls, - ) - report.append(entry) - - -def _resolve_null_parent_with_sibling_window( - *, - path_titles: tuple[str, ...], - title: str, - left: int, - right: int, - body_pages: list[int], - body_set: set[int], - page_texts: dict[int, str], - ctx: ToolContext | None, - out: dict[tuple[str, ...], TitleMatch], - entry: dict[str, Any], - report: list[dict[str, Any]], -) -> None: - scope_pages = [page for page in body_pages if left <= page <= right] - match = locate_title_normalized_strict( - title, - scope_pages=scope_pages, - page_texts=page_texts, - ) - visual_calls = 0 - if match is None and ctx is not None: - match, visual_calls = _visual_rtl_locate_parent( - title=title, - left=left, - right=right, - body_set=body_set, - ctx=ctx, - ) - _record_null_parent_outcome( - path_titles=path_titles, - title=title, - left=left, - right=right, - match=match, - visual_calls=visual_calls, - body_set=body_set, - out=out, - entry=entry, - report=report, - ) - - -def _resolve_null_parent_first_sibling( - *, - path_titles: tuple[str, ...], - title: str, - left: int, - right: int, - body_pages: list[int], - body_set: set[int], - page_texts: dict[int, str], - ctx: ToolContext | None, - out: dict[tuple[str, ...], TitleMatch], - entry: dict[str, Any], - report: list[dict[str, Any]], -) -> None: - """First-at-level null parent: capped text window, then ``scan_title_forward``.""" - from app.services.document_agent.calibration.scan import ( - DEFAULT_WINDOW_SCHEDULE, - scan_title_forward, - ) - - scope_pages = [page for page in body_pages if left <= page <= right] - match = locate_title_normalized_strict( - title, - scope_pages=scope_pages, - page_texts=page_texts, - ) - visual_calls = 0 - if match is None and ctx is not None: - scan = scan_title_forward( - ctx=ctx, - title=title, - start_page=left, - page_count=right, - window_schedule=DEFAULT_WINDOW_SCHEDULE, - ) - visual_calls = len(scan.scanned_pages) - if scan.found and scan.found_page is not None: - match = TitleMatch( - page=int(scan.found_page), - source="inspect_vlm", - matched_line="", - candidates=[int(scan.found_page)], - evidence={ - "accept": "scan_forward", - "null_page_parent_probe": True, - "scanned_pages": list(scan.scanned_pages), - }, - ) - _record_null_parent_outcome( - path_titles=path_titles, - title=title, - left=left, - right=right, - match=match, - visual_calls=visual_calls, - body_set=body_set, - out=out, - entry=entry, - report=report, - ) - - -def _visual_rtl_locate_parent( - *, - title: str, - left: int, - right: int, - body_set: set[int], - ctx: ToolContext, -) -> tuple[TitleMatch | None, int]: - """Confirm parent title from right boundary toward left via VLM verify.""" - visual_calls = 0 - for page in range(right, left - 1, -1): - if page not in body_set: - continue - candidate = TitleMatch( - page=page, - source="inspect_vlm", - matched_line="", - candidates=[page], - evidence={"null_page_parent_probe": True}, - ) - visual_calls += 1 - result = verify_section_page_choice( - ctx=ctx, - title=title, - candidate_matches=[candidate], - candidate_page_cap=1, - ) - selected = result.get("selected_page") - if selected != page: - continue - return ( - TitleMatch( - page=page, - source="inspect_vlm", - matched_line="", - candidates=[page], - evidence={ - "accept": "visual_rtl", - "reason": result.get("reason", ""), - "visual_verify_calls": visual_calls, - }, - ), - visual_calls, - ) - return None, visual_calls - - # ── Offset-guided bulk anchoring with recursive recalibrate (Phase-2) ─────── _MAX_RECALIBRATE_DEPTH = 5 @@ -1032,7 +701,6 @@ def apply_null_page_locates_and_prune( nodes: list[TitleNode], match_overrides: dict[tuple[str, ...], TitleMatch], body_pages: list[int], - page_texts: dict[int, str], ctx: ToolContext | None, structural_parent_paths: set[tuple[str, ...]], ) -> tuple[ @@ -1041,7 +709,7 @@ def apply_null_page_locates_and_prune( list[dict[str, Any]], int, ]: - """Shared Phase-2 tail: leaf ReAct → null-page parent locate → final prune. + """Shared Phase-2 tail: parent-first null-page ReAct → final prune. Callers collect ``structural_parent_paths`` and run ``prune(..., keep_null=True)`` before this helper. Regimes also runs printed-page parent backfill before @@ -1050,21 +718,13 @@ def apply_null_page_locates_and_prune( working = nodes overrides = dict(match_overrides) - overrides, leaf_report = locate_null_page_node_overrides( + overrides, null_page_report = locate_null_page_overrides( nodes=working, match_overrides=overrides, body_pages=body_pages, ctx=ctx, structural_parent_paths=structural_parent_paths, ) - overrides, parent_report = locate_null_page_parent_overrides( - nodes=working, - match_overrides=overrides, - page_texts=page_texts, - body_pages=body_pages, - ctx=ctx, - ) - null_page_report = [*leaf_report, *parent_report] working, failed_null_removed = prune_unanchored_toc_leaves( working, @@ -1116,8 +776,7 @@ def anchor_hierarchy_from_offset( locate_method = "offset_only" bulk_count = 0 - # Capture parents before prune: empty shells after keep_null prune must - # not enter leaf ReAct (H1). + # Capture parent identity before prune so empty shells retain ``kind=parent``. structural_parent_paths = { path for path, node in _iter_all_title_nodes(working) @@ -1137,7 +796,6 @@ def anchor_hierarchy_from_offset( nodes=working, match_overrides=match_overrides, body_pages=body_pages, - page_texts=page_texts, ctx=ctx, structural_parent_paths=structural_parent_paths, ) diff --git a/apps/worker/app/services/document_agent/structure/hierarchy_locator.py b/apps/worker/app/services/document_agent/structure/hierarchy_locator.py index 4ed61933a..8266232ed 100644 --- a/apps/worker/app/services/document_agent/structure/hierarchy_locator.py +++ b/apps/worker/app/services/document_agent/structure/hierarchy_locator.py @@ -1,11 +1,10 @@ """Locate hierarchy titles on PDF pages and resolve page ranges. Deterministic range assembly from PROFILE ``match_overrides``. Leaf starts -come only from those overrides. Null-page leaves are located upstream via -bounded grep ReAct + VLM; null-page parents use their sibling/first-child -window. They are then resolved here, including parent self-only spans for -interstitial pages. Parents without an override may still inherit start from -the earliest located descendant leaf. +come only from those overrides. Every null-page node is located upstream by +one parent-first whole-line ReAct + VLM path. They are then resolved here, +including parent self-only spans for interstitial pages. Parents without an +override may still inherit start from the earliest located descendant leaf. """ from __future__ import annotations @@ -14,11 +13,7 @@ from dataclasses import dataclass, field from typing import Any, Literal -from app.services.document_parser.structure.body_boundary import ( - clean_toc_title, - normalize_heading_label, - normalize_match_text, -) +from app.services.document_parser.structure.body_boundary import normalize_heading_label TitleMatchSource = Literal[ "anchored", @@ -26,7 +21,7 @@ "inspect_vlm", "inferred_descendant", "pdf_outline", - "react_normalized_grep_vlm", + "react_line_grep_vlm", ] @@ -136,46 +131,6 @@ class ResolvedHierarchyRange: evidence: dict[str, Any] = field(default_factory=dict) -def locate_title_normalized_strict( - title: str, - *, - scope_pages: list[int], - page_texts: dict[int, str], -) -> TitleMatch | None: - """Locate *title* after unified text normalization; accept one unique page. - - Query and page text both preserve one space between non-CJK words while - removing whitespace adjacent to CJK. Accept iff exactly one page in - ``scope_pages`` hits. - """ - needle = normalize_match_text(clean_toc_title(title) or title) - if not needle or not scope_pages: - return None - - hit_pages: list[int] = [] - matched_preview = "" - for page in scope_pages: - haystack = normalize_match_text(page_texts.get(page, "")) - if not haystack or needle not in haystack: - continue - hit_pages.append(page) - if not matched_preview: - matched_preview = needle[:160] - - unique_pages = sorted(set(hit_pages)) - if len(unique_pages) != 1: - return None - - page = unique_pages[0] - return TitleMatch( - page=page, - source="anchored", - matched_line=matched_preview, - candidates=[page], - evidence={"accept": "normalized_strict_unique"}, - ) - - def last_leaf_start_under( node: TitleNode, parent_titles: tuple[str, ...], diff --git a/apps/worker/app/services/document_agent/structure/null_page_react.py b/apps/worker/app/services/document_agent/structure/null_page_react.py index 1ef30cb11..14673bb60 100644 --- a/apps/worker/app/services/document_agent/structure/null_page_react.py +++ b/apps/worker/app/services/document_agent/structure/null_page_react.py @@ -1,19 +1,9 @@ -"""Bounded ReAct locate for TOC leaves with ``printed_page=None``. - -After Phase-2 printed-page bulk/bisect, null-page leaves use a serial probe: -text LLM plans a ``grep.text`` query inside a sibling window, then -``inspect.pages`` confirms only the first hit page (binary section-start check). -Later hits from the same grep are discarded; a reject advances to the next -planner turn. - -Budget (``REACT_PLANNER_GREP_BUDGET``): -- Seed full-title grep is free (does not consume the budget). -- Planner-chosen greps: at most ``REACT_PLANNER_GREP_BUDGET`` times. -- strip_header / strip_footer each auto re-grep the last query once for free. -- Planner turn cap = budget + 2 (room for the two strip actions). - -No offset seed, no fixed page-count cap, and no fallback to normalized-strict -unique hit / RTL / ``scan_title_forward``. +"""Parent-first ReAct locate for every TOC node with ``printed_page=None``. + +Each query uses normalized whole-line text only to nominate physical pages. +Candidate pages are then VLM-confirmed in non-overlapping 2/4/6/10 physical +windows. A text hit, including a single unique hit, never writes an override +without visual confirmation. """ from __future__ import annotations @@ -36,21 +26,17 @@ def react_budget() -> int: - """Planner grep-loop budget (and shared visual-confirm cap). - - Seed full-title grep does not consume this. Strip auto re-greps do not - consume it. Value is ``REACT_PLANNER_GREP_BUDGET``, independent of TOC - page-window constants. - """ + """Return the planner grep-loop budget.""" return int(REACT_PLANNER_GREP_BUDGET) + _REACT_INSTRUCTIONS = """\ -You are the search planner in a small ReAct loop. Propose the next action to -find the physical START page of a section. Grep collapses whitespace/newlines -to one space between non-CJK words, removes whitespace adjacent to CJK, and -matches case-insensitively. After each grep, a visual check confirms only the -first hit page (binary section-start). Later hits are discarded; a reject -means try a different query next. +You are the search planner in a small ReAct loop. Propose the next whole-line +text query to find the physical START page of a section. Grep normalizes each +PDF text line independently, collapses whitespace with CJK-aware spacing, and +matches the complete normalized line case-insensitively. It never accepts a +substring inside a longer line. Every candidate page, including a unique one, +must pass visual section-start confirmation. The system already grepped the full TOC title once before this loop (see previous_attempts). Do not repeat that exact full-title query. @@ -74,49 +60,39 @@ def react_budget() -> int: "Appendix " using the letter taken from the TOC label. Prefer this before inventing other phrases. 3. Only after the above: try other variants such as "Appendix " plus - the title body, a shorter distinctive fragment of the title, or another - structural prefix (chapter / part / section / annex) when supported by the - title or parent path. + the title body, a shorter distinctive complete-line title variant, or + another structural prefix (chapter / part / section / annex) when supported + by the title or parent path. 4. Prefer queries specific enough to avoid running headers and passing mentions. Do not guess page numbers. Reflection rules (mandatory): -- Read previous_attempts. Reflect on hit_count and observation before answering. -- If the last observation is no_normalized_hits, visual_rejected, +- Read previous_attempts. Reflect on query, hit_page_count, and observation + before answering. +- If the last observation is no_line_hits, visual_rejected, empty_normalized_query, grep_tool_error, or duplicate_normalized_query, you MUST change the query when choosing grep. Emitting the same grep query again - (same text after whitespace/case normalization) is invalid for planner-chosen - greps. -- Prefer strip_header or strip_footer when hits look scattered by running + after whitespace/case normalization is invalid for planner-chosen greps. +- Prefer strip_header or strip_footer when candidate pages look like running headers/footers. Each strip automatically re-greps the last query once for - free (same action; do not spend a planner turn to repeat that query). - Otherwise go to the next step in the ordered strategy (narrower / different - query). + free. Otherwise advance to the next ordered query strategy. - strip_header / strip_footer only update a temporary search view; they do not change stored page text. They do NOT consume react_budget. Call each at most once per locate. -- Planner greps consume react_budget (grep_loops_remaining). The automatic - full-title seed grep is free and does not consume it. -- no_normalized_hits / visual_rejected: advance to the next ordered strategy - step rather than repeating the same query. +- Planner greps consume react_budget. The automatic full-title seed grep and + strip auto re-greps are free. """ def _react_history_item(item: dict[str, Any]) -> dict[str, Any]: - hit_pages = [int(page) for page in (item.get("hit_pages") or [])] - out = { + """Planner-facing history only. Runtime audit fields stay on the attempt.""" + out: dict[str, Any] = { + "action": item.get("action"), "query": item.get("query"), "normalized_query": item.get("normalized_query"), - "hit_count": int(item.get("hit_count") or len(hit_pages)), + "hit_page_count": int(item.get("hit_page_count") or 0), "observation": item.get("observation"), - "visual_selected_page": item.get("visual_selected_page"), - "visual_reason": item.get("visual_reason"), - "visual_pages_checked": item.get("visual_pages_checked"), } - if item.get("seed_full_title"): - out["seed_full_title"] = True - if item.get("post_strip"): - out["post_strip"] = item.get("post_strip") return out @@ -127,17 +103,33 @@ def _next_located_bound( parent_titles: tuple[str, ...], overrides: dict[tuple[str, ...], TitleMatch], ) -> int | None: + """Return the next sibling's own or earliest descendant start.""" for later in sibling_nodes[index + 1 :]: path = (*parent_titles, later.title) - if path in overrides: - return int(overrides[path].page) - bound = first_leaf_start_under(later, parent_titles, overrides) - if bound is not None: - return int(bound) + own = overrides.get(path) + if own is not None: + return int(own.page) + descendant = first_leaf_start_under(later, parent_titles, overrides) + if descendant is not None: + return int(descendant) return None -def _normalized_grep( +def _last_confirmed_start_under( + *, + node: TitleNode, + parent_titles: tuple[str, ...], + overrides: dict[tuple[str, ...], TitleMatch], +) -> int | None: + """Return the last confirmed descendant leaf, or the node's own start.""" + descendant = last_leaf_start_under(node, parent_titles, overrides) + if descendant is not None: + return int(descendant) + own = overrides.get((*parent_titles, node.title)) + return int(own.page) if own is not None else None + + +def _whole_line_grep( *, ctx: ToolContext, query: str, @@ -145,11 +137,9 @@ def _normalized_grep( right: int, body_pages: list[int], ) -> tuple[str, str, list[int], int]: - """Grep only ``body_pages ∩ [left, right]``. + """Search only ``body_pages ∩ [left, right]`` using complete-line equality. - Returns ``(status, needle_or_error, hit_pages, match_count)``. - ``status`` is ``"ok"`` or ``"error"``; on error the second field is a short - error summary (not a normalized needle). + Returns ``(status, normalized_query_or_error, hit_pages, line_match_count)``. """ from app.services.document_agent.pdf_text import page_content_map from app.services.document_agent.tools.grep_text import grep_text @@ -176,28 +166,31 @@ def _normalized_grep( if not texts: return "ok", "", [], 0 - prev_view = ctx.blackboard.page_text_search_view + previous_view = ctx.blackboard.page_text_search_view ctx.blackboard.page_text_search_view = texts try: result = grep_text( ctx, { "query": query, + "whole_line": True, "start_page": min(texts), "end_page": max(texts), }, ) finally: - ctx.blackboard.page_text_search_view = prev_view + ctx.blackboard.page_text_search_view = previous_view if result.status != "ok": return "error", str(result.error or "grep.text failed"), [], 0 payload = result.payload or {} - hit_pages = [ - int(page) - for page in (payload.get("hit_pages") or []) - if int(page) in body_set - ] + hit_pages = sorted( + { + int(page) + for page in (payload.get("hit_pages") or []) + if int(page) in body_set + } + ) return ( "ok", str(payload.get("normalized_query") or ""), @@ -220,17 +213,8 @@ def _propose_react_query( "toc_title": title, "parent_path": list(parent_titles), "physical_search_scope": [left, right], - "react_budget": budget, - "grep_loops_used": grep_loops_used, "grep_loops_remaining": max(0, budget - grep_loops_used), - "visual_budget": budget, "previous_attempts": [_react_history_item(item) for item in attempts], - "note": ( - "Full TOC title was already grepped automatically before this loop " - "(free; does not consume react_budget). Planner greps consume " - "react_budget. strip_header/strip_footer are free and each " - "auto-retries the last grep query once for free." - ), } prompt = ( f"{_REACT_INSTRUCTIONS}\n\nCurrent state:\n" @@ -263,67 +247,44 @@ def _propose_react_query( } if action == "grep" and not query: return None, {"error": "planner returned empty grep query", "usage": usage} - return ( - { - "action": action, - "query": query, - }, - {"usage": usage}, - ) + return {"action": action, "query": query}, {"usage": usage} -def _verify_section_beginning_page( +def _verify_section_beginning_pages( *, ctx: ToolContext, title: str, - page: int, - query: str, -) -> tuple[bool, str, int]: - """Confirm one physical page as the section beginning.""" + pages: list[int], +) -> tuple[str, int | None, str, int]: + """Ask the VLM to select a section start from the supplied candidate pages.""" from app.services.document_agent.calibration.prompts import ( + SECTION_START_ANSWER_KEYS, + build_section_start_question, coerce_found, coerce_found_page, ) from app.services.document_agent.tools.inspect_pages import inspect_pages - question = ( - f"Does this page mark the physical BEGINNING of the document section " - f"corresponding to the TOC entry {title!r}? A cover page, section " - "title page, or first body-heading page can be the beginning. Allow " - "equivalent wording and added or omitted numbering, lettering, or " - "structural prefixes. Do not accept a table-of-contents line, running " - "header or footer, passing mention, or continuation page. " - f"The normalized text query that nominated this page was {query!r}. " - "Report the physical page number printed in the page label above the image." - ) - verify_result = inspect_pages( + result = inspect_pages( ctx, { - "pages": [page], - "page_cap": 1, - "question": question, - "answer_keys": { - "found": ( - "boolean, true only when this page is the physical beginning " - "of the requested section" - ), - "found_page": ( - "number|null, the physical page number where the section begins" - ), - }, + "pages": pages, + "page_cap": len(pages), + "question": build_section_start_question(title), + "answer_keys": SECTION_START_ANSWER_KEYS, "folder_name": "null_page_react_verify", "prefix": "verify", "usage_task": "document_agent.null_page_react_verify", }, ) - tokens = int(verify_result.tokens_used or 0) - if verify_result.status != "ok": - return False, str(verify_result.error or "inspect.pages failed"), tokens - fields = (verify_result.payload or {}).get("fields") or {} - found_page = coerce_found_page(fields.get("found_page"), pages=[page]) - ok = coerce_found(fields.get("found")) and found_page == page - reason = str((verify_result.payload or {}).get("answer") or "") - return ok, reason, tokens + tokens = int(result.tokens_used or 0) + if result.status != "ok": + return "error", None, str(result.error or "inspect.pages failed"), tokens + fields = (result.payload or {}).get("fields") or {} + found_page = coerce_found_page(fields.get("found_page"), pages=pages) + selected = found_page if coerce_found(fields.get("found")) else None + reason = str((result.payload or {}).get("answer") or "") + return "ok", selected, reason, tokens def _locate_with_react( @@ -338,76 +299,110 @@ def _locate_with_react( budget = react_budget() attempts: list[dict[str, Any]] = [] attempted_needles: set[str] = set() - visual_calls = 0 - visual_remaining = budget stripped: set[str] = set() last_grep_query: str | None = None - # Each locate starts from stored content (no cross-node strip leakage). - ctx.blackboard.page_text_search_view = None - # Seed full-title grep is free. Planner greps consume budget (≤ budget). - # strip_* each auto re-greps last query once for free. - # +2 planner turns: room for strip_header and strip_footer actions. + visual_calls = 0 grep_loops_used = 0 planner_turn = 0 max_planner_turns = budget + 2 + # Each node starts from stored page text; strip views never cross nodes. + ctx.blackboard.page_text_search_view = None + def _visual_confirm( *, query: str, hit_pages: list[int], attempt: dict[str, Any], ) -> TitleMatch | None: - """Confirm only the first hit page; discard the rest of this grep.""" - nonlocal visual_calls, visual_remaining - if visual_remaining <= 0: - attempt["observation"] = "visual_budget_exhausted" - attempts.append(attempt) - return None - - page = int(hit_pages[0]) - visual_remaining -= 1 - visual_calls += 1 - ok, reason, tokens = _verify_section_beginning_page( - ctx=ctx, - title=title, - page=page, - query=query, + nonlocal visual_calls + from app.services.document_agent.calibration.scan import ( + DEFAULT_WINDOW_SCHEDULE, + progressive_page_windows, ) - checked = [ - { - "page": page, - "confirmed": ok, - "reason": reason, - "tokens_used": tokens, + + checked_pages: list[int] = [] + rounds: list[dict[str, Any]] = [] + selected_page: int | None = None + selected_reason = "" + visual_error = False + + for physical_pages in progressive_page_windows( + start_page=hit_pages[0], + end_page=right, + window_schedule=DEFAULT_WINDOW_SCHEDULE, + ): + candidate_pages = [ + page + for page in hit_pages + if physical_pages[0] <= page <= physical_pages[-1] + ] + round_row: dict[str, Any] = { + "physical_window": [physical_pages[0], physical_pages[-1]], + "candidate_pages": candidate_pages, + "selected_page": None, } - ] - attempt["visual_pages_checked"] = checked - attempt["visual_selected_page"] = page if ok else None - attempt["visual_reason"] = reason - attempt["visual_budget_remaining_after"] = visual_remaining - if ok: - attempt["observation"] = "section_start_confirmed" - attempts.append(attempt) - return TitleMatch( - page=page, - source="react_normalized_grep_vlm", - matched_line=query, - candidates=[page], - evidence={ - "accept": "react_normalized_grep_vlm", - "null_page_react": True, - "loop": grep_loops_used, - "normalized_query": attempt.get("normalized_query"), - "visual_reason": reason, - "visual_pages_checked": [page], - "post_strip": attempt.get("post_strip"), - "seed_full_title": attempt.get("seed_full_title"), - }, + if not candidate_pages: + rounds.append(round_row) + continue + + visual_calls += 1 + checked_pages.extend(candidate_pages) + status, found_page, reason, tokens = _verify_section_beginning_pages( + ctx=ctx, + title=title, + pages=candidate_pages, + ) + round_row.update( + { + "status": status, + "selected_page": found_page, + "reason": reason, + "tokens_used": tokens, + } ) + rounds.append(round_row) + if status != "ok": + visual_error = True + selected_reason = reason + break + if found_page is None: + selected_reason = reason + continue + selected_page = int(found_page) + selected_reason = reason + break + + attempt["visual_rounds"] = rounds + attempt["visual_pages_checked"] = checked_pages + attempt["visual_selected_page"] = selected_page + attempt["visual_reason"] = selected_reason + if selected_page is None: + attempt["observation"] = ( + "visual_tool_error" if visual_error else "visual_rejected" + ) + attempts.append(attempt) + return None - attempt["observation"] = "visual_rejected" + attempt["observation"] = "section_start_confirmed" attempts.append(attempt) - return None + return TitleMatch( + page=selected_page, + source="react_line_grep_vlm", + matched_line=query, + candidates=list(hit_pages), + evidence={ + "accept": "react_line_grep_vlm", + "null_page_react": True, + "grep_loop": grep_loops_used, + "normalized_query": attempt.get("normalized_query"), + "candidate_pages": list(hit_pages), + "visual_pages_checked": checked_pages, + "visual_rounds": rounds, + "post_strip": attempt.get("post_strip"), + "seed_full_title": attempt.get("seed_full_title"), + }, + ) def _apply_grep_result( *, @@ -419,10 +414,9 @@ def _apply_grep_result( planner_meta: dict[str, Any], seed_full_title: bool = False, ) -> TitleMatch | None: - """Grep + classify. Appends to attempts; returns match on visual confirm.""" nonlocal grep_loops_used, last_grep_query - status, needle_or_error, hit_pages, match_count = _normalized_grep( + status, needle_or_error, hit_pages, line_match_count = _whole_line_grep( ctx=ctx, query=query, left=left, @@ -438,10 +432,9 @@ def _apply_grep_result( "action": "grep", "query": query, "normalized_query": needle, - "hit_count": len(hit_pages), + "hit_page_count": len(hit_pages), "hit_pages": hit_pages, - "match_count": match_count, - "visual_budget_remaining_before": visual_remaining, + "line_match_count": line_match_count, **planner_meta, } if post_strip is not None: @@ -454,12 +447,10 @@ def _apply_grep_result( attempt["error"] = needle_or_error attempts.append(attempt) return None - if not needle: attempt["observation"] = "empty_normalized_query" attempts.append(attempt) return None - if needle in attempted_needles and not allow_duplicate: attempt["observation"] = "duplicate_normalized_query" attempts.append(attempt) @@ -467,18 +458,15 @@ def _apply_grep_result( last_grep_query = query attempted_needles.add(needle) - if not hit_pages: attempt["observation"] = ( - "post_strip_no_normalized_hits" if post_strip else "no_normalized_hits" + "post_strip_no_line_hits" if post_strip else "no_line_hits" ) attempts.append(attempt) return None - return _visual_confirm(query=query, hit_pages=hit_pages, attempt=attempt) try: - # Free automatic probe: full TOC title (no planner, no react_budget). seed_match = _apply_grep_result( query=title, planner_turn_index=0, @@ -489,7 +477,7 @@ def _apply_grep_result( seed_full_title=True, ) if seed_match is not None: - return seed_match, attempts, visual_calls, "react_normalized_grep_vlm" + return seed_match, attempts, visual_calls, "react_line_grep_vlm" while grep_loops_used < budget and planner_turn < max_planner_turns: planner_turn += 1 @@ -508,8 +496,8 @@ def _apply_grep_result( "loop": planner_turn, "grep_loop": grep_loops_used, "action": "planner_error", - "hit_count": 0, - "hit_pages": [], + "hit_page_count": 0, + "line_match_count": 0, **planner_meta, } ) @@ -522,8 +510,8 @@ def _apply_grep_result( "loop": planner_turn, "grep_loop": grep_loops_used, **proposal, - "hit_count": 0, - "hit_pages": [], + "hit_page_count": 0, + "line_match_count": 0, **planner_meta, } ) @@ -542,8 +530,8 @@ def _apply_grep_result( "loop": planner_turn, "grep_loop": grep_loops_used, **proposal, - "hit_count": 0, - "hit_pages": [], + "hit_page_count": 0, + "line_match_count": 0, "observation": f"duplicate_strip_{which}", **planner_meta, } @@ -562,8 +550,8 @@ def _apply_grep_result( "loop": planner_turn, "grep_loop": grep_loops_used, **proposal, - "hit_count": 0, - "hit_pages": [], + "hit_page_count": 0, + "line_match_count": 0, "observation": ( f"stripped_{which}" if strip_ok @@ -577,7 +565,6 @@ def _apply_grep_result( if not strip_ok or not last_grep_query: continue - # Same action: auto re-grep last query on stripped view (no budget). from app.services.document_parser.structure.body_boundary import ( normalize_match_text, ) @@ -594,12 +581,11 @@ def _apply_grep_result( planner_meta={}, ) if match is not None: - return match, attempts, visual_calls, "react_normalized_grep_vlm" + return match, attempts, visual_calls, "react_line_grep_vlm" continue - query = proposal["query"] match = _apply_grep_result( - query=query, + query=proposal["query"], planner_turn_index=planner_turn, consume_budget=True, allow_duplicate=False, @@ -607,15 +593,14 @@ def _apply_grep_result( planner_meta=planner_meta, ) if match is not None: - return match, attempts, visual_calls, "react_normalized_grep_vlm" + return match, attempts, visual_calls, "react_line_grep_vlm" return None, attempts, visual_calls, "react_loop_limit" finally: - # Do not leak strip view across nodes / locate paths. ctx.blackboard.page_text_search_view = None -def locate_null_page_node_overrides( +def locate_null_page_overrides( *, nodes: list[TitleNode], match_overrides: dict[tuple[str, ...], TitleMatch], @@ -623,93 +608,25 @@ def locate_null_page_node_overrides( ctx: ToolContext | None, structural_parent_paths: set[tuple[str, ...]] | None = None, ) -> tuple[dict[tuple[str, ...], TitleMatch], list[dict[str, Any]]]: - """Locate null-page leaves with normalized grep ReAct + VLM. - - Grep reads ``ctx.blackboard.page_full_text_cache``. When ``ctx`` is None, - every null-page leaf is recorded as unresolved (no text-unique fallback). - - ``structural_parent_paths`` are paths that had children *before* - ``prune(keep_null=True)``. Empty-shell parents after prune must not be - leaf-probed; they go to parent locate instead. - """ + """Locate all null-page nodes in parent-first DFS order.""" if not nodes or not body_pages: return dict(match_overrides), [] - parent_paths = structural_parent_paths or set() out = dict(match_overrides) + parent_paths = structural_parent_paths or set() report: list[dict[str, Any]] = [] + from app.services.document_agent.calibration.scan import DEFAULT_WINDOW_SCHEDULE - def _skip_entry( - *, - node: TitleNode, - path: tuple[str, ...], - result: str, - failed_sibling: str | None = None, - ) -> dict[str, Any]: - entry: dict[str, Any] = { - "path_titles": list(path), - "title": node.title, - "kind": "leaf", - "printed_page": None, - "search_scope": None, - "result": result, - "page": None, - "accept": None, - "visual_verify_calls": 0, - "react_attempts": [], - } - if failed_sibling is not None: - entry["failed_sibling"] = failed_sibling - return entry - - def _record_unresolved_no_ctx( - sibling_nodes: list[TitleNode], - parent_titles: tuple[str, ...], - ) -> None: - for node in sibling_nodes: - path = (*parent_titles, node.title) - if ( - not node.children - and path not in parent_paths - and node.printed_page is None - and path not in out - ): - report.append( - { - "path_titles": list(path), - "title": node.title, - "kind": "leaf", - "printed_page": None, - "search_scope": None, - "result": "unresolved_no_ctx", - "page": None, - "accept": None, - "visual_verify_calls": 0, - "react_attempts": [], - } - ) - if node.children: - _record_unresolved_no_ctx(node.children, path) - - if ctx is None: - _record_unresolved_no_ctx(nodes, ()) - logger.info( - "[null_page_react] ctx is None: {} null-page leaf/leaves unresolved " - "(no LLM/VLM probe)", - len(report), - ) - return out, report + physical_window_limit = sum(DEFAULT_WINDOW_SCHEDULE) def walk( sibling_nodes: list[TitleNode], parent_titles: tuple[str, ...], - scope_start: int, - scope_end: int, + scope_left: int, + scope_right: int, + parent_has_printed_page: bool, ) -> None: - cursor = int(scope_start) - # A failed leaf only invalidates the serial cursor for later leaves at - # this level. Later parents keep their own walk (fresh flag). - failed_sibling: str | None = None + last_confirmed_at_level: int | None = None for index, node in enumerate(sibling_nodes): path_titles = (*parent_titles, node.title) next_bound = _next_located_bound( @@ -718,134 +635,118 @@ def walk( parent_titles=parent_titles, overrides=out, ) - node_scope_end = ( - min(int(next_bound), scope_end) + subtree_right = ( + min(int(next_bound), int(scope_right)) if next_bound is not None - else int(scope_end) + else int(scope_right) + ) + first_descendant = ( + first_leaf_start_under(node, parent_titles, out) + if node.children + else None + ) + probe_right = ( + min(int(first_descendant), subtree_right) + if first_descendant is not None + else subtree_right ) - if path_titles in out: - cursor = max(cursor, int(out[path_titles].page)) + parent_match = out.get(parent_titles) if parent_titles else None + if last_confirmed_at_level is not None: + probe_left = max(int(scope_left), last_confirmed_at_level) + elif parent_match is not None: + probe_left = max(int(scope_left), int(parent_match.page)) + elif parent_has_printed_page: + probe_left = int(scope_left) + else: + probe_left = max( + int(scope_left), + probe_right - physical_window_limit + 1, + ) + is_parent = bool(node.children) or path_titles in parent_paths needs_probe = ( - not node.children - and path_titles not in parent_paths - and node.printed_page is None + node.printed_page is None and path_titles not in out ) if needs_probe: - if failed_sibling is not None: - report.append( - _skip_entry( - node=node, - path=path_titles, - result="skipped_after_sibling_failure", - failed_sibling=failed_sibling, - ) - ) - continue - entry: dict[str, Any] = { "path_titles": list(path_titles), "title": node.title, - "kind": "leaf", + "kind": "parent" if is_parent else "leaf", "printed_page": None, - "search_scope": None, + "search_scope": [probe_left, probe_right], "result": "unresolved", "page": None, "accept": None, "visual_verify_calls": 0, "react_attempts": [], } - - left = int(cursor) - right = int(node_scope_end) - entry["search_scope"] = [left, right] scope_pages = [ - page for page in body_pages if left <= page <= right + page + for page in body_pages + if probe_left <= page <= probe_right ] - if right < left: + if probe_right < probe_left: entry["result"] = "skipped_bad_window" - report.append(entry) - failed_sibling = node.title - continue - - if not scope_pages: + elif not scope_pages: entry["result"] = "no_scope_pages" - report.append(entry) - failed_sibling = node.title - continue - - match, attempts, visual_calls, result = _locate_with_react( - path_titles=path_titles, - title=node.title, - left=left, - right=right, - body_pages=body_pages, - ctx=ctx, - ) - entry["react_attempts"] = attempts - entry["visual_verify_calls"] = visual_calls - entry["result"] = result - if match is not None: - out[path_titles] = match - entry["page"] = int(match.page) - entry["accept"] = match.evidence.get("accept") + elif ctx is None: + entry["result"] = "unresolved_no_ctx" + else: + match, attempts, visual_calls, result = _locate_with_react( + path_titles=path_titles, + title=node.title, + left=probe_left, + right=probe_right, + body_pages=body_pages, + ctx=ctx, + ) + entry["react_attempts"] = attempts + entry["visual_verify_calls"] = visual_calls + entry["result"] = result + if match is not None: + out[path_titles] = match + entry["page"] = int(match.page) + entry["accept"] = match.evidence.get("accept") report.append(entry) - if entry.get("page") is None: - failed_sibling = node.title - continue - - cursor = int(entry["page"]) - continue - if node.children: - child_scope_start = ( - int(out[path_titles].page) - if path_titles in out - else cursor + own_match = out.get(path_titles) + child_left = ( + int(own_match.page) + if own_match is not None + else probe_left ) walk( node.children, path_titles, - child_scope_start, - node_scope_end, + child_left, + subtree_right, + node.printed_page is not None, ) - last_under = last_leaf_start_under(node, parent_titles, out) - if last_under is not None: - cursor = max(cursor, int(last_under)) - elif path_titles in out: - cursor = max(cursor, int(out[path_titles].page)) - walk(nodes, (), body_pages[0], body_pages[-1]) + last_under = _last_confirmed_start_under( + node=node, + parent_titles=parent_titles, + overrides=out, + ) + if last_under is not None: + last_confirmed_at_level = ( + last_under + if last_confirmed_at_level is None + else max(last_confirmed_at_level, last_under) + ) + + walk(nodes, (), body_pages[0], body_pages[-1], False) located = sum(1 for row in report if row.get("page") is not None) - skipped = sum( - 1 - for row in report - if row.get("result") == "skipped_after_sibling_failure" - ) - budget = react_budget() logger.info( - "[null_page_react] serial null-page ReAct: attempted={} located={} " - "unresolved={} skipped_after_fail={} budget={}", + "[null_page_react] parent-first null-page ReAct: attempted={} " + "located={} unresolved={} planner_budget={} physical_window_limit={}", len(report), located, - sum( - 1 - for row in report - if row.get("result") - in { - "react_give_up", - "react_loop_limit", - "planner_error", - "unresolved", - "unresolved_no_ctx", - "skipped_bad_window", - "no_scope_pages", - } - ), - skipped, - budget, + len(report) - located, + react_budget(), + physical_window_limit, ) return out, report diff --git a/apps/worker/app/services/document_agent/tools/grep_text.py b/apps/worker/app/services/document_agent/tools/grep_text.py index 1c5bf173c..a831d807d 100644 --- a/apps/worker/app/services/document_agent/tools/grep_text.py +++ b/apps/worker/app/services/document_agent/tools/grep_text.py @@ -20,16 +20,17 @@ @register_tool( name="grep.text", description=( - "Search normalized PDF text for a substring or regex. Whitespace is " - "collapsed with CJK-aware spacing and matching is case-insensitive. " - "Uses page_text_search_view when set (after text.strip_*), else each " - "page's stored content field." + "Search normalized PDF text for a substring, regex, or complete line. " + "Whitespace is collapsed with CJK-aware spacing and matching is " + "case-insensitive. Uses page_text_search_view when set (after " + "text.strip_*), else each page's stored content field." ), parameters={ "type": "object", "properties": { "query": {"type": "string"}, "regex": {"type": "boolean", "default": False}, + "whole_line": {"type": "boolean", "default": False}, "max_results": {"type": "integer", "default": 30}, "context_chars": {"type": "integer", "default": 80}, "start_page": {"type": "integer"}, @@ -49,6 +50,7 @@ def grep_text(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: latency_ms=int((time.monotonic() - start) * 1000), ) use_regex = bool(args.get("regex", False)) + whole_line = bool(args.get("whole_line", False)) max_results = max(1, min(int(args.get("max_results") or 30), 100)) context_chars = max(20, min(int(args.get("context_chars") or 80), 300)) start_page = max(1, int(args.get("start_page") or 1)) @@ -74,27 +76,45 @@ def grep_text(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: for page, text in sorted(texts.items()): if page < start_page or (end_page and page > end_page): continue - normalized_text = normalize_match_text(str(text or "")) page_hit = False - for match in pattern.finditer(normalized_text): - hit_count += 1 - page_hit = True - if len(results) >= max_results: - continue - start_idx = max(match.start() - context_chars, 0) - end_idx = min(match.end() + context_chars, len(normalized_text)) - results.append( - { - "page": page, - "char_offset": match.start(), - "snippet": normalized_text[start_idx:end_idx], - } - ) + if whole_line: + for line_index, line in enumerate(str(text or "").splitlines()): + normalized_line = normalize_match_text(line) + if not normalized_line or pattern.fullmatch(normalized_line) is None: + continue + hit_count += 1 + page_hit = True + if len(results) < max_results: + results.append( + { + "page": page, + "line_index": line_index, + "char_offset": 0, + "snippet": normalized_line, + } + ) + else: + normalized_text = normalize_match_text(str(text or "")) + for match in pattern.finditer(normalized_text): + hit_count += 1 + page_hit = True + if len(results) >= max_results: + continue + start_idx = max(match.start() - context_chars, 0) + end_idx = min(match.end() + context_chars, len(normalized_text)) + results.append( + { + "page": page, + "char_offset": match.start(), + "snippet": normalized_text[start_idx:end_idx], + } + ) if page_hit: hit_pages.append(page) summary = { "query": query, "normalized_query": normalized_query, + "whole_line": whole_line, "hit_count": hit_count, "hit_page_count": len(hit_pages), "hit_pages": hit_pages, diff --git a/apps/worker/scripts/page_memory/debug_pm_null_page_react.py b/apps/worker/scripts/page_memory/debug_pm_null_page_react.py index 2e64c3561..6c8fe14cc 100644 --- a/apps/worker/scripts/page_memory/debug_pm_null_page_react.py +++ b/apps/worker/scripts/page_memory/debug_pm_null_page_react.py @@ -2,7 +2,7 @@ # ruff: noqa: E402 """Debug: dump production null-page locate report via Stage-2 anchoring. -Uses the live leaf ReAct + parent window locator path (no patches). +Uses the live parent-first unified ReAct locator (no patches). Usage: cd apps/worker @@ -95,7 +95,7 @@ def main() -> int: if isinstance(match, dict) else getattr(match, "source", None) ) - if source != "react_normalized_grep_vlm": + if source != "react_line_grep_vlm": continue titles = path if isinstance(path, (list, tuple)) else (path,) page = ( @@ -108,8 +108,10 @@ def main() -> int: payload = { "policy": { "prune_pre": "keep_null_page_nodes=True", - "leaf_probe": "null_page_react.locate_null_page_node_overrides", - "parent_probe": "anchoring_primitives.locate_null_page_parent_overrides", + "probe": "null_page_react.locate_null_page_overrides", + "traversal": "parent-first DFS", + "text_match": "normalized whole-line equality", + "visual_windows": [2, 4, 6, 10], "prune_post": "keep_null_page_nodes=False (drop unresolved)", "react_budget": react_budget(), "react_planner_grep_budget": REACT_PLANNER_GREP_BUDGET, @@ -136,14 +138,13 @@ def main() -> int: ) for row in report: logger.info( - " [{}] {} search_scope={} result={} page={} loops={} failed_sibling={}", + " [{}] {} search_scope={} result={} page={} loops={}", row.get("kind"), row.get("path_titles"), row.get("search_scope"), row.get("result"), row.get("page"), len(row.get("react_attempts") or []), - row.get("failed_sibling"), ) for hit in react_hits: logger.info(" OVERRIDE {} -> p{}", hit["path"], hit["page"]) diff --git a/apps/worker/tests/contract/test_calibration_scan_contract.py b/apps/worker/tests/contract/test_calibration_scan_contract.py index 83191311a..d56cfd69d 100644 --- a/apps/worker/tests/contract/test_calibration_scan_contract.py +++ b/apps/worker/tests/contract/test_calibration_scan_contract.py @@ -17,6 +17,7 @@ from app.services.document_agent.calibration import scan as scan_module from app.services.document_agent.calibration.scan import ( DEFAULT_WINDOW_SCHEDULE, + progressive_page_windows, scan_title_forward, ) from app.services.document_agent.manifest import ToolContext, ToolResult @@ -68,6 +69,14 @@ def _apply(fake: _FakeInspect) -> _FakeInspect: return _apply +def test_progressive_page_windows_are_non_overlapping_and_clipped() -> None: + assert progressive_page_windows(start_page=10, end_page=20) == [ + [10, 11], + [12, 13, 14, 15], + [16, 17, 18, 19, 20], + ] + + def test_first_round_opens_the_candidate_page_and_its_successor(patch_inspect) -> None: fake = patch_inspect(_FakeInspect(hit_page=10)) diff --git a/apps/worker/tests/contract/test_outline_short_circuit_contract.py b/apps/worker/tests/contract/test_outline_short_circuit_contract.py index bdbbc882b..053ee9bb3 100644 --- a/apps/worker/tests/contract/test_outline_short_circuit_contract.py +++ b/apps/worker/tests/contract/test_outline_short_circuit_contract.py @@ -117,7 +117,7 @@ def fake_calibrate(*args: Any, **kwargs: Any) -> Any: ), patch( "app.services.document_agent.structure.anchoring_primitives." - "locate_null_page_node_overrides", + "locate_null_page_overrides", side_effect=_no_null_parent_locate, ), ): @@ -153,7 +153,7 @@ def fake_judge(tool_ctx: ToolContext, args: dict[str, Any]) -> ToolResult: ), patch( "app.services.document_agent.structure.anchoring_primitives." - "locate_null_page_node_overrides", + "locate_null_page_overrides", side_effect=_no_null_parent_locate, ), ): diff --git a/apps/worker/tests/contract/test_structure_anchoring_contract.py b/apps/worker/tests/contract/test_structure_anchoring_contract.py index ccd7d2336..f58d70e93 100644 --- a/apps/worker/tests/contract/test_structure_anchoring_contract.py +++ b/apps/worker/tests/contract/test_structure_anchoring_contract.py @@ -74,10 +74,9 @@ def test_prune_out_of_scope_nodes_removes_overflow_leaves() -> None: assert [n.title for n in pruned] == ["A"] -def test_null_page_leaf_unresolved_without_ctx() -> None: - """No ctx → no leaf ReAct; null-page parent is not handled here.""" +def test_all_null_page_nodes_unresolved_without_ctx_in_parent_first_order() -> None: from app.services.document_agent.structure.null_page_react import ( - locate_null_page_node_overrides, + locate_null_page_overrides, ) parent = TitleNode( @@ -86,15 +85,18 @@ def test_null_page_leaf_unresolved_without_ctx() -> None: printed_page=None, children=[TitleNode(title="Orphan", level=2, printed_page=None, children=[])], ) - overrides, report = locate_null_page_node_overrides( + overrides, report = locate_null_page_overrides( nodes=[parent], match_overrides={}, body_pages=[1, 2, 3], ctx=None, ) assert overrides == {} - assert len(report) == 1 - assert report[0]["path_titles"] == ["Chapter", "Orphan"] + assert [row["path_titles"] for row in report] == [ + ["Chapter"], + ["Chapter", "Orphan"], + ] + assert [row["kind"] for row in report] == ["parent", "leaf"] assert {row["result"] for row in report} == {"unresolved_no_ctx"} @@ -118,10 +120,10 @@ def test_null_page_leaf_kept_by_pre_react_prune() -> None: assert [n.title for n in kept] == ["PrintedOk", "NullLeaf"] -def test_null_page_react_skips_later_siblings_after_failure() -> None: +def test_null_page_react_continues_later_siblings_after_failure() -> None: from app.services.document_agent.structure import null_page_react as npr from app.services.document_agent.structure.null_page_react import ( - locate_null_page_node_overrides, + locate_null_page_overrides, ) nodes = [ @@ -130,28 +132,39 @@ def test_null_page_react_skips_later_siblings_after_failure() -> None: ] ctx = _ctx() - def fail_a(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, str]: - assert kwargs["title"] == "A" - return None, [{"loop": 1, "observation": "react_give_up"}], 0, "react_give_up" + def probe(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, str]: + if kwargs["title"] == "A": + return None, [], 0, "react_give_up" + return ( + TitleMatch( + page=4, + source="react_line_grep_vlm", + matched_line="B", + candidates=[4], + evidence={"accept": "react_line_grep_vlm"}, + ), + [], + 1, + "react_line_grep_vlm", + ) - with patch.object(npr, "_locate_with_react", side_effect=fail_a): - overrides, report = locate_null_page_node_overrides( + with patch.object(npr, "_locate_with_react", side_effect=probe): + overrides, report = locate_null_page_overrides( nodes=nodes, match_overrides={}, body_pages=[1, 2, 3, 4, 5], ctx=ctx, ) - assert overrides == {} + assert overrides[("B",)].page == 4 assert report[0]["result"] == "react_give_up" - assert report[1]["result"] == "skipped_after_sibling_failure" + assert report[1]["result"] == "react_line_grep_vlm" -def test_null_page_react_still_walks_later_parent_subtree_after_failure() -> None: - """H5: a failed leaf must not block later parents' subtrees.""" +def test_null_page_react_parent_first_walk_survives_sibling_failure() -> None: from app.services.document_agent.structure import null_page_react as npr from app.services.document_agent.structure.null_page_react import ( - locate_null_page_node_overrides, + locate_null_page_overrides, ) nodes = [ @@ -175,45 +188,44 @@ def probe(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, probed.append(title) if title == "A": return None, [], 0, "react_give_up" - page = 10 if title == "B1" else 11 + page = {"B": 10, "B1": 10, "B2": 11, "C": 15}[title] return ( TitleMatch( page=page, - source="react_normalized_grep_vlm", + source="react_line_grep_vlm", matched_line=title, candidates=[page], - evidence={"accept": "react_normalized_grep_vlm"}, + evidence={"accept": "react_line_grep_vlm"}, ), [], 1, - "react_normalized_grep_vlm", + "react_line_grep_vlm", ) with patch.object(npr, "_locate_with_react", side_effect=probe): - overrides, report = locate_null_page_node_overrides( + overrides, report = locate_null_page_overrides( nodes=nodes, match_overrides={}, body_pages=list(range(1, 21)), ctx=ctx, ) - # A failed; B1/B2 under the later parent are still probed and located. - assert probed == ["A", "B1", "B2"] + assert probed == ["A", "B", "B1", "B2", "C"] + assert overrides[("B",)].page == 10 assert overrides[("B", "B1")].page == 10 assert overrides[("B", "B2")].page == 11 + assert overrides[("C",)].page == 15 by_path = {tuple(row["path_titles"]): row for row in report} assert by_path[("A",)]["result"] == "react_give_up" assert by_path[("B", "B1")]["page"] == 10 assert by_path[("B", "B2")]["page"] == 11 - # C is a later leaf at A's level: still skipped (serial cursor invalid). - assert by_path[("C",)]["result"] == "skipped_after_sibling_failure" - assert by_path[("C",)]["failed_sibling"] == "A" + assert by_path[("C",)]["page"] == 15 -def test_null_page_react_does_not_probe_parent() -> None: +def test_null_page_react_probes_parent_before_child() -> None: from app.services.document_agent.structure import null_page_react as npr from app.services.document_agent.structure.null_page_react import ( - locate_null_page_node_overrides, + locate_null_page_overrides, ) parent = TitleNode( @@ -226,12 +238,14 @@ def test_null_page_react_does_not_probe_parent() -> None: ) ctx = _ctx() - def fail_child(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, str]: - assert kwargs["title"] == "Child" + probed: list[str] = [] + + def fail_probe(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, str]: + probed.append(str(kwargs["title"])) return None, [], 0, "react_give_up" - with patch.object(npr, "_locate_with_react", side_effect=fail_child): - overrides, report = locate_null_page_node_overrides( + with patch.object(npr, "_locate_with_react", side_effect=fail_probe): + overrides, report = locate_null_page_overrides( nodes=[parent], match_overrides={}, body_pages=[1, 2, 3, 4, 5], @@ -239,15 +253,17 @@ def fail_child(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], ) assert overrides == {} - assert [row["path_titles"] for row in report] == [["Parent", "Child"]] - assert report[0]["result"] == "react_give_up" + assert probed == ["Parent", "Child"] + assert [row["path_titles"] for row in report] == [ + ["Parent"], + ["Parent", "Child"], + ] -def test_null_page_react_skips_empty_shell_structural_parent() -> None: - """H1: pre-prune parents that become childless must not leaf-probe.""" +def test_null_page_react_preserves_structural_parent_kind_after_prune() -> None: from app.services.document_agent.structure import null_page_react as npr from app.services.document_agent.structure.null_page_react import ( - locate_null_page_node_overrides, + locate_null_page_overrides, ) # After prune(keep_null=True): former parent survives as empty shell. @@ -261,7 +277,7 @@ def track(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, return None, [], 0, "react_give_up" with patch.object(npr, "_locate_with_react", side_effect=track): - overrides, report = locate_null_page_node_overrides( + overrides, report = locate_null_page_overrides( nodes=[shell, leaf], match_overrides={}, body_pages=[1, 2, 3, 4, 5], @@ -270,8 +286,8 @@ def track(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, ) assert overrides == {} - assert probed == ["RealLeaf"] - assert [row["path_titles"] for row in report] == [["RealLeaf"]] + assert probed == ["Appendix", "RealLeaf"] + assert [row["kind"] for row in report] == ["parent", "leaf"] def test_anchor_offset_collects_structural_parent_paths_before_prune() -> None: @@ -293,14 +309,8 @@ def fake_leaf(**kwargs: Any) -> tuple[dict, list]: captured["nodes"] = kwargs["nodes"] return dict(kwargs["match_overrides"]), [] - def fake_parent(**kwargs: Any) -> tuple[dict, list]: - return dict(kwargs["match_overrides"]), [] - - with ( - patch.object(anchoring, "locate_null_page_node_overrides", side_effect=fake_leaf), - patch.object( - anchoring, "locate_null_page_parent_overrides", side_effect=fake_parent - ), + with patch.object( + anchoring, "locate_null_page_overrides", side_effect=fake_leaf ): _working, _anchor = anchoring.anchor_hierarchy_from_offset( nodes=[parent], @@ -312,15 +322,15 @@ def fake_parent(**kwargs: Any) -> tuple[dict, list]: ) assert ("P",) in captured["structural_parent_paths"] - # PrintedMiss dropped by keep_null prune; empty-shell P still passed to leaf. + # PrintedMiss dropped by keep-null prune; structural kind remains available. assert len(captured["nodes"]) == 1 assert captured["nodes"][0].title == "P" assert captured["nodes"][0].children == [] -def test_null_page_normalized_grep_excludes_pages_outside_body() -> None: +def test_null_page_whole_line_grep_excludes_pages_outside_body() -> None: """H2: grep page map is body_pages ∩ [left, right] (TOC pages dropped).""" from app.services.document_agent.pdf_text import PageTextBands - from app.services.document_agent.structure.null_page_react import _normalized_grep + from app.services.document_agent.structure.null_page_react import _whole_line_grep ctx = _ctx() ctx.blackboard.page_count = 10 @@ -330,7 +340,7 @@ def test_null_page_normalized_grep_excludes_pages_outside_body() -> None: 2: PageTextBands(content="noise"), 5: PageTextBands(content="Appendix F Overview"), } - status, needle, hit_pages, _count = _normalized_grep( + status, needle, hit_pages, line_count = _whole_line_grep( ctx=ctx, query="Appendix F Overview", left=1, @@ -340,6 +350,33 @@ def test_null_page_normalized_grep_excludes_pages_outside_body() -> None: assert status == "ok" assert needle assert hit_pages == [5] + assert line_count == 1 + + +def test_null_page_react_planner_history_omits_runtime_audit_fields() -> None: + from app.services.document_agent.structure.null_page_react import _react_history_item + + history = _react_history_item( + { + "action": "grep", + "query": "Furniture Item Specific Guidelines", + "normalized_query": "furniture item specific guidelines", + "hit_page_count": 1, + "line_match_count": 2, + "observation": "visual_rejected", + "visual_selected_page": None, + "visual_pages_checked": [304], + "post_strip": "header", + "seed_full_title": True, + } + ) + assert history == { + "action": "grep", + "query": "Furniture Item Specific Guidelines", + "normalized_query": "furniture item specific guidelines", + "hit_page_count": 1, + "observation": "visual_rejected", + } def test_null_page_react_clears_page_text_search_view() -> None: @@ -357,7 +394,7 @@ def dirty_grep(**kwargs: Any) -> tuple[str, str, list[int], int]: return "ok", "x", [], 0 with ( - patch.object(npr, "_normalized_grep", side_effect=dirty_grep), + patch.object(npr, "_whole_line_grep", side_effect=dirty_grep), patch.object( npr, "_propose_react_query", @@ -378,50 +415,44 @@ def dirty_grep(**kwargs: Any) -> tuple[str, str, list[int], int]: assert ctx.blackboard.page_text_search_view is None -def test_null_page_react_confirms_only_first_hit_page() -> None: - """H6: each grep visually confirms only the first hit; rest are discarded.""" +def test_null_page_react_progresses_windows_until_candidate_is_confirmed() -> None: from app.services.document_agent.structure import null_page_react as npr from app.services.document_agent.structure.null_page_react import _locate_with_react ctx = _ctx() - verified_pages: list[int] = [] + verified_batches: list[list[int]] = [] def fake_grep(**kwargs: Any) -> tuple[str, str, list[int], int]: - return "ok", "appendix f", [10, 11, 12, 13, 14], 5 + return "ok", "appendix a", [260, 262, 280], 3 - def fake_verify(**kwargs: Any) -> tuple[bool, str, int]: - page = int(kwargs["page"]) - verified_pages.append(page) - return False, "not start", 0 + def fake_verify(**kwargs: Any) -> tuple[str, int | None, str, int]: + pages = list(kwargs["pages"]) + verified_batches.append(pages) + selected = 262 if 262 in pages else None + return "ok", selected, "checked", 0 with ( - patch.object(npr, "_normalized_grep", side_effect=fake_grep), - patch.object(npr, "_verify_section_beginning_page", side_effect=fake_verify), - patch.object( - npr, - "_propose_react_query", - return_value=({"action": "give_up", "query": ""}, {}), - ), + patch.object(npr, "_whole_line_grep", side_effect=fake_grep), + patch.object(npr, "_verify_section_beginning_pages", side_effect=fake_verify), ): match, attempts, visual_calls, result = _locate_with_react( - path_titles=("Appendix F",), - title="Appendix F", - left=1, - right=20, - body_pages=list(range(1, 21)), + path_titles=("Appendix A",), + title="Appendix A", + left=250, + right=300, + body_pages=list(range(250, 301)), ctx=ctx, ) - assert match is None - assert result == "react_give_up" - # Seed rejected first hit only; did not walk 11..14. - assert verified_pages == [10] - assert visual_calls == 1 - assert attempts[0]["observation"] == "visual_rejected" - assert attempts[0]["visual_pages_checked"] == [ - {"page": 10, "confirmed": False, "reason": "not start", "tokens_used": 0} - ] - assert "too_many_hits" not in {a.get("observation") for a in attempts} + assert match is not None + assert match.page == 262 + assert result == "react_line_grep_vlm" + assert verified_batches == [[260], [262]] + assert visual_calls == 2 + assert attempts[0]["observation"] == "section_start_confirmed" + assert attempts[0]["visual_pages_checked"] == [260, 262] + assert attempts[0]["visual_rounds"][0]["physical_window"] == [260, 261] + assert attempts[0]["visual_rounds"][1]["physical_window"] == [262, 265] def test_react_planner_grep_budget_is_not_page_constant() -> None: @@ -449,7 +480,7 @@ def fake_propose(**kwargs: Any) -> tuple[dict[str, Any] | None, dict[str, Any]]: with ( patch.object(npr, "react_budget", return_value=1), - patch.object(npr, "_normalized_grep", side_effect=fake_grep), + patch.object(npr, "_whole_line_grep", side_effect=fake_grep), patch.object(npr, "_propose_react_query", side_effect=fake_propose), ): _match, attempts, _visual, result = _locate_with_react( @@ -496,13 +527,13 @@ def fake_grep(**kwargs: Any) -> tuple[str, str, list[int], int]: def fake_propose(**kwargs: Any) -> tuple[dict[str, Any] | None, dict[str, Any]]: return next(proposals) - def fake_verify(**kwargs: Any) -> tuple[bool, str, int]: - return False, "reject", 0 + def fake_verify(**kwargs: Any) -> tuple[str, int | None, str, int]: + return "ok", None, "reject", 0 with ( - patch.object(npr, "_normalized_grep", side_effect=fake_grep), + patch.object(npr, "_whole_line_grep", side_effect=fake_grep), patch.object(npr, "_propose_react_query", side_effect=fake_propose), - patch.object(npr, "_verify_section_beginning_page", side_effect=fake_verify), + patch.object(npr, "_verify_section_beginning_pages", side_effect=fake_verify), patch.object(npr, "react_budget", return_value=5), ): _match, attempts, _visual, _result = _locate_with_react( @@ -529,7 +560,7 @@ def test_null_page_locate_summary_splits_leaf_and_parent() -> None: { "kind": "leaf", "page": 5, - "result": "react_normalized_grep_vlm", + "result": "react_line_grep_vlm", "visual_verify_calls": 1, }, { @@ -541,7 +572,7 @@ def test_null_page_locate_summary_splits_leaf_and_parent() -> None: { "kind": "parent", "page": 4, - "result": "visual_rtl", + "result": "react_line_grep_vlm", "visual_verify_calls": 2, }, ] @@ -561,18 +592,18 @@ def test_null_page_locate_summary_splits_leaf_and_parent() -> None: def test_null_page_react_hit_writes_override() -> None: from app.services.document_agent.structure import null_page_react as npr from app.services.document_agent.structure.null_page_react import ( - locate_null_page_node_overrides, + locate_null_page_overrides, ) leaf = TitleNode(title="Appendix B", level=1, printed_page=None, children=[]) ctx = _ctx() match = TitleMatch( page=12, - source="react_normalized_grep_vlm", + source="react_line_grep_vlm", matched_line="Appendix B", candidates=[12], evidence={ - "accept": "react_normalized_grep_vlm", + "accept": "react_line_grep_vlm", "null_page_react": True, "normalized_query": "appendix b", }, @@ -582,11 +613,11 @@ def fake_react(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], assert kwargs["left"] == 1 assert kwargs["right"] == 20 return match, [{"loop": 1, "observation": "section_start_confirmed"}], 1, ( - "react_normalized_grep_vlm" + "react_line_grep_vlm" ) with patch.object(npr, "_locate_with_react", side_effect=fake_react): - overrides, report = locate_null_page_node_overrides( + overrides, report = locate_null_page_overrides( nodes=[leaf], match_overrides={}, body_pages=list(range(1, 21)), @@ -594,31 +625,40 @@ def fake_react(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], ) assert overrides[("Appendix B",)].page == 12 - assert overrides[("Appendix B",)].source == "react_normalized_grep_vlm" + assert overrides[("Appendix B",)].source == "react_line_grep_vlm" assert report[0]["page"] == 12 - assert report[0]["result"] == "react_normalized_grep_vlm" + assert report[0]["result"] == "react_line_grep_vlm" -def test_null_page_parent_skipped_without_right_anchor() -> None: +def test_null_page_parent_without_descendant_anchor_uses_inherited_scope() -> None: + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_overrides, + ) + parent = TitleNode( title="Chapter", level=1, printed_page=None, children=[TitleNode(title="Orphan", level=2, printed_page=None, children=[])], ) - overrides, report = anchoring.locate_null_page_parent_overrides( + overrides, report = locate_null_page_overrides( nodes=[parent], match_overrides={}, - page_texts={1: "Chapter\nHello"}, body_pages=[1, 2, 3], ctx=None, ) assert overrides == {} - assert len(report) == 1 - assert report[0]["result"] == "skipped_no_right" + assert report[0]["path_titles"] == ["Chapter"] + assert report[0]["search_scope"] == [1, 3] + assert report[0]["result"] == "unresolved_no_ctx" -def test_null_page_parent_located_via_normalized_text() -> None: +def test_null_page_parent_uses_first_descendant_as_right_boundary() -> None: + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_overrides, + ) + child = TitleNode(title="1.1 Detail", level=2, printed_page=5, children=[]) parent = TitleNode( title="1 Overview", @@ -630,27 +670,41 @@ def test_null_page_parent_located_via_normalized_text() -> None: [(("1 Overview", "1.1 Detail"), child)], offset=0, ) - page_texts = { - 4: "noise", - 5: "1 Overview\n1.1 Detail\nbody", - 6: "more", - } - overrides, report = anchoring.locate_null_page_parent_overrides( - nodes=[parent], - match_overrides=leaf_match, - page_texts=page_texts, - body_pages=[4, 5, 6], - ctx=None, - ) + def probe(**kwargs: Any) -> tuple[TitleMatch, list[dict[str, Any]], int, str]: + assert kwargs["title"] == "1 Overview" + assert kwargs["left"] == 4 + assert kwargs["right"] == 5 + return ( + TitleMatch( + page=4, + source="react_line_grep_vlm", + matched_line="1 Overview", + candidates=[4], + evidence={"accept": "react_line_grep_vlm"}, + ), + [], + 1, + "react_line_grep_vlm", + ) + + with patch.object(npr, "_locate_with_react", side_effect=probe): + overrides, report = locate_null_page_overrides( + nodes=[parent], + match_overrides=leaf_match, + body_pages=[4, 5, 6], + ctx=_ctx(), + ) assert ("1 Overview",) in overrides - assert overrides[("1 Overview",)].page == 5 - assert report[0]["result"] != "unresolved" - assert report[0]["page"] == 5 - assert report[0]["window"] == [1, 5] + assert overrides[("1 Overview",)].page == 4 + assert report[0]["search_scope"] == [4, 5] -def test_first_sibling_null_parent_uses_scan_forward_not_wide_rtl() -> None: - """No left sibling: miss text → ``scan_title_forward`` within 2+4+6+10 budget.""" +def test_root_first_null_parent_uses_22_page_left_cap() -> None: + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_overrides, + ) + child = TitleNode(title="22.1 Intro", level=2, printed_page=278, children=[]) parent = TitleNode( title="Chapter 22", @@ -668,51 +722,77 @@ def test_first_sibling_null_parent_uses_scan_forward_not_wide_rtl() -> None: ) } body_pages = list(range(1, 301)) - page_texts = {page: "noise" for page in body_pages} - ctx = _ctx() - scanned_starts: list[int] = [] - - def fake_scan(**kwargs: Any) -> Any: - from app.services.document_agent.calibration.scan import TitleScanResult - - scanned_starts.append(int(kwargs["start_page"])) - assert int(kwargs["page_count"]) == 278 - assert int(kwargs["start_page"]) == anchoring._first_sibling_null_parent_scan_start( - 278 + def probe(**kwargs: Any) -> tuple[TitleMatch, list[dict[str, Any]], int, str]: + assert kwargs["left"] == 257 + assert kwargs["right"] == 278 + return ( + TitleMatch( + page=270, + source="react_line_grep_vlm", + matched_line="Chapter 22", + candidates=[270], + evidence={"accept": "react_line_grep_vlm"}, + ), + [], + 1, + "react_line_grep_vlm", ) - return TitleScanResult( - title=str(kwargs["title"]), - found=True, - found_page=270, - scanned_pages=list(range(int(kwargs["start_page"]), 271)), - next_start=271, + + with patch.object(npr, "_locate_with_react", side_effect=probe): + overrides, report = locate_null_page_overrides( + nodes=[parent], + match_overrides=leaf_match, + body_pages=body_pages, + ctx=_ctx(), ) - with patch( - "app.services.document_agent.calibration.scan.scan_title_forward", - side_effect=fake_scan, - ): - with patch.object(anchoring, "_visual_rtl_locate_parent") as rtl: - overrides, report = anchoring.locate_null_page_parent_overrides( - nodes=[parent], - match_overrides=leaf_match, - page_texts=page_texts, - body_pages=body_pages, - ctx=ctx, + assert overrides[("Chapter 22",)].page == 270 + assert report[0]["search_scope"] == [257, 278] + + +def test_null_child_of_unanchored_printed_parent_keeps_inherited_scope() -> None: + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_overrides, + ) + + parent = TitleNode( + title="Appendices", + level=1, + printed_page=261, + children=[ + TitleNode( + title="A City palette maps", + level=2, + printed_page=None, + children=[], ) - rtl.assert_not_called() + ], + ) - assert scanned_starts == [anchoring._first_sibling_null_parent_scan_start(278)] - assert overrides[("Chapter 22",)].page == 270 - assert report[0]["accept"] == "scan_forward" - assert report[0]["window"] == [ - anchoring._first_sibling_null_parent_scan_start(278), - 278, - ] + def probe(**kwargs: Any) -> tuple[None, list[dict[str, Any]], int, str]: + assert kwargs["left"] == 250 + assert kwargs["right"] == 443 + return None, [], 0, "react_give_up" + + with patch.object(npr, "_locate_with_react", side_effect=probe): + _overrides, report = locate_null_page_overrides( + nodes=[parent], + match_overrides={}, + body_pages=list(range(250, 444)), + ctx=_ctx(), + ) + assert report[0]["search_scope"] == [250, 443] + + +def test_null_page_parent_uses_previous_sibling_last_leaf_as_left() -> None: + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_overrides, + ) -def test_null_page_parent_with_left_sibling_still_uses_rtl() -> None: left_child = TitleNode(title="A.1", level=2, printed_page=10, children=[]) left = TitleNode(title="A", level=1, printed_page=10, children=[left_child]) right_child = TitleNode(title="B.1", level=2, printed_page=50, children=[]) @@ -740,70 +820,112 @@ def test_null_page_parent_with_left_sibling_still_uses_rtl() -> None: evidence={}, ), } - page_texts = {p: "noise" for p in range(1, 61)} - ctx = _ctx() - - def fake_rtl(**kwargs: Any) -> tuple[TitleMatch, int]: + def probe(**kwargs: Any) -> tuple[TitleMatch, list[dict[str, Any]], int, str]: assert kwargs["left"] == 10 assert kwargs["right"] == 50 return ( TitleMatch( page=40, - source="inspect_vlm", - matched_line="", + source="react_line_grep_vlm", + matched_line="B", candidates=[40], - evidence={"accept": "visual_rtl"}, + evidence={"accept": "react_line_grep_vlm"}, ), - 3, + [], + 1, + "react_line_grep_vlm", ) - with patch( - "app.services.document_agent.calibration.scan.scan_title_forward" - ) as scan: - with patch.object( - anchoring, "_visual_rtl_locate_parent", side_effect=fake_rtl - ): - overrides, report = anchoring.locate_null_page_parent_overrides( - nodes=[left, right], - match_overrides=overrides_in, - page_texts=page_texts, - body_pages=list(range(1, 61)), - ctx=ctx, - ) - scan.assert_not_called() + with patch.object(npr, "_locate_with_react", side_effect=probe): + overrides, report = locate_null_page_overrides( + nodes=[left, right], + match_overrides=overrides_in, + body_pages=list(range(1, 61)), + ctx=_ctx(), + ) assert overrides[("B",)].page == 40 - assert report[0]["accept"] == "visual_rtl" + assert report[0]["search_scope"] == [10, 50] -def test_null_page_leaf_runs_before_parent_in_production_flow() -> None: - child = TitleNode(title="Child", level=2, printed_page=None, children=[]) +def test_confirmed_null_parent_becomes_child_left_boundary() -> None: + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_overrides, + ) + + detail = TitleNode(title="3.1.1 Detail", level=3, printed_page=304, children=[]) + child = TitleNode( + title="3.1 Street Furniture with Advertising", + level=2, + printed_page=None, + children=[detail], + ) parent = TitleNode( - title="Parent", + title="3 Advertising", level=1, printed_page=None, children=[child], ) - calls: list[str] = [] - - def fake_leaf(**kwargs: Any) -> tuple[dict[tuple[str, ...], TitleMatch], list[dict[str, Any]]]: - calls.append("leaf") - overrides = dict(kwargs["match_overrides"]) - overrides[("Parent", "Child")] = TitleMatch( - page=5, + overrides_in = { + ("3 Advertising", "3.1 Street Furniture with Advertising", "3.1.1 Detail"): TitleMatch( + page=304, source="anchored", - matched_line="Child", - candidates=[5], + matched_line="", + candidates=[304], evidence={}, ) - return overrides, [{"kind": "leaf", "page": 5}] + } + scopes: list[tuple[str, int, int]] = [] + + def probe(**kwargs: Any) -> tuple[TitleMatch, list[dict[str, Any]], int, str]: + title = str(kwargs["title"]) + scopes.append((title, int(kwargs["left"]), int(kwargs["right"]))) + return ( + TitleMatch( + page=304, + source="react_line_grep_vlm", + matched_line=title, + candidates=[304], + evidence={"accept": "react_line_grep_vlm"}, + ), + [], + 1, + "react_line_grep_vlm", + ) + + with patch.object(npr, "_locate_with_react", side_effect=probe): + overrides, report = locate_null_page_overrides( + nodes=[parent], + match_overrides=overrides_in, + body_pages=list(range(280, 330)), + ctx=_ctx(), + ) + + assert scopes == [ + ("3 Advertising", 283, 304), + ("3.1 Street Furniture with Advertising", 304, 304), + ] + assert overrides[("3 Advertising",)].page == 304 + assert overrides[("3 Advertising", "3.1 Street Furniture with Advertising")].page == 304 + assert [row["kind"] for row in report] == ["parent", "parent"] + + +def test_production_flow_calls_one_parent_first_null_page_walker() -> None: + child = TitleNode(title="Child", level=2, printed_page=None, children=[]) + parent = TitleNode( + title="Parent", + level=1, + printed_page=None, + children=[child], + ) + calls: list[str] = [] - def fake_parent( + def fake_locate( **kwargs: Any, ) -> tuple[dict[tuple[str, ...], TitleMatch], list[dict[str, Any]]]: - calls.append("parent") + calls.append("unified") overrides = dict(kwargs["match_overrides"]) - assert ("Parent", "Child") in overrides overrides[("Parent",)] = TitleMatch( page=4, source="anchored", @@ -811,11 +933,20 @@ def fake_parent( candidates=[4], evidence={}, ) - return overrides, [{"kind": "parent", "page": 4}] + overrides[("Parent", "Child")] = TitleMatch( + page=5, + source="anchored", + matched_line="Child", + candidates=[5], + evidence={}, + ) + return overrides, [ + {"kind": "parent", "page": 4}, + {"kind": "leaf", "page": 5}, + ] - with ( - patch.object(anchoring, "locate_null_page_node_overrides", side_effect=fake_leaf), - patch.object(anchoring, "locate_null_page_parent_overrides", side_effect=fake_parent), + with patch.object( + anchoring, "locate_null_page_overrides", side_effect=fake_locate ): resolved, anchor = anchoring.anchor_hierarchy_from_offset( nodes=[parent], @@ -827,27 +958,10 @@ def fake_parent( ctx=None, ) - assert calls == ["leaf", "parent"] + assert calls == ["unified"] assert [node.title for node in resolved] == ["Parent"] assert set(anchor.match_overrides) == {("Parent",), ("Parent", "Child")} - assert [row["kind"] for row in anchor.null_page_report] == ["leaf", "parent"] - - -def test_normalized_title_match_preserves_english_word_boundary() -> None: - from app.services.document_agent.structure.hierarchy_locator import ( - locate_title_normalized_strict, - ) - - match = locate_title_normalized_strict( - "附录 A OVERVIEW", - scope_pages=[7], - page_texts={7: "附录\nA OVERVIEW"}, - ) - - assert match is not None - assert match.page == 7 - assert match.matched_line == "附录a overview" - assert match.evidence["accept"] == "normalized_strict_unique" + assert [row["kind"] for row in anchor.null_page_report] == ["parent", "leaf"] def test_phase2_bulk_via_mocked_offset() -> None: @@ -999,7 +1113,7 @@ def fake_verify(**kwargs: Any) -> dict[str, Any]: def test_regimes_bulk_count_excludes_null_page_react_hits() -> None: - """H4: bulk_count frozen before leaf/parent ReAct; ReAct hits stay in report only.""" + """Bulk count freezes before unified ReAct; its hits stay in the report.""" from app.services.document_agent.calibration import procedure as proc from app.services.document_agent.calibration.procedure import ( anchor_hierarchy_from_regimes, @@ -1036,10 +1150,10 @@ def test_regimes_bulk_count_excludes_null_page_react_hits() -> None: ) react_match = TitleMatch( page=20, - source="react_normalized_grep_vlm", + source="react_line_grep_vlm", matched_line="NullLeaf", candidates=[20], - evidence={"accept": "react_normalized_grep_vlm"}, + evidence={"accept": "react_line_grep_vlm"}, ) def fake_offset(**kwargs: Any) -> dict[tuple[str, ...], TitleMatch]: @@ -1056,7 +1170,7 @@ def fake_apply(**kwargs: Any) -> tuple[list, dict, list, int]: "path_titles": ["NullLeaf"], "kind": "leaf", "page": 20, - "result": "react_normalized_grep_vlm", + "result": "react_line_grep_vlm", } ], 0, @@ -1379,10 +1493,10 @@ def test_parent_backfill_ignores_react_located_children_for_offset() -> None: # ReAct hit comes first in document order and must not set the offset. ("Section A", "Intro"): TitleMatch( page=99, - source="react_normalized_grep_vlm", + source="react_line_grep_vlm", matched_line="Intro", candidates=[99], - evidence={"accept": "react_normalized_grep_vlm", "null_page_react": True}, + evidence={"accept": "react_line_grep_vlm", "null_page_react": True}, ), **primitives.bulk_offset_matches( [(("Section A", "Body"), printed_child)], 5 @@ -1408,10 +1522,10 @@ def test_parent_backfill_unresolved_when_only_react_children() -> None: matches = { ("Section A", "Intro"): TitleMatch( page=99, - source="react_normalized_grep_vlm", + source="react_line_grep_vlm", matched_line="Intro", candidates=[99], - evidence={"accept": "react_normalized_grep_vlm", "null_page_react": True}, + evidence={"accept": "react_line_grep_vlm", "null_page_react": True}, ) } diff --git a/apps/worker/tests/contract/test_tool_registry_smoke_contract.py b/apps/worker/tests/contract/test_tool_registry_smoke_contract.py index 61785a191..a153d9979 100644 --- a/apps/worker/tests/contract/test_tool_registry_smoke_contract.py +++ b/apps/worker/tests/contract/test_tool_registry_smoke_contract.py @@ -64,6 +64,39 @@ def test_grep_text_normalizes_query_and_corpus_by_default() -> None: assert result.payload["hit_pages"] == [2] +def test_grep_text_whole_line_rejects_body_substring_and_dedupes_pages() -> None: + blackboard = ProfileBlackboard(page_count=2) + blackboard.page_full_text_cache = { + 1: ( + "The street furniture with advertising program continues.\n" + "Street Furniture with Advertising\n" + "Street Furniture with Advertising" + ), + 2: "No heading here", + } + ctx = ToolContext( + pdf_path="/tmp/doc.pdf", + job_id="grep-whole-line", + blackboard=blackboard, + trace=None, + settings={}, + ) + + result = grep_text( + ctx, + { + "query": "Street Furniture with Advertising", + "whole_line": True, + }, + ) + + assert result.status == "ok" + assert result.payload["whole_line"] is True + assert result.payload["hit_count"] == 2 + assert result.payload["hit_page_count"] == 1 + assert result.payload["hit_pages"] == [1] + + def test_strip_footer_updates_search_view_for_grep() -> None: from app.services.document_agent.pdf_text import PageTextBands from app.services.document_agent.tools.text_strip_margins import strip_footer From ace6b31f17fe7819d11c98130f6cfca09cf4cf57 Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Sat, 22 Aug 2026 22:42:44 +0800 Subject: [PATCH 6/7] fix: backfill printed TOC pages and drop 22-page probe left cap Anchor all remaining printed nodes from nearest regime offset before null-page ReAct, and use preorder max confirmed pages for probe left bounds so nested appendices keep a usable search window. Co-authored-by: Cursor --- .../document_agent/calibration/procedure.py | 40 ++--- .../structure/anchoring_primitives.py | 139 +++++++++++------- .../structure/hierarchy_locator.py | 16 -- .../structure/null_page_react.py | 80 +++------- .../page_memory/debug_pm_null_page_react.py | 2 + .../test_structure_anchoring_contract.py | 78 ++++++---- 6 files changed, 173 insertions(+), 182 deletions(-) diff --git a/apps/worker/app/services/document_agent/calibration/procedure.py b/apps/worker/app/services/document_agent/calibration/procedure.py index b59684674..868654760 100644 --- a/apps/worker/app/services/document_agent/calibration/procedure.py +++ b/apps/worker/app/services/document_agent/calibration/procedure.py @@ -4,7 +4,7 @@ 1. Builds TitleNodes the same way production does 2. Runs Phase-2 **per regime** (prune → bulk/bisect → recalibrate) 3. Merges physical-page ``match_overrides`` across regimes -4. Locates null-page leaves, then null-page parents, then final prune +4. Backfills printed-page nodes, runs unified null-page ReAct, then final prune Returns production ``SkeletonAnchor`` plus regime diagnostics for debug payloads. """ @@ -34,7 +34,6 @@ from app.services.document_agent.structure.anchoring_primitives import ( SkeletonAnchor, apply_null_page_locates_and_prune, - backfill_parent_offset_matches, prune_unanchored_toc_leaves, serialize_skeleton_anchor, ) @@ -374,34 +373,25 @@ def anchor_hierarchy_from_regimes( if path in surviving_paths } - parent_matches = backfill_parent_offset_matches( + ( + working, + match_overrides, + null_page_report, + failed_null_removed, + pre_react_override_count, + ) = apply_null_page_locates_and_prune( nodes=working, - matches=merged, + match_overrides=merged, + body_pages=body_pages, page_count=page_count, - ) - if parent_matches: - merged.update(parent_matches) - logger.info( - "[calibration.phase2] parent backfill: {} printed-page TOC parents " - "anchored from descendant offset", - len(parent_matches), - ) - - # Freeze bulk before unified null-page ReAct (same as offset path: - # offset_guided → len(overrides then); else 0). Never recount after ReAct. - bulk_count = len(merged) if regime_bulk > 0 else 0 - - working, match_overrides, null_page_report, failed_null_removed = ( - apply_null_page_locates_and_prune( - nodes=working, - match_overrides=merged, - body_pages=body_pages, - ctx=ctx, - structural_parent_paths=structural_parent_paths, - ) + ctx=ctx, + structural_parent_paths=structural_parent_paths, ) total_pruned += failed_null_removed + # Freeze bulk after printed backfill and before ReAct (ReAct hits stay out). + bulk_count = pre_react_override_count if regime_bulk > 0 else 0 + primary = pick_primary_offset(result) if primary is None and usable_regimes: primary = int(usable_regimes[0].offset) # type: ignore[arg-type] diff --git a/apps/worker/app/services/document_agent/structure/anchoring_primitives.py b/apps/worker/app/services/document_agent/structure/anchoring_primitives.py index ab580f1d0..6068f8b65 100644 --- a/apps/worker/app/services/document_agent/structure/anchoring_primitives.py +++ b/apps/worker/app/services/document_agent/structure/anchoring_primitives.py @@ -318,35 +318,46 @@ def bulk_offset_matches( return matches -def _iter_printed_page_parents( - nodes: list[TitleNode], +def _regime_offset_for_printed_node( *, - parent_titles: tuple[str, ...] = (), -) -> list[tuple[tuple[str, ...], TitleNode]]: - """DFS non-leaf nodes that print their own page in the TOC.""" - parents: list[tuple[tuple[str, ...], TitleNode]] = [] - for node in nodes: - path_titles = (*parent_titles, node.title) - if not node.children: - continue - if node.printed_page is not None: - parents.append((path_titles, node)) - parents.extend( - _iter_printed_page_parents(node.children, parent_titles=path_titles) - ) - return parents - - -def _descendant_regime_offset( - node: TitleNode, path_titles: tuple[str, ...], + ordered_paths: list[tuple[str, ...]], matches: dict[tuple[str, ...], TitleMatch], ) -> int | None: - """Offset of the parent's first anchored descendant leaf, i.e. its regime.""" - for leaf_path, _leaf in iter_leaf_title_nodes( - node.children, parent_titles=path_titles - ): - match = matches.get(leaf_path) + """Resolve the calibration-segment offset for one printed TOC node. + + Prefer an offset-bearing match under the node (same binary/regime segment as + its descendants). If the subtree has none, take the nearest preceding + offset-bearing match; if still none, the first subsequent one. + """ + depth = len(path_titles) + for path in ordered_paths: + if path == path_titles or len(path) <= depth: + continue + if path[:depth] != path_titles: + continue + match = matches.get(path) + if match is None: + continue + offset = match.evidence.get("offset") + if offset is not None: + return int(offset) + + try: + index = ordered_paths.index(path_titles) + except ValueError: + return None + for prev in reversed(ordered_paths[:index]): + match = matches.get(prev) + if match is None: + continue + offset = match.evidence.get("offset") + if offset is not None: + return int(offset) + for nxt in ordered_paths[index + 1 :]: + if len(nxt) > depth and nxt[:depth] == path_titles: + continue + match = matches.get(nxt) if match is None: continue offset = match.evidence.get("offset") @@ -355,24 +366,29 @@ def _descendant_regime_offset( return None -def backfill_parent_offset_matches( +def backfill_printed_offset_matches( *, nodes: list[TitleNode], matches: dict[tuple[str, ...], TitleMatch], page_count: int, ) -> dict[tuple[str, ...], TitleMatch]: - """Anchor TOC parents that print a page, reusing their descendant's offset. + """Blind-write physical pages for every still-unanchored printed TOC node. - Bulk anchoring consumes leaves only, so a TOC whose section headings carry - printed pages leaves every parent without a physical page. The parent shares - the calibration regime of its first anchored descendant, so ``printed_page + - that regime's offset`` is the parent's physical page. + Bulk/bisect only consume printed leaves. After that pass, any remaining node + with ``printed_page`` (parent or leaf) reuses its calibration-segment offset: + ``printed_page + offset``, clamped to ``1..page_count``. """ + ordered = _iter_all_title_nodes(nodes) + ordered_paths = [path for path, _node in ordered] by_offset: dict[int, list[tuple[tuple[str, ...], TitleNode]]] = {} - for path_titles, node in _iter_printed_page_parents(nodes): - if path_titles in matches: + for path_titles, node in ordered: + if node.printed_page is None or path_titles in matches: continue - offset = _descendant_regime_offset(node, path_titles, matches) + offset = _regime_offset_for_printed_node( + path_titles=path_titles, + ordered_paths=ordered_paths, + matches=matches, + ) if offset is None: continue by_offset.setdefault(offset, []).append((path_titles, node)) @@ -383,7 +399,7 @@ def backfill_parent_offset_matches( if 1 <= match.page <= page_count: out[path_titles] = replace( match, - evidence={**match.evidence, "parent_backfill": True}, + evidence={**match.evidence, "printed_offset_backfill": True}, ) return out @@ -701,6 +717,7 @@ def apply_null_page_locates_and_prune( nodes: list[TitleNode], match_overrides: dict[tuple[str, ...], TitleMatch], body_pages: list[int], + page_count: int, ctx: ToolContext | None, structural_parent_paths: set[tuple[str, ...]], ) -> tuple[ @@ -708,15 +725,28 @@ def apply_null_page_locates_and_prune( dict[tuple[str, ...], TitleMatch], list[dict[str, Any]], int, + int, ]: - """Shared Phase-2 tail: parent-first null-page ReAct → final prune. + """Shared Phase-2 tail: printed backfill → null-page ReAct → final prune. Callers collect ``structural_parent_paths`` and run ``prune(..., keep_null=True)`` - before this helper. Regimes also runs printed-page parent backfill before - calling; the offset path intentionally does not (known asymmetry). + before this helper. Returns ``pre_react_override_count`` so callers can freeze + ``bulk_count`` after backfill and before ReAct hits are added. """ working = nodes overrides = dict(match_overrides) + backfilled = backfill_printed_offset_matches( + nodes=working, + matches=overrides, + page_count=page_count, + ) + if backfilled: + overrides.update(backfilled) + logger.info( + "[structure_anchoring] printed-offset backfill: {} nodes", + len(backfilled), + ) + pre_react_override_count = len(overrides) overrides, null_page_report = locate_null_page_overrides( nodes=working, @@ -732,7 +762,13 @@ def apply_null_page_locates_and_prune( keep_null_page_nodes=False, ) overrides = _filter_overrides_to_tree(working, overrides) - return working, overrides, null_page_report, failed_null_removed + return ( + working, + overrides, + null_page_report, + failed_null_removed, + pre_react_override_count, + ) def anchor_hierarchy_from_offset( @@ -770,11 +806,9 @@ def anchor_hierarchy_from_offset( if offset_matches is not None: match_overrides = offset_matches locate_method = "offset_guided_bulk" - bulk_count = len(offset_matches) else: match_overrides = seed_overrides locate_method = "offset_only" - bulk_count = 0 # Capture parent identity before prune so empty shells retain ``kind=parent``. structural_parent_paths = { @@ -790,17 +824,22 @@ def anchor_hierarchy_from_offset( pruned_count += unanchored_removed match_overrides = _filter_overrides_to_tree(working, match_overrides) - # Offset path does not run printed-page parent backfill (regimes does). - working, match_overrides, null_page_report, failed_null_removed = ( - apply_null_page_locates_and_prune( - nodes=working, - match_overrides=match_overrides, - body_pages=body_pages, - ctx=ctx, - structural_parent_paths=structural_parent_paths, - ) + ( + working, + match_overrides, + null_page_report, + failed_null_removed, + pre_react_override_count, + ) = apply_null_page_locates_and_prune( + nodes=working, + match_overrides=match_overrides, + body_pages=body_pages, + page_count=page_count, + ctx=ctx, + structural_parent_paths=structural_parent_paths, ) pruned_count += failed_null_removed + bulk_count = pre_react_override_count if offset_matches is not None else 0 if offset_hint is None: offset_status = "failed" if ctx is not None else "skipped" diff --git a/apps/worker/app/services/document_agent/structure/hierarchy_locator.py b/apps/worker/app/services/document_agent/structure/hierarchy_locator.py index 8266232ed..512689944 100644 --- a/apps/worker/app/services/document_agent/structure/hierarchy_locator.py +++ b/apps/worker/app/services/document_agent/structure/hierarchy_locator.py @@ -131,22 +131,6 @@ class ResolvedHierarchyRange: evidence: dict[str, Any] = field(default_factory=dict) -def last_leaf_start_under( - node: TitleNode, - parent_titles: tuple[str, ...], - match_overrides: dict[tuple[str, ...], TitleMatch], -) -> int | None: - """Max start page among located leaves under *node*; None if none located.""" - max_page: int | None = None - for leaf_path, _leaf in iter_leaf_title_nodes([node], parent_titles=parent_titles): - match = match_overrides.get(leaf_path) - if match is None: - continue - if max_page is None or match.page > max_page: - max_page = match.page - return max_page - - def first_leaf_start_under( node: TitleNode, parent_titles: tuple[str, ...], diff --git a/apps/worker/app/services/document_agent/structure/null_page_react.py b/apps/worker/app/services/document_agent/structure/null_page_react.py index 14673bb60..29ed7b166 100644 --- a/apps/worker/app/services/document_agent/structure/null_page_react.py +++ b/apps/worker/app/services/document_agent/structure/null_page_react.py @@ -18,7 +18,6 @@ TitleMatch, TitleNode, first_leaf_start_under, - last_leaf_start_under, ) # Planner grep rounds after the free seed full-title probe. Not a page count. @@ -115,20 +114,6 @@ def _next_located_bound( return None -def _last_confirmed_start_under( - *, - node: TitleNode, - parent_titles: tuple[str, ...], - overrides: dict[tuple[str, ...], TitleMatch], -) -> int | None: - """Return the last confirmed descendant leaf, or the node's own start.""" - descendant = last_leaf_start_under(node, parent_titles, overrides) - if descendant is not None: - return int(descendant) - own = overrides.get((*parent_titles, node.title)) - return int(own.page) if own is not None else None - - def _whole_line_grep( *, ctx: ToolContext, @@ -608,25 +593,27 @@ def locate_null_page_overrides( ctx: ToolContext | None, structural_parent_paths: set[tuple[str, ...]] | None = None, ) -> tuple[dict[tuple[str, ...], TitleMatch], list[dict[str, Any]]]: - """Locate all null-page nodes in parent-first DFS order.""" + """Locate all null-page nodes in parent-first DFS order. + + Left bound is the max confirmed page among all preorder predecessors + (fallback: body start). Right bound is the earliest of the node's first + located descendant, the next located sibling/subtree start, and the + inherited scope right. No fixed page-window cap on either side. + """ if not nodes or not body_pages: return dict(match_overrides), [] out = dict(match_overrides) parent_paths = structural_parent_paths or set() report: list[dict[str, Any]] = [] - from app.services.document_agent.calibration.scan import DEFAULT_WINDOW_SCHEDULE - - physical_window_limit = sum(DEFAULT_WINDOW_SCHEDULE) + cursor = int(body_pages[0]) def walk( sibling_nodes: list[TitleNode], parent_titles: tuple[str, ...], - scope_left: int, scope_right: int, - parent_has_printed_page: bool, ) -> None: - last_confirmed_at_level: int | None = None + nonlocal cursor for index, node in enumerate(sibling_nodes): path_titles = (*parent_titles, node.title) next_bound = _next_located_bound( @@ -650,19 +637,7 @@ def walk( if first_descendant is not None else subtree_right ) - - parent_match = out.get(parent_titles) if parent_titles else None - if last_confirmed_at_level is not None: - probe_left = max(int(scope_left), last_confirmed_at_level) - elif parent_match is not None: - probe_left = max(int(scope_left), int(parent_match.page)) - elif parent_has_printed_page: - probe_left = int(scope_left) - else: - probe_left = max( - int(scope_left), - probe_right - physical_window_limit + 1, - ) + probe_left = cursor is_parent = bool(node.children) or path_titles in parent_paths needs_probe = ( @@ -711,42 +686,21 @@ def walk( entry["accept"] = match.evidence.get("accept") report.append(entry) - if node.children: - own_match = out.get(path_titles) - child_left = ( - int(own_match.page) - if own_match is not None - else probe_left - ) - walk( - node.children, - path_titles, - child_left, - subtree_right, - node.printed_page is not None, - ) + own_match = out.get(path_titles) + if own_match is not None: + cursor = max(cursor, int(own_match.page)) - last_under = _last_confirmed_start_under( - node=node, - parent_titles=parent_titles, - overrides=out, - ) - if last_under is not None: - last_confirmed_at_level = ( - last_under - if last_confirmed_at_level is None - else max(last_confirmed_at_level, last_under) - ) + if node.children: + walk(node.children, path_titles, subtree_right) - walk(nodes, (), body_pages[0], body_pages[-1], False) + walk(nodes, (), body_pages[-1]) located = sum(1 for row in report if row.get("page") is not None) logger.info( "[null_page_react] parent-first null-page ReAct: attempted={} " - "located={} unresolved={} planner_budget={} physical_window_limit={}", + "located={} unresolved={} planner_budget={}", len(report), located, len(report) - located, react_budget(), - physical_window_limit, ) return out, report diff --git a/apps/worker/scripts/page_memory/debug_pm_null_page_react.py b/apps/worker/scripts/page_memory/debug_pm_null_page_react.py index 6c8fe14cc..126474e01 100644 --- a/apps/worker/scripts/page_memory/debug_pm_null_page_react.py +++ b/apps/worker/scripts/page_memory/debug_pm_null_page_react.py @@ -108,8 +108,10 @@ def main() -> int: payload = { "policy": { "prune_pre": "keep_null_page_nodes=True", + "printed_backfill": "anchoring_primitives.backfill_printed_offset_matches", "probe": "null_page_react.locate_null_page_overrides", "traversal": "parent-first DFS", + "left_bound": "preorder max confirmed page (body start fallback)", "text_match": "normalized whole-line equality", "visual_windows": [2, 4, 6, 10], "prune_post": "keep_null_page_nodes=False (drop unresolved)", diff --git a/apps/worker/tests/contract/test_structure_anchoring_contract.py b/apps/worker/tests/contract/test_structure_anchoring_contract.py index f58d70e93..0f842b357 100644 --- a/apps/worker/tests/contract/test_structure_anchoring_contract.py +++ b/apps/worker/tests/contract/test_structure_anchoring_contract.py @@ -699,7 +699,7 @@ def probe(**kwargs: Any) -> tuple[TitleMatch, list[dict[str, Any]], int, str]: assert report[0]["search_scope"] == [4, 5] -def test_root_first_null_parent_uses_22_page_left_cap() -> None: +def test_root_first_null_parent_uses_body_start_as_left() -> None: from app.services.document_agent.structure import null_page_react as npr from app.services.document_agent.structure.null_page_react import ( locate_null_page_overrides, @@ -724,7 +724,7 @@ def test_root_first_null_parent_uses_22_page_left_cap() -> None: body_pages = list(range(1, 301)) def probe(**kwargs: Any) -> tuple[TitleMatch, list[dict[str, Any]], int, str]: - assert kwargs["left"] == 257 + assert kwargs["left"] == 1 assert kwargs["right"] == 278 return ( TitleMatch( @@ -748,15 +748,16 @@ def probe(**kwargs: Any) -> tuple[TitleMatch, list[dict[str, Any]], int, str]: ) assert overrides[("Chapter 22",)].page == 270 - assert report[0]["search_scope"] == [257, 278] + assert report[0]["search_scope"] == [1, 278] -def test_null_child_of_unanchored_printed_parent_keeps_inherited_scope() -> None: +def test_null_child_of_unanchored_printed_parent_uses_preorder_cursor() -> None: from app.services.document_agent.structure import null_page_react as npr from app.services.document_agent.structure.null_page_react import ( locate_null_page_overrides, ) + prior = TitleNode(title="E11 Utilities", level=1, printed_page=250, children=[]) parent = TitleNode( title="Appendices", level=1, @@ -778,8 +779,16 @@ def probe(**kwargs: Any) -> tuple[None, list[dict[str, Any]], int, str]: with patch.object(npr, "_locate_with_react", side_effect=probe): _overrides, report = locate_null_page_overrides( - nodes=[parent], - match_overrides={}, + nodes=[prior, parent], + match_overrides={ + ("E11 Utilities",): TitleMatch( + page=250, + source="bulk_offset", + matched_line="", + candidates=[250], + evidence={"offset": 0, "printed_page": 250}, + ) + }, body_pages=list(range(250, 444)), ctx=_ctx(), ) @@ -903,7 +912,7 @@ def probe(**kwargs: Any) -> tuple[TitleMatch, list[dict[str, Any]], int, str]: ) assert scopes == [ - ("3 Advertising", 283, 304), + ("3 Advertising", 280, 304), ("3.1 Street Furniture with Advertising", 304, 304), ] assert overrides[("3 Advertising",)].page == 304 @@ -1159,8 +1168,9 @@ def test_regimes_bulk_count_excludes_null_page_react_hits() -> None: def fake_offset(**kwargs: Any) -> dict[tuple[str, ...], TitleMatch]: return {("Ch1",): bulk_match} - def fake_apply(**kwargs: Any) -> tuple[list, dict, list, int]: + def fake_apply(**kwargs: Any) -> tuple[list, dict, list, int, int]: out = dict(kwargs["match_overrides"]) + pre_react = len(out) out[("NullLeaf",)] = react_match return ( list(kwargs["nodes"]), @@ -1174,6 +1184,7 @@ def fake_apply(**kwargs: Any) -> tuple[list, dict, list, int]: } ], 0, + pre_react, ) with ( @@ -1454,7 +1465,7 @@ def _printed_page_parent(title: str, printed: int, child: TitleNode) -> TitleNod return TitleNode(title=title, level=1, printed_page=printed, children=[child]) -def test_parent_backfill_uses_descendant_regime_offset() -> None: +def test_printed_backfill_uses_nearest_doc_order_offset() -> None: from app.services.document_agent.structure import anchoring_primitives as primitives early_child = TitleNode(title="A.1", level=2, printed_page=12, children=[]) @@ -1466,7 +1477,7 @@ def test_parent_backfill_uses_descendant_regime_offset() -> None: **primitives.bulk_offset_matches([(("Section B", "B.1"), late_child)], 9), } - parents = primitives.backfill_parent_offset_matches( + parents = primitives.backfill_printed_offset_matches( nodes=[section_a, section_b], matches=matches, page_count=60, @@ -1474,11 +1485,11 @@ def test_parent_backfill_uses_descendant_regime_offset() -> None: assert parents[("Section A",)].page == 15 assert parents[("Section B",)].page == 49 - assert parents[("Section A",)].evidence["parent_backfill"] is True + assert parents[("Section A",)].evidence["printed_offset_backfill"] is True -def test_parent_backfill_ignores_react_located_children_for_offset() -> None: - """ReAct leaves have no printed page, so they carry no offset for the parent.""" +def test_printed_backfill_ignores_react_hits_without_offset() -> None: + """ReAct leaves carry no offset; nearest offset-bearing neighbor wins.""" from app.services.document_agent.structure import anchoring_primitives as primitives react_child = TitleNode(title="Intro", level=2, printed_page=None, children=[]) @@ -1490,7 +1501,6 @@ def test_parent_backfill_ignores_react_located_children_for_offset() -> None: children=[react_child, printed_child], ) matches = { - # ReAct hit comes first in document order and must not set the offset. ("Section A", "Intro"): TitleMatch( page=99, source="react_line_grep_vlm", @@ -1503,7 +1513,7 @@ def test_parent_backfill_ignores_react_located_children_for_offset() -> None: ), } - parents = primitives.backfill_parent_offset_matches( + parents = primitives.backfill_printed_offset_matches( nodes=[section], matches=matches, page_count=200, @@ -1513,8 +1523,7 @@ def test_parent_backfill_ignores_react_located_children_for_offset() -> None: assert parents[("Section A",)].evidence["offset"] == 5 -def test_parent_backfill_unresolved_when_only_react_children() -> None: - """No printed-page descendant → no offset → parent left to its own locate.""" +def test_printed_backfill_unresolved_when_no_offset_bearing_neighbor() -> None: from app.services.document_agent.structure import anchoring_primitives as primitives react_child = TitleNode(title="Intro", level=2, printed_page=None, children=[]) @@ -1529,7 +1538,7 @@ def test_parent_backfill_unresolved_when_only_react_children() -> None: ) } - parents = primitives.backfill_parent_offset_matches( + parents = primitives.backfill_printed_offset_matches( nodes=[section], matches=matches, page_count=200, @@ -1538,22 +1547,35 @@ def test_parent_backfill_unresolved_when_only_react_children() -> None: assert parents == {} -def test_parent_backfill_skips_unanchored_and_out_of_range() -> None: +def test_printed_backfill_uses_preceding_neighbor_and_skips_out_of_range() -> None: from app.services.document_agent.structure import anchoring_primitives as primitives - tail_child = TitleNode(title="Tail.1", level=2, printed_page=96, children=[]) - ghost_child = TitleNode(title="Ghost.1", level=2, printed_page=11, children=[]) - tail = _printed_page_parent("Tail", 95, tail_child) - ghost = _printed_page_parent("Ghost", 10, ghost_child) - matches = primitives.bulk_offset_matches([(("Tail", "Tail.1"), tail_child)], 8) + prior_leaf = TitleNode(title="E11", level=1, printed_page=250, children=[]) + appendices = TitleNode( + title="Appendices", + level=1, + printed_page=261, + children=[ + TitleNode(title="A maps", level=2, printed_page=None, children=[]), + ], + ) + tail_child = TitleNode(title="Tail.1", level=2, printed_page=440, children=[]) + tail = _printed_page_parent("Tail", 439, tail_child) + matches = { + **primitives.bulk_offset_matches([(("E11",), prior_leaf)], 0), + **primitives.bulk_offset_matches([(("Tail", "Tail.1"), tail_child)], 8), + } - parents = primitives.backfill_parent_offset_matches( - nodes=[tail, ghost], + filled = primitives.backfill_printed_offset_matches( + nodes=[prior_leaf, appendices, tail], matches=matches, - page_count=100, + page_count=443, ) - assert parents == {} + assert filled[("Appendices",)].page == 261 + assert filled[("Appendices",)].evidence["offset"] == 0 + # 439 + 8 = 447 exceeds page_count + assert ("Tail",) not in filled def test_multi_regime_phase2_merges_physical_overrides() -> None: From eb7dfcb94658adeee901131ef1f0265664f17dbc Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Sat, 22 Aug 2026 23:04:10 +0800 Subject: [PATCH 7/7] test: reimport grep_text in strip_footer contract after module clear Avoid PageTextBands class-identity mismatch when a prior contract fixture clears app.* modules mid-suite. Co-authored-by: Cursor --- .../contract/test_tool_registry_smoke_contract.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/worker/tests/contract/test_tool_registry_smoke_contract.py b/apps/worker/tests/contract/test_tool_registry_smoke_contract.py index a153d9979..6316f76ff 100644 --- a/apps/worker/tests/contract/test_tool_registry_smoke_contract.py +++ b/apps/worker/tests/contract/test_tool_registry_smoke_contract.py @@ -98,7 +98,10 @@ def test_grep_text_whole_line_rejects_body_substring_and_dedupes_pages() -> None def test_strip_footer_updates_search_view_for_grep() -> None: + # Import inside the test so PageTextBands / grep_text share one module + # identity after worker_contract_environment clears app.* mid-suite. from app.services.document_agent.pdf_text import PageTextBands + from app.services.document_agent.tools.grep_text import grep_text as grep_text_fn from app.services.document_agent.tools.text_strip_margins import strip_footer blackboard = ProfileBlackboard(page_count=1) @@ -117,7 +120,9 @@ def test_strip_footer_updates_search_view_for_grep() -> None: settings={}, ) - before = grep_text(ctx, {"query": "Public Domain Manual", "start_page": 1, "end_page": 1}) + before = grep_text_fn( + ctx, {"query": "Public Domain Manual", "start_page": 1, "end_page": 1} + ) assert before.status == "ok" assert before.payload["hit_count"] == 1 @@ -126,7 +131,9 @@ def test_strip_footer_updates_search_view_for_grep() -> None: assert strip.payload["pages_updated"] == 1 assert ctx.blackboard.page_text_search_view[1] == "Section Start\n" - after = grep_text(ctx, {"query": "Public Domain Manual", "start_page": 1, "end_page": 1}) + after = grep_text_fn( + ctx, {"query": "Public Domain Manual", "start_page": 1, "end_page": 1} + ) assert after.status == "ok" assert after.payload["hit_count"] == 0