diff --git a/apps/worker/app/services/document_agent/calibration/phase1.py b/apps/worker/app/services/document_agent/calibration/phase1.py index 87ae08bdf..caff9f871 100644 --- a/apps/worker/app/services/document_agent/calibration/phase1.py +++ b/apps/worker/app/services/document_agent/calibration/phase1.py @@ -64,7 +64,7 @@ def _regime_probes( ) -> dict[str, list[_Probe]]: """Keep leaf probes from the first distinct printed pages of each kind.""" from app.services.document_parser.structure.body_boundary import ( - normalize_heading_text, + normalize_heading_label, ) probes: dict[str, list[_Probe]] = {} @@ -82,7 +82,7 @@ def _regime_probes( continue if printed in seen_printed.get(kind, set()): continue - title = normalize_heading_text(str(entry.get("heading") or "")) + title = normalize_heading_label(str(entry.get("heading") or "")) if not title: continue probes.setdefault(kind, []).append(_Probe(title=title, printed=printed)) diff --git a/apps/worker/app/services/document_agent/calibration/procedure.py b/apps/worker/app/services/document_agent/calibration/procedure.py index 90dbc78ac..868654760 100644 --- a/apps/worker/app/services/document_agent/calibration/procedure.py +++ b/apps/worker/app/services/document_agent/calibration/procedure.py @@ -4,7 +4,7 @@ 1. Builds TitleNodes the same way production does 2. Runs Phase-2 **per regime** (prune → bulk/bisect → recalibrate) 3. Merges physical-page ``match_overrides`` across regimes -4. Runs null-page parent locate once on the combined tree +4. Backfills printed-page nodes, runs unified null-page ReAct, then final prune Returns production ``SkeletonAnchor`` plus regime diagnostics for debug payloads. """ @@ -33,8 +33,7 @@ ) from app.services.document_agent.structure.anchoring_primitives import ( SkeletonAnchor, - backfill_parent_offset_matches, - locate_null_page_parent_overrides, + apply_null_page_locates_and_prune, prune_unanchored_toc_leaves, serialize_skeleton_anchor, ) @@ -142,7 +141,7 @@ def _entry_titles_for_regime( ) -> set[str] | None: """Titles belonging to this regime; None means fall back to page_kind match.""" from app.services.document_parser.structure.body_boundary import ( - normalize_heading_text, + normalize_heading_label, ) kind = normalize_kind(regime.kind) @@ -160,7 +159,7 @@ def _entry_titles_for_regime( if idx < 0 or idx >= len(entries): continue heading = entries[idx].get("heading") - title = normalize_heading_text(str(heading or "")) + title = normalize_heading_label(str(heading or "")) if title: titles.add(title) return titles or None @@ -350,9 +349,17 @@ def anchor_hierarchy_from_regimes( len(regime_seed), ) - # Failed suffix / never-confirmed printed leaves → drop from TOC tree. + # Capture parent identity before prune so empty shells retain ``kind=parent``. + structural_parent_paths = { + path + for path, node in _iter_all_title_nodes(working) + if node.children + } + # Failed printed-page leaves → drop; keep null-page nodes for ReAct. working, unanchored_removed = prune_unanchored_toc_leaves( - working, match_overrides=merged + working, + match_overrides=merged, + keep_null_page_nodes=True, ) total_pruned += unanchored_removed if working: @@ -366,26 +373,24 @@ def anchor_hierarchy_from_regimes( if path in surviving_paths } - parent_matches = backfill_parent_offset_matches( - nodes=working, - matches=merged, - page_count=page_count, - ) - if parent_matches: - merged.update(parent_matches) - logger.info( - "[calibration.phase2] parent backfill: {} printed-page TOC parents " - "anchored from descendant offset", - len(parent_matches), - ) - - match_overrides, null_page_report = locate_null_page_parent_overrides( + ( + working, + match_overrides, + null_page_report, + failed_null_removed, + pre_react_override_count, + ) = apply_null_page_locates_and_prune( nodes=working, match_overrides=merged, - page_texts=page_texts, body_pages=body_pages, + page_count=page_count, ctx=ctx, + structural_parent_paths=structural_parent_paths, ) + total_pruned += failed_null_removed + + # Freeze bulk after printed backfill and before ReAct (ReAct hits stay out). + bulk_count = pre_react_override_count if regime_bulk > 0 else 0 primary = pick_primary_offset(result) if primary is None and usable_regimes: @@ -401,7 +406,6 @@ def anchor_hierarchy_from_regimes( if match_overrides and (regime_bulk > 0 or seed) else "offset_only" ) - bulk_count = len(match_overrides) return working, SkeletonAnchor( offset=primary, @@ -445,10 +449,10 @@ def _annotate_regimes_from_anchor( continue heading = str(entries[idx].get("heading") or "") from app.services.document_parser.structure.body_boundary import ( - normalize_heading_text, + normalize_heading_label, ) - title = normalize_heading_text(heading) + title = normalize_heading_label(heading) path = path_by_title.get(title) if path is not None and path in (anchor.match_overrides or {}): ok_indices.append(idx) diff --git a/apps/worker/app/services/document_agent/calibration/scan.py b/apps/worker/app/services/document_agent/calibration/scan.py index 95b468e75..5cf0399e1 100644 --- a/apps/worker/app/services/document_agent/calibration/scan.py +++ b/apps/worker/app/services/document_agent/calibration/scan.py @@ -25,6 +25,27 @@ DEFAULT_WINDOW_SCHEDULE: tuple[int, ...] = (2, 4, 6, 10) +def progressive_page_windows( + *, + start_page: int, + end_page: int, + window_schedule: tuple[int, ...] = DEFAULT_WINDOW_SCHEDULE, +) -> list[list[int]]: + """Return non-overlapping physical-page windows clipped to ``end_page``.""" + cursor = max(int(start_page), 1) + last_page = int(end_page) + windows: list[list[int]] = [] + for size in window_schedule: + if cursor > last_page: + break + pages = list(range(cursor, min(cursor + int(size), last_page + 1))) + if not pages: + break + windows.append(pages) + cursor = pages[-1] + 1 + return windows + + @dataclass class ScanRound: pages: list[int] @@ -76,17 +97,15 @@ def scan_title_forward( Each round opens ``window_schedule[i]`` consecutive pages starting at the cursor left by the previous round, so no page is inspected twice. """ - cursor = max(int(start_page), 1) scanned: list[int] = [] rounds: list[ScanRound] = [] + next_start = max(int(start_page), 1) - for size in window_schedule: - if cursor > page_count: - break - pages = [page for page in range(cursor, cursor + size) if page <= page_count] - if not pages: - break - + for pages in progressive_page_windows( + start_page=start_page, + end_page=page_count, + window_schedule=window_schedule, + ): result = inspect_pages( ctx, { @@ -99,7 +118,7 @@ def scan_title_forward( "usage_task": "calibration.scan_title_forward", }, ) - cursor = pages[-1] + 1 + next_start = pages[-1] + 1 scanned.extend(pages) if result.status != "ok": @@ -128,7 +147,7 @@ def scan_title_forward( found=True, found_page=found_page, scanned_pages=scanned, - next_start=cursor if cursor <= page_count else None, + next_start=next_start if next_start <= page_count else None, rounds=rounds, ) @@ -142,6 +161,6 @@ def scan_title_forward( found=False, found_page=None, scanned_pages=scanned, - next_start=cursor if cursor <= page_count else None, + next_start=next_start if next_start <= page_count else None, rounds=rounds, ) diff --git a/apps/worker/app/services/document_agent/calibration/service.py b/apps/worker/app/services/document_agent/calibration/service.py index 39a609d77..7c3bae11f 100644 --- a/apps/worker/app/services/document_agent/calibration/service.py +++ b/apps/worker/app/services/document_agent/calibration/service.py @@ -16,6 +16,7 @@ CalibrationResult, ) from app.services.document_agent.manifest import ToolContext +from app.services.document_agent.pdf_text import page_bands_map def calibrate_offset( @@ -44,7 +45,7 @@ def calibrate_offset( if page_count and not ctx.blackboard.page_count: ctx.blackboard.page_count = int(page_count) if page_texts and not ctx.blackboard.page_full_text_cache: - ctx.blackboard.page_full_text_cache = dict(page_texts) + ctx.blackboard.page_full_text_cache = page_bands_map(page_texts) try: phase1 = run_calibration_phase1( diff --git a/apps/worker/app/services/document_agent/coordinator.py b/apps/worker/app/services/document_agent/coordinator.py index f9a43b15d..7cb899279 100644 --- a/apps/worker/app/services/document_agent/coordinator.py +++ b/apps/worker/app/services/document_agent/coordinator.py @@ -20,7 +20,7 @@ ToolContext, ToolResult, ) -from app.services.document_agent.pdf_text import read_page_texts +from app.services.document_agent.pdf_text import read_page_text_bands from app.services.document_agent.persist import build_anatomy_map, persist_anatomy_map from app.services.document_agent.coarse_profile import CoarseProfiler from app.services.document_agent.registry import REGISTRY @@ -297,6 +297,7 @@ def _run_text_scan(self) -> None: pages = list(range(1, page_count + 1)) if not pages: self.blackboard.page_full_text_cache = {} + self.blackboard.page_text_search_view = None return if profile.is_scanned: result = REGISTRY.dispatch("ocr.pages", self.ctx, {"pages": pages}) @@ -312,19 +313,30 @@ def _run_text_scan(self) -> None: raise RuntimeError(result.error or "ocr.pages failed") self.round_index += 1 return - texts = read_page_texts(self.ctx.pdf_path, pages, timeout=300) - self.blackboard.page_full_text_cache = texts + bands = read_page_text_bands( + self.ctx.pdf_path, + pages, + header_y=profile.header_y, + footer_y=profile.footer_y, + timeout=300, + ) + self.blackboard.page_full_text_cache = bands + self.blackboard.page_text_search_view = None self.trace.record_step( round_index=self.round_index, - actor="scan:read_page_texts", + actor="scan:read_page_text_bands", action_type="scan", result=ToolResult( status="ok", - payload={"page_count": len(texts)}, - output_summary={"page_count": len(texts)}, + payload={"page_count": len(bands)}, + output_summary={"page_count": len(bands)}, ), - tool_name="read_page_texts", - tool_args={"pages": pages}, + tool_name="read_page_text_bands", + tool_args={ + "pages": pages, + "header_y": profile.header_y, + "footer_y": profile.footer_y, + }, ) self.round_index += 1 diff --git a/apps/worker/app/services/document_agent/pdf_text.py b/apps/worker/app/services/document_agent/pdf_text.py index 7a6da6522..12994405b 100644 --- a/apps/worker/app/services/document_agent/pdf_text.py +++ b/apps/worker/app/services/document_agent/pdf_text.py @@ -3,16 +3,186 @@ from __future__ import annotations import gc -from typing import Any +from dataclasses import dataclass +from typing import Any, Literal from app.services.document_parser.formats.pdf.pymupdf_subprocess import ( run_in_child_process, worker, ) +from app.services.document_parser.structure.body_boundary import normalize_heading_label -def normalize_spaces(text: str) -> str: - return " ".join((text or "").split()) +@dataclass(frozen=True) +class PageTextBands: + """Per-page text from one span pass: full content plus edge-band extracts. + + ``content`` is the full page text (includes header/footer span text). + ``header`` / ``footer`` are the same spans whose vertical centers fall in the + coarse ``header_y`` / ``footer_y`` bands. Strip tools remove those extracts + from a search view; they do not rewrite the stored ``content``. + """ + + content: str + header: str = "" + footer: str = "" + + def to_dict(self) -> dict[str, str]: + return { + "content": self.content, + "header": self.header, + "footer": self.footer, + } + + @classmethod + def from_any(cls, value: Any) -> "PageTextBands": + if isinstance(value, PageTextBands): + return value + if isinstance(value, str): + return cls(content=value) + if isinstance(value, dict): + content = value.get("content") + if content is None and "text" in value: + content = value.get("text") + return cls( + content=str(content or ""), + header=str(value.get("header") or ""), + footer=str(value.get("footer") or ""), + ) + return cls(content=str(value or "")) + + +def page_content(value: Any) -> str: + return PageTextBands.from_any(value).content + + +def page_content_map(raw: Any) -> dict[int, str]: + """Map page -> content string (legacy-compatible plain cache shape).""" + if not isinstance(raw, dict): + return {} + out: dict[int, str] = {} + for page, value in raw.items(): + out[int(page)] = page_content(value) + return out + + +def page_bands_map(raw: Any) -> dict[int, PageTextBands]: + if not isinstance(raw, dict): + return {} + return {int(page): PageTextBands.from_any(value) for page, value in raw.items()} + + +def strip_margin_text( + content: str, + margin: str, + *, + edge: Literal["header", "footer"], +) -> str: + """Remove a header/footer band extract from page content for search view. + + Header is removed only from the content prefix; footer only from the + content suffix. Never uses a whole-string first-occurrence replace (that + can delete a same-looking body span). Falls back to edge-aligned line + fragments when the full margin blob is not contiguous at that edge. + """ + if not content or not margin: + return content + + if edge == "header": + if content.startswith(margin): + return content[len(margin) :] + out = content + for frag in margin.split("\n"): + if not frag: + continue + if out.startswith(frag + "\n"): + out = out[len(frag) + 1 :] + elif out.startswith(frag): + out = out[len(frag) :] + else: + break + return out + + if content.endswith(margin): + return content[: -len(margin)] + out = content + for frag in reversed([part for part in margin.split("\n") if part]): + if out.endswith("\n" + frag): + out = out[: -(len(frag) + 1)] + elif out.endswith(frag): + out = out[: -len(frag)] + else: + break + return out + + +def _band_for_center( + cy: float, + *, + page_h: float, + header_y: float | None, + footer_y: float | None, +) -> str: + if page_h <= 0: + return "content" + if header_y is not None and cy < float(header_y) * page_h: + return "header" + if footer_y is not None and cy > float(footer_y) * page_h: + return "footer" + return "content" + + +def _extract_page_bands_from_pymupdf_page( + page: Any, + *, + header_y: float | None, + footer_y: float | None, +) -> PageTextBands: + """Build content/header/footer from the same span walk (line-join with ``\\n``).""" + page_h = float(getattr(page.rect, "height", 0) or 0) + data = page.get_text("dict") or {} + content_lines: list[str] = [] + header_chunks: list[str] = [] + footer_chunks: list[str] = [] + + for block in data.get("blocks") or []: + if int(block.get("type") or 0) != 0: + continue + for line in block.get("lines") or []: + line_content_parts: list[str] = [] + line_header_parts: list[str] = [] + line_footer_parts: list[str] = [] + for span in line.get("spans") or []: + text = str(span.get("text") or "") + if not text: + continue + bbox = span.get("bbox") or (0, 0, 0, 0) + try: + y0 = float(bbox[1]) + y1 = float(bbox[3]) + except (TypeError, ValueError, IndexError): + y0, y1 = 0.0, 0.0 + cy = (y0 + y1) / 2.0 + band = _band_for_center( + cy, page_h=page_h, header_y=header_y, footer_y=footer_y + ) + line_content_parts.append(text) + if band == "header": + line_header_parts.append(text) + elif band == "footer": + line_footer_parts.append(text) + if line_content_parts: + content_lines.append("".join(line_content_parts)) + if line_header_parts: + header_chunks.append("".join(line_header_parts)) + if line_footer_parts: + footer_chunks.append("".join(line_footer_parts)) + + return PageTextBands( + content="\n".join(content_lines), + header="\n".join(header_chunks), + footer="\n".join(footer_chunks), + ) @worker @@ -35,10 +205,40 @@ def _read_page_texts_worker(queue, pdf_path: str, pages: list[int]) -> None: queue.put({"ok": True, "texts": texts}) +@worker +def _read_page_text_bands_worker( + queue, + pdf_path: str, + pages: list[int], + header_y: float | None, + footer_y: float | None, +) -> None: + import pymupdf # type: ignore[import] + + bands: dict[int, dict[str, str]] = {} + try: + doc = pymupdf.open(pdf_path) + for page in pages: + idx = page - 1 + if 0 <= idx < doc.page_count: + record = _extract_page_bands_from_pymupdf_page( + doc[idx], + header_y=header_y, + footer_y=footer_y, + ) + bands[page] = record.to_dict() + finally: + try: + doc.close() + except Exception: + pass + gc.collect() + queue.put({"ok": True, "bands": bands}) + + def coerce_page_text_cache(raw: Any) -> dict[int, str]: - if not isinstance(raw, dict): - return {} - return {int(page): str(text) for page, text in raw.items()} + """Legacy helper: page -> content string only.""" + return page_content_map(raw) def read_page_texts( @@ -47,14 +247,47 @@ def read_page_texts( *, timeout: int = 180, ) -> dict[int, str]: + """Plain ``get_text()`` map for call sites that need a one-shot full dump. + + PROFILE text-scan uses :func:`read_page_text_bands` instead. + """ if not pages: return {} - result = run_in_child_process(_read_page_texts_worker, pdf_path, pages, timeout=timeout) + result = run_in_child_process( + _read_page_texts_worker, pdf_path, pages, timeout=timeout + ) return {int(k): str(v) for k, v in (result.get("texts") or {}).items()} +def read_page_text_bands( + pdf_path: str, + pages: list[int], + *, + header_y: float | None = None, + footer_y: float | None = None, + timeout: int = 180, +) -> dict[int, PageTextBands]: + """Span-homogeneous content/header/footer for PROFILE text scan.""" + if not pages: + return {} + result = run_in_child_process( + _read_page_text_bands_worker, + pdf_path, + pages, + header_y, + footer_y, + timeout=timeout, + ) + raw_bands = result.get("bands") or {} + return { + int(page): PageTextBands.from_any(value) + for page, value in raw_bands.items() + } + + def meaningful_lines(text: str) -> list[str]: - return [normalize_spaces(line) for line in text.splitlines() if normalize_spaces(line)] + lines = [normalize_heading_label(line) for line in text.splitlines()] + return [line for line in lines if line] def top_lines(text: str, *, max_lines: int = 20) -> list[str]: diff --git a/apps/worker/app/services/document_agent/state.py b/apps/worker/app/services/document_agent/state.py index 67b434022..8f1e703b6 100644 --- a/apps/worker/app/services/document_agent/state.py +++ b/apps/worker/app/services/document_agent/state.py @@ -42,5 +42,8 @@ class ProfileBlackboard: shard_plan: ShardPlan | None = None validation_report: dict[str, Any] | None = None verdict: ProfileVerdict | None = None - page_full_text_cache: dict[int, str] = field(default_factory=dict) + # Values are PageTextBands (or legacy plain str / {"content","header","footer"}). + page_full_text_cache: dict[int, Any] = field(default_factory=dict) + # Optional temporary grep view after text.strip_*; None means use each page's content. + page_text_search_view: dict[int, str] | None = None global_signals: dict[str, Any] = field(default_factory=dict) diff --git a/apps/worker/app/services/document_agent/structure/anchoring_primitives.py b/apps/worker/app/services/document_agent/structure/anchoring_primitives.py index 91a11b7b4..6068f8b65 100644 --- a/apps/worker/app/services/document_agent/structure/anchoring_primitives.py +++ b/apps/worker/app/services/document_agent/structure/anchoring_primitives.py @@ -12,10 +12,10 @@ from app.services.document_agent.structure.hierarchy_locator import ( TitleMatch, TitleNode, - first_leaf_start_under, iter_leaf_title_nodes, - last_leaf_start_under, - locate_title_compact_strict, +) +from app.services.document_agent.structure.null_page_react import ( + locate_null_page_overrides, ) from app.services.document_agent.structure.section_page_verify import ( verify_section_page_choice, @@ -23,17 +23,6 @@ from loguru import logger -def _first_sibling_null_parent_scan_start(right: int) -> int: - """Left edge for first-at-level null parents: at most one 2+4+6+10 budget. - - Does not inherit a wider parent/body scope. Floors at document page 1. - """ - from app.services.document_agent.calibration.scan import DEFAULT_WINDOW_SCHEDULE - - budget = sum(DEFAULT_WINDOW_SCHEDULE) - return max(1, int(right) - budget + 1) - - def prune_out_of_scope_nodes( nodes: list[TitleNode], *, @@ -88,16 +77,30 @@ def prune_unanchored_toc_leaves( nodes: list[TitleNode], *, match_overrides: dict[tuple[str, ...], TitleMatch], + keep_null_page_nodes: bool = False, ) -> tuple[list[TitleNode], int]: - """Remove TOC leaves that have no physical ``match_overrides`` entry. + """Remove TOC nodes that have no physical ``match_overrides`` entry. Implements Phase-2 ``suffix = no TOC``: after bulk/bisect/recalibrate, any leaf that was not successfully anchored is dropped from the coarse tree - instead of sticky ``inherited_unlocated`` ranges. Childless parents are - removed unless they themselves have an override. + instead of sticky ``inherited_unlocated`` ranges. + + Parents that still have at least one surviving child after recursive prune + are kept even without their own override, so later descendant-based infer + can use the subtree. Childless parents are removed unless they themselves + have an override. + + When ``keep_null_page_nodes`` is True (pre null-page ReAct), nodes with + ``printed_page is None`` are retained so they can be probed. Call again with + the default after locate to drop still-unanchored null-page nodes. """ removed = 0 + def _keep(path: tuple[str, ...], node: TitleNode) -> bool: + if path in match_overrides: + return True + return keep_null_page_nodes and node.printed_page is None + def _prune( node: TitleNode, parent_titles: tuple[str, ...] ) -> TitleNode | None: @@ -111,11 +114,11 @@ def _prune( children.append(kept) if children: return replace(node, children=children) - if path in match_overrides: + if _keep(path, node): return replace(node, children=[]) removed += 1 return None - if path in match_overrides: + if _keep(path, node): return node removed += 1 return None @@ -129,8 +132,9 @@ def _prune( if removed: logger.info( "[structure_anchoring] pruned {} unanchored TOC nodes " - "(suffix / no match_overrides → no TOC)", + "(keep_null_page_nodes={} → no TOC)", removed, + keep_null_page_nodes, ) return out, removed @@ -155,322 +159,6 @@ def toc_range_end(hierarchy: dict[str, Any]) -> int | None: return None -# ── Null-page parent locate (sibling window / first-sibling scan) ─────────── - - -def locate_null_page_parent_overrides( - *, - nodes: list[TitleNode], - match_overrides: dict[tuple[str, ...], TitleMatch], - page_texts: dict[int, str], - body_pages: list[int], - ctx: ToolContext | None, -) -> tuple[dict[tuple[str, ...], TitleMatch], list[dict[str, Any]]]: - """Locate TOC parents with ``printed_page=None`` into ``match_overrides``. - - Window for parent P with a previous same-level sibling: ``[last leaf under - that sibling, first leaf under P]``; text then RTL visual verify. - - First-at-level parents (no left sibling) do **not** inherit a wider parent - or body scope. Left edge is one Phase-1 ``2+4+6+10`` budget before the first - child (floor page 1). Text runs in that window; on miss, reuse - ``scan_title_forward`` (same schedule, early exit). Miss → unresolved. - - Returns ``(overrides, report)`` where *report* lists every null-page parent - attempt (for debug / LLM-call accounting). - """ - if not nodes or not body_pages: - return dict(match_overrides), [] - - out = dict(match_overrides) - body_set = set(body_pages) - parent_scope_start = body_pages[0] - report: list[dict[str, Any]] = [] - - def walk( - sibling_nodes: list[TitleNode], - parent_titles: tuple[str, ...], - scope_start: int, - ) -> None: - for index, node in enumerate(sibling_nodes): - path_titles = (*parent_titles, node.title) - if ( - node.children - and node.printed_page is None - and path_titles not in out - ): - right = first_leaf_start_under(node, parent_titles, out) - entry: dict[str, Any] = { - "path_titles": list(path_titles), - "title": node.title, - "printed_page": None, - "window": None, - "result": "skipped_no_right", - "page": None, - "accept": None, - "visual_verify_calls": 0, - } - if right is None: - report.append(entry) - logger.info( - "[structure_anchoring] null-page parent skipped: " - "title={!r} reason=no_located_first_child", - node.title, - ) - elif index > 0: - left = last_leaf_start_under( - sibling_nodes[index - 1], parent_titles, out - ) - if left is None: - left = scope_start - if right < left: - report.append(entry) - logger.info( - "[structure_anchoring] null-page parent skipped: " - "title={!r} reason=no_located_first_child left={}", - node.title, - left, - ) - else: - _resolve_null_parent_with_sibling_window( - path_titles=path_titles, - title=node.title, - left=left, - right=right, - body_pages=body_pages, - body_set=body_set, - page_texts=page_texts, - ctx=ctx, - out=out, - entry=entry, - report=report, - ) - else: - left = _first_sibling_null_parent_scan_start(right) - _resolve_null_parent_first_sibling( - path_titles=path_titles, - title=node.title, - left=left, - right=right, - body_pages=body_pages, - body_set=body_set, - page_texts=page_texts, - ctx=ctx, - out=out, - entry=entry, - report=report, - ) - if node.children: - child_scope_start = ( - out[path_titles].page if path_titles in out else scope_start - ) - walk(node.children, path_titles, child_scope_start) - - walk(nodes, (), parent_scope_start) - logger.info( - "[structure_anchoring] null-page parent locate summary: " - "attempted={} located={} unresolved={} visual_verify_calls={}", - len(report), - sum(1 for row in report if row.get("page") is not None), - sum(1 for row in report if row.get("result") == "unresolved"), - sum(int(row.get("visual_verify_calls") or 0) for row in report), - ) - return out, report - - -def _record_null_parent_outcome( - *, - path_titles: tuple[str, ...], - title: str, - left: int, - right: int, - match: TitleMatch | None, - visual_calls: int, - body_set: set[int], - out: dict[tuple[str, ...], TitleMatch], - entry: dict[str, Any], - report: list[dict[str, Any]], -) -> None: - entry["window"] = [left, right] - entry["visual_verify_calls"] = visual_calls - if match is not None and match.page in body_set: - out[path_titles] = match - entry["result"] = str(match.evidence.get("accept") or match.source) - entry["page"] = match.page - entry["accept"] = match.evidence.get("accept") - logger.info( - "[structure_anchoring] null-page parent located: " - "title={!r} page={} window={} accept={} visual_calls={}", - title, - match.page, - [left, right], - match.evidence.get("accept"), - visual_calls, - ) - else: - entry["result"] = "unresolved" - logger.info( - "[structure_anchoring] null-page parent unresolved: " - "title={!r} window={} visual_calls={}", - title, - [left, right], - visual_calls, - ) - report.append(entry) - - -def _resolve_null_parent_with_sibling_window( - *, - path_titles: tuple[str, ...], - title: str, - left: int, - right: int, - body_pages: list[int], - body_set: set[int], - page_texts: dict[int, str], - ctx: ToolContext | None, - out: dict[tuple[str, ...], TitleMatch], - entry: dict[str, Any], - report: list[dict[str, Any]], -) -> None: - scope_pages = [page for page in body_pages if left <= page <= right] - match = locate_title_compact_strict( - title, - scope_pages=scope_pages, - page_texts=page_texts, - ) - visual_calls = 0 - if match is None and ctx is not None: - match, visual_calls = _visual_rtl_locate_parent( - title=title, - left=left, - right=right, - body_set=body_set, - ctx=ctx, - ) - _record_null_parent_outcome( - path_titles=path_titles, - title=title, - left=left, - right=right, - match=match, - visual_calls=visual_calls, - body_set=body_set, - out=out, - entry=entry, - report=report, - ) - - -def _resolve_null_parent_first_sibling( - *, - path_titles: tuple[str, ...], - title: str, - left: int, - right: int, - body_pages: list[int], - body_set: set[int], - page_texts: dict[int, str], - ctx: ToolContext | None, - out: dict[tuple[str, ...], TitleMatch], - entry: dict[str, Any], - report: list[dict[str, Any]], -) -> None: - """First-at-level null parent: capped text window, then ``scan_title_forward``.""" - from app.services.document_agent.calibration.scan import ( - DEFAULT_WINDOW_SCHEDULE, - scan_title_forward, - ) - - scope_pages = [page for page in body_pages if left <= page <= right] - match = locate_title_compact_strict( - title, - scope_pages=scope_pages, - page_texts=page_texts, - ) - visual_calls = 0 - if match is None and ctx is not None: - scan = scan_title_forward( - ctx=ctx, - title=title, - start_page=left, - page_count=right, - window_schedule=DEFAULT_WINDOW_SCHEDULE, - ) - visual_calls = len(scan.scanned_pages) - if scan.found and scan.found_page is not None: - match = TitleMatch( - page=int(scan.found_page), - source="inspect_vlm", - matched_line="", - candidates=[int(scan.found_page)], - evidence={ - "accept": "scan_forward", - "null_page_parent_probe": True, - "scanned_pages": list(scan.scanned_pages), - }, - ) - _record_null_parent_outcome( - path_titles=path_titles, - title=title, - left=left, - right=right, - match=match, - visual_calls=visual_calls, - body_set=body_set, - out=out, - entry=entry, - report=report, - ) - - -def _visual_rtl_locate_parent( - *, - title: str, - left: int, - right: int, - body_set: set[int], - ctx: ToolContext, -) -> tuple[TitleMatch | None, int]: - """Confirm parent title from right boundary toward left via VLM verify.""" - visual_calls = 0 - for page in range(right, left - 1, -1): - if page not in body_set: - continue - candidate = TitleMatch( - page=page, - source="inspect_vlm", - matched_line="", - candidates=[page], - evidence={"null_page_parent_probe": True}, - ) - visual_calls += 1 - result = verify_section_page_choice( - ctx=ctx, - title=title, - candidate_matches=[candidate], - candidate_page_cap=1, - ) - selected = result.get("selected_page") - if selected != page: - continue - return ( - TitleMatch( - page=page, - source="inspect_vlm", - matched_line="", - candidates=[page], - evidence={ - "accept": "visual_rtl", - "reason": result.get("reason", ""), - "visual_verify_calls": visual_calls, - }, - ), - visual_calls, - ) - return None, visual_calls - - # ── Offset-guided bulk anchoring with recursive recalibrate (Phase-2) ─────── _MAX_RECALIBRATE_DEPTH = 5 @@ -630,35 +318,46 @@ def bulk_offset_matches( return matches -def _iter_printed_page_parents( - nodes: list[TitleNode], +def _regime_offset_for_printed_node( *, - parent_titles: tuple[str, ...] = (), -) -> list[tuple[tuple[str, ...], TitleNode]]: - """DFS non-leaf nodes that print their own page in the TOC.""" - parents: list[tuple[tuple[str, ...], TitleNode]] = [] - for node in nodes: - path_titles = (*parent_titles, node.title) - if not node.children: - continue - if node.printed_page is not None: - parents.append((path_titles, node)) - parents.extend( - _iter_printed_page_parents(node.children, parent_titles=path_titles) - ) - return parents - - -def _descendant_regime_offset( - node: TitleNode, path_titles: tuple[str, ...], + ordered_paths: list[tuple[str, ...]], matches: dict[tuple[str, ...], TitleMatch], ) -> int | None: - """Offset of the parent's first anchored descendant leaf, i.e. its regime.""" - for leaf_path, _leaf in iter_leaf_title_nodes( - node.children, parent_titles=path_titles - ): - match = matches.get(leaf_path) + """Resolve the calibration-segment offset for one printed TOC node. + + Prefer an offset-bearing match under the node (same binary/regime segment as + its descendants). If the subtree has none, take the nearest preceding + offset-bearing match; if still none, the first subsequent one. + """ + depth = len(path_titles) + for path in ordered_paths: + if path == path_titles or len(path) <= depth: + continue + if path[:depth] != path_titles: + continue + match = matches.get(path) + if match is None: + continue + offset = match.evidence.get("offset") + if offset is not None: + return int(offset) + + try: + index = ordered_paths.index(path_titles) + except ValueError: + return None + for prev in reversed(ordered_paths[:index]): + match = matches.get(prev) + if match is None: + continue + offset = match.evidence.get("offset") + if offset is not None: + return int(offset) + for nxt in ordered_paths[index + 1 :]: + if len(nxt) > depth and nxt[:depth] == path_titles: + continue + match = matches.get(nxt) if match is None: continue offset = match.evidence.get("offset") @@ -667,24 +366,29 @@ def _descendant_regime_offset( return None -def backfill_parent_offset_matches( +def backfill_printed_offset_matches( *, nodes: list[TitleNode], matches: dict[tuple[str, ...], TitleMatch], page_count: int, ) -> dict[tuple[str, ...], TitleMatch]: - """Anchor TOC parents that print a page, reusing their descendant's offset. + """Blind-write physical pages for every still-unanchored printed TOC node. - Bulk anchoring consumes leaves only, so a TOC whose section headings carry - printed pages leaves every parent without a physical page. The parent shares - the calibration regime of its first anchored descendant, so ``printed_page + - that regime's offset`` is the parent's physical page. + Bulk/bisect only consume printed leaves. After that pass, any remaining node + with ``printed_page`` (parent or leaf) reuses its calibration-segment offset: + ``printed_page + offset``, clamped to ``1..page_count``. """ + ordered = _iter_all_title_nodes(nodes) + ordered_paths = [path for path, _node in ordered] by_offset: dict[int, list[tuple[tuple[str, ...], TitleNode]]] = {} - for path_titles, node in _iter_printed_page_parents(nodes): - if path_titles in matches: + for path_titles, node in ordered: + if node.printed_page is None or path_titles in matches: continue - offset = _descendant_regime_offset(node, path_titles, matches) + offset = _regime_offset_for_printed_node( + path_titles=path_titles, + ordered_paths=ordered_paths, + matches=matches, + ) if offset is None: continue by_offset.setdefault(offset, []).append((path_titles, node)) @@ -695,7 +399,7 @@ def backfill_parent_offset_matches( if 1 <= match.page <= page_count: out[path_titles] = replace( match, - evidence={**match.evidence, "parent_backfill": True}, + evidence={**match.evidence, "printed_offset_backfill": True}, ) return out @@ -980,6 +684,93 @@ def deserialize_skeleton_anchor(data: dict[str, Any]) -> SkeletonAnchor: ) +def _iter_all_title_nodes( + nodes: list[TitleNode], + *, + parent_titles: tuple[str, ...] = (), +) -> list[tuple[tuple[str, ...], TitleNode]]: + rows: list[tuple[tuple[str, ...], TitleNode]] = [] + for node in nodes: + path = (*parent_titles, node.title) + rows.append((path, node)) + if node.children: + rows.extend(_iter_all_title_nodes(node.children, parent_titles=path)) + return rows + + +def _filter_overrides_to_tree( + nodes: list[TitleNode], + match_overrides: dict[tuple[str, ...], TitleMatch], +) -> dict[tuple[str, ...], TitleMatch]: + if not nodes: + return {} + surviving = {path for path, _node in _iter_all_title_nodes(nodes)} + return { + path: match + for path, match in match_overrides.items() + if path in surviving + } + + +def apply_null_page_locates_and_prune( + *, + nodes: list[TitleNode], + match_overrides: dict[tuple[str, ...], TitleMatch], + body_pages: list[int], + page_count: int, + ctx: ToolContext | None, + structural_parent_paths: set[tuple[str, ...]], +) -> tuple[ + list[TitleNode], + dict[tuple[str, ...], TitleMatch], + list[dict[str, Any]], + int, + int, +]: + """Shared Phase-2 tail: printed backfill → null-page ReAct → final prune. + + Callers collect ``structural_parent_paths`` and run ``prune(..., keep_null=True)`` + before this helper. Returns ``pre_react_override_count`` so callers can freeze + ``bulk_count`` after backfill and before ReAct hits are added. + """ + working = nodes + overrides = dict(match_overrides) + backfilled = backfill_printed_offset_matches( + nodes=working, + matches=overrides, + page_count=page_count, + ) + if backfilled: + overrides.update(backfilled) + logger.info( + "[structure_anchoring] printed-offset backfill: {} nodes", + len(backfilled), + ) + pre_react_override_count = len(overrides) + + overrides, null_page_report = locate_null_page_overrides( + nodes=working, + match_overrides=overrides, + body_pages=body_pages, + ctx=ctx, + structural_parent_paths=structural_parent_paths, + ) + + working, failed_null_removed = prune_unanchored_toc_leaves( + working, + match_overrides=overrides, + keep_null_page_nodes=False, + ) + overrides = _filter_overrides_to_tree(working, overrides) + return ( + working, + overrides, + null_page_report, + failed_null_removed, + pre_react_override_count, + ) + + def anchor_hierarchy_from_offset( *, nodes: list[TitleNode], @@ -990,7 +781,7 @@ def anchor_hierarchy_from_offset( page_count: int, ctx: ToolContext | None, ) -> tuple[list[TitleNode], SkeletonAnchor]: - """Production prune → bulk → null-page given a precomputed offset. + """Production prune → bulk → null-page ReAct → final prune. Phase-2 entry after ``calibrate_offset`` (Phase-1). """ @@ -1015,24 +806,40 @@ def anchor_hierarchy_from_offset( if offset_matches is not None: match_overrides = offset_matches locate_method = "offset_guided_bulk" - bulk_count = len(offset_matches) else: match_overrides = seed_overrides locate_method = "offset_only" - bulk_count = 0 + # Capture parent identity before prune so empty shells retain ``kind=parent``. + structural_parent_paths = { + path + for path, node in _iter_all_title_nodes(working) + if node.children + } working, unanchored_removed = prune_unanchored_toc_leaves( - working, match_overrides=match_overrides + working, + match_overrides=match_overrides, + keep_null_page_nodes=True, ) pruned_count += unanchored_removed - - match_overrides, null_page_report = locate_null_page_parent_overrides( + match_overrides = _filter_overrides_to_tree(working, match_overrides) + + ( + working, + match_overrides, + null_page_report, + failed_null_removed, + pre_react_override_count, + ) = apply_null_page_locates_and_prune( nodes=working, match_overrides=match_overrides, - page_texts=page_texts, body_pages=body_pages, + page_count=page_count, ctx=ctx, + structural_parent_paths=structural_parent_paths, ) + pruned_count += failed_null_removed + bulk_count = pre_react_override_count if offset_matches is not None else 0 if offset_hint is None: offset_status = "failed" if ctx is not None else "skipped" diff --git a/apps/worker/app/services/document_agent/structure/hierarchy_locator.py b/apps/worker/app/services/document_agent/structure/hierarchy_locator.py index bf94692b8..512689944 100644 --- a/apps/worker/app/services/document_agent/structure/hierarchy_locator.py +++ b/apps/worker/app/services/document_agent/structure/hierarchy_locator.py @@ -1,10 +1,10 @@ """Locate hierarchy titles on PDF pages and resolve page ranges. Deterministic range assembly from PROFILE ``match_overrides``. Leaf starts -come only from those overrides. Null-page parents are located upstream via -compact-strict (cross-line) + optional VLM, then resolved here including -parent self-only spans for interstitial pages. Parents without an override -may still inherit start from the earliest located descendant leaf. +come only from those overrides. Every null-page node is located upstream by +one parent-first whole-line ReAct + VLM path. They are then resolved here, +including parent self-only spans for interstitial pages. Parents without an +override may still inherit start from the earliest located descendant leaf. """ from __future__ import annotations @@ -13,10 +13,7 @@ from dataclasses import dataclass, field from typing import Any, Literal -from app.services.document_parser.structure.body_boundary import ( - clean_toc_title, - normalize_heading_text, -) +from app.services.document_parser.structure.body_boundary import normalize_heading_label TitleMatchSource = Literal[ "anchored", @@ -24,6 +21,7 @@ "inspect_vlm", "inferred_descendant", "pdf_outline", + "react_line_grep_vlm", ] @@ -133,62 +131,6 @@ class ResolvedHierarchyRange: evidence: dict[str, Any] = field(default_factory=dict) -def locate_title_compact_strict( - title: str, - *, - scope_pages: list[int], - page_texts: dict[int, str], -) -> TitleMatch | None: - """Locate *title* after cross-line compact cleanup; accept only a unique page. - - Pipeline: compact(page text) → contiguous strict match of compact(title) → - accept iff exactly one page in ``scope_pages`` hits. Handles PyMuPDF line - splits; does not use token/normalized weak matching. - """ - needle = _compact_match_text(clean_toc_title(title) or title) - if not needle or not scope_pages: - return None - - hit_pages: list[int] = [] - matched_preview = "" - for page in scope_pages: - haystack = _compact_match_text(page_texts.get(page, "")) - if not haystack or needle not in haystack: - continue - hit_pages.append(page) - if not matched_preview: - matched_preview = needle[:160] - - unique_pages = sorted(set(hit_pages)) - if len(unique_pages) != 1: - return None - - page = unique_pages[0] - return TitleMatch( - page=page, - source="anchored", - matched_line=matched_preview, - candidates=[page], - evidence={"accept": "compact_strict_unique"}, - ) - - -def last_leaf_start_under( - node: TitleNode, - parent_titles: tuple[str, ...], - match_overrides: dict[tuple[str, ...], TitleMatch], -) -> int | None: - """Max start page among located leaves under *node*; None if none located.""" - max_page: int | None = None - for leaf_path, _leaf in iter_leaf_title_nodes([node], parent_titles=parent_titles): - match = match_overrides.get(leaf_path) - if match is None: - continue - if max_page is None or match.page > max_page: - max_page = match.page - return max_page - - def first_leaf_start_under( node: TitleNode, parent_titles: tuple[str, ...], @@ -403,7 +345,7 @@ def _locate_match_for_node( if match is not None: return match if node.children: - # Parent active locate is upstream (compact-strict / visual). + # Parent active locate is upstream (null_page_react / backfill). return _infer_start_from_descendant_overrides( node, parent_titles=path_titles[:-1], match_overrides=match_overrides, scope_pages=scope_pages, @@ -550,10 +492,6 @@ def _allowed_pages_between(start: int, end: int, allowed_pages: set[int]) -> lis return [page for page in range(start, end + 1) if page in allowed_pages] -def _compact_match_text(text: str) -> str: - return re.sub(r"\s+", "", normalize_heading_text(text)).casefold() - - def _extract_flat_entries(payload: Any) -> list[dict[str, Any]]: if isinstance(payload, list): return [ @@ -617,8 +555,8 @@ def _entries_to_tree(entries: list[dict[str, Any]]) -> list[TitleNode]: for entry in entries: # Keep original TOC heading (incl. numbering). Prefix stripping belongs - # only in text/compact match helpers used for null-page parents. - title = normalize_heading_text(str(entry.get("heading") or "")) + # only in normalized text-match helpers used for null-page parents. + title = normalize_heading_label(str(entry.get("heading") or "")) level = _safe_int(entry.get("level")) or 1 if not title or len(title) < 2: continue diff --git a/apps/worker/app/services/document_agent/structure/null_page_react.py b/apps/worker/app/services/document_agent/structure/null_page_react.py new file mode 100644 index 000000000..29ed7b166 --- /dev/null +++ b/apps/worker/app/services/document_agent/structure/null_page_react.py @@ -0,0 +1,706 @@ +"""Parent-first ReAct locate for every TOC node with ``printed_page=None``. + +Each query uses normalized whole-line text only to nominate physical pages. +Candidate pages are then VLM-confirmed in non-overlapping 2/4/6/10 physical +windows. A text hit, including a single unique hit, never writes an override +without visual confirmation. +""" + +from __future__ import annotations + +import json +from typing import Any, cast + +from loguru import logger + +from app.services.document_agent.manifest import ToolContext +from app.services.document_agent.structure.hierarchy_locator import ( + TitleMatch, + TitleNode, + first_leaf_start_under, +) + +# Planner grep rounds after the free seed full-title probe. Not a page count. +REACT_PLANNER_GREP_BUDGET = 5 + + +def react_budget() -> int: + """Return the planner grep-loop budget.""" + return int(REACT_PLANNER_GREP_BUDGET) + + +_REACT_INSTRUCTIONS = """\ +You are the search planner in a small ReAct loop. Propose the next whole-line +text query to find the physical START page of a section. Grep normalizes each +PDF text line independently, collapses whitespace with CJK-aware spacing, and +matches the complete normalized line case-insensitively. It never accepts a +substring inside a longer line. Every candidate page, including a unique one, +must pass visual section-start confirmation. + +The system already grepped the full TOC title once before this loop (see +previous_attempts). Do not repeat that exact full-title query. + +Return one strict json object with action one of (no other keys): +{"action":"grep","query":"..."} +{"action":"strip_header","query":""} +{"action":"strip_footer","query":""} +{"action":"give_up","query":""} + +Do not include a reason field. Use give_up only when no useful untried query +or strip remains. + +Ordered query strategy after the automatic full-title grep (follow this order; +skip a step only if already tried or not applicable to the TOC title / parent +path). Pattern-level only — do not invent document-specific titles: +1. Remove the leading number / letter / punctuation prefix from the TOC title + and grep the remaining title body. +2. When the parent path indicates appendices/annexes (or the TOC label is a + lettered appendix-style entry): grep the structural form + "Appendix " using the letter taken from the TOC label. Prefer this + before inventing other phrases. +3. Only after the above: try other variants such as "Appendix " plus + the title body, a shorter distinctive complete-line title variant, or + another structural prefix (chapter / part / section / annex) when supported + by the title or parent path. +4. Prefer queries specific enough to avoid running headers and passing mentions. + Do not guess page numbers. + +Reflection rules (mandatory): +- Read previous_attempts. Reflect on query, hit_page_count, and observation + before answering. +- If the last observation is no_line_hits, visual_rejected, + empty_normalized_query, grep_tool_error, or duplicate_normalized_query, you + MUST change the query when choosing grep. Emitting the same grep query again + after whitespace/case normalization is invalid for planner-chosen greps. +- Prefer strip_header or strip_footer when candidate pages look like running + headers/footers. Each strip automatically re-greps the last query once for + free. Otherwise advance to the next ordered query strategy. +- strip_header / strip_footer only update a temporary search view; they do not + change stored page text. They do NOT consume react_budget. Call each at most + once per locate. +- Planner greps consume react_budget. The automatic full-title seed grep and + strip auto re-greps are free. +""" + + +def _react_history_item(item: dict[str, Any]) -> dict[str, Any]: + """Planner-facing history only. Runtime audit fields stay on the attempt.""" + out: dict[str, Any] = { + "action": item.get("action"), + "query": item.get("query"), + "normalized_query": item.get("normalized_query"), + "hit_page_count": int(item.get("hit_page_count") or 0), + "observation": item.get("observation"), + } + return out + + +def _next_located_bound( + *, + sibling_nodes: list[TitleNode], + index: int, + parent_titles: tuple[str, ...], + overrides: dict[tuple[str, ...], TitleMatch], +) -> int | None: + """Return the next sibling's own or earliest descendant start.""" + for later in sibling_nodes[index + 1 :]: + path = (*parent_titles, later.title) + own = overrides.get(path) + if own is not None: + return int(own.page) + descendant = first_leaf_start_under(later, parent_titles, overrides) + if descendant is not None: + return int(descendant) + return None + + +def _whole_line_grep( + *, + ctx: ToolContext, + query: str, + left: int, + right: int, + body_pages: list[int], +) -> tuple[str, str, list[int], int]: + """Search only ``body_pages ∩ [left, right]`` using complete-line equality. + + Returns ``(status, normalized_query_or_error, hit_pages, line_match_count)``. + """ + from app.services.document_agent.pdf_text import page_content_map + from app.services.document_agent.tools.grep_text import grep_text + + scope_pages = [page for page in body_pages if left <= page <= right] + if not scope_pages: + return "ok", "", [], 0 + + body_set = set(scope_pages) + view = ctx.blackboard.page_text_search_view + if view is not None: + texts = { + int(page): str(view[page]) + for page in scope_pages + if page in view + } + else: + full = page_content_map(ctx.blackboard.page_full_text_cache) + texts = { + int(page): str(full[page]) + for page in scope_pages + if page in full + } + if not texts: + return "ok", "", [], 0 + + previous_view = ctx.blackboard.page_text_search_view + ctx.blackboard.page_text_search_view = texts + try: + result = grep_text( + ctx, + { + "query": query, + "whole_line": True, + "start_page": min(texts), + "end_page": max(texts), + }, + ) + finally: + ctx.blackboard.page_text_search_view = previous_view + + if result.status != "ok": + return "error", str(result.error or "grep.text failed"), [], 0 + payload = result.payload or {} + hit_pages = sorted( + { + int(page) + for page in (payload.get("hit_pages") or []) + if int(page) in body_set + } + ) + return ( + "ok", + str(payload.get("normalized_query") or ""), + hit_pages, + int(payload.get("hit_count") or 0), + ) + + +def _propose_react_query( + *, + title: str, + parent_titles: tuple[str, ...], + left: int, + right: int, + attempts: list[dict[str, Any]], + budget: int, + grep_loops_used: int, +) -> tuple[dict[str, Any] | None, dict[str, Any]]: + state = { + "toc_title": title, + "parent_path": list(parent_titles), + "physical_search_scope": [left, right], + "grep_loops_remaining": max(0, budget - grep_loops_used), + "previous_attempts": [_react_history_item(item) for item in attempts], + } + prompt = ( + f"{_REACT_INSTRUCTIONS}\n\nCurrent state:\n" + f"{json.dumps(state, ensure_ascii=False)}" + ) + + try: + from shared.services.ai.llm_overrides import get_text_client + + client, model = get_text_client() + raw, usage = client.chat_completion_with_usage( + messages=cast(Any, [{"role": "user", "content": prompt}]), + model=model, + temperature=0.0, + max_tokens=120, + response_format={"type": "json_object"}, + usage_task="document_agent.null_page_title_react", + ) + payload = json.loads(raw) if raw else {} + except Exception as exc: + logger.warning("[null_page_react] planner failed for {!r}: {}", title, exc) + return None, {"error": f"planner failed: {exc}"} + + action = str(payload.get("action") or "").strip().lower() + query = str(payload.get("query") or "").strip() + if action not in {"grep", "give_up", "strip_header", "strip_footer"}: + return None, { + "error": f"unknown planner action: {action!r}", + "usage": usage, + } + if action == "grep" and not query: + return None, {"error": "planner returned empty grep query", "usage": usage} + return {"action": action, "query": query}, {"usage": usage} + + +def _verify_section_beginning_pages( + *, + ctx: ToolContext, + title: str, + pages: list[int], +) -> tuple[str, int | None, str, int]: + """Ask the VLM to select a section start from the supplied candidate pages.""" + from app.services.document_agent.calibration.prompts import ( + SECTION_START_ANSWER_KEYS, + build_section_start_question, + coerce_found, + coerce_found_page, + ) + from app.services.document_agent.tools.inspect_pages import inspect_pages + + result = inspect_pages( + ctx, + { + "pages": pages, + "page_cap": len(pages), + "question": build_section_start_question(title), + "answer_keys": SECTION_START_ANSWER_KEYS, + "folder_name": "null_page_react_verify", + "prefix": "verify", + "usage_task": "document_agent.null_page_react_verify", + }, + ) + tokens = int(result.tokens_used or 0) + if result.status != "ok": + return "error", None, str(result.error or "inspect.pages failed"), tokens + fields = (result.payload or {}).get("fields") or {} + found_page = coerce_found_page(fields.get("found_page"), pages=pages) + selected = found_page if coerce_found(fields.get("found")) else None + reason = str((result.payload or {}).get("answer") or "") + return "ok", selected, reason, tokens + + +def _locate_with_react( + *, + path_titles: tuple[str, ...], + title: str, + left: int, + right: int, + body_pages: list[int], + ctx: ToolContext, +) -> tuple[TitleMatch | None, list[dict[str, Any]], int, str]: + budget = react_budget() + attempts: list[dict[str, Any]] = [] + attempted_needles: set[str] = set() + stripped: set[str] = set() + last_grep_query: str | None = None + visual_calls = 0 + grep_loops_used = 0 + planner_turn = 0 + max_planner_turns = budget + 2 + + # Each node starts from stored page text; strip views never cross nodes. + ctx.blackboard.page_text_search_view = None + + def _visual_confirm( + *, + query: str, + hit_pages: list[int], + attempt: dict[str, Any], + ) -> TitleMatch | None: + nonlocal visual_calls + from app.services.document_agent.calibration.scan import ( + DEFAULT_WINDOW_SCHEDULE, + progressive_page_windows, + ) + + checked_pages: list[int] = [] + rounds: list[dict[str, Any]] = [] + selected_page: int | None = None + selected_reason = "" + visual_error = False + + for physical_pages in progressive_page_windows( + start_page=hit_pages[0], + end_page=right, + window_schedule=DEFAULT_WINDOW_SCHEDULE, + ): + candidate_pages = [ + page + for page in hit_pages + if physical_pages[0] <= page <= physical_pages[-1] + ] + round_row: dict[str, Any] = { + "physical_window": [physical_pages[0], physical_pages[-1]], + "candidate_pages": candidate_pages, + "selected_page": None, + } + if not candidate_pages: + rounds.append(round_row) + continue + + visual_calls += 1 + checked_pages.extend(candidate_pages) + status, found_page, reason, tokens = _verify_section_beginning_pages( + ctx=ctx, + title=title, + pages=candidate_pages, + ) + round_row.update( + { + "status": status, + "selected_page": found_page, + "reason": reason, + "tokens_used": tokens, + } + ) + rounds.append(round_row) + if status != "ok": + visual_error = True + selected_reason = reason + break + if found_page is None: + selected_reason = reason + continue + selected_page = int(found_page) + selected_reason = reason + break + + attempt["visual_rounds"] = rounds + attempt["visual_pages_checked"] = checked_pages + attempt["visual_selected_page"] = selected_page + attempt["visual_reason"] = selected_reason + if selected_page is None: + attempt["observation"] = ( + "visual_tool_error" if visual_error else "visual_rejected" + ) + attempts.append(attempt) + return None + + attempt["observation"] = "section_start_confirmed" + attempts.append(attempt) + return TitleMatch( + page=selected_page, + source="react_line_grep_vlm", + matched_line=query, + candidates=list(hit_pages), + evidence={ + "accept": "react_line_grep_vlm", + "null_page_react": True, + "grep_loop": grep_loops_used, + "normalized_query": attempt.get("normalized_query"), + "candidate_pages": list(hit_pages), + "visual_pages_checked": checked_pages, + "visual_rounds": rounds, + "post_strip": attempt.get("post_strip"), + "seed_full_title": attempt.get("seed_full_title"), + }, + ) + + def _apply_grep_result( + *, + query: str, + planner_turn_index: int, + consume_budget: bool, + allow_duplicate: bool, + post_strip: str | None, + planner_meta: dict[str, Any], + seed_full_title: bool = False, + ) -> TitleMatch | None: + nonlocal grep_loops_used, last_grep_query + + status, needle_or_error, hit_pages, line_match_count = _whole_line_grep( + ctx=ctx, + query=query, + left=left, + right=right, + body_pages=body_pages, + ) + needle = needle_or_error if status == "ok" else "" + if consume_budget and status == "ok" and needle and needle not in attempted_needles: + grep_loops_used += 1 + attempt: dict[str, Any] = { + "loop": planner_turn_index, + "grep_loop": grep_loops_used, + "action": "grep", + "query": query, + "normalized_query": needle, + "hit_page_count": len(hit_pages), + "hit_pages": hit_pages, + "line_match_count": line_match_count, + **planner_meta, + } + if post_strip is not None: + attempt["post_strip"] = post_strip + if seed_full_title: + attempt["seed_full_title"] = True + + if status != "ok": + attempt["observation"] = "grep_tool_error" + attempt["error"] = needle_or_error + attempts.append(attempt) + return None + if not needle: + attempt["observation"] = "empty_normalized_query" + attempts.append(attempt) + return None + if needle in attempted_needles and not allow_duplicate: + attempt["observation"] = "duplicate_normalized_query" + attempts.append(attempt) + return None + + last_grep_query = query + attempted_needles.add(needle) + if not hit_pages: + attempt["observation"] = ( + "post_strip_no_line_hits" if post_strip else "no_line_hits" + ) + attempts.append(attempt) + return None + return _visual_confirm(query=query, hit_pages=hit_pages, attempt=attempt) + + try: + seed_match = _apply_grep_result( + query=title, + planner_turn_index=0, + consume_budget=False, + allow_duplicate=False, + post_strip=None, + planner_meta={}, + seed_full_title=True, + ) + if seed_match is not None: + return seed_match, attempts, visual_calls, "react_line_grep_vlm" + + while grep_loops_used < budget and planner_turn < max_planner_turns: + planner_turn += 1 + proposal, planner_meta = _propose_react_query( + title=title, + parent_titles=path_titles[:-1], + left=left, + right=right, + attempts=attempts, + budget=budget, + grep_loops_used=grep_loops_used, + ) + if proposal is None: + attempts.append( + { + "loop": planner_turn, + "grep_loop": grep_loops_used, + "action": "planner_error", + "hit_page_count": 0, + "line_match_count": 0, + **planner_meta, + } + ) + continue + + action = proposal["action"] + if action == "give_up": + attempts.append( + { + "loop": planner_turn, + "grep_loop": grep_loops_used, + **proposal, + "hit_page_count": 0, + "line_match_count": 0, + **planner_meta, + } + ) + return None, attempts, visual_calls, "react_give_up" + + if action in {"strip_header", "strip_footer"}: + from app.services.document_agent.tools.text_strip_margins import ( + strip_footer, + strip_header, + ) + + which = "header" if action == "strip_header" else "footer" + if which in stripped: + attempts.append( + { + "loop": planner_turn, + "grep_loop": grep_loops_used, + **proposal, + "hit_page_count": 0, + "line_match_count": 0, + "observation": f"duplicate_strip_{which}", + **planner_meta, + } + ) + continue + strip_fn = strip_header if which == "header" else strip_footer + strip_result = strip_fn( + ctx, + {"start_page": left, "end_page": right}, + ) + stripped.add(which) + payload = strip_result.payload or {} + strip_ok = strip_result.status == "ok" + attempts.append( + { + "loop": planner_turn, + "grep_loop": grep_loops_used, + **proposal, + "hit_page_count": 0, + "line_match_count": 0, + "observation": ( + f"stripped_{which}" + if strip_ok + else f"strip_{which}_failed" + ), + "pages_updated": int(payload.get("pages_updated") or 0), + "strip_error": strip_result.error, + **planner_meta, + } + ) + if not strip_ok or not last_grep_query: + continue + + from app.services.document_parser.structure.body_boundary import ( + normalize_match_text, + ) + + prior_needle = normalize_match_text(last_grep_query) + if prior_needle: + attempted_needles.discard(prior_needle) + match = _apply_grep_result( + query=last_grep_query, + planner_turn_index=planner_turn, + consume_budget=False, + allow_duplicate=True, + post_strip=which, + planner_meta={}, + ) + if match is not None: + return match, attempts, visual_calls, "react_line_grep_vlm" + continue + + match = _apply_grep_result( + query=proposal["query"], + planner_turn_index=planner_turn, + consume_budget=True, + allow_duplicate=False, + post_strip=None, + planner_meta=planner_meta, + ) + if match is not None: + return match, attempts, visual_calls, "react_line_grep_vlm" + + return None, attempts, visual_calls, "react_loop_limit" + finally: + ctx.blackboard.page_text_search_view = None + + +def locate_null_page_overrides( + *, + nodes: list[TitleNode], + match_overrides: dict[tuple[str, ...], TitleMatch], + body_pages: list[int], + ctx: ToolContext | None, + structural_parent_paths: set[tuple[str, ...]] | None = None, +) -> tuple[dict[tuple[str, ...], TitleMatch], list[dict[str, Any]]]: + """Locate all null-page nodes in parent-first DFS order. + + Left bound is the max confirmed page among all preorder predecessors + (fallback: body start). Right bound is the earliest of the node's first + located descendant, the next located sibling/subtree start, and the + inherited scope right. No fixed page-window cap on either side. + """ + if not nodes or not body_pages: + return dict(match_overrides), [] + + out = dict(match_overrides) + parent_paths = structural_parent_paths or set() + report: list[dict[str, Any]] = [] + cursor = int(body_pages[0]) + + def walk( + sibling_nodes: list[TitleNode], + parent_titles: tuple[str, ...], + scope_right: int, + ) -> None: + nonlocal cursor + for index, node in enumerate(sibling_nodes): + path_titles = (*parent_titles, node.title) + next_bound = _next_located_bound( + sibling_nodes=sibling_nodes, + index=index, + parent_titles=parent_titles, + overrides=out, + ) + subtree_right = ( + min(int(next_bound), int(scope_right)) + if next_bound is not None + else int(scope_right) + ) + first_descendant = ( + first_leaf_start_under(node, parent_titles, out) + if node.children + else None + ) + probe_right = ( + min(int(first_descendant), subtree_right) + if first_descendant is not None + else subtree_right + ) + probe_left = cursor + + is_parent = bool(node.children) or path_titles in parent_paths + needs_probe = ( + node.printed_page is None + and path_titles not in out + ) + if needs_probe: + entry: dict[str, Any] = { + "path_titles": list(path_titles), + "title": node.title, + "kind": "parent" if is_parent else "leaf", + "printed_page": None, + "search_scope": [probe_left, probe_right], + "result": "unresolved", + "page": None, + "accept": None, + "visual_verify_calls": 0, + "react_attempts": [], + } + scope_pages = [ + page + for page in body_pages + if probe_left <= page <= probe_right + ] + if probe_right < probe_left: + entry["result"] = "skipped_bad_window" + elif not scope_pages: + entry["result"] = "no_scope_pages" + elif ctx is None: + entry["result"] = "unresolved_no_ctx" + else: + match, attempts, visual_calls, result = _locate_with_react( + path_titles=path_titles, + title=node.title, + left=probe_left, + right=probe_right, + body_pages=body_pages, + ctx=ctx, + ) + entry["react_attempts"] = attempts + entry["visual_verify_calls"] = visual_calls + entry["result"] = result + if match is not None: + out[path_titles] = match + entry["page"] = int(match.page) + entry["accept"] = match.evidence.get("accept") + report.append(entry) + + own_match = out.get(path_titles) + if own_match is not None: + cursor = max(cursor, int(own_match.page)) + + if node.children: + walk(node.children, path_titles, subtree_right) + + walk(nodes, (), body_pages[-1]) + located = sum(1 for row in report if row.get("page") is not None) + logger.info( + "[null_page_react] parent-first null-page ReAct: attempted={} " + "located={} unresolved={} planner_budget={}", + len(report), + located, + len(report) - located, + react_budget(), + ) + return out, report diff --git a/apps/worker/app/services/document_agent/structure/outline_check.py b/apps/worker/app/services/document_agent/structure/outline_check.py index d3975af8e..1cbaf2d06 100644 --- a/apps/worker/app/services/document_agent/structure/outline_check.py +++ b/apps/worker/app/services/document_agent/structure/outline_check.py @@ -2,12 +2,11 @@ from __future__ import annotations -import re from typing import Any from app.services.document_parser.structure.body_boundary import ( clean_toc_title, - normalize_heading_text, + normalize_match_text, ) _DEFAULT_DIGEST_TITLE_CHARS = 80 @@ -94,12 +93,8 @@ def build_tree_digest_from_entries( def _title_on_page(title: str, page: int, page_texts: dict[int, str]) -> bool: - needle = _compact(clean_toc_title(title) or title) + needle = normalize_match_text(clean_toc_title(title) or title) if not needle: return False - haystack = _compact(page_texts.get(page, "")) + haystack = normalize_match_text(page_texts.get(page, "")) return bool(haystack) and needle in haystack - - -def _compact(text: str) -> str: - return re.sub(r"\s+", "", normalize_heading_text(text)).casefold() diff --git a/apps/worker/app/services/document_agent/structure/toc_anchoring.py b/apps/worker/app/services/document_agent/structure/toc_anchoring.py index 74eff2f09..1e9ccae0b 100644 --- a/apps/worker/app/services/document_agent/structure/toc_anchoring.py +++ b/apps/worker/app/services/document_agent/structure/toc_anchoring.py @@ -9,6 +9,7 @@ from loguru import logger from app.services.document_agent.manifest import ToolContext +from app.services.document_agent.pdf_text import page_content_map from app.services.document_agent.structure.anchoring_primitives import ( SkeletonAnchor, anchor_hierarchy_from_offset, @@ -24,14 +25,29 @@ TitleNode, collapse_intermediate_single_child_chains, extract_toc_nodes, - iter_leaf_title_nodes, - normalize_heading_text, ) +from app.services.document_parser.structure.body_boundary import normalize_heading_label _LOG_PREFIX = "[profile.toc_anchoring]" PENDING_TOC_CALIBRATION_CONCURRENCY = 10 +def fork_ctx_for_pending(ctx: ToolContext) -> ToolContext: + """Shallow-fork ctx for one pending TOC job. + + Shares ``page_full_text_cache`` and other blackboard fields by reference; + isolates ``page_text_search_view`` so concurrent strip/grep cannot race. + """ + return ToolContext( + pdf_path=ctx.pdf_path, + job_id=ctx.job_id, + blackboard=replace(ctx.blackboard, page_text_search_view=None), + trace=ctx.trace, + output_dir=ctx.output_dir, + settings=ctx.settings, + ) + + def run_toc_anchoring(ctx: ToolContext) -> None: """Anchor TOC structure onto the profile blackboard. @@ -48,7 +64,7 @@ def run_toc_anchoring(ctx: ToolContext) -> None: if page_count <= 0: return - page_texts = dict(ctx.blackboard.page_full_text_cache) + page_texts = page_content_map(ctx.blackboard.page_full_text_cache) if not page_texts: raise ValueError( "page_full_text_cache missing; run text scan before TOC anchoring" @@ -253,7 +269,7 @@ def outline_physical_overrides( for entry in entries: if not isinstance(entry, dict): continue - title = normalize_heading_text(str(entry.get("heading") or "")) + title = normalize_heading_label(str(entry.get("heading") or "")) if not title or len(title) < 2: continue try: @@ -509,19 +525,6 @@ def _calibrate_one_pending_toc( pending_toc.get("toc_range"), ) return None - if not any( - node.printed_page is not None - for _path, node in iter_leaf_title_nodes(nodes) - ): - logger.info( - "{} pending TOC toc_range={}: no printed pages, unresolvable", - _LOG_PREFIX, - pending_toc.get("toc_range"), - ) - return { - "toc": pending_toc, - "relationship": "unresolvable", - } resolve_nodes, skeleton_anchor, _finalized = finalize_calibration_result( result=phase1, entries=list(pending_toc.get("toc_with_level") or []), @@ -576,7 +579,7 @@ def _calibrate_pending_tocs( index=i, pending_toc=pending_toc, pending_tocs=pending_tocs, - ctx=ctx, + ctx=fork_ctx_for_pending(ctx), page_texts=page_texts, page_count=page_count, body_pages=body_pages, diff --git a/apps/worker/app/services/document_agent/tools/__init__.py b/apps/worker/app/services/document_agent/tools/__init__.py index bc94a8fda..72eee350b 100644 --- a/apps/worker/app/services/document_agent/tools/__init__.py +++ b/apps/worker/app/services/document_agent/tools/__init__.py @@ -11,6 +11,7 @@ from . import probe_links as probe_links # noqa: F401 from . import probe_outline as probe_outline # noqa: F401 from . import propose_shard_plan as propose_shard_plan # noqa: F401 +from . import text_strip_margins as text_strip_margins # noqa: F401 from . import validate_anatomy_map as validate_anatomy_map # noqa: F401 from . import verdict as verdict # noqa: F401 diff --git a/apps/worker/app/services/document_agent/tools/find_toc_anchor_pages.py b/apps/worker/app/services/document_agent/tools/find_toc_anchor_pages.py index cd161a5c7..dc272cbc8 100644 --- a/apps/worker/app/services/document_agent/tools/find_toc_anchor_pages.py +++ b/apps/worker/app/services/document_agent/tools/find_toc_anchor_pages.py @@ -9,7 +9,9 @@ from typing import Any from app.services.document_agent.manifest import TocAnchorPage, ToolContext, ToolResult +from app.services.document_agent.pdf_text import page_content_map from app.services.document_agent.registry import has_page_full_text, has_page_labels, register_tool +from app.services.document_parser.structure.body_boundary import normalize_match_text from app.services.document_parser.formats.pdf.pymupdf_subprocess import ( run_in_child_process, worker, @@ -17,7 +19,7 @@ from loguru import logger # CJK and English TOC keywords used for first-pass anchor detection. -TOC_KEYWORDS = frozenset({"目录", "目次", "contents", "tableofcontents"}) +TOC_KEYWORDS = frozenset({"目录", "目次", "contents", "table of contents"}) # If a TOC keyword fingerprint appears on more than this fraction of total # pages, it is treated as a recurring navigation element (header/footer link) @@ -33,9 +35,19 @@ _MAX_KEYWORD_SPLIT_LINES = max(len(keyword) for keyword in TOC_KEYWORDS) -def _normalize_for_toc(text: str) -> str: - """Collapse whitespace for keyword matching.""" - return text.replace(" ", "").replace("\u3000", "").lower() +def _match_toc_keyword_parts(parts: list[str]) -> str | None: + candidates = {normalize_match_text(parts[0])} if parts else set() + for part in parts[1:]: + next_candidates: set[str] = set() + for prefix in candidates: + for separator in ("", " "): + candidate = normalize_match_text(f"{prefix}{separator}{part}") + if any(keyword.startswith(candidate) for keyword in TOC_KEYWORDS): + next_candidates.add(candidate) + candidates = next_candidates + if not candidates: + return None + return next((candidate for candidate in candidates if candidate in TOC_KEYWORDS), None) def _meaningful_text_lines(text: str) -> list[str]: @@ -57,9 +69,10 @@ def _merge_keyword_split_lines( upper = min(_MAX_KEYWORD_SPLIT_LINES, len(lines) - index) for part_count in range(upper, 1, -1): parts = [lines[index + offset].strip() for offset in range(part_count)] - if _normalize_for_toc("".join(parts)) not in TOC_KEYWORDS: + keyword = _match_toc_keyword_parts(parts) + if keyword is None: continue - joined = ("".join(parts), index, index + part_count - 1) + joined = (keyword, index, index + part_count - 1) break if joined is not None: merged.append(joined) @@ -74,7 +87,7 @@ def _find_toc_text_matches(lines: list[str]) -> list[dict[str, Any]]: """Match TOC keywords only as whole lines after keyword-split repair.""" matches: list[dict[str, Any]] = [] for raw_line, start_idx, end_idx in _merge_keyword_split_lines(lines): - keyword = _normalize_for_toc(raw_line) + keyword = normalize_match_text(raw_line) if keyword not in TOC_KEYWORDS: continue matches.append( @@ -202,7 +215,7 @@ def find_toc_anchor_pages(ctx: ToolContext, _args: dict[str, Any]) -> ToolResult total_pages = ctx.blackboard.page_count keyword_matches = _scan_toc_from_page_texts( - ctx.blackboard.page_full_text_cache, + page_content_map(ctx.blackboard.page_full_text_cache), page_count=total_pages, ) raw_hit_pages = {int(match["page"]) for match in keyword_matches} diff --git a/apps/worker/app/services/document_agent/tools/grep_text.py b/apps/worker/app/services/document_agent/tools/grep_text.py index 3663c8268..a831d807d 100644 --- a/apps/worker/app/services/document_agent/tools/grep_text.py +++ b/apps/worker/app/services/document_agent/tools/grep_text.py @@ -7,25 +7,34 @@ from typing import Any from app.services.document_agent.manifest import ToolContext, ToolResult +from app.services.document_agent.pdf_text import page_content_map from app.services.document_agent.registry import ( has_page_features, has_page_full_text, not_is_scanned, register_tool, ) +from app.services.document_parser.structure.body_boundary import normalize_match_text @register_tool( name="grep.text", - description="Search full PDF text for a substring or regex. Available only for native PDFs.", + description=( + "Search normalized PDF text for a substring, regex, or complete line. " + "Whitespace is collapsed with CJK-aware spacing and matching is " + "case-insensitive. Uses page_text_search_view when set (after " + "text.strip_*), else each page's stored content field." + ), parameters={ "type": "object", "properties": { "query": {"type": "string"}, "regex": {"type": "boolean", "default": False}, - "case_sensitive": {"type": "boolean", "default": False}, + "whole_line": {"type": "boolean", "default": False}, "max_results": {"type": "integer", "default": 30}, "context_chars": {"type": "integer", "default": 80}, + "start_page": {"type": "integer"}, + "end_page": {"type": "integer"}, }, "required": ["query"], }, @@ -41,34 +50,83 @@ def grep_text(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: latency_ms=int((time.monotonic() - start) * 1000), ) use_regex = bool(args.get("regex", False)) - case_sensitive = bool(args.get("case_sensitive", False)) + whole_line = bool(args.get("whole_line", False)) max_results = max(1, min(int(args.get("max_results") or 30), 100)) context_chars = max(20, min(int(args.get("context_chars") or 80), 300)) - flags = 0 if case_sensitive else re.IGNORECASE - pattern = re.compile(query if use_regex else re.escape(query), flags) - results: list[dict[str, Any]] = [] - for page, text in sorted(ctx.blackboard.page_full_text_cache.items()): - for match in pattern.finditer(text): - start_idx = max(match.start() - context_chars, 0) - end_idx = min(match.end() + context_chars, len(text)) - results.append( - { - "page": page, - "char_offset": match.start(), - "snippet": text[start_idx:end_idx].replace("\n", " "), - } - ) - if len(results) >= max_results: - break - if len(results) >= max_results: - break - summary = {"query": query, "hit_count": len(results), "results": results} - ctx.blackboard.global_signals.setdefault("grep_history", []).append( - {"query": query, "hit_count": len(results), "sample_pages": [item["page"] for item in results[:10]]} + start_page = max(1, int(args.get("start_page") or 1)) + end_page = int(args.get("end_page") or ctx.blackboard.page_count or 0) + normalized_query = normalize_match_text(query) + if not normalized_query: + return ToolResult( + status="error", + error="grep.text normalized query is empty", + latency_ms=int((time.monotonic() - start) * 1000), + ) + pattern = re.compile( + normalized_query if use_regex else re.escape(normalized_query) ) + view = ctx.blackboard.page_text_search_view + if view is None: + texts = page_content_map(ctx.blackboard.page_full_text_cache) + else: + texts = {int(page): str(text) for page, text in view.items()} + results: list[dict[str, Any]] = [] + hit_count = 0 + hit_pages: list[int] = [] + for page, text in sorted(texts.items()): + if page < start_page or (end_page and page > end_page): + continue + page_hit = False + if whole_line: + for line_index, line in enumerate(str(text or "").splitlines()): + normalized_line = normalize_match_text(line) + if not normalized_line or pattern.fullmatch(normalized_line) is None: + continue + hit_count += 1 + page_hit = True + if len(results) < max_results: + results.append( + { + "page": page, + "line_index": line_index, + "char_offset": 0, + "snippet": normalized_line, + } + ) + else: + normalized_text = normalize_match_text(str(text or "")) + for match in pattern.finditer(normalized_text): + hit_count += 1 + page_hit = True + if len(results) >= max_results: + continue + start_idx = max(match.start() - context_chars, 0) + end_idx = min(match.end() + context_chars, len(normalized_text)) + results.append( + { + "page": page, + "char_offset": match.start(), + "snippet": normalized_text[start_idx:end_idx], + } + ) + if page_hit: + hit_pages.append(page) + summary = { + "query": query, + "normalized_query": normalized_query, + "whole_line": whole_line, + "hit_count": hit_count, + "hit_page_count": len(hit_pages), + "hit_pages": hit_pages, + "results": results, + } return ToolResult( status="ok", payload=summary, latency_ms=int((time.monotonic() - start) * 1000), - output_summary={"query": query, "hit_count": len(results)}, + output_summary={ + "query": query, + "hit_count": hit_count, + "hit_page_count": len(hit_pages), + }, ) diff --git a/apps/worker/app/services/document_agent/tools/judge_toc_source.py b/apps/worker/app/services/document_agent/tools/judge_toc_source.py index 8f053ac27..a535846aa 100644 --- a/apps/worker/app/services/document_agent/tools/judge_toc_source.py +++ b/apps/worker/app/services/document_agent/tools/judge_toc_source.py @@ -14,6 +14,7 @@ from loguru import logger from app.services.document_agent.manifest import ToolContext, ToolResult +from app.services.document_agent.pdf_text import page_content_map from app.services.document_agent.registry import has_page_full_text, register_tool OUTLINE_CHOICE = "outline" @@ -118,7 +119,7 @@ def judge_toc_source(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: latency_ms=int((time.monotonic() - start) * 1000), ) - cache = dict(ctx.blackboard.page_full_text_cache) + cache = page_content_map(ctx.blackboard.page_full_text_cache) printed_toc = merge_printed_toc_texts([cache.get(page, "") for page in pages]) if not printed_toc.strip(): return ToolResult( diff --git a/apps/worker/app/services/document_agent/tools/ocr_pages.py b/apps/worker/app/services/document_agent/tools/ocr_pages.py index 737a8c377..01ff59d14 100644 --- a/apps/worker/app/services/document_agent/tools/ocr_pages.py +++ b/apps/worker/app/services/document_agent/tools/ocr_pages.py @@ -6,6 +6,7 @@ from typing import Any from app.services.document_agent.manifest import ToolContext, ToolResult +from app.services.document_agent.pdf_text import PageTextBands from app.services.document_agent.registry import has_page_features, register_tool from app.services.document_agent.visual import render_pages @@ -73,6 +74,7 @@ def ocr_pages(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: engine = RapidOCR() page_texts: dict[int, str] = {} + page_bands: dict[int, PageTextBands] = {} page_lines: dict[int, list[dict[str, Any]]] = {} for page in pages: image_path = png_by_page.get(page) @@ -89,11 +91,15 @@ def ocr_pages(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: } ) page_lines[page] = lines - page_texts[page] = "\n".join(line["text"] for line in lines if line["text"]) + content = "\n".join(line["text"] for line in lines if line["text"]) + page_texts[page] = content + # OCR has no reliable Y bands; content only, empty header/footer. + page_bands[page] = PageTextBands(content=content) cache = dict(ctx.blackboard.page_full_text_cache) - cache.update(page_texts) + cache.update(page_bands) ctx.blackboard.page_full_text_cache = cache + ctx.blackboard.page_text_search_view = None return ToolResult( status="ok", payload={"page_texts": page_texts, "page_lines": page_lines}, diff --git a/apps/worker/app/services/document_agent/tools/text_strip_margins.py b/apps/worker/app/services/document_agent/tools/text_strip_margins.py new file mode 100644 index 000000000..cb5f3b9f3 --- /dev/null +++ b/apps/worker/app/services/document_agent/tools/text_strip_margins.py @@ -0,0 +1,132 @@ +"""Temporary search-view strip of stored header/footer page bands.""" + +from __future__ import annotations + +import time +from typing import Any, Literal + +from app.services.document_agent.manifest import ToolContext, ToolResult +from app.services.document_agent.pdf_text import ( + page_bands_map, + page_content_map, + strip_margin_text, +) +from app.services.document_agent.registry import ( + has_page_features, + has_page_full_text, + not_is_scanned, + register_tool, +) + + +def _resolve_page_range( + ctx: ToolContext, args: dict[str, Any] +) -> tuple[int, int] | ToolResult: + start_page = max(1, int(args.get("start_page") or 1)) + end_page = int(args.get("end_page") or ctx.blackboard.page_count or 0) + if end_page < start_page: + return ToolResult( + status="error", + error="end_page must be >= start_page", + latency_ms=0, + ) + return start_page, end_page + + +def _apply_strip( + ctx: ToolContext, + *, + which: Literal["header", "footer"], + start_page: int, + end_page: int, +) -> dict[str, Any]: + bands = page_bands_map(ctx.blackboard.page_full_text_cache) + view = ctx.blackboard.page_text_search_view + if view is None: + view = page_content_map(bands) + else: + view = dict(view) + + pages_updated = 0 + for page in range(start_page, end_page + 1): + record = bands.get(page) + if record is None: + continue + margin = record.header if which == "header" else record.footer + before = view.get(page, record.content) + after = strip_margin_text(before, margin, edge=which) + view[page] = after + if after != before: + pages_updated += 1 + + ctx.blackboard.page_text_search_view = view + return { + "strip": which, + "start_page": start_page, + "end_page": end_page, + "pages_updated": pages_updated, + "view_active": True, + } + + +def _strip_tool( + ctx: ToolContext, + args: dict[str, Any], + *, + which: Literal["header", "footer"], +) -> ToolResult: + start = time.monotonic() + resolved = _resolve_page_range(ctx, args) + if isinstance(resolved, ToolResult): + resolved.latency_ms = int((time.monotonic() - start) * 1000) + return resolved + start_page, end_page = resolved + payload = _apply_strip( + ctx, which=which, start_page=start_page, end_page=end_page + ) + return ToolResult( + status="ok", + payload=payload, + latency_ms=int((time.monotonic() - start) * 1000), + output_summary={ + "strip": which, + "pages_updated": payload["pages_updated"], + }, + ) + + +_STRIP_PARAMS = { + "type": "object", + "properties": { + "start_page": {"type": "integer"}, + "end_page": {"type": "integer"}, + }, +} + + +@register_tool( + name="text.strip_header", + description=( + "Temporarily remove stored header-band text from the grep search view " + "for the given page range. Does not mutate page_full_text_cache. " + "Subsequent grep.text uses the stripped view until cleared." + ), + parameters=_STRIP_PARAMS, + preconditions=(has_page_features, has_page_full_text, not_is_scanned), +) +def strip_header(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: + return _strip_tool(ctx, args, which="header") + + +@register_tool( + name="text.strip_footer", + description=( + "Temporarily remove stored footer-band text from the grep search view " + "for the given page range. Does not mutate page_full_text_cache. " + "Subsequent grep.text uses the stripped view until cleared." + ), + parameters=_STRIP_PARAMS, + preconditions=(has_page_features, has_page_full_text, not_is_scanned), +) +def strip_footer(ctx: ToolContext, args: dict[str, Any]) -> ToolResult: + return _strip_tool(ctx, args, which="footer") diff --git a/apps/worker/app/services/document_parser/profiling/doc_profiler.py b/apps/worker/app/services/document_parser/profiling/doc_profiler.py index 612d73fbe..03bbd4c44 100644 --- a/apps/worker/app/services/document_parser/profiling/doc_profiler.py +++ b/apps/worker/app/services/document_parser/profiling/doc_profiler.py @@ -8,6 +8,7 @@ from loguru import logger from app.services.document_agent.coordinator import ProfileCoordinator +from app.services.document_agent.pdf_text import page_content_map from app.services.document_agent.visual import purge_debug_visual_dirs, visual_debug_enabled from app.services.document_parser.orchestration.oversized_pdf_policy import ( build_oversized_pdf_profile_failed_exception, @@ -159,7 +160,9 @@ def _profile_pdf_with_db( {}, ), }, - page_full_text_cache=dict(coordinator.blackboard.page_full_text_cache), + page_full_text_cache=page_content_map( + coordinator.blackboard.page_full_text_cache + ), ) if profile.page_count > settings.MAX_PDF_PAGE_LIMIT: if oversized_policy != "page_memory": diff --git a/apps/worker/app/services/document_parser/structure/body_boundary.py b/apps/worker/app/services/document_parser/structure/body_boundary.py index f0ce15e42..b3d4a6122 100644 --- a/apps/worker/app/services/document_parser/structure/body_boundary.py +++ b/apps/worker/app/services/document_parser/structure/body_boundary.py @@ -26,13 +26,49 @@ _PAGE_SUFFIX_RE = re.compile(r"[\s\.\-·…]+\d+\s*$") -def normalize_heading_text(text: str) -> str: - """Normalize text for fuzzy heading matching.""" +def normalize_heading_label(text: str) -> str: + """Normalize heading text while preserving display casing.""" text = unicodedata.normalize("NFKC", text or "") text = re.sub(r"\s+", " ", text).strip() return text +def _is_cjk_char(char: str) -> bool: + codepoint = ord(char) + return ( + 0x3000 <= codepoint <= 0x303F + or 0x3040 <= codepoint <= 0x30FF + or 0x31F0 <= codepoint <= 0x31FF + or 0x3400 <= codepoint <= 0x4DBF + or 0x4E00 <= codepoint <= 0x9FFF + or 0xF900 <= codepoint <= 0xFAFF + or 0x1100 <= codepoint <= 0x11FF + or 0x3130 <= codepoint <= 0x318F + or 0xAC00 <= codepoint <= 0xD7AF + or 0x20000 <= codepoint <= 0x2FA1F + ) + + +def normalize_match_text(text: str) -> str: + """Normalize query and corpus text for every deterministic text match. + + Whitespace becomes one space between non-CJK text, and disappears whenever + either adjacent character is CJK. Matching is case-insensitive. + """ + normalized = unicodedata.normalize("NFKC", text or "").casefold().strip() + parts = re.split(r"\s+", normalized) + if not parts or not parts[0]: + return "" + + output = parts[0] + for part in parts[1:]: + if not part: + continue + separator = "" if _is_cjk_char(output[-1]) or _is_cjk_char(part[0]) else " " + output = f"{output}{separator}{part}" + return output + + def clean_toc_title(title: str) -> str: """Remove leading numbering/hashes and trailing page numbers from a TOC title.""" cleaned = _PAGE_SUFFIX_RE.sub("", title or "").strip() @@ -78,15 +114,15 @@ def find_first_body_boundary( ) -> int | None: """Return the first line index matching a TOC level-1 title, if any.""" normalized_titles = [ - normalize_heading_text(title) + normalize_match_text(title) for title in level1_titles - if normalize_heading_text(title) + if normalize_match_text(title) ] if not normalized_titles: return None for index, line in enumerate(lines): - normalized_line = normalize_heading_text(line.lstrip("#").strip()) + normalized_line = normalize_match_text(line.lstrip("#").strip()) if any(title in normalized_line for title in normalized_titles): return index return None diff --git a/apps/worker/app/services/document_parser/structure/toc_hierarchy.py b/apps/worker/app/services/document_parser/structure/toc_hierarchy.py index 439223b38..6f921477c 100644 --- a/apps/worker/app/services/document_parser/structure/toc_hierarchy.py +++ b/apps/worker/app/services/document_parser/structure/toc_hierarchy.py @@ -1,10 +1,10 @@ from __future__ import annotations import pandas as pd +from app.services.document_parser.structure.body_boundary import normalize_match_text from app.services.document_parser.structure.layout_parser import hiearchy_llm from app.services.document_parser.support.stage_profiler import stage_timer from app.services.document_parser.tables.table_text_parser import df2md -from app.services.document_parser.support.text_helpers import normalize_md from loguru import logger from pandas import Index @@ -109,14 +109,14 @@ def build_toc_hierarchy_payload( def eval_toc_levels( toc_lines: list[str], model_name: str | None = None, max_depth: int = 6 ) -> tuple[str, dict]: - toc_title_keywords = {"目录", "目次", "tableofcontents", "contents"} + toc_title_keywords = {"目录", "目次", "table of contents", "contents"} valid_data = [] for index, line in enumerate(toc_lines): heading = line.strip() if not heading: continue - if normalize_md(heading) in toc_title_keywords: + if normalize_match_text(heading.lstrip("#").strip()) in toc_title_keywords: logger.debug( f"eval_toc_levels: skipping TOC keyword title line id={index}: {heading[:60]}" ) diff --git a/apps/worker/app/services/document_parser/structure/toc_parser.py b/apps/worker/app/services/document_parser/structure/toc_parser.py index 884437421..d6a28d39a 100644 --- a/apps/worker/app/services/document_parser/structure/toc_parser.py +++ b/apps/worker/app/services/document_parser/structure/toc_parser.py @@ -12,8 +12,9 @@ import gevent import pandas as pd +from app.services.document_parser.structure.body_boundary import normalize_match_text from app.services.document_parser.structure.toc_hierarchy import eval_toc_levels -from app.services.document_parser.support.text_helpers import normalize_md, truncate_text_by_tokens +from app.services.document_parser.support.text_helpers import truncate_text_by_tokens from app.services.document_parser.support.stage_profiler import stage_timer from app.services.document_parser.tables.table_text_parser import df2md from gevent.pool import Pool as GeventPool @@ -111,12 +112,12 @@ def detect_toc_candidates(md_lines: list, limit_: int = 100) -> tuple: - area_ranges: List[(start_idx, end_idx)] - full raw md_lines ranges for later filtering """ - toc_keywords = {"目录", "目次", "tableofcontents", "contents"} + toc_keywords = {"目录", "目次", "table of contents", "contents"} # Step 1: find all TOC keywords start_indices = [] for i, line in enumerate(md_lines): - if normalize_md(line) in toc_keywords: + if normalize_match_text(line.lstrip("#").strip()) in toc_keywords: start_indices.append(i) # Step 2: if no TOC keywords found, use the first line as the candidate area diff --git a/apps/worker/app/services/document_parser/support/text_helpers.py b/apps/worker/app/services/document_parser/support/text_helpers.py index b11fdc772..b87af9af5 100644 --- a/apps/worker/app/services/document_parser/support/text_helpers.py +++ b/apps/worker/app/services/document_parser/support/text_helpers.py @@ -8,14 +8,6 @@ EN_START_LIMIT = 15 CN_RATIO_THRESHOLD = 0.3 - -def normalize_md(text: str) -> str: - """Normalize markdown string for comparison.""" - text = re.sub(r"^\s*#+\s*", "", text) - text = re.sub(r"\s+", "", text) - return text.lower() - - def truncate_text(text: str, start_limit: int, end_limit: int) -> str: """Truncate text by raw character count, keeping start and end parts.""" text = str(text) diff --git a/apps/worker/app/services/page_memory/fine_hierarchy.py b/apps/worker/app/services/page_memory/fine_hierarchy.py index 96fd18de4..57db6362b 100644 --- a/apps/worker/app/services/page_memory/fine_hierarchy.py +++ b/apps/worker/app/services/page_memory/fine_hierarchy.py @@ -21,6 +21,7 @@ from loguru import logger +from app.services.document_parser.structure.body_boundary import normalize_match_text from app.services.page_memory.page_tagger import PageTagResult from app.services.page_memory.skeleton_extractor import SectionSkeleton from app.services.page_memory._utils import page_scope_info, sort_skeletons @@ -461,7 +462,7 @@ def _exclusive_end(skeletons: list[SectionSkeleton], index: int) -> int: def _title_key(title: str | None) -> str: - normalized = re.sub(r"\s+", "", str(title or "")).casefold() + normalized = normalize_match_text(str(title or "")) normalized = re.sub(r"[^\w\u4e00-\u9fff]+", "", normalized) return normalized diff --git a/apps/worker/app/services/page_memory/skeleton_extractor.py b/apps/worker/app/services/page_memory/skeleton_extractor.py index f7faa8420..07b366277 100644 --- a/apps/worker/app/services/page_memory/skeleton_extractor.py +++ b/apps/worker/app/services/page_memory/skeleton_extractor.py @@ -34,6 +34,24 @@ ) +def _null_page_locate_bucket( + null_page_report: list[dict[str, Any]], + *, + kind: str, +) -> dict[str, Any]: + """Summarize leaf or parent null-page locate rows (M2: no mixed bucket).""" + rows = [row for row in null_page_report if row.get("kind") == kind] + return { + "attempted": len(rows), + "located": sum(1 for row in rows if row.get("page") is not None), + "unresolved": sum(1 for row in rows if row.get("result") == "unresolved"), + "visual_verify_calls": sum( + int(row.get("visual_verify_calls") or 0) for row in rows + ), + "entries": rows, + } + + @dataclass(frozen=True) class SectionSkeleton: section_path: str @@ -143,17 +161,12 @@ def extract_section_skeletons( "reason": "offset_guided_anchoring_skipped_or_empty", "pruned_out_of_scope": skeleton_anchor.pruned_count, } - locate_summary["null_page_parent_locate"] = { - "attempted": len(null_page_report), - "located": sum(1 for row in null_page_report if row.get("page") is not None), - "unresolved": sum( - 1 for row in null_page_report if row.get("result") == "unresolved" - ), - "visual_verify_calls": sum( - int(row.get("visual_verify_calls") or 0) for row in null_page_report - ), - "entries": null_page_report, - } + locate_summary["null_page_leaf_locate"] = _null_page_locate_bucket( + null_page_report, kind="leaf" + ) + locate_summary["null_page_parent_locate"] = _null_page_locate_bucket( + null_page_report, kind="parent" + ) ranges = resolve_hierarchy_page_ranges( resolve_nodes, @@ -332,19 +345,12 @@ def _resolve_pending_tocs( "pruned_out_of_scope": skeleton_anchor.pruned_count, "toc_relationship": relationship, } - locate_summary["null_page_parent_locate"] = { - "attempted": len(null_page_report), - "located": sum( - 1 for row in null_page_report if row.get("page") is not None - ), - "unresolved": sum( - 1 for row in null_page_report if row.get("result") == "unresolved" - ), - "visual_verify_calls": sum( - int(row.get("visual_verify_calls") or 0) for row in null_page_report - ), - "entries": null_page_report, - } + locate_summary["null_page_leaf_locate"] = _null_page_locate_bucket( + null_page_report, kind="leaf" + ) + locate_summary["null_page_parent_locate"] = _null_page_locate_bucket( + null_page_report, kind="parent" + ) ranges = resolve_hierarchy_page_ranges( resolve_nodes, diff --git a/apps/worker/scripts/page_memory/_debug_pm_shared.py b/apps/worker/scripts/page_memory/_debug_pm_shared.py index 13270c720..be93e6030 100644 --- a/apps/worker/scripts/page_memory/_debug_pm_shared.py +++ b/apps/worker/scripts/page_memory/_debug_pm_shared.py @@ -462,8 +462,10 @@ def persist_stage0_state(out_dir: Path, coordinator) -> Path: doc_agent_dir = out_dir / "_doc_agent" doc_agent_dir.mkdir(parents=True, exist_ok=True) + from app.services.document_agent.pdf_text import PageTextBands + texts = { - str(page): text + str(page): PageTextBands.from_any(text).to_dict() for page, text in dict(coordinator.blackboard.page_full_text_cache or {}).items() } write_debug_json(page_text_cache_path(out_dir), texts) @@ -505,7 +507,9 @@ def load_stage0_into_coordinator(coordinator, out_dir: Path) -> None: hint="Stage-0 page_full_text_cache.json missing; re-run Stage 0", ) raw_texts = json.loads(text_path.read_text(encoding="utf-8")) - page_texts = {int(page): str(text) for page, text in dict(raw_texts or {}).items()} + from app.services.document_agent.pdf_text import page_bands_map + + page_texts = page_bands_map(raw_texts) bb = coordinator.blackboard bb.page_count = int(state.get("page_count") or 0) @@ -515,6 +519,7 @@ def load_stage0_into_coordinator(coordinator, out_dir: Path) -> None: bb.doc_stats = dict(state.get("doc_stats") or {}) bb.global_signals = dict(state.get("global_signals") or {}) bb.page_full_text_cache = page_texts + bb.page_text_search_view = None # Stage-1 owns TOC + assets from here. bb.toc_result = None bb.toc_hierarchies = None diff --git a/apps/worker/scripts/page_memory/debug_pm_null_page_react.py b/apps/worker/scripts/page_memory/debug_pm_null_page_react.py new file mode 100644 index 000000000..126474e01 --- /dev/null +++ b/apps/worker/scripts/page_memory/debug_pm_null_page_react.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +# ruff: noqa: E402 +"""Debug: dump production null-page locate report via Stage-2 anchoring. + +Uses the live parent-first unified ReAct locator (no patches). + +Usage: + cd apps/worker + uv run python scripts/page_memory/debug_pm_null_page_react.py \\ + --file "/path/to/doc.pdf" +""" + +from __future__ import annotations + +import sys +import time +from pathlib import Path as _Path + +sys.path.insert(0, str(_Path(__file__).resolve().parent)) + +from loguru import logger + +from _debug_pm_shared import ( + _build_debug_coordinator, + base_argparser, + load_anatomy_cache, + load_stage0_into_coordinator, + page_text_cache_path, + require_file, + resolve_anatomy_cache_path, + resolve_paths, + stage0_state_path, + write_debug_json, +) + +from app.services.document_agent.structure.null_page_react import ( + REACT_PLANNER_GREP_BUDGET, + react_budget, +) + + +def main() -> int: + parser = base_argparser( + "Debug: production null-page locate via run_toc_anchoring (Stage 2)" + ) + args = parser.parse_args() + + from app.services.document_agent.structure.toc_anchoring import run_toc_anchoring + from app.services.document_agent.validators import single_shard_plan + from shared.core.config import settings + + pdf_path, filename, out_dir = resolve_paths(args) + anatomy_cache = resolve_anatomy_cache_path(out_dir) + require_file(stage0_state_path(out_dir), hint="Run Stage 0 first") + require_file(page_text_cache_path(out_dir), hint="Re-run Stage 0") + require_file(anatomy_cache, hint="Run Stage 1 first") + + anatomy = load_anatomy_cache(anatomy_cache, pdf_path, filename) + page_count = int(anatomy.page_count or 0) + hierarchies = list(getattr(anatomy, "toc_hierarchies", None) or []) + + logger.info("█" * 70) + logger.info(" Production null-page locate dump — {}", filename) + logger.info(" OUTPUT: {}", out_dir) + logger.info("█" * 70) + + t0 = time.time() + previous_image_model = settings.IMAGE_MODEL + try: + coordinator = _build_debug_coordinator( + pdf_path=pdf_path, + job_id=filename, + out_dir=out_dir, + model=None if args.no_vlm else args.model, + settings_extra={"skip_toc_anchoring": False}, + ) + load_stage0_into_coordinator(coordinator, out_dir) + bb = coordinator.blackboard + bb.toc_result = anatomy.toc_result + bb.toc_hierarchies = hierarchies + bb.shard_plan = anatomy.shard_plan or single_shard_plan(page_count) + bb.skeleton_anchor = None + bb.skeleton_nodes = None + bb.pending_skeleton_anchors = [] + + run_toc_anchoring(coordinator.ctx) + + anchor = bb.skeleton_anchor or {} + report = list(anchor.get("null_page_report") or []) + overrides = dict(anchor.get("match_overrides") or {}) + react_hits = [] + for path, match in overrides.items(): + source = ( + match.get("source") + if isinstance(match, dict) + else getattr(match, "source", None) + ) + if source != "react_line_grep_vlm": + continue + titles = path if isinstance(path, (list, tuple)) else (path,) + page = ( + match.get("page") + if isinstance(match, dict) + else getattr(match, "page", None) + ) + react_hits.append({"path": list(titles), "page": page}) + + payload = { + "policy": { + "prune_pre": "keep_null_page_nodes=True", + "printed_backfill": "anchoring_primitives.backfill_printed_offset_matches", + "probe": "null_page_react.locate_null_page_overrides", + "traversal": "parent-first DFS", + "left_bound": "preorder max confirmed page (body start fallback)", + "text_match": "normalized whole-line equality", + "visual_windows": [2, 4, 6, 10], + "prune_post": "keep_null_page_nodes=False (drop unresolved)", + "react_budget": react_budget(), + "react_planner_grep_budget": REACT_PLANNER_GREP_BUDGET, + "seed_full_title_grep": "free", + "strip_auto_regrep": "free", + "max_planner_turns": REACT_PLANNER_GREP_BUDGET + 2, + }, + "offset": anchor.get("offset"), + "pruned_count": anchor.get("pruned_count"), + "bulk_count": anchor.get("bulk_count"), + "override_count": len(overrides), + "null_page_report": report, + "react_override_hits": react_hits, + "elapsed_s": round(time.time() - t0, 2), + } + + out_path = out_dir / "_doc_agent" / "null_page_react_report.json" + write_debug_json(out_path, payload) + logger.info("wrote {}", out_path) + logger.info( + "null_page rows={} react_override_hits={}", + len(report), + len(react_hits), + ) + for row in report: + logger.info( + " [{}] {} search_scope={} result={} page={} loops={}", + row.get("kind"), + row.get("path_titles"), + row.get("search_scope"), + row.get("result"), + row.get("page"), + len(row.get("react_attempts") or []), + ) + for hit in react_hits: + logger.info(" OVERRIDE {} -> p{}", hit["path"], hit["page"]) + finally: + if args.model: + settings.IMAGE_MODEL = previous_image_model + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/apps/worker/scripts/page_memory/debug_pm_stage2_calibration.py b/apps/worker/scripts/page_memory/debug_pm_stage2_calibration.py index 5034d429d..775e25b9c 100644 --- a/apps/worker/scripts/page_memory/debug_pm_stage2_calibration.py +++ b/apps/worker/scripts/page_memory/debug_pm_stage2_calibration.py @@ -148,7 +148,9 @@ def main() -> int: except FileNotFoundError: pass - page_texts = dict(bb.page_full_text_cache or {}) + from app.services.document_agent.pdf_text import page_content_map + + page_texts = page_content_map(bb.page_full_text_cache or {}) skeletons = extract_section_skeletons( anatomy=anchored, filename=filename, diff --git a/apps/worker/tests/contract/test_body_boundary_contract.py b/apps/worker/tests/contract/test_body_boundary_contract.py index 3512c9f89..56b9906d3 100644 --- a/apps/worker/tests/contract/test_body_boundary_contract.py +++ b/apps/worker/tests/contract/test_body_boundary_contract.py @@ -14,6 +14,7 @@ from app.services.document_parser.structure.body_boundary import ( extract_level1_titles, find_first_body_boundary, + normalize_match_text, ) from app.services.document_parser.structure.layout_parser import ( _supports_multi_toc_zones, @@ -40,6 +41,13 @@ def test_extract_level1_titles_reads_toc_with_level_not_toc_tree() -> None: assert titles == ["Overview", "Requirements"] +def test_normalize_match_text_uses_cjk_aware_spacing_and_lowercase() -> None: + assert normalize_match_text("附录 A OVERVIEW") == "附录a overview" + assert normalize_match_text("Public\n Domain\tManual") == "public domain manual" + assert normalize_match_text("目\n录") == "目录" + assert normalize_match_text("Chapter 1 概述") == "chapter 1概述" + + def test_extract_level1_titles_ignores_empty_or_non_list_payloads() -> None: assert extract_level1_titles([]) == [] assert extract_level1_titles([{"toc_with_level": None}]) == [] diff --git a/apps/worker/tests/contract/test_calibration_scan_contract.py b/apps/worker/tests/contract/test_calibration_scan_contract.py index 83191311a..d56cfd69d 100644 --- a/apps/worker/tests/contract/test_calibration_scan_contract.py +++ b/apps/worker/tests/contract/test_calibration_scan_contract.py @@ -17,6 +17,7 @@ from app.services.document_agent.calibration import scan as scan_module from app.services.document_agent.calibration.scan import ( DEFAULT_WINDOW_SCHEDULE, + progressive_page_windows, scan_title_forward, ) from app.services.document_agent.manifest import ToolContext, ToolResult @@ -68,6 +69,14 @@ def _apply(fake: _FakeInspect) -> _FakeInspect: return _apply +def test_progressive_page_windows_are_non_overlapping_and_clipped() -> None: + assert progressive_page_windows(start_page=10, end_page=20) == [ + [10, 11], + [12, 13, 14, 15], + [16, 17, 18, 19, 20], + ] + + def test_first_round_opens_the_candidate_page_and_its_successor(patch_inspect) -> None: fake = patch_inspect(_FakeInspect(hit_page=10)) diff --git a/apps/worker/tests/contract/test_doc_profile_anatomy_contract.py b/apps/worker/tests/contract/test_doc_profile_anatomy_contract.py index a8c27f2dc..b73f1aa26 100644 --- a/apps/worker/tests/contract/test_doc_profile_anatomy_contract.py +++ b/apps/worker/tests/contract/test_doc_profile_anatomy_contract.py @@ -100,7 +100,7 @@ def test_toc_anchor_text_scan_whole_line_keyword_and_split_repair() -> None: assert late_matches[0]["line_index"] == 60 assert late_matches[0]["match_kind"] == "keyword:目录" - assert split_matches[0]["match_kind"] == "keyword:tableofcontents" + assert split_matches[0]["match_kind"] == "keyword:table of contents" assert split_matches[0]["line_index"] == 0 assert split_matches[0]["line_end_index"] == 2 assert false_matches == [] @@ -618,10 +618,12 @@ def fake_finalize() -> None: assert anatomy.toc_result.toc_pages == [17] -def test_run_text_scan_native_uses_read_page_texts( +def test_run_text_scan_native_uses_read_page_text_bands( monkeypatch, tmp_path: Path, ) -> None: + from app.services.document_agent.pdf_text import PageTextBands + coordinator = ProfileCoordinator( pdf_path=str(tmp_path / "doc.pdf"), job_id="job-scan-native", @@ -632,21 +634,41 @@ def test_run_text_scan_native_uses_read_page_texts( is_scanned=False, category="Report", routing_category=PdfRoutingCategory.GENERIC.value, + header_y=0.06, + footer_y=0.95, ) - def fake_read(_pdf_path: str, pages: list[int], timeout: int = 300) -> dict[int, str]: + def fake_read( + _pdf_path: str, + pages: list[int], + *, + header_y: float | None = None, + footer_y: float | None = None, + timeout: int = 300, + ) -> dict[int, PageTextBands]: assert pages == [1, 2] - return {1: "a", 2: "b"} + assert header_y == 0.06 + assert footer_y == 0.95 + return { + 1: PageTextBands(content="a", header="h1", footer="f1"), + 2: PageTextBands(content="b", header="", footer="f2"), + } - monkeypatch.setattr(coordinator_module, "read_page_texts", fake_read) + monkeypatch.setattr(coordinator_module, "read_page_text_bands", fake_read) coordinator._run_text_scan() - assert coordinator.blackboard.page_full_text_cache == {1: "a", 2: "b"} + assert coordinator.blackboard.page_full_text_cache == { + 1: PageTextBands(content="a", header="h1", footer="f1"), + 2: PageTextBands(content="b", header="", footer="f2"), + } + assert coordinator.blackboard.page_text_search_view is None def test_run_text_scan_scanned_dispatches_ocr_pages( monkeypatch, tmp_path: Path, ) -> None: + from app.services.document_agent.pdf_text import PageTextBands + coordinator = ProfileCoordinator( pdf_path=str(tmp_path / "doc.pdf"), job_id="job-scan-ocr", @@ -662,12 +684,18 @@ def test_run_text_scan_scanned_dispatches_ocr_pages( def fake_dispatch(name: str, ctx, args): assert name == "ocr.pages" assert args == {"pages": [1, 2]} - ctx.blackboard.page_full_text_cache = {1: "ocr-1", 2: "ocr-2"} + ctx.blackboard.page_full_text_cache = { + 1: PageTextBands(content="ocr-1"), + 2: PageTextBands(content="ocr-2"), + } return ToolResult(status="ok", payload={}) monkeypatch.setattr(coordinator_module.REGISTRY, "dispatch", fake_dispatch) coordinator._run_text_scan() - assert coordinator.blackboard.page_full_text_cache == {1: "ocr-1", 2: "ocr-2"} + assert coordinator.blackboard.page_full_text_cache == { + 1: PageTextBands(content="ocr-1"), + 2: PageTextBands(content="ocr-2"), + } def test_anchor_confirmation_failure_requires_one_strict_retry(tmp_path: Path) -> None: diff --git a/apps/worker/tests/contract/test_ocr_pages_contract.py b/apps/worker/tests/contract/test_ocr_pages_contract.py index d55254046..efb28a8ce 100644 --- a/apps/worker/tests/contract/test_ocr_pages_contract.py +++ b/apps/worker/tests/contract/test_ocr_pages_contract.py @@ -72,5 +72,8 @@ def fake_render(*_args, **_kwargs): result = ocr_pages(ctx, {"pages": [1]}) assert result.status == "ok" - assert ctx.blackboard.page_full_text_cache[1] == "Hello" + bands = ctx.blackboard.page_full_text_cache[1] + assert getattr(bands, "content", None) == "Hello" + assert getattr(bands, "header", None) == "" + assert getattr(bands, "footer", None) == "" assert result.payload["page_lines"][1][0]["text"] == "Hello" diff --git a/apps/worker/tests/contract/test_outline_short_circuit_contract.py b/apps/worker/tests/contract/test_outline_short_circuit_contract.py index cb038742f..053ee9bb3 100644 --- a/apps/worker/tests/contract/test_outline_short_circuit_contract.py +++ b/apps/worker/tests/contract/test_outline_short_circuit_contract.py @@ -117,7 +117,7 @@ def fake_calibrate(*args: Any, **kwargs: Any) -> Any: ), patch( "app.services.document_agent.structure.anchoring_primitives." - "locate_null_page_parent_overrides", + "locate_null_page_overrides", side_effect=_no_null_parent_locate, ), ): @@ -153,7 +153,7 @@ def fake_judge(tool_ctx: ToolContext, args: dict[str, Any]) -> ToolResult: ), patch( "app.services.document_agent.structure.anchoring_primitives." - "locate_null_page_parent_overrides", + "locate_null_page_overrides", side_effect=_no_null_parent_locate, ), ): diff --git a/apps/worker/tests/contract/test_profile_skeleton_anchor_wiring_contract.py b/apps/worker/tests/contract/test_profile_skeleton_anchor_wiring_contract.py index fc10dcaa8..19bc37727 100644 --- a/apps/worker/tests/contract/test_profile_skeleton_anchor_wiring_contract.py +++ b/apps/worker/tests/contract/test_profile_skeleton_anchor_wiring_contract.py @@ -307,16 +307,26 @@ def fake_finalize(**kwargs): assert records[0]["nodes"][0]["title"] == "App" -def test_profile_skips_finalize_for_unresolvable_pending_toc() -> None: +def test_profile_pending_toc_without_printed_pages_still_finalizes() -> None: + """H7a: all-null printed leaves still run finalize; unresolvable only if no span.""" ctx = _ctx(page_count=30) hierarchies = _pending_tocs() hierarchies[1]["toc_with_level"] = [{"heading": "App", "level": 1}] ctx.blackboard.toc_hierarchies = hierarchies ctx.blackboard.toc_result = TocResult(method="vlm_batch", toc_pages=[1, 20, 21]) ctx.blackboard.page_full_text_cache = {page: "body" for page in range(1, 31)} + finalize_calls: list[object] = [] - def _boom(*_args, **_kwargs): - raise AssertionError("unresolvable pending TOC must not finalize") + def fake_finalize(**kwargs): + finalize_calls.append(kwargs) + empty = SkeletonAnchor( + offset=0, + offset_status="ok", + match_overrides={}, + null_page_report=[], + bulk_count=0, + ) + return [], empty, True with ( patch( @@ -333,11 +343,12 @@ def _boom(*_args, **_kwargs): ), patch( "app.services.document_agent.calibration.procedure.finalize_calibration_result", - side_effect=_boom, + side_effect=fake_finalize, ), ): run_toc_anchoring(ctx) + assert finalize_calls, "H7a: finalize must run even when leaves lack printed pages" records = ctx.blackboard.pending_skeleton_anchors assert len(records) == 1 assert records[0]["relationship"] == "unresolvable" @@ -345,6 +356,30 @@ def _boom(*_args, **_kwargs): assert "skeleton_anchor" not in records[0] +def test_fork_ctx_for_pending_isolates_search_view() -> None: + """H7b: each pending job gets its own page_text_search_view.""" + from app.services.document_agent.structure.toc_anchoring import fork_ctx_for_pending + + ctx = _ctx(page_count=10) + ctx.blackboard.page_full_text_cache = {1: "shared"} + ctx.blackboard.page_text_search_view = {1: "parent-stale"} + + job_a = fork_ctx_for_pending(ctx) + job_b = fork_ctx_for_pending(ctx) + + assert job_a.blackboard.page_text_search_view is None + assert job_b.blackboard.page_text_search_view is None + # Shared cache by reference; search views independent. + assert job_a.blackboard.page_full_text_cache is ctx.blackboard.page_full_text_cache + assert job_b.blackboard.page_full_text_cache is ctx.blackboard.page_full_text_cache + + job_a.blackboard.page_text_search_view = {1: "a-strip"} + job_b.blackboard.page_text_search_view = {1: "b-strip"} + assert job_a.blackboard.page_text_search_view == {1: "a-strip"} + assert job_b.blackboard.page_text_search_view == {1: "b-strip"} + assert ctx.blackboard.page_text_search_view == {1: "parent-stale"} + + def test_c4_uses_persisted_pending_relationship_and_does_not_classify() -> None: pending_toc = _pending_tocs()[1] anatomy = _anatomy(with_anchor=True) diff --git a/apps/worker/tests/contract/test_structure_anchoring_contract.py b/apps/worker/tests/contract/test_structure_anchoring_contract.py index e87104b4a..0f842b357 100644 --- a/apps/worker/tests/contract/test_structure_anchoring_contract.py +++ b/apps/worker/tests/contract/test_structure_anchoring_contract.py @@ -74,26 +74,591 @@ def test_prune_out_of_scope_nodes_removes_overflow_leaves() -> None: assert [n.title for n in pruned] == ["A"] -def test_null_page_parent_skipped_without_right_anchor() -> None: +def test_all_null_page_nodes_unresolved_without_ctx_in_parent_first_order() -> None: + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_overrides, + ) + parent = TitleNode( title="Chapter", level=1, printed_page=None, children=[TitleNode(title="Orphan", level=2, printed_page=None, children=[])], ) - overrides, report = anchoring.locate_null_page_parent_overrides( + overrides, report = locate_null_page_overrides( nodes=[parent], match_overrides={}, - page_texts={1: "Chapter\nHello"}, body_pages=[1, 2, 3], ctx=None, ) assert overrides == {} - assert len(report) == 1 - assert report[0]["result"] == "skipped_no_right" + assert [row["path_titles"] for row in report] == [ + ["Chapter"], + ["Chapter", "Orphan"], + ] + assert [row["kind"] for row in report] == ["parent", "leaf"] + assert {row["result"] for row in report} == {"unresolved_no_ctx"} + + +def test_null_page_leaf_kept_by_pre_react_prune() -> None: + """Printed unanchored leaves drop; null-page leaves survive until ReAct.""" + nodes = [ + _leaf("PrintedOk", 1), + _leaf("PrintedMiss", 10), + TitleNode(title="NullLeaf", level=1, printed_page=None, children=[]), + ] + overrides = anchoring.bulk_offset_matches( + [(("PrintedOk",), nodes[0])], + offset=0, + ) + kept, removed = anchoring.prune_unanchored_toc_leaves( + nodes, + match_overrides=overrides, + keep_null_page_nodes=True, + ) + assert removed == 1 + assert [n.title for n in kept] == ["PrintedOk", "NullLeaf"] -def test_null_page_parent_located_via_compact_text() -> None: +def test_null_page_react_continues_later_siblings_after_failure() -> None: + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_overrides, + ) + + nodes = [ + TitleNode(title="A", level=1, printed_page=None, children=[]), + TitleNode(title="B", level=1, printed_page=None, children=[]), + ] + ctx = _ctx() + + def probe(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, str]: + if kwargs["title"] == "A": + return None, [], 0, "react_give_up" + return ( + TitleMatch( + page=4, + source="react_line_grep_vlm", + matched_line="B", + candidates=[4], + evidence={"accept": "react_line_grep_vlm"}, + ), + [], + 1, + "react_line_grep_vlm", + ) + + with patch.object(npr, "_locate_with_react", side_effect=probe): + overrides, report = locate_null_page_overrides( + nodes=nodes, + match_overrides={}, + body_pages=[1, 2, 3, 4, 5], + ctx=ctx, + ) + + assert overrides[("B",)].page == 4 + assert report[0]["result"] == "react_give_up" + assert report[1]["result"] == "react_line_grep_vlm" + + +def test_null_page_react_parent_first_walk_survives_sibling_failure() -> None: + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_overrides, + ) + + nodes = [ + TitleNode(title="A", level=1, printed_page=None, children=[]), + TitleNode( + title="B", + level=1, + printed_page=None, + children=[ + TitleNode(title="B1", level=2, printed_page=None, children=[]), + TitleNode(title="B2", level=2, printed_page=None, children=[]), + ], + ), + TitleNode(title="C", level=1, printed_page=None, children=[]), + ] + ctx = _ctx() + probed: list[str] = [] + + def probe(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, str]: + title = str(kwargs["title"]) + probed.append(title) + if title == "A": + return None, [], 0, "react_give_up" + page = {"B": 10, "B1": 10, "B2": 11, "C": 15}[title] + return ( + TitleMatch( + page=page, + source="react_line_grep_vlm", + matched_line=title, + candidates=[page], + evidence={"accept": "react_line_grep_vlm"}, + ), + [], + 1, + "react_line_grep_vlm", + ) + + with patch.object(npr, "_locate_with_react", side_effect=probe): + overrides, report = locate_null_page_overrides( + nodes=nodes, + match_overrides={}, + body_pages=list(range(1, 21)), + ctx=ctx, + ) + + assert probed == ["A", "B", "B1", "B2", "C"] + assert overrides[("B",)].page == 10 + assert overrides[("B", "B1")].page == 10 + assert overrides[("B", "B2")].page == 11 + assert overrides[("C",)].page == 15 + by_path = {tuple(row["path_titles"]): row for row in report} + assert by_path[("A",)]["result"] == "react_give_up" + assert by_path[("B", "B1")]["page"] == 10 + assert by_path[("B", "B2")]["page"] == 11 + assert by_path[("C",)]["page"] == 15 + + +def test_null_page_react_probes_parent_before_child() -> None: + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_overrides, + ) + + parent = TitleNode( + title="Parent", + level=1, + printed_page=None, + children=[ + TitleNode(title="Child", level=2, printed_page=None, children=[]), + ], + ) + ctx = _ctx() + + probed: list[str] = [] + + def fail_probe(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, str]: + probed.append(str(kwargs["title"])) + return None, [], 0, "react_give_up" + + with patch.object(npr, "_locate_with_react", side_effect=fail_probe): + overrides, report = locate_null_page_overrides( + nodes=[parent], + match_overrides={}, + body_pages=[1, 2, 3, 4, 5], + ctx=ctx, + ) + + assert overrides == {} + assert probed == ["Parent", "Child"] + assert [row["path_titles"] for row in report] == [ + ["Parent"], + ["Parent", "Child"], + ] + + +def test_null_page_react_preserves_structural_parent_kind_after_prune() -> None: + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_overrides, + ) + + # After prune(keep_null=True): former parent survives as empty shell. + shell = TitleNode(title="Appendix", level=1, printed_page=None, children=[]) + leaf = TitleNode(title="RealLeaf", level=1, printed_page=None, children=[]) + ctx = _ctx() + probed: list[str] = [] + + def track(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, str]: + probed.append(str(kwargs["title"])) + return None, [], 0, "react_give_up" + + with patch.object(npr, "_locate_with_react", side_effect=track): + overrides, report = locate_null_page_overrides( + nodes=[shell, leaf], + match_overrides={}, + body_pages=[1, 2, 3, 4, 5], + ctx=ctx, + structural_parent_paths={("Appendix",)}, + ) + + assert overrides == {} + assert probed == ["Appendix", "RealLeaf"] + assert [row["kind"] for row in report] == ["parent", "leaf"] + + +def test_anchor_offset_collects_structural_parent_paths_before_prune() -> None: + """H1 production wiring: paths collected before keep_null prune.""" + parent = TitleNode( + title="P", + level=1, + printed_page=None, + children=[ + TitleNode(title="PrintedMiss", level=2, printed_page=10, children=[]), + ], + ) + captured: dict[str, Any] = {} + + def fake_leaf(**kwargs: Any) -> tuple[dict, list]: + captured["structural_parent_paths"] = set( + kwargs.get("structural_parent_paths") or set() + ) + captured["nodes"] = kwargs["nodes"] + return dict(kwargs["match_overrides"]), [] + + with patch.object( + anchoring, "locate_null_page_overrides", side_effect=fake_leaf + ): + _working, _anchor = anchoring.anchor_hierarchy_from_offset( + nodes=[parent], + offset_hint=None, + page_texts={1: "x"}, + body_pages=[1, 2, 3], + page_count=3, + ctx=_ctx(), + ) + + assert ("P",) in captured["structural_parent_paths"] + # PrintedMiss dropped by keep-null prune; structural kind remains available. + assert len(captured["nodes"]) == 1 + assert captured["nodes"][0].title == "P" + assert captured["nodes"][0].children == [] + +def test_null_page_whole_line_grep_excludes_pages_outside_body() -> None: + """H2: grep page map is body_pages ∩ [left, right] (TOC pages dropped).""" + from app.services.document_agent.pdf_text import PageTextBands + from app.services.document_agent.structure.null_page_react import _whole_line_grep + + ctx = _ctx() + ctx.blackboard.page_count = 10 + # Page 1 is TOC-excluded; title only there would inflate hits without filter. + ctx.blackboard.page_full_text_cache = { + 1: PageTextBands(content="Appendix F Overview\nAppendix F Overview"), + 2: PageTextBands(content="noise"), + 5: PageTextBands(content="Appendix F Overview"), + } + status, needle, hit_pages, line_count = _whole_line_grep( + ctx=ctx, + query="Appendix F Overview", + left=1, + right=10, + body_pages=[2, 3, 4, 5, 6], + ) + assert status == "ok" + assert needle + assert hit_pages == [5] + assert line_count == 1 + + +def test_null_page_react_planner_history_omits_runtime_audit_fields() -> None: + from app.services.document_agent.structure.null_page_react import _react_history_item + + history = _react_history_item( + { + "action": "grep", + "query": "Furniture Item Specific Guidelines", + "normalized_query": "furniture item specific guidelines", + "hit_page_count": 1, + "line_match_count": 2, + "observation": "visual_rejected", + "visual_selected_page": None, + "visual_pages_checked": [304], + "post_strip": "header", + "seed_full_title": True, + } + ) + assert history == { + "action": "grep", + "query": "Furniture Item Specific Guidelines", + "normalized_query": "furniture item specific guidelines", + "hit_page_count": 1, + "observation": "visual_rejected", + } + + +def test_null_page_react_clears_page_text_search_view() -> None: + """H3: locate always clears strip view in finally (no cross-node leak).""" + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import _locate_with_react + + ctx = _ctx() + ctx.blackboard.page_count = 5 + ctx.blackboard.page_full_text_cache = {page: "X" for page in range(1, 6)} + ctx.blackboard.page_text_search_view = {1: "stale-before"} + + def dirty_grep(**kwargs: Any) -> tuple[str, str, list[int], int]: + ctx.blackboard.page_text_search_view = {1: "dirty-during"} + return "ok", "x", [], 0 + + with ( + patch.object(npr, "_whole_line_grep", side_effect=dirty_grep), + patch.object( + npr, + "_propose_react_query", + return_value=({"action": "give_up", "query": ""}, {}), + ), + ): + match, _attempts, _visual, result = _locate_with_react( + path_titles=("T",), + title="T", + left=1, + right=5, + body_pages=[1, 2, 3, 4, 5], + ctx=ctx, + ) + + assert match is None + assert result == "react_give_up" + assert ctx.blackboard.page_text_search_view is None + + +def test_null_page_react_progresses_windows_until_candidate_is_confirmed() -> None: + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import _locate_with_react + + ctx = _ctx() + verified_batches: list[list[int]] = [] + + def fake_grep(**kwargs: Any) -> tuple[str, str, list[int], int]: + return "ok", "appendix a", [260, 262, 280], 3 + + def fake_verify(**kwargs: Any) -> tuple[str, int | None, str, int]: + pages = list(kwargs["pages"]) + verified_batches.append(pages) + selected = 262 if 262 in pages else None + return "ok", selected, "checked", 0 + + with ( + patch.object(npr, "_whole_line_grep", side_effect=fake_grep), + patch.object(npr, "_verify_section_beginning_pages", side_effect=fake_verify), + ): + match, attempts, visual_calls, result = _locate_with_react( + path_titles=("Appendix A",), + title="Appendix A", + left=250, + right=300, + body_pages=list(range(250, 301)), + ctx=ctx, + ) + + assert match is not None + assert match.page == 262 + assert result == "react_line_grep_vlm" + assert verified_batches == [[260], [262]] + assert visual_calls == 2 + assert attempts[0]["observation"] == "section_start_confirmed" + assert attempts[0]["visual_pages_checked"] == [260, 262] + assert attempts[0]["visual_rounds"][0]["physical_window"] == [260, 261] + assert attempts[0]["visual_rounds"][1]["physical_window"] == [262, 265] + + +def test_react_planner_grep_budget_is_not_page_constant() -> None: + """M5: dedicated grep budget; seed free; max turns = budget + 2 (strips).""" + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + REACT_PLANNER_GREP_BUDGET, + _locate_with_react, + react_budget, + ) + + assert REACT_PLANNER_GREP_BUDGET == 5 + assert react_budget() == REACT_PLANNER_GREP_BUDGET + + ctx = _ctx() + planner_turns = {"n": 0} + + def fake_grep(**kwargs: Any) -> tuple[str, str, list[int], int]: + # Seed + any strip re-grep: empty hits, never consume via consume_budget=False path + return "ok", "needle", [], 0 + + def fake_propose(**kwargs: Any) -> tuple[dict[str, Any] | None, dict[str, Any]]: + planner_turns["n"] += 1 + return None, {"error": "planner failed"} + + with ( + patch.object(npr, "react_budget", return_value=1), + patch.object(npr, "_whole_line_grep", side_effect=fake_grep), + patch.object(npr, "_propose_react_query", side_effect=fake_propose), + ): + _match, attempts, _visual, result = _locate_with_react( + path_titles=("T",), + title="T", + left=1, + right=5, + body_pages=[1, 2, 3, 4, 5], + ctx=ctx, + ) + + # max_planner_turns = budget + 2 = 3 (not budget + 2 + budget). + assert planner_turns["n"] == 3 + assert result == "react_loop_limit" + assert all(a.get("action") == "planner_error" or a.get("action") == "grep" for a in attempts) + +def test_null_page_grep_observations_distinguish_error_empty_duplicate() -> None: + """M1: tool error / empty needle / true duplicate are distinct labels.""" + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import _locate_with_react + + ctx = _ctx() + calls = {"n": 0} + + def fake_grep(**kwargs: Any) -> tuple[str, str, list[int], int]: + calls["n"] += 1 + if calls["n"] == 1: + return "error", "grep.text failed", [], 0 + if calls["n"] == 2: + return "ok", "", [], 0 + if calls["n"] == 3: + return "ok", "same needle", [5], 1 + return "ok", "same needle", [5], 1 + + proposals = iter( + [ + ({"action": "grep", "query": "q2"}, {}), + ({"action": "grep", "query": "q3"}, {}), + ({"action": "grep", "query": "q3 again"}, {}), + ({"action": "give_up", "query": ""}, {}), + ] + ) + + def fake_propose(**kwargs: Any) -> tuple[dict[str, Any] | None, dict[str, Any]]: + return next(proposals) + + def fake_verify(**kwargs: Any) -> tuple[str, int | None, str, int]: + return "ok", None, "reject", 0 + + with ( + patch.object(npr, "_whole_line_grep", side_effect=fake_grep), + patch.object(npr, "_propose_react_query", side_effect=fake_propose), + patch.object(npr, "_verify_section_beginning_pages", side_effect=fake_verify), + patch.object(npr, "react_budget", return_value=5), + ): + _match, attempts, _visual, _result = _locate_with_react( + path_titles=("T",), + title="T", + left=1, + right=10, + body_pages=list(range(1, 11)), + ctx=ctx, + ) + + observations = [a.get("observation") for a in attempts if a.get("action") == "grep"] + assert observations[0] == "grep_tool_error" + assert observations[1] == "empty_normalized_query" + assert observations[2] == "visual_rejected" + assert observations[3] == "duplicate_normalized_query" + + +def test_null_page_locate_summary_splits_leaf_and_parent() -> None: + """M2: C4 locate_summary keeps leaf and parent buckets separate.""" + from app.services.page_memory.skeleton_extractor import _null_page_locate_bucket + + report = [ + { + "kind": "leaf", + "page": 5, + "result": "react_line_grep_vlm", + "visual_verify_calls": 1, + }, + { + "kind": "leaf", + "page": None, + "result": "unresolved", + "visual_verify_calls": 0, + }, + { + "kind": "parent", + "page": 4, + "result": "react_line_grep_vlm", + "visual_verify_calls": 2, + }, + ] + leaf = _null_page_locate_bucket(report, kind="leaf") + parent = _null_page_locate_bucket(report, kind="parent") + assert leaf["attempted"] == 2 + assert leaf["located"] == 1 + assert leaf["unresolved"] == 1 + assert leaf["visual_verify_calls"] == 1 + assert parent["attempted"] == 1 + assert parent["located"] == 1 + assert parent["visual_verify_calls"] == 2 + assert [e["kind"] for e in leaf["entries"]] == ["leaf", "leaf"] + assert [e["kind"] for e in parent["entries"]] == ["parent"] + + +def test_null_page_react_hit_writes_override() -> None: + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_overrides, + ) + + leaf = TitleNode(title="Appendix B", level=1, printed_page=None, children=[]) + ctx = _ctx() + match = TitleMatch( + page=12, + source="react_line_grep_vlm", + matched_line="Appendix B", + candidates=[12], + evidence={ + "accept": "react_line_grep_vlm", + "null_page_react": True, + "normalized_query": "appendix b", + }, + ) + + def fake_react(**kwargs: Any) -> tuple[TitleMatch | None, list[dict[str, Any]], int, str]: + assert kwargs["left"] == 1 + assert kwargs["right"] == 20 + return match, [{"loop": 1, "observation": "section_start_confirmed"}], 1, ( + "react_line_grep_vlm" + ) + + with patch.object(npr, "_locate_with_react", side_effect=fake_react): + overrides, report = locate_null_page_overrides( + nodes=[leaf], + match_overrides={}, + body_pages=list(range(1, 21)), + ctx=ctx, + ) + + assert overrides[("Appendix B",)].page == 12 + assert overrides[("Appendix B",)].source == "react_line_grep_vlm" + assert report[0]["page"] == 12 + assert report[0]["result"] == "react_line_grep_vlm" + + +def test_null_page_parent_without_descendant_anchor_uses_inherited_scope() -> None: + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_overrides, + ) + + parent = TitleNode( + title="Chapter", + level=1, + printed_page=None, + children=[TitleNode(title="Orphan", level=2, printed_page=None, children=[])], + ) + overrides, report = locate_null_page_overrides( + nodes=[parent], + match_overrides={}, + body_pages=[1, 2, 3], + ctx=None, + ) + assert overrides == {} + assert report[0]["path_titles"] == ["Chapter"] + assert report[0]["search_scope"] == [1, 3] + assert report[0]["result"] == "unresolved_no_ctx" + + +def test_null_page_parent_uses_first_descendant_as_right_boundary() -> None: + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_overrides, + ) + child = TitleNode(title="1.1 Detail", level=2, printed_page=5, children=[]) parent = TitleNode( title="1 Overview", @@ -105,27 +670,41 @@ def test_null_page_parent_located_via_compact_text() -> None: [(("1 Overview", "1.1 Detail"), child)], offset=0, ) - page_texts = { - 4: "noise", - 5: "1 Overview\n1.1 Detail\nbody", - 6: "more", - } - overrides, report = anchoring.locate_null_page_parent_overrides( - nodes=[parent], - match_overrides=leaf_match, - page_texts=page_texts, - body_pages=[4, 5, 6], - ctx=None, - ) + def probe(**kwargs: Any) -> tuple[TitleMatch, list[dict[str, Any]], int, str]: + assert kwargs["title"] == "1 Overview" + assert kwargs["left"] == 4 + assert kwargs["right"] == 5 + return ( + TitleMatch( + page=4, + source="react_line_grep_vlm", + matched_line="1 Overview", + candidates=[4], + evidence={"accept": "react_line_grep_vlm"}, + ), + [], + 1, + "react_line_grep_vlm", + ) + + with patch.object(npr, "_locate_with_react", side_effect=probe): + overrides, report = locate_null_page_overrides( + nodes=[parent], + match_overrides=leaf_match, + body_pages=[4, 5, 6], + ctx=_ctx(), + ) assert ("1 Overview",) in overrides - assert overrides[("1 Overview",)].page == 5 - assert report[0]["result"] != "unresolved" - assert report[0]["page"] == 5 - assert report[0]["window"] == [1, 5] + assert overrides[("1 Overview",)].page == 4 + assert report[0]["search_scope"] == [4, 5] -def test_first_sibling_null_parent_uses_scan_forward_not_wide_rtl() -> None: - """No left sibling: miss text → ``scan_title_forward`` within 2+4+6+10 budget.""" +def test_root_first_null_parent_uses_body_start_as_left() -> None: + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_overrides, + ) + child = TitleNode(title="22.1 Intro", level=2, printed_page=278, children=[]) parent = TitleNode( title="Chapter 22", @@ -136,58 +715,93 @@ def test_first_sibling_null_parent_uses_scan_forward_not_wide_rtl() -> None: leaf_match = { ("Chapter 22", "22.1 Intro"): TitleMatch( page=278, - source="test", + source="anchored", matched_line="", candidates=[278], evidence={}, ) } body_pages = list(range(1, 301)) - page_texts = {page: "noise" for page in body_pages} - ctx = _ctx() - - scanned_starts: list[int] = [] - - def fake_scan(**kwargs: Any) -> Any: - from app.services.document_agent.calibration.scan import TitleScanResult - scanned_starts.append(int(kwargs["start_page"])) - assert int(kwargs["page_count"]) == 278 - assert int(kwargs["start_page"]) == anchoring._first_sibling_null_parent_scan_start( - 278 + def probe(**kwargs: Any) -> tuple[TitleMatch, list[dict[str, Any]], int, str]: + assert kwargs["left"] == 1 + assert kwargs["right"] == 278 + return ( + TitleMatch( + page=270, + source="react_line_grep_vlm", + matched_line="Chapter 22", + candidates=[270], + evidence={"accept": "react_line_grep_vlm"}, + ), + [], + 1, + "react_line_grep_vlm", ) - return TitleScanResult( - title=str(kwargs["title"]), - found=True, - found_page=270, - scanned_pages=list(range(int(kwargs["start_page"]), 271)), - next_start=271, + + with patch.object(npr, "_locate_with_react", side_effect=probe): + overrides, report = locate_null_page_overrides( + nodes=[parent], + match_overrides=leaf_match, + body_pages=body_pages, + ctx=_ctx(), ) - with patch( - "app.services.document_agent.calibration.scan.scan_title_forward", - side_effect=fake_scan, - ): - with patch.object(anchoring, "_visual_rtl_locate_parent") as rtl: - overrides, report = anchoring.locate_null_page_parent_overrides( - nodes=[parent], - match_overrides=leaf_match, - page_texts=page_texts, - body_pages=body_pages, - ctx=ctx, + assert overrides[("Chapter 22",)].page == 270 + assert report[0]["search_scope"] == [1, 278] + + +def test_null_child_of_unanchored_printed_parent_uses_preorder_cursor() -> None: + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_overrides, + ) + + prior = TitleNode(title="E11 Utilities", level=1, printed_page=250, children=[]) + parent = TitleNode( + title="Appendices", + level=1, + printed_page=261, + children=[ + TitleNode( + title="A City palette maps", + level=2, + printed_page=None, + children=[], ) - rtl.assert_not_called() + ], + ) - assert scanned_starts == [anchoring._first_sibling_null_parent_scan_start(278)] - assert overrides[("Chapter 22",)].page == 270 - assert report[0]["accept"] == "scan_forward" - assert report[0]["window"] == [ - anchoring._first_sibling_null_parent_scan_start(278), - 278, - ] + def probe(**kwargs: Any) -> tuple[None, list[dict[str, Any]], int, str]: + assert kwargs["left"] == 250 + assert kwargs["right"] == 443 + return None, [], 0, "react_give_up" + + with patch.object(npr, "_locate_with_react", side_effect=probe): + _overrides, report = locate_null_page_overrides( + nodes=[prior, parent], + match_overrides={ + ("E11 Utilities",): TitleMatch( + page=250, + source="bulk_offset", + matched_line="", + candidates=[250], + evidence={"offset": 0, "printed_page": 250}, + ) + }, + body_pages=list(range(250, 444)), + ctx=_ctx(), + ) + assert report[0]["search_scope"] == [250, 443] + + +def test_null_page_parent_uses_previous_sibling_last_leaf_as_left() -> None: + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_overrides, + ) -def test_null_page_parent_with_left_sibling_still_uses_rtl() -> None: left_child = TitleNode(title="A.1", level=2, printed_page=10, children=[]) left = TitleNode(title="A", level=1, printed_page=10, children=[left_child]) right_child = TitleNode(title="B.1", level=2, printed_page=50, children=[]) @@ -195,60 +809,168 @@ def test_null_page_parent_with_left_sibling_still_uses_rtl() -> None: overrides_in = { ("A",): TitleMatch( page=10, - source="test", + source="anchored", matched_line="", candidates=[10], evidence={}, ), ("A", "A.1"): TitleMatch( page=10, - source="test", + source="anchored", matched_line="", candidates=[10], evidence={}, ), ("B", "B.1"): TitleMatch( page=50, - source="test", + source="anchored", matched_line="", candidates=[50], evidence={}, ), } - page_texts = {p: "noise" for p in range(1, 61)} - ctx = _ctx() - - def fake_rtl(**kwargs: Any) -> tuple[TitleMatch, int]: + def probe(**kwargs: Any) -> tuple[TitleMatch, list[dict[str, Any]], int, str]: assert kwargs["left"] == 10 assert kwargs["right"] == 50 return ( TitleMatch( page=40, - source="inspect_vlm", - matched_line="", + source="react_line_grep_vlm", + matched_line="B", candidates=[40], - evidence={"accept": "visual_rtl"}, + evidence={"accept": "react_line_grep_vlm"}, ), - 3, + [], + 1, + "react_line_grep_vlm", ) - with patch( - "app.services.document_agent.calibration.scan.scan_title_forward" - ) as scan: - with patch.object( - anchoring, "_visual_rtl_locate_parent", side_effect=fake_rtl - ): - overrides, report = anchoring.locate_null_page_parent_overrides( - nodes=[left, right], - match_overrides=overrides_in, - page_texts=page_texts, - body_pages=list(range(1, 61)), - ctx=ctx, - ) - scan.assert_not_called() + with patch.object(npr, "_locate_with_react", side_effect=probe): + overrides, report = locate_null_page_overrides( + nodes=[left, right], + match_overrides=overrides_in, + body_pages=list(range(1, 61)), + ctx=_ctx(), + ) assert overrides[("B",)].page == 40 - assert report[0]["accept"] == "visual_rtl" + assert report[0]["search_scope"] == [10, 50] + + +def test_confirmed_null_parent_becomes_child_left_boundary() -> None: + from app.services.document_agent.structure import null_page_react as npr + from app.services.document_agent.structure.null_page_react import ( + locate_null_page_overrides, + ) + + detail = TitleNode(title="3.1.1 Detail", level=3, printed_page=304, children=[]) + child = TitleNode( + title="3.1 Street Furniture with Advertising", + level=2, + printed_page=None, + children=[detail], + ) + parent = TitleNode( + title="3 Advertising", + level=1, + printed_page=None, + children=[child], + ) + overrides_in = { + ("3 Advertising", "3.1 Street Furniture with Advertising", "3.1.1 Detail"): TitleMatch( + page=304, + source="anchored", + matched_line="", + candidates=[304], + evidence={}, + ) + } + scopes: list[tuple[str, int, int]] = [] + + def probe(**kwargs: Any) -> tuple[TitleMatch, list[dict[str, Any]], int, str]: + title = str(kwargs["title"]) + scopes.append((title, int(kwargs["left"]), int(kwargs["right"]))) + return ( + TitleMatch( + page=304, + source="react_line_grep_vlm", + matched_line=title, + candidates=[304], + evidence={"accept": "react_line_grep_vlm"}, + ), + [], + 1, + "react_line_grep_vlm", + ) + + with patch.object(npr, "_locate_with_react", side_effect=probe): + overrides, report = locate_null_page_overrides( + nodes=[parent], + match_overrides=overrides_in, + body_pages=list(range(280, 330)), + ctx=_ctx(), + ) + + assert scopes == [ + ("3 Advertising", 280, 304), + ("3.1 Street Furniture with Advertising", 304, 304), + ] + assert overrides[("3 Advertising",)].page == 304 + assert overrides[("3 Advertising", "3.1 Street Furniture with Advertising")].page == 304 + assert [row["kind"] for row in report] == ["parent", "parent"] + + +def test_production_flow_calls_one_parent_first_null_page_walker() -> None: + child = TitleNode(title="Child", level=2, printed_page=None, children=[]) + parent = TitleNode( + title="Parent", + level=1, + printed_page=None, + children=[child], + ) + calls: list[str] = [] + + def fake_locate( + **kwargs: Any, + ) -> tuple[dict[tuple[str, ...], TitleMatch], list[dict[str, Any]]]: + calls.append("unified") + overrides = dict(kwargs["match_overrides"]) + overrides[("Parent",)] = TitleMatch( + page=4, + source="anchored", + matched_line="Parent", + candidates=[4], + evidence={}, + ) + overrides[("Parent", "Child")] = TitleMatch( + page=5, + source="anchored", + matched_line="Child", + candidates=[5], + evidence={}, + ) + return overrides, [ + {"kind": "parent", "page": 4}, + {"kind": "leaf", "page": 5}, + ] + + with patch.object( + anchoring, "locate_null_page_overrides", side_effect=fake_locate + ): + resolved, anchor = anchoring.anchor_hierarchy_from_offset( + nodes=[parent], + offset_hint=None, + calibration_overrides={}, + page_texts={page: "noise" for page in range(1, 11)}, + body_pages=list(range(1, 11)), + page_count=10, + ctx=None, + ) + + assert calls == ["unified"] + assert [node.title for node in resolved] == ["Parent"] + assert set(anchor.match_overrides) == {("Parent",), ("Parent", "Child")} + assert [row["kind"] for row in anchor.null_page_report] == ["parent", "leaf"] def test_phase2_bulk_via_mocked_offset() -> None: @@ -399,6 +1121,98 @@ def fake_verify(**kwargs: Any) -> dict[str, Any]: assert bp == -1 +def test_regimes_bulk_count_excludes_null_page_react_hits() -> None: + """Bulk count freezes before unified ReAct; its hits stay in the report.""" + from app.services.document_agent.calibration import procedure as proc + from app.services.document_agent.calibration.procedure import ( + anchor_hierarchy_from_regimes, + ) + from app.services.document_agent.calibration.types import ( + CalibrationRegime, + CalibrationResult, + CalibrationSample, + ) + + nodes = [ + _leaf("Ch1", 1), + TitleNode(title="NullLeaf", level=1, printed_page=None, children=[]), + ] + phase1 = CalibrationResult( + status="ok", + offset=10, + regimes=[ + CalibrationRegime( + kind="decimal", + offset=10, + offset_status="ok", + entry_indices=[0, 1], + samples=[CalibrationSample(title="Ch1", physical=11)], + ) + ], + ) + bulk_match = TitleMatch( + page=11, + source="bulk_offset", + matched_line="Ch1", + candidates=[11], + evidence={}, + ) + react_match = TitleMatch( + page=20, + source="react_line_grep_vlm", + matched_line="NullLeaf", + candidates=[20], + evidence={"accept": "react_line_grep_vlm"}, + ) + + def fake_offset(**kwargs: Any) -> dict[tuple[str, ...], TitleMatch]: + return {("Ch1",): bulk_match} + + def fake_apply(**kwargs: Any) -> tuple[list, dict, list, int, int]: + out = dict(kwargs["match_overrides"]) + pre_react = len(out) + out[("NullLeaf",)] = react_match + return ( + list(kwargs["nodes"]), + out, + [ + { + "path_titles": ["NullLeaf"], + "kind": "leaf", + "page": 20, + "result": "react_line_grep_vlm", + } + ], + 0, + pre_react, + ) + + with ( + patch.object(proc, "offset_guided_anchoring", side_effect=fake_offset), + patch.object(proc, "apply_null_page_locates_and_prune", side_effect=fake_apply), + ): + _working, anchor = anchor_hierarchy_from_regimes( + nodes=nodes, + result=phase1, + entries=[ + {"heading": "Ch1", "level": 1, "page_number": 1}, + {"heading": "NullLeaf", "level": 1, "page_number": None}, + ], + page_texts={11: "Ch1", 20: "NullLeaf"}, + body_pages=list(range(1, 30)), + page_count=30, + ctx=_ctx(), + ) + + assert ("Ch1",) in anchor.match_overrides + assert ("NullLeaf",) in anchor.match_overrides + assert len(anchor.match_overrides) == 2 + assert anchor.bulk_count == 1 + assert any( + row.get("path_titles") == ["NullLeaf"] for row in anchor.null_page_report + ) + + def test_phase2_all_bisect_fail_does_not_invent_first_leaf() -> None: """When every Phase-2 probe fails, do not bulk-anchor the first TOC leaf.""" from app.services.document_agent.calibration.procedure import ( @@ -651,7 +1465,7 @@ def _printed_page_parent(title: str, printed: int, child: TitleNode) -> TitleNod return TitleNode(title=title, level=1, printed_page=printed, children=[child]) -def test_parent_backfill_uses_descendant_regime_offset() -> None: +def test_printed_backfill_uses_nearest_doc_order_offset() -> None: from app.services.document_agent.structure import anchoring_primitives as primitives early_child = TitleNode(title="A.1", level=2, printed_page=12, children=[]) @@ -663,7 +1477,7 @@ def test_parent_backfill_uses_descendant_regime_offset() -> None: **primitives.bulk_offset_matches([(("Section B", "B.1"), late_child)], 9), } - parents = primitives.backfill_parent_offset_matches( + parents = primitives.backfill_printed_offset_matches( nodes=[section_a, section_b], matches=matches, page_count=60, @@ -671,27 +1485,99 @@ def test_parent_backfill_uses_descendant_regime_offset() -> None: assert parents[("Section A",)].page == 15 assert parents[("Section B",)].page == 49 - assert parents[("Section A",)].evidence["parent_backfill"] is True + assert parents[("Section A",)].evidence["printed_offset_backfill"] is True + + +def test_printed_backfill_ignores_react_hits_without_offset() -> None: + """ReAct leaves carry no offset; nearest offset-bearing neighbor wins.""" + from app.services.document_agent.structure import anchoring_primitives as primitives + + react_child = TitleNode(title="Intro", level=2, printed_page=None, children=[]) + printed_child = TitleNode(title="Body", level=2, printed_page=12, children=[]) + section = TitleNode( + title="Section A", + level=1, + printed_page=10, + children=[react_child, printed_child], + ) + matches = { + ("Section A", "Intro"): TitleMatch( + page=99, + source="react_line_grep_vlm", + matched_line="Intro", + candidates=[99], + evidence={"accept": "react_line_grep_vlm", "null_page_react": True}, + ), + **primitives.bulk_offset_matches( + [(("Section A", "Body"), printed_child)], 5 + ), + } + + parents = primitives.backfill_printed_offset_matches( + nodes=[section], + matches=matches, + page_count=200, + ) + + assert parents[("Section A",)].page == 15 + assert parents[("Section A",)].evidence["offset"] == 5 -def test_parent_backfill_skips_unanchored_and_out_of_range() -> None: +def test_printed_backfill_unresolved_when_no_offset_bearing_neighbor() -> None: from app.services.document_agent.structure import anchoring_primitives as primitives - tail_child = TitleNode(title="Tail.1", level=2, printed_page=96, children=[]) - ghost_child = TitleNode(title="Ghost.1", level=2, printed_page=11, children=[]) - tail = _printed_page_parent("Tail", 95, tail_child) - ghost = _printed_page_parent("Ghost", 10, ghost_child) - matches = primitives.bulk_offset_matches([(("Tail", "Tail.1"), tail_child)], 8) + react_child = TitleNode(title="Intro", level=2, printed_page=None, children=[]) + section = _printed_page_parent("Section A", 10, react_child) + matches = { + ("Section A", "Intro"): TitleMatch( + page=99, + source="react_line_grep_vlm", + matched_line="Intro", + candidates=[99], + evidence={"accept": "react_line_grep_vlm", "null_page_react": True}, + ) + } - parents = primitives.backfill_parent_offset_matches( - nodes=[tail, ghost], + parents = primitives.backfill_printed_offset_matches( + nodes=[section], matches=matches, - page_count=100, + page_count=200, ) assert parents == {} +def test_printed_backfill_uses_preceding_neighbor_and_skips_out_of_range() -> None: + from app.services.document_agent.structure import anchoring_primitives as primitives + + prior_leaf = TitleNode(title="E11", level=1, printed_page=250, children=[]) + appendices = TitleNode( + title="Appendices", + level=1, + printed_page=261, + children=[ + TitleNode(title="A maps", level=2, printed_page=None, children=[]), + ], + ) + tail_child = TitleNode(title="Tail.1", level=2, printed_page=440, children=[]) + tail = _printed_page_parent("Tail", 439, tail_child) + matches = { + **primitives.bulk_offset_matches([(("E11",), prior_leaf)], 0), + **primitives.bulk_offset_matches([(("Tail", "Tail.1"), tail_child)], 8), + } + + filled = primitives.backfill_printed_offset_matches( + nodes=[prior_leaf, appendices, tail], + matches=matches, + page_count=443, + ) + + assert filled[("Appendices",)].page == 261 + assert filled[("Appendices",)].evidence["offset"] == 0 + # 439 + 8 = 447 exceeds page_count + assert ("Tail",) not in filled + + def test_multi_regime_phase2_merges_physical_overrides() -> None: """Roman + decimal + prefixed each apply their own offset → physical pages.""" from app.services.document_agent.calibration.procedure import ( diff --git a/apps/worker/tests/contract/test_tool_registry_smoke_contract.py b/apps/worker/tests/contract/test_tool_registry_smoke_contract.py index 031266c02..6316f76ff 100644 --- a/apps/worker/tests/contract/test_tool_registry_smoke_contract.py +++ b/apps/worker/tests/contract/test_tool_registry_smoke_contract.py @@ -12,7 +12,10 @@ os.environ.setdefault("S3_TEMP_PATH", "/tmp") import app.services.document_agent.tools as _tools # noqa: F401 +from app.services.document_agent.manifest import ToolContext from app.services.document_agent.registry import REGISTRY +from app.services.document_agent.state import ProfileBlackboard +from app.services.document_agent.tools.grep_text import grep_text from app.services.document_agent.tools.inspect_pages import inspect_pages @@ -24,6 +27,8 @@ def test_probe_and_inspect_registered() -> None: "inspect.pages", "ocr.pages", "grep.text", + "text.strip_header", + "text.strip_footer", ): assert REGISTRY.get(name) is not None, name @@ -34,5 +39,123 @@ def test_inspect_pages_handler_is_same_callable() -> None: assert spec.handler is inspect_pages +def test_grep_text_normalizes_query_and_corpus_by_default() -> None: + blackboard = ProfileBlackboard(page_count=2) + blackboard.page_full_text_cache = { + 1: "Public\nDomain Manual", + 2: "附录\nA OVERVIEW", + } + ctx = ToolContext( + pdf_path="/tmp/doc.pdf", + job_id="grep-normalization", + blackboard=blackboard, + trace=None, + settings={}, + ) + + result = grep_text( + ctx, + {"query": "附录 A OVERVIEW", "start_page": 2, "end_page": 2}, + ) + + assert result.status == "ok" + assert result.payload["normalized_query"] == "附录a overview" + assert result.payload["hit_page_count"] == 1 + assert result.payload["hit_pages"] == [2] + + +def test_grep_text_whole_line_rejects_body_substring_and_dedupes_pages() -> None: + blackboard = ProfileBlackboard(page_count=2) + blackboard.page_full_text_cache = { + 1: ( + "The street furniture with advertising program continues.\n" + "Street Furniture with Advertising\n" + "Street Furniture with Advertising" + ), + 2: "No heading here", + } + ctx = ToolContext( + pdf_path="/tmp/doc.pdf", + job_id="grep-whole-line", + blackboard=blackboard, + trace=None, + settings={}, + ) + + result = grep_text( + ctx, + { + "query": "Street Furniture with Advertising", + "whole_line": True, + }, + ) + + assert result.status == "ok" + assert result.payload["whole_line"] is True + assert result.payload["hit_count"] == 2 + assert result.payload["hit_page_count"] == 1 + assert result.payload["hit_pages"] == [1] + + +def test_strip_footer_updates_search_view_for_grep() -> None: + # Import inside the test so PageTextBands / grep_text share one module + # identity after worker_contract_environment clears app.* mid-suite. + from app.services.document_agent.pdf_text import PageTextBands + from app.services.document_agent.tools.grep_text import grep_text as grep_text_fn + from app.services.document_agent.tools.text_strip_margins import strip_footer + + blackboard = ProfileBlackboard(page_count=1) + blackboard.page_full_text_cache = { + 1: PageTextBands( + content="Section Start\nPublic Domain Manual", + header="", + footer="Public Domain Manual", + ) + } + ctx = ToolContext( + pdf_path="/tmp/doc.pdf", + job_id="strip-footer", + blackboard=blackboard, + trace=None, + settings={}, + ) + + before = grep_text_fn( + ctx, {"query": "Public Domain Manual", "start_page": 1, "end_page": 1} + ) + assert before.status == "ok" + assert before.payload["hit_count"] == 1 + + strip = strip_footer(ctx, {"start_page": 1, "end_page": 1}) + assert strip.status == "ok" + assert strip.payload["pages_updated"] == 1 + assert ctx.blackboard.page_text_search_view[1] == "Section Start\n" + + after = grep_text_fn( + ctx, {"query": "Public Domain Manual", "start_page": 1, "end_page": 1} + ) + assert after.status == "ok" + assert after.payload["hit_count"] == 0 + + # Stored cache must remain untouched. + assert blackboard.page_full_text_cache[1].content == "Section Start\nPublic Domain Manual" + + +def test_strip_margin_text_edge_aligned_not_first_occurrence() -> None: + """M4: footer/header strip only the matching edge, not body duplicates.""" + from app.services.document_agent.pdf_text import strip_margin_text + + body_and_footer = "Public Domain Manual\nSection body\nPublic Domain Manual" + assert ( + strip_margin_text(body_and_footer, "Public Domain Manual", edge="footer") + == "Public Domain Manual\nSection body\n" + ) + body_and_header = "Running Header\nSection body\nRunning Header" + assert ( + strip_margin_text(body_and_header, "Running Header", edge="header") + == "\nSection body\nRunning Header" + ) + + def test_openai_specs_removed() -> None: - assert not hasattr(REGISTRY, "openai_specs") + assert not hasattr(REGISTRY, "openai_specs") \ No newline at end of file