diff --git a/src/bigos/eval/omnidocbench.py b/src/bigos/eval/omnidocbench.py index c50d39d..8981f1c 100644 --- a/src/bigos/eval/omnidocbench.py +++ b/src/bigos/eval/omnidocbench.py @@ -113,7 +113,11 @@ def _merge_truncated_json2md( return sorted(annos, key=lambda x: int(x.get("order") or 0)) relations = extra.get("relation") if not isinstance(relations, list): - return annos + # Reference ``tools/json2md.py`` always sorts by ``order`` after the + # truncated merge step. When ``extra`` is present but has no usable + # ``relation`` list, fall through to the no-op sort path so the GT + # markdown still reflects reading order rather than manifest order. + return sorted(annos, key=lambda x: int(x.get("order") or 0)) truncated_all: dict[str, dict[str, Any]] = {} related_truncated: list[list[str]] = [] diff --git a/tests/test_eval_omnidocbench.py b/tests/test_eval_omnidocbench.py index ba0daac..cf09bbd 100644 --- a/tests/test_eval_omnidocbench.py +++ b/tests/test_eval_omnidocbench.py @@ -180,6 +180,45 @@ def test_gt_markdown_for_eval_prefers_explicit_fields() -> None: assert _gt_markdown_for_eval(row, "json2md") == "FROM_ROW" +def test_extra_without_relation_still_sorts_by_order() -> None: + """Regression: if ``row['extra']`` is a dict without a usable ``relation`` + list (e.g. only other keys, or ``relation`` set to ``None``), the json2md + GT must still sort blocks by ``order``. Previously this branch returned + annos in manifest order, breaking GT reading order and skewing CER/NED.""" + row_no_relation_key = { + "page_info": {"image_path": "p.png"}, + "layout_dets": [ + {"category_type": "text_block", "text": "second", "order": 2}, + {"category_type": "text_block", "text": "first", "order": 1}, + ], + "extra": {"unrelated_key": "x"}, + } + gt = gt_markdown_json2md(row_no_relation_key) + assert gt.index("first") < gt.index("second") + + row_relation_none = { + "page_info": {"image_path": "p.png"}, + "layout_dets": [ + {"category_type": "text_block", "text": "second", "order": 2}, + {"category_type": "text_block", "text": "first", "order": 1}, + ], + "extra": {"relation": None}, + } + gt2 = gt_markdown_json2md(row_relation_none) + assert gt2.index("first") < gt2.index("second") + + row_relation_dict = { + "page_info": {"image_path": "p.png"}, + "layout_dets": [ + {"category_type": "text_block", "text": "second", "order": 2}, + {"category_type": "text_block", "text": "first", "order": 1}, + ], + "extra": {"relation": {"not": "a list"}}, + } + gt3 = gt_markdown_json2md(row_relation_dict) + assert gt3.index("first") < gt3.index("second") + + def test_truncated_merge_joins_text() -> None: row = { "page_info": {"image_path": "p.png"},