From 17b73ac4f5587b46b441f0899858412929ed7f39 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 May 2026 22:18:18 +0000
Subject: [PATCH] fix(eval): sort json2md GT when extra has no usable relation
 list
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

_merge_truncated_json2md skipped the final sort-by-order step when row['extra']
was present but row['extra']['relation'] was missing, None, or not a list.
Real OmniDocBench manifest rows can carry an 'extra' dict that holds keys
other than 'relation' (or sets 'relation' to a non-list value), so this
branch is reachable. When triggered, prepare_annos_json2md returned blocks
in manifest order rather than reading order, which means:

- gt_markdown_json2md emitted GT markdown out of reading order.
- _gt_markdown_for_eval (json2md strategy) fed misordered GT into NED/CER
  for those rows, biasing baseline-*.json results.
- _compute_per_category_metrics zipped pred blocks against misordered GT
  annos, attributing wrong text/table/formula NED/TEDS scores to the wrong
  GT blocks.

The reference tools/json2md.py from OmniDocBench always finishes with
sorted(merged_annos, key=lambda x: x['order']) (see
docs/eval/json2md-reference.md). Mirror that ordering in the no-relation
branch by sorting before returning.

Add a regression test covering three triggers:
- extra dict with only unrelated keys
- extra['relation'] explicitly None
- extra['relation'] set to a dict instead of a list

Co-authored-by: Bartłomiej Rosa <bartrosa@users.noreply.github.com>
---
 src/bigos/eval/omnidocbench.py  |  6 ++++-
 tests/test_eval_omnidocbench.py | 39 +++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/src/bigos/eval/omnidocbench.py b/src/bigos/eval/omnidocbench.py
index c50d39d..8981f1c 100644
--- a/src/bigos/eval/omnidocbench.py
+++ b/src/bigos/eval/omnidocbench.py
@@ -113,7 +113,11 @@ def _merge_truncated_json2md(
         return sorted(annos, key=lambda x: int(x.get("order") or 0))
     relations = extra.get("relation")
     if not isinstance(relations, list):
-        return annos
+        # Reference ``tools/json2md.py`` always sorts by ``order`` after the
+        # truncated merge step. When ``extra`` is present but has no usable
+        # ``relation`` list, fall through to the no-op sort path so the GT
+        # markdown still reflects reading order rather than manifest order.
+        return sorted(annos, key=lambda x: int(x.get("order") or 0))
 
     truncated_all: dict[str, dict[str, Any]] = {}
     related_truncated: list[list[str]] = []
diff --git a/tests/test_eval_omnidocbench.py b/tests/test_eval_omnidocbench.py
index ba0daac..cf09bbd 100644
--- a/tests/test_eval_omnidocbench.py
+++ b/tests/test_eval_omnidocbench.py
@@ -180,6 +180,45 @@ def test_gt_markdown_for_eval_prefers_explicit_fields() -> None:
     assert _gt_markdown_for_eval(row, "json2md") == "FROM_ROW"
 
 
+def test_extra_without_relation_still_sorts_by_order() -> None:
+    """Regression: if ``row['extra']`` is a dict without a usable ``relation``
+    list (e.g. only other keys, or ``relation`` set to ``None``), the json2md
+    GT must still sort blocks by ``order``. Previously this branch returned
+    annos in manifest order, breaking GT reading order and skewing CER/NED."""
+    row_no_relation_key = {
+        "page_info": {"image_path": "p.png"},
+        "layout_dets": [
+            {"category_type": "text_block", "text": "second", "order": 2},
+            {"category_type": "text_block", "text": "first", "order": 1},
+        ],
+        "extra": {"unrelated_key": "x"},
+    }
+    gt = gt_markdown_json2md(row_no_relation_key)
+    assert gt.index("first") < gt.index("second")
+
+    row_relation_none = {
+        "page_info": {"image_path": "p.png"},
+        "layout_dets": [
+            {"category_type": "text_block", "text": "second", "order": 2},
+            {"category_type": "text_block", "text": "first", "order": 1},
+        ],
+        "extra": {"relation": None},
+    }
+    gt2 = gt_markdown_json2md(row_relation_none)
+    assert gt2.index("first") < gt2.index("second")
+
+    row_relation_dict = {
+        "page_info": {"image_path": "p.png"},
+        "layout_dets": [
+            {"category_type": "text_block", "text": "second", "order": 2},
+            {"category_type": "text_block", "text": "first", "order": 1},
+        ],
+        "extra": {"relation": {"not": "a list"}},
+    }
+    gt3 = gt_markdown_json2md(row_relation_dict)
+    assert gt3.index("first") < gt3.index("second")
+
+
 def test_truncated_merge_joins_text() -> None:
     row = {
         "page_info": {"image_path": "p.png"},