From b0900e2db99dce3021511384c91903349b194a40 Mon Sep 17 00:00:00 2001 From: Jiangang Han Date: Fri, 17 Jul 2026 13:06:49 +0800 Subject: [PATCH] Fix last-screenshot selection: os.listdir order is filesystem-dependent os.listdir returns entries in arbitrary order, so taking [-1] as the "last" screenshot silently picks a wrong step on some filesystems (observed 0.png, the initial page, on macOS/APFS), causing the judge to fail tasks for "no visual evidence in final screenshot". Even lexicographic order breaks past step 9 ('9.png' > '15.png'). Sort by numeric step index instead. Co-Authored-By: Claude Fable 5 --- src/eval/naveval.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/eval/naveval.py b/src/eval/naveval.py index cef9497..847d0b1 100644 --- a/src/eval/naveval.py +++ b/src/eval/naveval.py @@ -320,7 +320,9 @@ def eval_task(dir_path, save_base_dir): task = task.split("### key points")[0] url_trajectory = list(dict.fromkeys(result_data.get("urls"))) - image_list = os.listdir(f'{dir_path}/trajectory') + # os.listdir order is arbitrary; must sort by numeric step index + # (lexicographic order would also break: '9.png' > '15.png') + image_list = sorted(os.listdir(f'{dir_path}/trajectory'), key=lambda x: int(x.split('.')[0])) last_img = f"{dir_path}/trajectory/{image_list[-1]}" with open(last_img, "rb") as f: screenshot_b64 = base64.b64encode(f.read()).decode("utf-8")