demo-data.json

{
  "meta": {
    "title": "AdaSpatial-MLLM Adaptive Latent Lab",
    "mode": "real experiment replay",
    "checkpoint": "jankin123/3DThinker-Mindcube Stage-1",
    "benchmark": "MindCube-Tiny",
    "holdout_samples": 950,
    "epsilon": 0.04,
    "patience": 2,
    "seed": 42
  },
  "cases": [
    {
      "slug": "viewpoint-left",
      "id": "among_15131f022f3c6bac68f29de62891226d69b905809770dffe1efb69a327f3f9f7_q3_5_1",
      "label": "Adaptive 9 步答对",
      "note": "Adaptive 在 hidden-state 收敛后退出，fixed-12 反而改成了错误选项。",
      "question": "Based on these four images (image 1, 2, 3, and 4) showing the coffee table from different viewpoints (front, left, back, and right), with each camera aligned with room walls and partially capturing the surroundings: From the viewpoint presented in image 4, what is to the left of the coffee table?",
      "options": [
        {
          "key": "A",
          "text": "Two blue leather armchairs"
        },
        {
          "key": "B",
          "text": "Cream-colored cabinet"
        },
        {
          "key": "C",
          "text": "Blue bin"
        },
        {
          "key": "D",
          "text": "Table and white chair"
        }
      ],
      "ground_truth": "C",
      "images": [
        "assets/sample-coffee/front_137.png",
        "assets/sample-coffee/left_171.png",
        "assets/sample-coffee/back_196.png",
        "assets/sample-coffee/right_222.png"
      ],
      "runs": {
        "fixed4": {
          "prediction": "A",
          "correct": false,
          "steps": 4,
          "forward_steps": 5,
          "exit_reason": "max_steps",
          "latency_s": 9.015515419188887,
          "output_tokens": 465,
          "peak_vram_gib": 8.134821891784668,
          "distances": [
            0.20351839065551758,
            0.06738436222076416,
            0.32673633098602295,
            0.17436909675598145
          ],
          "reasoning": "I am mentally constructing the 3D layout of this room based on the four images provided.\n\n- The central object is a coffee table, placed roughly in the middle of the scene.\n- Image 1 shows the front view: facing the table, we see a fridge and kitchen cabinets in the background, as well as two blue leather armchairs to the right.\n- Image 2 is the left view: from here, we see the two blue armchairs directly in front, and on the right side of the image there is a table and white chair.\n- Image 3 is the back view: the camera faces away from the kitchen/fridge, showing the blue armchairs on the left, and on the right a bit of the table and white chair visible.\n- Image 4 is the right view: here, we see the table and white chair in the foreground, and on the left edge of the image, two blue leather armchairs.\n\nNow, the question asks: \"From the viewpoint presented in image 4, what is to the left of the coffee table?\"\n\nTo answer this, I need to first orient myself within the 3D scene. In image 4, the camera is positioned at the right side of the table, looking towards the left. Therefore, to the left of the coffee table from this image's perspective, spatially, would be along the wall where the two blue leather armchairs are located (as seen in image 2 and partially in image 3).\n\nThis matches Answer A: Two blue leather armchairs. This is confirmed by the room's arrangement—armchairs are on one long wall, and from the right viewpoint (image 4), they are to the left of the table.\n\nOther options do not spatially align with the left-of-table position from image 4's perspective:\n- Cream-colored cabinet is on the opposite end (seen in image 1).\n- Blue bin and more tables/white chairs are nearer the back (image 3).\n- Table and white chair are in the foreground (image 4).\n\nTherefore, the answer must be the two blue leather armchairs.",
          "answer": "A. Two blue leather armchairs"
        },
        "fixed8": {
          "prediction": "A",
          "correct": false,
          "steps": 8,
          "forward_steps": 9,
          "exit_reason": "max_steps",
          "latency_s": 7.280281116720289,
          "output_tokens": 350,
          "peak_vram_gib": 8.134821891784668,
          "distances": [
            0.20351839065551758,
            0.06738436222076416,
            0.32673633098602295,
            0.17436909675598145,
            0.09666585922241211,
            0.11970794200897217,
            0.07713139057159424,
            0.006371438503265381
          ],
          "reasoning": "I am constructing a 3D layout of the room based on the four images provided, each taken from one side of the coffee table, showing context clues surrounding the furniture.\n\nImage 1 (front): Shows the coffee table in the center, with two blue leather armchairs behind it, and a fridge to the left side of the image. Cabinets are visible to the left and behind the table.\n\nImage 2 (left): The viewpoint has shifted so that we see the coffee table from the left, with the two blue armchairs now directly behind the table, and more of the room's perimeter visible.\n\nImage 3 (back): This perspective shows the rear of the coffee table, with the two blue armchairs now to the left, and a table with a chair and some electronics visible to the right.\n\nImage 4 (right): The camera is now viewing the coffee table from the right side, facing toward the table and white chair, with part of the fridge visible on the far right. This allows me to mentally place the objects in the 3D room: the white chair and table are now in front, and based on the previous images, the blue armchairs are positioned to the left of this viewpoint (relative to the table).\n\nGiven this 3D arrangement, if I stand at the position shown in image 4 and look at the coffee table, the left side relative to my viewpoint would be occupied by the area where the two blue leather armchairs are located. Thus, the answer is A. Two blue leather armchairs.",
          "answer": "A. Two blue leather armchairs"
        },
        "fixed12": {
          "prediction": "A",
          "correct": false,
          "steps": 12,
          "forward_steps": 13,
          "exit_reason": "max_steps",
          "latency_s": 8.046036839019507,
          "output_tokens": 419,
          "peak_vram_gib": 8.134821891784668,
          "distances": [
            0.20351839065551758,
            0.06738436222076416,
            0.32673633098602295,
            0.17436909675598145,
            0.09666585922241211,
            0.11970794200897217,
            0.07713139057159424,
            0.006371438503265381,
            0.006605923175811768,
            0.004562258720397949,
            0.009433269500732422,
            0.00787508487701416
          ],
          "reasoning": "I am constructing a 3D layout of the room based on the four images provided, each taken from one side of the coffee table, showing notable furniture arrangements around the table and its surroundings.\n\nImage 1 shows the front view, with the table centered and facing the background wall, which has a fridge, cabinets, and two blue chairs on the right side. There's also a chair on the left side next to a table with electronics.\n\nImage 2 is the left-side view, showing the table in the foreground and two blue armchairs along the wall behind it, plus a partial view of a round bin and more of the seating area.\n\nImage 3 is the back view, where the table is in the foreground and the background features two blue armchairs directly facing the table, a bin to the right, and part of a work desk and chair to the further right.\n\nImage 4, the right-side view, shows the table in the foreground with a single blue chair and a cream-colored cabinet to the right, and a desk with a white chair behind.\n\nThe question asks: From the viewpoint presented in image 4 (right side of the table), what is to the left of the coffee table?\n\nTo solve this, I need to mentally rotate myself into the position used for image 4, facing the table from that side. In this orientation, the \"left\" of the table (from my perspective as the camera in image 4) will be towards the back wall seen in images 2 and 3. In image 3, on the left of the table (from the right-side viewpoint), we see two blue armchairs clustered together. This matches option A.\n\nThus, aligning the 3D scene and mentally walking to the described position, the objects to the left of the table as seen from image 4 are the two blue leather armchairs.",
          "answer": "A. Two blue leather armchairs"
        },
        "adaptive": {
          "prediction": "C",
          "correct": true,
          "steps": 9,
          "forward_steps": 10,
          "exit_reason": "converged",
          "latency_s": 8.241917691193521,
          "output_tokens": 394,
          "peak_vram_gib": 8.134821891784668,
          "distances": [
            0.20351839065551758,
            0.06738436222076416,
            0.32673633098602295,
            0.17436909675598145,
            0.09666585922241211,
            0.11970794200897217,
            0.07713139057159424,
            0.006371438503265381,
            0.006605923175811768
          ],
          "reasoning": "I am constructing a 3D layout of the room based on the four images provided, each taken from one side of the coffee table, showing context clues surrounding the furniture.\n\nImage 1 (front): Shows the coffee table in the center, with two blue leather armchairs behind it (one on each side), a fridge to the left, and cabinetry/sink to the far side.\nImage 2 (left): Shows the coffee table directly, with the blue armchairs flanking either side, and a table and white chair to the right.\nImage 3 (back): Shows the rear of the coffee table, with the blue armchairs again visible (one on each side), and a bin and part of the white chair and desk to the right.\nImage 4 (right): Shows the coffee table directly, with a view of the desk and chair to the right, and the edge of a cabinet to the left.\n\nTo answer the question: \"From the viewpoint presented in image 4, what is to the left of the coffee table?\", I need to mentally place myself at the location/camera angle of image 4 (right side of the table).\n\nIf I face the table from this right-side position (image 4), the left of the table (from my viewpoint) points toward the space that is captured by image 3 (back) and image 2 (left). In image 2 (left), we see the blue bin positioned directly to the left of the coffee table, along with the table and white chair. This means, relative to the right-side viewpoint, the blue bin is on the left side of the table.\n\nIn summary: From the right viewpoint (image 4), the blue bin is located to the left of the coffee table, matching option C.",
          "answer": "C. Blue bin"
        }
      }
    },
    {
      "slug": "behind-camera",
      "id": "among_15131f022f3c6bac68f29de62891226d69b905809770dffe1efb69a327f3f9f7_q1_2_1",
      "label": "Adaptive 7 步答对",
      "note": "视角变换题中，adaptive 用 7 步找到正确空间关系，fixed-12 失败。",
      "question": "Based on these four images (image 1, 2, 3, and 4) showing the coffee table from different viewpoints (front, left, back, and right), with each camera aligned with room walls and partially capturing the surroundings: If I am standing at the same spot and facing the same direction as shown in image 2, what is behind me?",
      "options": [
        {
          "key": "A",
          "text": "Blue bin"
        },
        {
          "key": "B",
          "text": "Table and white chair"
        },
        {
          "key": "C",
          "text": "Two blue leather armchairs"
        },
        {
          "key": "D",
          "text": "Cream-colored cabinet"
        }
      ],
      "ground_truth": "B",
      "images": [
        "assets/sample-coffee/front_137.png",
        "assets/sample-coffee/left_171.png",
        "assets/sample-coffee/back_196.png",
        "assets/sample-coffee/right_222.png"
      ],
      "runs": {
        "fixed4": {
          "prediction": "B",
          "correct": true,
          "steps": 4,
          "forward_steps": 5,
          "exit_reason": "max_steps",
          "latency_s": 10.543854353018105,
          "output_tokens": 559,
          "peak_vram_gib": 8.134863376617432,
          "distances": [
            0.211686372756958,
            0.07620573043823242,
            0.1882057785987854,
            0.07269465923309326
          ],
          "reasoning": "Let's mentally reconstruct the 3D layout of this room using the four images:\n\n- In image 1, the viewpoint is looking toward a blue bin on the floor, a coffee table above it, and two armchairs to the right and left. There's a fridge and cabinets in the background.\n- In image 2, the viewpoint is facing directly toward the two armchairs (one blue, one brown), with the coffee table in the foreground. To the right, we see part of a table and a white chair.\n- In image 3, the viewpoint is turned 180° from image 1. We see the blue bin, the coffee table, and now a white chair and a TV stand with a monitor to the right. Behind the blue bin, there is a garbage bin.\n- In image 4, the viewpoint is turned another 90° from image 3. Now, the blue bin is to the left, and a table with a white chair is in the foreground.\n\nBased on the question, image 2 is the reference view (\"If I am standing at the same spot and facing the same direction as shown in image 2\"). In image 2, you are facing directly toward the two armchairs. \n\nTo determine what is \"behind me\" from this position, we need to imagine rotating the mental scene 180°, so you are now facing away from the armchairs, toward the opposite wall. Retracing the images in relation to one another:\n- If image 2 is facing the armchairs, then image 1 is facing the side with the fridge and cabinets (the \"front\" direction). \n- Image 3 is facing the TV and bin (the \"back\" direction).\n- Image 4 is facing the table and white chair (the \"right\" direction).\n\nTherefore, if you are facing the armchairs (image 2), behind you is the side with the blue bin and, farther ahead, the table and white chair as seen in image 4.\n\nMatching this to the options:\nA. Blue bin — yes, but also matchs image 3 for context.\nB. Table and white chair — matches image 4, which is directly behind you when facing the armchairs.\nC. Two blue leather armchairs — you are already facing these.\nD. Cream-colored cabinet — not visible in these images as options include more colorful furniture, and the cabinets are under everything.\n\nSo option B (\"Table and white chair\") fits the scene behind you when facing the armchairs (image 2).",
          "answer": "B. Table and white chair"
        },
        "fixed8": {
          "prediction": "C",
          "correct": false,
          "steps": 8,
          "forward_steps": 9,
          "exit_reason": "max_steps",
          "latency_s": 9.953594063874334,
          "output_tokens": 550,
          "peak_vram_gib": 8.13486385345459,
          "distances": [
            0.211686372756958,
            0.07620573043823242,
            0.1882057785987854,
            0.07269465923309326,
            0.0961298942565918,
            0.01564037799835205,
            0.0072591304779052734,
            0.005102932453155518
          ],
          "reasoning": "To construct the 3D mental layout, let's analyze the four images:\n\n- Image 1 shows the coffee table from the \"front\" view, with a pair of blue chairs to the right, a fridge and cabinets to the left rear, and the table centered in the room.\n- Image 2 is the \"left\" view, showing the table directly in front, with two blue leather armchairs on either side, and a bit of the room to the right including a round water dispenser and a waste bin.\n- Image 3 is the \"back\" view, showing the table from the opposite side, with the blue bin visible straight ahead, a waste bin and a couple of chairs and tables around, and a television screen to the right.\n- Image 4 is the \"right\" view, showing the table and a white chair in the background, plus a cabinet/fridge to the right side.\n\nFrom the question, if I am standing at the same spot and facing the same direction as in image 2, I am looking toward the two blue leather armchairs, with the table in front of me. To figure out what is behind me, I need to rotate mentally 180 degrees to face the opposite direction. Based on our earlier analysis, image 4 is that perspective, showing the table and a white chair, but the white chair is not visible in image 2, and the table dominates the foreground. However, the answer provided is C: Two blue leather armchairs.\n\nLet's verify: In image 2, facing the armchairs, what's directly behind me? Since the images are taken in cardinal directions around the table, and image 2 is the \"left\" view, the opposite side (\"right\" view) will be image 4, which shows the table and white chair. But the answer says it's the two blue leather armchairs, which matches what is pictured in image 2.\n\nTherefore, if I am standing facing the armchairs (image 2), behind me (on the opposite side of the table) is the setup captured in image 4. But since the question asks \"what is behind me,\" and the provided answer is C. Two blue leather armchairs, it means that when facing the table and the white chair (as in image 4), those are now behind me relative to the original context. Thus, the two blue leather armchairs (image 2) are behind me. This agrees with the answer choice C: Two blue leather armchairs.",
          "answer": "C. Two blue leather armchairs"
        },
        "fixed12": {
          "prediction": "D",
          "correct": false,
          "steps": 12,
          "forward_steps": 13,
          "exit_reason": "max_steps",
          "latency_s": 6.070493276696652,
          "output_tokens": 321,
          "peak_vram_gib": 8.13486385345459,
          "distances": [
            0.211686372756958,
            0.07620573043823242,
            0.1882057785987854,
            0.07269465923309326,
            0.0961298942565918,
            0.01564037799835205,
            0.0072591304779052734,
            0.005102932453155518,
            0.0032422542572021484,
            0.008281946182250977,
            0.006819486618041992,
            0.006731569766998291
          ],
          "reasoning": "To construct the 3D mental layout, let's analyze the four images:\n\n- Image 1 shows the coffee table from the \"front\" viewpoint. In this image, the background includes a fridge, a kitchen counter with drawers, and two blue leather armchairs on the right.\n- Image 2 provides the \"left\" viewpoint, where the background directly behind the table consists of the two blue leather armchairs and a bit of the wall.\n- Image 3 displays the \"back\" viewpoint, revealing a blue bin and a section of the table and white chair from image 1, plus a few more seats.\n- Image 4 serves as the \"right\" viewpoint, showing the table and white chair in the background along with a cream-colored cabinet.\n\nFrom the question, we are to imagine standing at the same spot and facing the same direction as in image 2 (the left view, facing the armchairs). Therefore, behind us is the opposite direction—the \"right\" perspective, which is image 4. In image 4, directly behind the table and white chair, there is a cream-colored cabinet visible.\n\nSo, mentally arranging the space:\n- Facing the armchairs (image 2)—directly behind is the cream-colored cabinet (image 4)—not the fridge/kitchen or the table/bin.\n\nThus, the correct answer behind the viewpoint of image 2 is D. Cream-colored cabinet.",
          "answer": "D. Cream-colored cabinet"
        },
        "adaptive": {
          "prediction": "B",
          "correct": true,
          "steps": 7,
          "forward_steps": 8,
          "exit_reason": "converged",
          "latency_s": 7.772809813264757,
          "output_tokens": 426,
          "peak_vram_gib": 8.13486385345459,
          "distances": [
            0.211686372756958,
            0.07620573043823242,
            0.1882057785987854,
            0.07269465923309326,
            0.0961298942565918,
            0.01564037799835205,
            0.0072591304779052734
          ],
          "reasoning": "Let's mentally construct the 3D layout of this room using the four images:\n\n- Image 1: Shows the coffee table in the center, with a fridge and microwave on the left side, a row of blue chairs and cabinets in the background, and a blue bin on the far right.\n- Image 2: This is a centered view of two blue leather armchairs (likely part of the blue chair row seen in image 1), with a small table or desk in front of them. To the left is a wall, to the right is a partial view of a glass/partitioned area.\n- Image 3: Shows the same blue chairs and, beyond them, a cream-colored cabinet and a trash/recycling bin. To the left is the blue chair row, to the right is the cabinet.\n- Image 4: Shows the coffee table from an angle, with a white chair and a desk visible behind the table, and part of the fridge/microwave on the right.\n\nTo determine which view is which:\n- Image 1: Front view, facing the kitchen area and blue chairs.\n- Image 2: Left view, facing the two blue armchairs.\n- Image 3: Back view, facing the cabinets and bin.\n- Image 4: Right view, facing the white chair and desk.\n\nThe question asks: \"If I am standing at the same spot and facing the same direction as shown in image 2, what is behind me?\"\n\nSo, if you are facing the direction from image 2 (the left view, looking at the blue chairs and desk), directly behind you would be the view from the opposite side (the right side), which is image 4. Image 4 shows the table and white chair, matching option B.\n\nTherefore, constructing the 3D spatial logic, standing facing the blue chairs/desk (image 2), directly behind is the table and white chair (image 4).",
          "answer": "B. Table and white chair"
        }
      }
    },
    {
      "slug": "turn-and-move",
      "id": "among_15131f022f3c6bac68f29de62891226d69b905809770dffe1efb69a327f3f9f7_q2_2_3",
      "label": "两者都答对",
      "note": "两种方案预测一致，adaptive 只使用 7 个 latent pads。",
      "question": "Based on these four images (image 1, 2, 3, and 4) showing the coffee table from different viewpoints (front, left, back, and right), with each camera aligned with room walls and partially capturing the surroundings: If I am standing at the same spot and facing the same direction as shown in image 3, then I turn right and move forward, will I get closer to the table and white chair?",
      "options": [
        {
          "key": "A",
          "text": "Yes"
        },
        {
          "key": "B",
          "text": "No"
        }
      ],
      "ground_truth": "A",
      "images": [
        "assets/sample-coffee/front_137.png",
        "assets/sample-coffee/left_171.png",
        "assets/sample-coffee/back_196.png",
        "assets/sample-coffee/right_222.png"
      ],
      "runs": {
        "fixed4": {
          "prediction": "A",
          "correct": true,
          "steps": 4,
          "forward_steps": 5,
          "exit_reason": "max_steps",
          "latency_s": 10.071882104035467,
          "output_tokens": 520,
          "peak_vram_gib": 8.134851932525635,
          "distances": [
            0.21500647068023682,
            0.1779475212097168,
            0.12557458877563477,
            0.07499885559082031
          ],
          "reasoning": "First, I mentally construct the 3D layout of the room using the four images.\n\n- Image 1 shows the view from the front of the coffee table. It reveals a room with kitchen cabinets in the background, a fridge on the left, a table with two chairs in the center foreground, and armchairs on both sides.\n- Image 2 is from the left side of the table. It shows the two armchairs directly ahead, the same table/chairs from the foreground, and more of the room layout (including a water dispenser and cabinet).\n- Image 3 is from the back of the table. Now, the fridge and previous cabinets are to the right, while the armchairs are now on the left. There's also a white chair next to the armchairs, visible in the foreground left.\n- Image 4 is from the right side. It shows the white chair and armchairs in the foreground, and the table/chairs are more distant.\n\nThe question asks: If you stand at the position/viewpoint of image 3 (back of the table), face the same direction (towards the front of the table, image 1's perspective), then turn right and move forward, will you get closer to the table and white chair?\n\nLet's break down the reasoning:\n- In image 3, you are at the rear of the table, facing towards the front of the table (the direction of image 1). The white chair and table are then on your left (visible in the left side image 4 and part of image 3).\n- Turning right from the viewpoint of image 3 puts you in the direction of movement shown in image 1 (towards the kitchen area and fridge).\n- But the question is about getting closer to the table and white chair, which are located on/near the side, not in the path directly ahead in image 1 but rather in the left region of image 3 and especially in image 4, where the white chair is seen directly next to the table.\n- When you turn right from the image 3 viewpoint and step forward, you end up moving closer to the area where the table and white chair are positioned (since the white chair is beside the table on that side).\n\nThus, imagining the 3D space, if you turn right from the back view and walk forward, you approach the table and white chair.\n\nTherefore, the answer is A. Yes.",
          "answer": "A. Yes"
        },
        "fixed8": {
          "prediction": "A",
          "correct": true,
          "steps": 8,
          "forward_steps": 9,
          "exit_reason": "max_steps",
          "latency_s": 7.811705379281193,
          "output_tokens": 427,
          "peak_vram_gib": 8.134851932525635,
          "distances": [
            0.21500647068023682,
            0.1779475212097168,
            0.12557458877563477,
            0.07499885559082031,
            0.05146050453186035,
            0.009487271308898926,
            0.008476734161376953,
            0.008779525756835938
          ],
          "reasoning": "First, I mentally construct the 3D layout of the room using the four images.\n\n- The central focus in all images is a small square coffee table, placed near the middle of the room.\n- Surrounding the table are various seating options: blue chairs, regular tables, and a white chair/table combination.\n\nImage 1:\nLooking at the table from one side, a fridge and kitchenette area are visible in the background. A blue chair is seen to the right of the table, and a regular table with a white chair is to the left.\nImage 2:\nViewed from the opposite side, two blue chairs are directly behind the table, and a white chair/table combination is to the left. The wall is behind the chairs.\nImage 3:\nFrom the \"back\" viewpoint, the fridge and white chair are now on the right side, while the two blue chairs are on the left. The table is in the center foreground.\nImage 4:\nFrom the \"right\" viewpoint, the white chair and table are now to the left, while the fridge and two blue chairs are on the right. The table is in the foreground.\n\nIf you are standing at the same spot and facing the same direction as in image 3, the \"back\" viewpoint, you are facing toward the fridge and white chair set (with the white chair slightly to the left). Turning right would rotate you toward the direction seen in image 4, which is now facing the white chair and fridge (since in image 3, the white chair and table are slightly to the left, and in image 4, they become more central/right). Moving forward from this new orientation would bring you closer to the white chair and table, which are now more directly ahead.\n\nThus, mentally walking in the 3D scene from the 'back' view (image 3), turning right, and moving forward, you approach the table and white chair area.\n\nThe answer is: A. Yes",
          "answer": "A. Yes"
        },
        "fixed12": {
          "prediction": "A",
          "correct": true,
          "steps": 12,
          "forward_steps": 13,
          "exit_reason": "max_steps",
          "latency_s": 7.568796406965703,
          "output_tokens": 392,
          "peak_vram_gib": 8.134851932525635,
          "distances": [
            0.21500647068023682,
            0.1779475212097168,
            0.12557458877563477,
            0.07499885559082031,
            0.05146050453186035,
            0.009487271308898926,
            0.008476734161376953,
            0.008779525756835938,
            0.0028560757637023926,
            0.0016444921493530273,
            0.0020551085472106934,
            0.001931309700012207
          ],
          "reasoning": "First, I mentally construct the 3D layout of the room using the four images.\n\n- Image 1 shows the view from the front of the coffee table. It reveals a room with kitchen cabinets in the background, a fridge on the left, a table with two chairs in the center foreground, and blue chairs to the right.\n- Image 2 is taken from the left side of the table. Now, we see the table and chairs more directly, with the blue chair receding on the left edge and part of a group of seats and furniture.\n- Image 3 is the back view, showing the table from the side opposite to image 1. Here, the two blue chairs are now to the left, and we see a white chair next to the table on the right.\n- Image 4 captures the right-side view, with the table, white chair, and then the kitchen and fridge further away.\n\nPositioning myself as in image 3 (behind the table looking toward the blue chairs and the table+white chair area), I conceptually arrange the elements:\n- The table is central, flanked by the blue chairs on two sides and a single white chair also flanking the table on the other side.\n- To \"turn right\" from the perspective of image 3 means pivoting clockwise 90 degrees.\n- Doing so, I would now be facing toward where the white chair and table are located (as seen in image 4).\n- If I move forward from this new orientation, I walk closer to the table and the white chair.\n\nThus, visualizing the spatial arrangement confirms that turning right (from the back view) and moving forward brings me nearer to the table and white chair shown in image 4.\n\nTherefore, the correct answer is A. Yes.",
          "answer": "A. Yes"
        },
        "adaptive": {
          "prediction": "A",
          "correct": true,
          "steps": 7,
          "forward_steps": 8,
          "exit_reason": "converged",
          "latency_s": 6.2878158823587,
          "output_tokens": 340,
          "peak_vram_gib": 8.134851932525635,
          "distances": [
            0.21500647068023682,
            0.1779475212097168,
            0.12557458877563477,
            0.07499885559082031,
            0.05146050453186035,
            0.009487271308898926,
            0.008476734161376953
          ],
          "reasoning": "First, I mentally construct the 3D layout of the room using the four images.\n\n- The central focus in all images is a small rectangular coffee table, placed roughly in the middle of the room.\n- Around the table, there are several seating arrangements and tables, mainly chairs arranged in pair(s), and some tables with computers or devices.\n- Image 1 (likely \"front\" view) shows, toward the back wall, a kitchen area with a fridge, counter, and some seating to the right.\n- Image 2 (likely \"left\" view) directly faces two blue chairs alongside a table, with additional seating and a water dispenser visible.\n- Image 3 (likely \"back\" view) shows the opposite side of the kitchen area, plus a white chair next to the table on the right.\n- Image 4 (likely \"right\" view) provides a perspective toward a desk or workstation area with chairs and other furniture.\n\nIf I am standing at the spot and facing the same direction as shown in image 3, I am looking toward the kitchen/fridge area, with the table and white chair slightly to my right. Turning right from this orientation would have me now facing towards the desk/workstation area (as shown in image 4). If I move forward from this new direction, I would be heading toward that table and the white chair, getting closer to them.\n\nTherefore, based on the spatial arrangement and the directions given, turning right from image 3's viewpoint and moving forward indeed brings me closer to the table and white chair.",
          "answer": "A. Yes"
        }
      }
    },
    {
      "slug": "full-budget",
      "id": "among_15131f022f3c6bac68f29de62891226d69b905809770dffe1efb69a327f3f9f7_q1_5_1",
      "label": "困难样本保留 12 步",
      "note": "收敛条件未满足时，adaptive 保留全部预算，不强行早退出。",
      "question": "Based on these four images (image 1, 2, 3, and 4) showing the coffee table from different viewpoints (front, left, back, and right), with each camera aligned with room walls and partially capturing the surroundings: From the viewpoint presented in image 2, what is to the left of the coffee table?",
      "options": [
        {
          "key": "A",
          "text": "Two blue leather armchairs"
        },
        {
          "key": "B",
          "text": "Table and white chair"
        },
        {
          "key": "C",
          "text": "Blue bin"
        },
        {
          "key": "D",
          "text": "Cream-colored cabinet"
        }
      ],
      "ground_truth": "D",
      "images": [
        "assets/sample-coffee/front_137.png",
        "assets/sample-coffee/left_171.png",
        "assets/sample-coffee/back_196.png",
        "assets/sample-coffee/right_222.png"
      ],
      "runs": {
        "fixed4": {
          "prediction": "B",
          "correct": false,
          "steps": 4,
          "forward_steps": 5,
          "exit_reason": "max_steps",
          "latency_s": 5.956688871141523,
          "output_tokens": 308,
          "peak_vram_gib": 8.134832859039307,
          "distances": [
            0.19842422008514404,
            0.05439513921737671,
            0.3591086268424988,
            0.20730948448181152
          ],
          "reasoning": "I am constructing a mental 3D layout of the room based on the four provided images showing the coffee table from different perspectives.\n\n- Image 1 shows the view facing the front of the table, with a fridge and kitchen cabinets in the background, and two blue chairs on either side.\n- Image 2 is from the left of the table, showing the two blue chairs directly behind the table, and part of a table and white chair to the right.\n- Image 3 is from the back of the table, showing the blue chairs and a trash bin in the background.\n- Image 4 is from the right side, showing the table and a white chair closely behind it, as well as more of the room's corner.\n\nFrom image 2's perspective, I am \"looking\" across the table from the left side toward the blue chairs (which are directly behind the table). To the left of the table from this viewpoint, I notice a section of the wall and what appears to be a table and white chair (partially visible). If I cross-check with image 4, where the white chair is visible directly behind the table, it confirms that the table and white chair are located to the left of the coffee table from image 2's viewpoint.\n\nTherefore, based on my 3D spatial understanding of the room and the camera positions, the correct answer is B. Table and white chair.",
          "answer": "B. Table and white chair"
        },
        "fixed8": {
          "prediction": "B",
          "correct": false,
          "steps": 8,
          "forward_steps": 9,
          "exit_reason": "max_steps",
          "latency_s": 9.211586812045425,
          "output_tokens": 503,
          "peak_vram_gib": 8.13483190536499,
          "distances": [
            0.19842422008514404,
            0.05439513921737671,
            0.3591086268424988,
            0.20730948448181152,
            0.22422486543655396,
            0.1675845980644226,
            0.12926441431045532,
            0.14116060733795166
          ],
          "reasoning": "I am constructing a mental 3D layout of the room based on the four provided images showing the coffee table from different viewpoints.\n\n- Image 1 shows the view facing the front of the table, with a fridge and kitchen cabinets directly behind it, and two blue chairs on either side.\n- Image 2 is from the left of the table, showing the two blue chairs along the wall directly behind the table, and part of the table and chairs extending to the left.\n- Image 3 is from the back of the table, showing a blue bin and some additional furniture toward the left side of the image, with the table in the foreground.\n- Image 4 is from the right side of the table, facing the kitchen area, table, and a single white chair.\n\nFrom these perspectives, the rectangular coffee table is centrally located in the room, with seating and kitchen areas varying according to viewpoint. To answer the question, I need to determine, from the viewpoint of image 2 (the left view), what is to the left of the coffee table.\n\nIn image 2, the viewpoint faces the side wall where the two blue chairs are placed. The left of the table from this perspective is toward the center of the room (moving away from the wall with the chairs). Checking image 1 for context, the kitchen area is straight ahead from the front view, while image 4 shows the right side of the table close to the white chair and cabinets.\n\nIn my mental 3D scene, if I'm standing at the position of image 2 (left side of the table, facing the chairs), \"to the left\" of the table means the direction extending deeper into the room—toward the area shown in image 1 (kitchen and fridge). However, examining all options and the visual cues, it's clear that the white chair, visible in image 4 and situated to the left of the table when viewed from image 2, is actually the most direct feature to the left. The blue chair is directly behind the table (not to the left as per this viewpoint), the bin is farther back, and the table-and-white-chair configuration is consistent with the spatial arrangement.\n\nThus, Option B (\"Table and white chair\") is the correct answer for what is to the left of the coffee table from the viewpoint of image 2.",
          "answer": "B. Table and white chair"
        },
        "fixed12": {
          "prediction": "D",
          "correct": true,
          "steps": 12,
          "forward_steps": 13,
          "exit_reason": "max_steps",
          "latency_s": 9.833671837113798,
          "output_tokens": 540,
          "peak_vram_gib": 8.134832382202148,
          "distances": [
            0.19842422008514404,
            0.05439513921737671,
            0.3591086268424988,
            0.20730948448181152,
            0.22422486543655396,
            0.1675845980644226,
            0.12926441431045532,
            0.14116060733795166,
            0.3697303533554077,
            0.4165208339691162,
            0.272794246673584,
            0.20071566104888916
          ],
          "reasoning": "",
          "answer": "D. Cream-colored cabinet"
        },
        "adaptive": {
          "prediction": "D",
          "correct": true,
          "steps": 12,
          "forward_steps": 13,
          "exit_reason": "max_steps",
          "latency_s": 10.13031991571188,
          "output_tokens": 540,
          "peak_vram_gib": 8.134832859039307,
          "distances": [
            0.19842422008514404,
            0.05439513921737671,
            0.3591086268424988,
            0.20730948448181152,
            0.22422486543655396,
            0.1675845980644226,
            0.12926441431045532,
            0.14116060733795166,
            0.3697303533554077,
            0.4165208339691162,
            0.272794246673584,
            0.20071566104888916
          ],
          "reasoning": "",
          "answer": "D. Cream-colored cabinet"
        }
      }
    }
  ],
  "overall": [
    {
      "condition": "fixed4",
      "samples": 950,
      "correct": 627,
      "accuracy": 0.66,
      "wilson95": [
        0.6292864721703727,
        0.6894247740107883
      ],
      "mean_latent_steps": 4.0,
      "early_exit_fraction": 0.0,
      "truncated": 0,
      "exit_reasons": {
        "max_steps": 950
      }
    },
    {
      "condition": "fixed8",
      "samples": 950,
      "correct": 600,
      "accuracy": 0.631578947368421,
      "wilson95": [
        0.6004321406641764,
        0.6616659236295415
      ],
      "mean_latent_steps": 7.997894736842105,
      "early_exit_fraction": 0.0,
      "truncated": 0,
      "exit_reasons": {
        "max_steps": 947,
        "model_end": 3
      }
    },
    {
      "condition": "fixed12",
      "samples": 950,
      "correct": 638,
      "accuracy": 0.671578947368421,
      "wilson95": [
        0.6410759648019856,
        0.7006999110370226
      ],
      "mean_latent_steps": 11.968421052631578,
      "early_exit_fraction": 0.0,
      "truncated": 0,
      "exit_reasons": {
        "max_steps": 887,
        "model_end": 63
      }
    },
    {
      "condition": "adaptive",
      "samples": 950,
      "correct": 636,
      "accuracy": 0.6694736842105263,
      "wilson95": [
        0.6389309163147582,
        0.6986513904955506
      ],
      "mean_latent_steps": 9.692631578947369,
      "early_exit_fraction": 0.5778947368421052,
      "truncated": 0,
      "exit_reasons": {
        "converged": 549,
        "max_steps": 353,
        "model_end": 48
      }
    }
  ],
  "tasks": [
    {
      "condition": "fixed4",
      "task": "among",
      "samples": 550,
      "correct": 390,
      "accuracy": 0.7090909090909091,
      "wilson95": [
        0.6697873321988682,
        0.745493965972171
      ],
      "mean_latent_steps": 4.0,
      "early_exit_fraction": 0.0,
      "truncated": 0,
      "exit_reasons": {
        "max_steps": 550
      }
    },
    {
      "condition": "fixed4",
      "task": "rotation",
      "samples": 175,
      "correct": 85,
      "accuracy": 0.4857142857142857,
      "wilson95": [
        0.41277065946101177,
        0.5592716173523696
      ],
      "mean_latent_steps": 4.0,
      "early_exit_fraction": 0.0,
      "truncated": 0,
      "exit_reasons": {
        "max_steps": 175
      }
    },
    {
      "condition": "fixed4",
      "task": "around",
      "samples": 225,
      "correct": 152,
      "accuracy": 0.6755555555555556,
      "wilson95": [
        0.6118799653031943,
        0.7333372012326103
      ],
      "mean_latent_steps": 4.0,
      "early_exit_fraction": 0.0,
      "truncated": 0,
      "exit_reasons": {
        "max_steps": 225
      }
    },
    {
      "condition": "fixed8",
      "task": "among",
      "samples": 550,
      "correct": 374,
      "accuracy": 0.68,
      "wilson95": [
        0.6398820157380027,
        0.7176210148614137
      ],
      "mean_latent_steps": 7.996363636363636,
      "early_exit_fraction": 0.0,
      "truncated": 0,
      "exit_reasons": {
        "max_steps": 548,
        "model_end": 2
      }
    },
    {
      "condition": "fixed8",
      "task": "rotation",
      "samples": 175,
      "correct": 71,
      "accuracy": 0.4057142857142857,
      "wilson95": [
        0.3357459469817683,
        0.47973307998654935
      ],
      "mean_latent_steps": 8.0,
      "early_exit_fraction": 0.0,
      "truncated": 0,
      "exit_reasons": {
        "max_steps": 174,
        "model_end": 1
      }
    },
    {
      "condition": "fixed8",
      "task": "around",
      "samples": 225,
      "correct": 155,
      "accuracy": 0.6888888888888889,
      "wilson95": [
        0.6256534297780915,
        0.7457827620642298
      ],
      "mean_latent_steps": 8.0,
      "early_exit_fraction": 0.0,
      "truncated": 0,
      "exit_reasons": {
        "max_steps": 225
      }
    },
    {
      "condition": "fixed12",
      "task": "among",
      "samples": 550,
      "correct": 385,
      "accuracy": 0.7,
      "wilson95": [
        0.6604226151234737,
        0.7368029744314334
      ],
      "mean_latent_steps": 11.972727272727273,
      "early_exit_fraction": 0.0,
      "truncated": 0,
      "exit_reasons": {
        "max_steps": 532,
        "model_end": 18
      }
    },
    {
      "condition": "fixed12",
      "task": "rotation",
      "samples": 175,
      "correct": 90,
      "accuracy": 0.5142857142857142,
      "wilson95": [
        0.44072838264763026,
        0.5872293405389881
      ],
      "mean_latent_steps": 11.96,
      "early_exit_fraction": 0.0,
      "truncated": 0,
      "exit_reasons": {
        "max_steps": 151,
        "model_end": 24
      }
    },
    {
      "condition": "fixed12",
      "task": "around",
      "samples": 225,
      "correct": 163,
      "accuracy": 0.7244444444444444,
      "wilson95": [
        0.6626664072966939,
        0.7786871853630055
      ],
      "mean_latent_steps": 11.964444444444444,
      "early_exit_fraction": 0.0,
      "truncated": 0,
      "exit_reasons": {
        "max_steps": 204,
        "model_end": 21
      }
    },
    {
      "condition": "adaptive",
      "task": "among",
      "samples": 550,
      "correct": 394,
      "accuracy": 0.7163636363636363,
      "wilson95": [
        0.6772921872365991,
        0.7524336778273457
      ],
      "mean_latent_steps": 9.5,
      "early_exit_fraction": 0.6472727272727272,
      "truncated": 0,
      "exit_reasons": {
        "converged": 356,
        "max_steps": 177,
        "model_end": 17
      }
    },
    {
      "condition": "adaptive",
      "task": "rotation",
      "samples": 175,
      "correct": 86,
      "accuracy": 0.49142857142857144,
      "wilson95": [
        0.4183434677171528,
        0.5648818983708761
      ],
      "mean_latent_steps": 11.765714285714285,
      "early_exit_fraction": 0.11428571428571428,
      "truncated": 0,
      "exit_reasons": {
        "converged": 20,
        "max_steps": 131,
        "model_end": 24
      }
    },
    {
      "condition": "adaptive",
      "task": "around",
      "samples": 225,
      "correct": 156,
      "accuracy": 0.6933333333333334,
      "wilson95": [
        0.6302571186815388,
        0.7499187482629547
      ],
      "mean_latent_steps": 8.551111111111112,
      "early_exit_fraction": 0.7688888888888888,
      "truncated": 0,
      "exit_reasons": {
        "converged": 173,
        "max_steps": 45,
        "model_end": 7
      }
    }
  ],
  "timing": [
    {
      "condition": "fixed4",
      "samples": 50,
      "accuracy": 0.64,
      "mean_latency_s": 6.055293409740552,
      "median_latency_s": 5.71184821263887,
      "p95_latency_s": 8.079614785360173,
      "latency_std_s": 1.0436078110164513,
      "mean_output_tokens": 365.84,
      "mean_latent_steps": 4.0,
      "max_peak_vram_gib": 12.636942386627197
    },
    {
      "condition": "fixed8",
      "samples": 50,
      "accuracy": 0.58,
      "mean_latency_s": 6.164726730296389,
      "median_latency_s": 5.865758160594851,
      "p95_latency_s": 8.84056845693849,
      "latency_std_s": 1.1942140950039266,
      "mean_output_tokens": 369.92,
      "mean_latent_steps": 8.0,
      "max_peak_vram_gib": 12.636942386627197
    },
    {
      "condition": "fixed12",
      "samples": 50,
      "accuracy": 0.6,
      "mean_latency_s": 6.305841272091493,
      "median_latency_s": 6.065744594903663,
      "p95_latency_s": 8.188560345605946,
      "latency_std_s": 1.1502105733026176,
      "mean_output_tokens": 372.8,
      "mean_latent_steps": 12.0,
      "max_peak_vram_gib": 12.636942386627197
    },
    {
      "condition": "adaptive",
      "samples": 50,
      "accuracy": 0.7,
      "mean_latency_s": 6.103865592340007,
      "median_latency_s": 5.828633384080604,
      "p95_latency_s": 8.117950263922102,
      "latency_std_s": 1.2210961056214655,
      "mean_output_tokens": 363.04,
      "mean_latent_steps": 10.06,
      "max_peak_vram_gib": 12.636942386627197
    }
  ],
  "paired": {
    "comparison": "adaptive_vs_fixed12",
    "paired_samples": 950,
    "baseline_accuracy": 0.671578947368421,
    "treatment_accuracy": 0.6694736842105263,
    "accuracy_delta_percentage_points": -0.21052631578947212,
    "improvements": 75,
    "regressions": 77,
    "mcnemar_exact_p": 0.9353893704780424,
    "prediction_agreement": 0.8347368421052631,
    "baseline_mean_latent_steps": 11.968421052631578,
    "treatment_mean_latent_steps": 9.692631578947369,
    "latent_step_reduction": 0.19014951627088827,
    "mean_intervention_cosine": null,
    "mean_intervention_relative_l2": null
  }
}