diff --git a/data/rf/task-69391d8d1ce51c407be1e531/tests/evaluate_rubrics.py b/data/rf/task-69391d8d1ce51c407be1e531/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69391d8d1ce51c407be1e531/tests/evaluate_rubrics.py +++ b/data/rf/task-69391d8d1ce51c407be1e531/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69391d8d1ce51c407be1e533/tests/evaluate_rubrics.py b/data/rf/task-69391d8d1ce51c407be1e533/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69391d8d1ce51c407be1e533/tests/evaluate_rubrics.py +++ b/data/rf/task-69391d8d1ce51c407be1e533/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-694b4b99829f00e24fd11885/tests/evaluate_rubrics.py b/data/rf/task-694b4b99829f00e24fd11885/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-694b4b99829f00e24fd11885/tests/evaluate_rubrics.py +++ b/data/rf/task-694b4b99829f00e24fd11885/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-694b4b99829f00e24fd11889/tests/evaluate_rubrics.py b/data/rf/task-694b4b99829f00e24fd11889/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-694b4b99829f00e24fd11889/tests/evaluate_rubrics.py +++ b/data/rf/task-694b4b99829f00e24fd11889/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-694b4b99829f00e24fd11891/tests/evaluate_rubrics.py b/data/rf/task-694b4b99829f00e24fd11891/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-694b4b99829f00e24fd11891/tests/evaluate_rubrics.py +++ b/data/rf/task-694b4b99829f00e24fd11891/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-694b4b99829f00e24fd118a1/tests/evaluate_rubrics.py b/data/rf/task-694b4b99829f00e24fd118a1/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-694b4b99829f00e24fd118a1/tests/evaluate_rubrics.py +++ b/data/rf/task-694b4b99829f00e24fd118a1/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b40c/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b40c/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b40c/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b40c/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b419/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b419/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b419/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b419/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b426/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b426/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b426/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b426/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b428/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b428/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b428/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b428/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b434/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b434/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b434/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b434/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b435/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b435/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b435/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b435/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b436/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b436/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b436/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b436/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b439/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b439/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b439/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b439/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b43d/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b43d/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b43d/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b43d/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b43e/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b43e/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b43e/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b43e/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b440/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b440/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b440/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b440/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b441/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b441/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b441/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b441/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b445/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b445/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b445/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b445/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b446/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b446/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b446/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b446/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b447/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b447/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b447/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b447/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b449/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b449/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b449/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b449/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b44b/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b44b/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b44b/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b44b/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b451/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b451/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b451/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b451/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b455/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b455/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b455/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b455/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b458/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b458/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b458/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b458/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b459/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b459/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b459/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b459/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b45b/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b45b/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b45b/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b45b/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b45f/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b45f/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b45f/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b45f/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b462/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b462/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b462/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b462/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b471/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b471/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b471/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b471/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b473/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b473/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b473/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b473/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-696719205599a51110d4b476/tests/evaluate_rubrics.py b/data/rf/task-696719205599a51110d4b476/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-696719205599a51110d4b476/tests/evaluate_rubrics.py +++ b/data/rf/task-696719205599a51110d4b476/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-697e7458be1623d850a8882d/tests/evaluate_rubrics.py b/data/rf/task-697e7458be1623d850a8882d/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-697e7458be1623d850a8882d/tests/evaluate_rubrics.py +++ b/data/rf/task-697e7458be1623d850a8882d/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-697e7458be1623d850a8882e/tests/evaluate_rubrics.py b/data/rf/task-697e7458be1623d850a8882e/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-697e7458be1623d850a8882e/tests/evaluate_rubrics.py +++ b/data/rf/task-697e7458be1623d850a8882e/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-697e7458be1623d850a88830/tests/evaluate_rubrics.py b/data/rf/task-697e7458be1623d850a88830/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-697e7458be1623d850a88830/tests/evaluate_rubrics.py +++ b/data/rf/task-697e7458be1623d850a88830/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-697e7458be1623d850a88832/tests/evaluate_rubrics.py b/data/rf/task-697e7458be1623d850a88832/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-697e7458be1623d850a88832/tests/evaluate_rubrics.py +++ b/data/rf/task-697e7458be1623d850a88832/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-697e7458be1623d850a88838/tests/evaluate_rubrics.py b/data/rf/task-697e7458be1623d850a88838/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-697e7458be1623d850a88838/tests/evaluate_rubrics.py +++ b/data/rf/task-697e7458be1623d850a88838/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-697e7458be1623d850a8883b/tests/evaluate_rubrics.py b/data/rf/task-697e7458be1623d850a8883b/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-697e7458be1623d850a8883b/tests/evaluate_rubrics.py +++ b/data/rf/task-697e7458be1623d850a8883b/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-697e7458be1623d850a88841/tests/evaluate_rubrics.py b/data/rf/task-697e7458be1623d850a88841/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-697e7458be1623d850a88841/tests/evaluate_rubrics.py +++ b/data/rf/task-697e7458be1623d850a88841/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-697e7458be1623d850a88844/tests/evaluate_rubrics.py b/data/rf/task-697e7458be1623d850a88844/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-697e7458be1623d850a88844/tests/evaluate_rubrics.py +++ b/data/rf/task-697e7458be1623d850a88844/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-697e7458be1623d850a8885b/tests/evaluate_rubrics.py b/data/rf/task-697e7458be1623d850a8885b/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-697e7458be1623d850a8885b/tests/evaluate_rubrics.py +++ b/data/rf/task-697e7458be1623d850a8885b/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-697e7458be1623d850a8885d/tests/evaluate_rubrics.py b/data/rf/task-697e7458be1623d850a8885d/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-697e7458be1623d850a8885d/tests/evaluate_rubrics.py +++ b/data/rf/task-697e7458be1623d850a8885d/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-697e7458be1623d850a88862/tests/evaluate_rubrics.py b/data/rf/task-697e7458be1623d850a88862/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-697e7458be1623d850a88862/tests/evaluate_rubrics.py +++ b/data/rf/task-697e7458be1623d850a88862/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69b7c2a04b6f8ff9ed98812c/tests/evaluate_rubrics.py b/data/rf/task-69b7c2a04b6f8ff9ed98812c/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69b7c2a04b6f8ff9ed98812c/tests/evaluate_rubrics.py +++ b/data/rf/task-69b7c2a04b6f8ff9ed98812c/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69b7c2a04b6f8ff9ed98812d/tests/evaluate_rubrics.py b/data/rf/task-69b7c2a04b6f8ff9ed98812d/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69b7c2a04b6f8ff9ed98812d/tests/evaluate_rubrics.py +++ b/data/rf/task-69b7c2a04b6f8ff9ed98812d/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69b7c2a04b6f8ff9ed98812e/tests/evaluate_rubrics.py b/data/rf/task-69b7c2a04b6f8ff9ed98812e/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69b7c2a04b6f8ff9ed98812e/tests/evaluate_rubrics.py +++ b/data/rf/task-69b7c2a04b6f8ff9ed98812e/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69b7c2a04b6f8ff9ed98812f/tests/evaluate_rubrics.py b/data/rf/task-69b7c2a04b6f8ff9ed98812f/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69b7c2a04b6f8ff9ed98812f/tests/evaluate_rubrics.py +++ b/data/rf/task-69b7c2a04b6f8ff9ed98812f/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69b7c2a04b6f8ff9ed988131/tests/evaluate_rubrics.py b/data/rf/task-69b7c2a04b6f8ff9ed988131/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69b7c2a04b6f8ff9ed988131/tests/evaluate_rubrics.py +++ b/data/rf/task-69b7c2a04b6f8ff9ed988131/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69b7c2a04b6f8ff9ed988134/tests/evaluate_rubrics.py b/data/rf/task-69b7c2a04b6f8ff9ed988134/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69b7c2a04b6f8ff9ed988134/tests/evaluate_rubrics.py +++ b/data/rf/task-69b7c2a04b6f8ff9ed988134/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69b7c2a04b6f8ff9ed988137/tests/evaluate_rubrics.py b/data/rf/task-69b7c2a04b6f8ff9ed988137/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69b7c2a04b6f8ff9ed988137/tests/evaluate_rubrics.py +++ b/data/rf/task-69b7c2a04b6f8ff9ed988137/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69b7c2a04b6f8ff9ed98813a/tests/evaluate_rubrics.py b/data/rf/task-69b7c2a04b6f8ff9ed98813a/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69b7c2a04b6f8ff9ed98813a/tests/evaluate_rubrics.py +++ b/data/rf/task-69b7c2a04b6f8ff9ed98813a/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69b7c2a04b6f8ff9ed98813b/tests/evaluate_rubrics.py b/data/rf/task-69b7c2a04b6f8ff9ed98813b/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69b7c2a04b6f8ff9ed98813b/tests/evaluate_rubrics.py +++ b/data/rf/task-69b7c2a04b6f8ff9ed98813b/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69b7c2a04b6f8ff9ed98813d/tests/evaluate_rubrics.py b/data/rf/task-69b7c2a04b6f8ff9ed98813d/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69b7c2a04b6f8ff9ed98813d/tests/evaluate_rubrics.py +++ b/data/rf/task-69b7c2a04b6f8ff9ed98813d/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69d196f015a150488265afb4/tests/evaluate_rubrics.py b/data/rf/task-69d196f015a150488265afb4/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69d196f015a150488265afb4/tests/evaluate_rubrics.py +++ b/data/rf/task-69d196f015a150488265afb4/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69d196f015a150488265afb5/tests/evaluate_rubrics.py b/data/rf/task-69d196f015a150488265afb5/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69d196f015a150488265afb5/tests/evaluate_rubrics.py +++ b/data/rf/task-69d196f015a150488265afb5/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69d196f015a150488265afb6/tests/evaluate_rubrics.py b/data/rf/task-69d196f015a150488265afb6/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69d196f015a150488265afb6/tests/evaluate_rubrics.py +++ b/data/rf/task-69d196f015a150488265afb6/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69d196f015a150488265afb7/tests/evaluate_rubrics.py b/data/rf/task-69d196f015a150488265afb7/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69d196f015a150488265afb7/tests/evaluate_rubrics.py +++ b/data/rf/task-69d196f015a150488265afb7/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69d196f015a150488265afb8/tests/evaluate_rubrics.py b/data/rf/task-69d196f015a150488265afb8/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69d196f015a150488265afb8/tests/evaluate_rubrics.py +++ b/data/rf/task-69d196f015a150488265afb8/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69d196f015a150488265afb9/tests/evaluate_rubrics.py b/data/rf/task-69d196f015a150488265afb9/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69d196f015a150488265afb9/tests/evaluate_rubrics.py +++ b/data/rf/task-69d196f015a150488265afb9/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69d196f015a150488265afba/tests/evaluate_rubrics.py b/data/rf/task-69d196f015a150488265afba/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69d196f015a150488265afba/tests/evaluate_rubrics.py +++ b/data/rf/task-69d196f015a150488265afba/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69d196f015a150488265afbb/tests/evaluate_rubrics.py b/data/rf/task-69d196f015a150488265afbb/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69d196f015a150488265afbb/tests/evaluate_rubrics.py +++ b/data/rf/task-69d196f015a150488265afbb/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69d196f015a150488265afbc/tests/evaluate_rubrics.py b/data/rf/task-69d196f015a150488265afbc/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69d196f015a150488265afbc/tests/evaluate_rubrics.py +++ b/data/rf/task-69d196f015a150488265afbc/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69d196f015a150488265afbd/tests/evaluate_rubrics.py b/data/rf/task-69d196f015a150488265afbd/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69d196f015a150488265afbd/tests/evaluate_rubrics.py +++ b/data/rf/task-69d196f015a150488265afbd/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69d196f015a150488265afbe/tests/evaluate_rubrics.py b/data/rf/task-69d196f015a150488265afbe/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69d196f015a150488265afbe/tests/evaluate_rubrics.py +++ b/data/rf/task-69d196f015a150488265afbe/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69d196f015a150488265afc0/tests/evaluate_rubrics.py b/data/rf/task-69d196f015a150488265afc0/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69d196f015a150488265afc0/tests/evaluate_rubrics.py +++ b/data/rf/task-69d196f015a150488265afc0/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69d196f015a150488265afc1/tests/evaluate_rubrics.py b/data/rf/task-69d196f015a150488265afc1/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69d196f015a150488265afc1/tests/evaluate_rubrics.py +++ b/data/rf/task-69d196f015a150488265afc1/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69d196f015a150488265afc2/tests/evaluate_rubrics.py b/data/rf/task-69d196f015a150488265afc2/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69d196f015a150488265afc2/tests/evaluate_rubrics.py +++ b/data/rf/task-69d196f015a150488265afc2/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69d196f015a150488265afc3/tests/evaluate_rubrics.py b/data/rf/task-69d196f015a150488265afc3/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69d196f015a150488265afc3/tests/evaluate_rubrics.py +++ b/data/rf/task-69d196f015a150488265afc3/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content): diff --git a/data/rf/task-69d196f015a150488265afc4/tests/evaluate_rubrics.py b/data/rf/task-69d196f015a150488265afc4/tests/evaluate_rubrics.py index 3f4d3fb..e10a453 100644 --- a/data/rf/task-69d196f015a150488265afc4/tests/evaluate_rubrics.py +++ b/data/rf/task-69d196f015a150488265afc4/tests/evaluate_rubrics.py @@ -41,30 +41,55 @@ def _parse_llm_response(text): if not text: return None text = text.strip() - if "```json" in text: - after = text[text.find("```json") + 7:] - end = after.find("```") - if end != -1: - text = after[:end].strip() - if not text.startswith("{"): - for pattern in ['{"ratings"', '{ "ratings"']: - start = text.find(pattern) - if start != -1: - text = text[start:] - brace_count = 0 - for i, char in enumerate(text): - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - text = text[:i + 1] - break + + # The reply may be bare JSON, wrapped in a ```json fence, or prose with the + # object embedded. Fences and braces also occur *inside* justification + # strings, so no single delimiter search is reliable on its own. Collect + # every plausible slice and let json.loads decide which one is really JSON. + candidates = [text] + + marker = "```json" + fence_start = text.find(marker) + if fence_start != -1: + body = text[fence_start + len(marker):] + # Closing-fence candidates outermost first: a ```python block quoted + # inside a justification would truncate a first-match search. + end = len(body) + while True: + end = body.rfind("```", 0, end) + if end == -1: break - try: - return json.loads(text) - except json.JSONDecodeError: - return None + candidates.append(body[:end].strip()) + candidates.append(body.strip()) + + for pattern in ['{"ratings"', '{ "ratings"']: + start = text.find(pattern) + if start == -1: + continue + brace_count = 0 + for i in range(start, len(text)): + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + if brace_count == 0: + candidates.append(text[start:i + 1]) + break + break + + # Prefer the rubric object itself. A top-level array parses cleanly but the + # caller's `"ratings" in parsed` check then silently skips the rubric. + fallback = None + for candidate in candidates: + try: + value = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(value, dict) and "ratings" in value: + return value + if fallback is None: + fallback = value + return fallback def llm_call(client, model, system_prompt, user_content):