From 23706da71cd05131e84230049549a2eff9adcbde Mon Sep 17 00:00:00 2001 From: SonAIengine Date: Sun, 19 Jul 2026 16:40:48 +0900 Subject: [PATCH] test: add bfcl sweep milestone gate --- benchmarks/bfcl_tool_selection/sweep.py | 246 +++++++++++++++++++++++- docs/benchmarks.md | 24 +++ docs/research/validation-loop.md | 8 + docs/research/xgen-tool-graph-goals.md | 7 + tests/test_bfcl_tool_selection_sweep.py | 79 ++++++++ 5 files changed, 362 insertions(+), 2 deletions(-) diff --git a/benchmarks/bfcl_tool_selection/sweep.py b/benchmarks/bfcl_tool_selection/sweep.py index 7b553a9..ecca578 100644 --- a/benchmarks/bfcl_tool_selection/sweep.py +++ b/benchmarks/bfcl_tool_selection/sweep.py @@ -30,6 +30,17 @@ ) from benchmarks.xgen_tool_graph.llm_loop import DEFAULT_OLLAMA_URL +DEFAULT_MILESTONE_PROFILE = "xgen-0.27" +MILESTONE_PROFILES: dict[str, dict[str, float | int | str]] = { + "xgen-0.27": { + "target_top_k": 5, + "min_retrieved_exact": 0.85, + "min_retrieval_recall": 0.95, + "min_row_source_preservation": 0.94, + "min_parallel_multiple_exact": 0.75, + } +} + def run_sweep( *, @@ -52,6 +63,7 @@ def run_sweep( concurrency: int = 1, progress: bool = False, progress_every: int = 25, + milestone_profile: str = DEFAULT_MILESTONE_PROFILE, ) -> dict[str, Any]: selected_categories = categories or list(DEFAULT_CATEGORIES) selected_top_ks = top_ks or [3, 5, 10] @@ -109,13 +121,19 @@ def run_sweep( "official_model_name": official_model_name if evaluator == "official" else "", "cache_dir": str(cache_dir) if cache_dir else "", "bfcl_ref": ref, + "milestone_profile": milestone_profile, "runs": runs, - "summary": _summarize_sweep(runs), + "summary": _summarize_sweep(runs, milestone_profile=milestone_profile), } -def _summarize_sweep(runs: list[dict[str, Any]]) -> dict[str, Any]: +def _summarize_sweep( + runs: list[dict[str, Any]], + *, + milestone_profile: str = DEFAULT_MILESTONE_PROFILE, +) -> dict[str, Any]: rows: list[dict[str, Any]] = [] + category_rows: list[dict[str, Any]] = [] failure_totals: Counter[str] = Counter() for run in runs: report = run["report"] @@ -135,15 +153,45 @@ def _summarize_sweep(runs: list[dict[str, Any]]) -> dict[str, Any]: "failure_breakdown": summary.get("failure_breakdown") or {}, } ) + category_rows.extend(_category_summary_rows(run)) return { "run_count": len(runs), "rows": rows, + "category_rows": category_rows, "repeat_groups": _summarize_repeat_groups(rows), + "category_repeat_groups": _summarize_category_repeat_groups(category_rows), "failure_breakdown": dict(sorted(failure_totals.items())), "best_retrieved": _best_retrieved(rows), + "milestone_gate": _evaluate_milestone_gate( + rows, + category_rows, + profile_name=milestone_profile, + ), } +def _category_summary_rows(run: dict[str, Any]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for category in run["report"].get("categories") or []: + summary = category.get("summary") or {} + rows.append( + { + "repeat": run["repeat"], + "tool_source": run["tool_source"], + "top_k": run["top_k"], + "category": category.get("category") or "", + "cases": summary.get("cases", category.get("case_count", 0)), + "retrieval_recall_at_k": summary.get("retrieval_recall_at_k", 0.0), + "model_tool_call_rate": summary.get("model_tool_call_rate", 0.0), + "strict_exact_match": summary.get("strict_exact_match", 0.0), + "evaluator_exact_match": summary.get("evaluator_exact_match", 0.0), + "avg_latency_ms": summary.get("avg_latency_ms", 0.0), + "failure_breakdown": summary.get("failure_breakdown") or {}, + } + ) + return rows + + def _best_retrieved(rows: list[dict[str, Any]]) -> dict[str, Any]: retrieved_rows = [row for row in rows if row["tool_source"] == "retrieved"] if not retrieved_rows: @@ -158,6 +206,36 @@ def _best_retrieved(rows: list[dict[str, Any]]) -> dict[str, Any]: ) +def _summarize_category_repeat_groups(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + grouped: dict[tuple[str, int, str], list[dict[str, Any]]] = {} + for row in rows: + key = (str(row["tool_source"]), int(row["top_k"]), str(row["category"])) + grouped.setdefault(key, []).append(row) + + summaries: list[dict[str, Any]] = [] + for (tool_source, top_k, category), group_rows in sorted(grouped.items()): + summaries.append( + { + "tool_source": tool_source, + "top_k": top_k, + "category": category, + "repeat_count": len(group_rows), + "cases_per_repeat": [int(row["cases"]) for row in group_rows], + "retrieval_recall_at_k": _metric_stats( + row["retrieval_recall_at_k"] for row in group_rows + ), + "evaluator_exact_match": _metric_stats( + row["evaluator_exact_match"] for row in group_rows + ), + "strict_exact_match": _metric_stats( + row["strict_exact_match"] for row in group_rows + ), + "avg_latency_ms": _metric_stats(row["avg_latency_ms"] for row in group_rows), + } + ) + return summaries + + def _summarize_repeat_groups(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: grouped: dict[tuple[str, int], list[dict[str, Any]]] = {} for row in rows: @@ -199,6 +277,140 @@ def _metric_stats(values: Any) -> dict[str, float]: } +def _mean(values: Any) -> float: + vals = [float(value) for value in values] + if not vals: + return 0.0 + return round(statistics.mean(vals), 6) + + +def _evaluate_milestone_gate( + rows: list[dict[str, Any]], + category_rows: list[dict[str, Any]], + *, + profile_name: str, +) -> dict[str, Any]: + if profile_name in {"", "none"}: + return {} + profile = MILESTONE_PROFILES.get(profile_name) + if profile is None: + return { + "profile": profile_name, + "status": "unknown_profile", + "missing_metrics": [f"unknown milestone profile: {profile_name}"], + "failed_gates": [], + } + + target_top_k = int(profile["target_top_k"]) + retrieved = _mean_run_row(rows, tool_source="retrieved", top_k=target_top_k) + row_source = _mean_run_row(rows, tool_source="row", top_k=target_top_k) + parallel_multiple = _mean_category_row( + category_rows, + tool_source="retrieved", + top_k=target_top_k, + category="parallel_multiple", + ) + + row_exact = _value_or_none(row_source, "evaluator_exact_match") + retrieved_exact = _value_or_none(retrieved, "evaluator_exact_match") + preservation = ( + round(retrieved_exact / row_exact, 6) + if retrieved_exact is not None and row_exact is not None and row_exact > 0 + else None + ) + metrics = { + f"retrieved_exact_at_{target_top_k}": retrieved_exact, + f"retrieval_recall_at_{target_top_k}": _value_or_none(retrieved, "retrieval_recall_at_k"), + f"row_source_exact_at_{target_top_k}": row_exact, + "row_source_upper_bound_preservation": preservation, + f"parallel_multiple_exact_at_{target_top_k}": _value_or_none( + parallel_multiple, "evaluator_exact_match" + ), + } + thresholds = { + f"retrieved_exact_at_{target_top_k}": profile["min_retrieved_exact"], + f"retrieval_recall_at_{target_top_k}": profile["min_retrieval_recall"], + "row_source_upper_bound_preservation": profile["min_row_source_preservation"], + f"parallel_multiple_exact_at_{target_top_k}": profile["min_parallel_multiple_exact"], + } + failed_gates = [ + { + "metric": metric, + "actual": actual, + "threshold": threshold, + } + for metric, threshold in thresholds.items() + if (actual := metrics.get(metric)) is not None and float(actual) < float(threshold) + ] + missing_metrics = [ + metric for metric, value in metrics.items() if value is None and metric in thresholds + ] + status = "pass" + if missing_metrics: + status = "incomplete" + elif failed_gates: + status = "fail" + + return { + "profile": profile_name, + "status": status, + "target_top_k": target_top_k, + "metrics": metrics, + "thresholds": thresholds, + "failed_gates": failed_gates, + "missing_metrics": missing_metrics, + } + + +def _mean_run_row( + rows: list[dict[str, Any]], + *, + tool_source: str, + top_k: int, +) -> dict[str, Any]: + selected = [ + row for row in rows if row["tool_source"] == tool_source and int(row["top_k"]) == top_k + ] + return _mean_selected_rows(selected) + + +def _mean_category_row( + rows: list[dict[str, Any]], + *, + tool_source: str, + top_k: int, + category: str, +) -> dict[str, Any]: + selected = [ + row + for row in rows + if row["tool_source"] == tool_source + and int(row["top_k"]) == top_k + and row["category"] == category + ] + return _mean_selected_rows(selected) + + +def _mean_selected_rows(rows: list[dict[str, Any]]) -> dict[str, Any]: + if not rows: + return {} + return { + "cases": int(round(_mean(row["cases"] for row in rows))), + "retrieval_recall_at_k": _mean(row["retrieval_recall_at_k"] for row in rows), + "model_tool_call_rate": _mean(row["model_tool_call_rate"] for row in rows), + "strict_exact_match": _mean(row["strict_exact_match"] for row in rows), + "evaluator_exact_match": _mean(row["evaluator_exact_match"] for row in rows), + "avg_latency_ms": _mean(row["avg_latency_ms"] for row in rows), + } + + +def _value_or_none(row: dict[str, Any], key: str) -> float | None: + if not row: + return None + value = row.get(key) + return None if value is None else float(value) + + def write_sweep_bfcl_result_files( report: dict[str, Any], output_dir: Path, @@ -279,6 +491,9 @@ def print_report(report: dict[str, Any]) -> None: latency_mean=latency["mean"], ) ) + gate = report["summary"].get("milestone_gate") or {} + if gate: + print(_format_milestone_gate(gate)) def _format_failure_breakdown(breakdown: dict[str, int]) -> str: @@ -287,6 +502,26 @@ def _format_failure_breakdown(breakdown: dict[str, int]) -> str: return ",".join(f"{name}:{count}" for name, count in sorted(breakdown.items())) +def _format_milestone_gate(gate: dict[str, Any]) -> str: + metrics = gate.get("metrics") or {} + top_k = gate.get("target_top_k", "?") + retrieved_exact = _format_optional_metric(metrics.get(f"retrieved_exact_at_{top_k}")) + retrieval = _format_optional_metric(metrics.get(f"retrieval_recall_at_{top_k}")) + preservation = _format_optional_metric(metrics.get("row_source_upper_bound_preservation")) + parallel = _format_optional_metric(metrics.get(f"parallel_multiple_exact_at_{top_k}")) + return ( + f"milestone {gate.get('profile')} status={gate.get('status')} " + f"retrieved_exact@{top_k}={retrieved_exact} " + f"retrieval@{top_k}={retrieval} " + f"row_preservation={preservation} " + f"parallel_multiple={parallel}" + ) + + +def _format_optional_metric(value: Any) -> str: + return "n/a" if value is None else f"{float(value):.3f}" + + def _parse_ints(value: str) -> list[int]: return [int(part.strip()) for part in value.split(",") if part.strip()] @@ -318,6 +553,12 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--repeats", type=int, default=1) parser.add_argument("--timeout", type=int, default=180) parser.add_argument("--disable-thinking", action="store_true") + parser.add_argument( + "--milestone-profile", + choices=[*MILESTONE_PROFILES.keys(), "none"], + default=DEFAULT_MILESTONE_PROFILE, + help="Add a milestone gate summary to the sweep artifact.", + ) parser.add_argument( "--concurrency", type=int, @@ -372,6 +613,7 @@ def main(argv: list[str] | None = None) -> int: concurrency=args.concurrency, progress=args.progress, progress_every=args.progress_every, + milestone_profile=args.milestone_profile, ) except ImportError as exc: parser.error(str(exc)) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 4fc773b..75a60fb 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -407,6 +407,30 @@ greater than one, the JSON `summary.repeat_groups` and text output include mean/std/min/max for exact match, strict exact match, retrieval recall, and latency by `(tool_source, top_k)`. +The sweep artifact also includes `summary.milestone_gate`. By default it uses +the `xgen-0.27` profile and checks the next product-readiness target directly: +retrieved-source exact match at `k=5`, retrieval recall at `k=5`, row-source +upper-bound preservation, and `parallel_multiple` exact match. This is the +fastest way to decide whether a full expensive run produced evidence strong +enough for the next XGEN milestone, or whether work should return to a smaller +failure subset first. + +```bash +poetry run python -m benchmarks.bfcl_tool_selection.sweep \ + --categories simple_python,multiple,parallel,parallel_multiple \ + --tool-sources row,retrieved \ + --top-ks 5 \ + --model qwen3.6-27b \ + --llm-url http://127.0.0.1:8000/v1 \ + --disable-thinking \ + --milestone-profile xgen-0.27 \ + --output /tmp/gtc-bfcl-xgen-027-sweep.json +``` + +For diagnostic sweeps that do not include the row-source baseline or +`parallel_multiple`, pass `--milestone-profile none` or treat the gate status +`incomplete` as expected. + Recommended smoke with a native function-calling endpoint: ```bash diff --git a/docs/research/validation-loop.md b/docs/research/validation-loop.md index 3a0f2f1..96c2e8b 100644 --- a/docs/research/validation-loop.md +++ b/docs/research/validation-loop.md @@ -29,6 +29,14 @@ graph-tool-call search 연구는 full model benchmark를 매번 돌리면 속도 T0-T1은 매일 자주 돌린다. T2-T3는 실험 branch에서만 선택적으로 돌린다. T4는 milestone 또는 publish candidate에서만 허용한다. +BFCL model sweep artifact에는 `summary.milestone_gate`가 포함된다. 기본 +profile은 `xgen-0.27`이며, retrieved `k=5` exact, retrieval recall, row-source +upper-bound preservation, `parallel_multiple` exact를 한 번에 판정한다. 이 gate가 +`fail`이면 full run 숫자를 더 오래 읽지 말고 `failure_breakdown`, +`category_rows`, hard-case bundle로 돌아가 작은 subset을 먼저 고친다. row-source +baseline이나 `parallel_multiple`를 일부러 제외한 smoke에서는 `incomplete`가 정상일 +수 있다. + ## 실행 타깃 ```bash diff --git a/docs/research/xgen-tool-graph-goals.md b/docs/research/xgen-tool-graph-goals.md index 4080775..5573fd3 100644 --- a/docs/research/xgen-tool-graph-goals.md +++ b/docs/research/xgen-tool-graph-goals.md @@ -244,6 +244,13 @@ Required work: Artifact는 `/tmp/gtc-x2bee-sweep-top1-ambiguity.json`이고, `TOP_KS=3,5,10` 기준 `hit@3=1.00`, `expected recall@3=1.00`, `top-1 hit=1.00`, `top-3 hit=1.00`, `mean MRR=1.00`이다. +- BFCL model sweep artifact가 0.27 milestone gate를 직접 포함한다. + - `2026-07-19`: `benchmarks.bfcl_tool_selection.sweep`의 + `summary.milestone_gate`에 `xgen-0.27` profile을 추가했다. Gate는 + retrieved `k=5` exact, retrieval recall, row-source upper-bound + preservation, `parallel_multiple` exact를 판정한다. 이 값이 `fail`이면 full + run을 반복하지 않고 `category_rows`와 hard-case bundle로 돌아가 작은 subset을 + 먼저 고친다. ## Paper-Ready Target diff --git a/tests/test_bfcl_tool_selection_sweep.py b/tests/test_bfcl_tool_selection_sweep.py index 94135ad..12cf03a 100644 --- a/tests/test_bfcl_tool_selection_sweep.py +++ b/tests/test_bfcl_tool_selection_sweep.py @@ -69,6 +69,45 @@ def fake_run_model_benchmark(**kwargs): "min": 0.5, "max": 0.5, } + assert report["summary"]["milestone_gate"]["status"] == "incomplete" + assert "parallel_multiple_exact_at_5" in report["summary"]["milestone_gate"]["missing_metrics"] + + +def test_sweep_milestone_gate_reports_xgen_027_bottlenecks(): + summary = sweep._summarize_sweep( + [ + _sweep_run( + tool_source="row", + top_k=5, + exact=0.90, + retrieval=0.91, + parallel_multiple_exact=0.80, + ), + _sweep_run( + tool_source="retrieved", + top_k=5, + exact=0.84, + retrieval=0.96, + parallel_multiple_exact=0.74, + ), + ] + ) + + gate = summary["milestone_gate"] + + assert gate["profile"] == "xgen-0.27" + assert gate["status"] == "fail" + assert gate["metrics"]["retrieved_exact_at_5"] == 0.84 + assert gate["metrics"]["retrieval_recall_at_5"] == 0.96 + assert gate["metrics"]["row_source_exact_at_5"] == 0.9 + assert gate["metrics"]["row_source_upper_bound_preservation"] == 0.933333 + assert gate["metrics"]["parallel_multiple_exact_at_5"] == 0.74 + assert {row["metric"] for row in gate["failed_gates"]} == { + "retrieved_exact_at_5", + "row_source_upper_bound_preservation", + "parallel_multiple_exact_at_5", + } + assert summary["category_repeat_groups"][0]["category"] == "parallel_multiple" def test_write_sweep_bfcl_result_files_separates_runs(tmp_path): @@ -121,3 +160,43 @@ def _single_case_report(tool_source: str): } ], } + + +def _sweep_run( + *, + tool_source: str, + top_k: int, + exact: float, + retrieval: float, + parallel_multiple_exact: float, +): + return { + "repeat": 1, + "tool_source": tool_source, + "top_k": top_k, + "report": { + "summary": { + "cases": 10, + "retrieval_recall_at_k": retrieval, + "model_tool_call_rate": 1.0, + "strict_exact_match": exact, + "evaluator_exact_match": exact, + "avg_latency_ms": 100.0, + "failure_breakdown": {"pass": int(exact * 10)}, + }, + "categories": [ + { + "category": "parallel_multiple", + "summary": { + "cases": 5, + "retrieval_recall_at_k": retrieval, + "model_tool_call_rate": 1.0, + "strict_exact_match": parallel_multiple_exact, + "evaluator_exact_match": parallel_multiple_exact, + "avg_latency_ms": 100.0, + "failure_breakdown": {"pass": int(parallel_multiple_exact * 5)}, + }, + } + ], + }, + }