From 2e435612c18b079e2c71bccbd09050e7ba380424 Mon Sep 17 00:00:00 2001 From: parth Date: Sun, 11 Oct 2026 23:15:17 +0530 Subject: [PATCH 1/3] Add an output-token cap and reasoning effort for the LLM client LLM_MAX_OUTPUT_TOKENS caps each reply (sent as max_tokens, the name every OpenAI-compatible API accepts). Cerebras reserves the cap against its rate limits before running a request, so an unset cap reserves the model maximum. LLM_REASONING_EFFORT (low|medium|high) is passed to reasoning models such as gpt-oss. Both are sent only when set, so existing setups are unchanged. Evaluation runs record the LLM host and reasoning effort, the report shows them, a resumed run must keep the same effort, and token estimates count only earlier runs with the same effort. .env.example documents Cerebras. --- .env.example | 8 ++++++++ backend/app/config.py | 5 +++++ backend/app/llm/client.py | 11 +++++++++++ backend/app/main.py | 8 +++++++- backend/tests/test_llm_client.py | 24 ++++++++++++++++++++++++ evaluation/report.py | 7 ++++++- evaluation/run.py | 25 ++++++++++++++++++++----- evaluation/tests/test_run.py | 13 ++++++++++--- 8 files changed, 91 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index 84ceb1f..a53826e 100644 --- a/.env.example +++ b/.env.example @@ -45,6 +45,14 @@ LLM_MODEL=gemini-3.8-flash # Groq free tier (key: https://console.groq.com/keys): # LLM_BASE_URL=https://api.groq.com/openai/v1 LLM_MODEL=openai/gpt-oss-120b # (the evaluated model; free tier: 8K tokens/minute, 200K tokens/day) +# Cerebras (key: https://cloud.cerebras.ai; trial: $5 credit for 30 days, 5 requests/minute, +# 1M tokens/day): +# LLM_BASE_URL=https://api.cerebras.ai/v1 LLM_MODEL=gpt-oss-120b LLM_MAX_OUTPUT_TOKENS=4096 +# Optional: cap on each reply, reasoning included (Cerebras reserves the cap against its rate +# limits before running a request), and reasoning effort for reasoning models (low|medium|high). +# Both unset: the provider's defaults. +# LLM_MAX_OUTPUT_TOKENS= +# LLM_REASONING_EFFORT= # --- Host ports published by docker compose (change when 5432 / 8000 are already in use; # then use the same port in DATABASE_URL and ADMIN_DATABASE_URL) --- diff --git a/backend/app/config.py b/backend/app/config.py index 04ceb08..abf170c 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -33,6 +33,11 @@ class Settings(BaseSettings): llm_base_url: str = "https://generativelanguage.googleapis.com/v1beta/openai/" llm_api_key: SecretStr | None = None llm_model: str = "gemini-3.8-flash" + # Cap on each reply, reasoning included. Unset: the provider's default. Set it where the + # provider reserves the cap against its rate limits before running a request (Cerebras). + llm_max_output_tokens: int | None = Field(default=None, ge=256, le=65_536) + # For reasoning models (gpt-oss: low, medium, high). Unset: the provider's default. + llm_reasoning_effort: Literal["low", "medium", "high"] | None = None # Limits on POST /api/query, the endpoint that spends LLM quota (see app/api/rate_limit.py). # 0 turns a limit off. Per process: several workers or instances each enforce their own. diff --git a/backend/app/llm/client.py b/backend/app/llm/client.py index 2498834..497ffcc 100644 --- a/backend/app/llm/client.py +++ b/backend/app/llm/client.py @@ -130,8 +130,15 @@ def __init__( max_wait_seconds: float = 8.0, transport: httpx.BaseTransport | None = None, sleep: Callable[[float], None] = time.sleep, + max_output_tokens: int | None = None, + reasoning_effort: str | None = None, ) -> None: + """max_output_tokens caps each reply, reasoning included (some providers reserve the cap + against their rate limits before a request runs); reasoning_effort ("low", "medium", + "high") is passed to reasoning models. Either is sent only when set.""" self.model = model + self.max_output_tokens = max_output_tokens + self.reasoning_effort = reasoning_effort self.max_attempts = max_attempts self.max_wait_seconds = max_wait_seconds self._sleep = sleep @@ -151,6 +158,10 @@ def complete_json( "response_format": {"type": "json_object"}, "messages": [{"role": "system", "content": system}, {"role": "user", "content": user}], } + if self.max_output_tokens is not None: + payload["max_tokens"] = self.max_output_tokens # the name every compatible API accepts + if self.reasoning_effort is not None: + payload["reasoning_effort"] = self.reasoning_effort started = time.perf_counter() attempt = 0 while True: diff --git a/backend/app/main.py b/backend/app/main.py index e185261..3da9019 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -28,7 +28,13 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: ) app.state.registry = registry app.state.llm = ( - OpenAICompatibleClient(settings.llm_base_url, settings.llm_api_key, settings.llm_model) + OpenAICompatibleClient( + settings.llm_base_url, + settings.llm_api_key, + settings.llm_model, + max_output_tokens=settings.llm_max_output_tokens, + reasoning_effort=settings.llm_reasoning_effort, + ) if settings.llm_api_key and settings.llm_api_key.get_secret_value() else None ) diff --git a/backend/tests/test_llm_client.py b/backend/tests/test_llm_client.py index c00d655..546581e 100644 --- a/backend/tests/test_llm_client.py +++ b/backend/tests/test_llm_client.py @@ -54,6 +54,28 @@ def handler(request: httpx.Request) -> httpx.Response: assert [m["role"] for m in seen["body"]["messages"]] == ["system", "user"] +def test_output_cap_and_reasoning_effort_are_sent_only_when_set() -> None: + bodies: list[dict[str, Any]] = [] + + def handler(request: httpx.Request) -> httpx.Response: + bodies.append(json.loads(request.content)) + return httpx.Response(200, json=_reply('{"sql": "SELECT 1", "explanation": "x"}')) + + _client(handler).complete_json("s", "u", GeneratedSQL) + configured = OpenAICompatibleClient( + "https://llm.example.test/v1", + SecretStr("sk-secret"), + "test-model", + transport=httpx.MockTransport(handler), + max_output_tokens=4096, + reasoning_effort="high", + ) + configured.complete_json("s", "u", GeneratedSQL) + assert "max_tokens" not in bodies[0] and "reasoning_effort" not in bodies[0] + assert (bodies[1]["max_tokens"], bodies[1]["reasoning_effort"]) == (4096, "high") + assert "max_completion_tokens" not in bodies[1] # an alias of max_tokens: never send both + + def test_http_errors_do_not_leak_the_response_body_or_key() -> None: client = _client(lambda request: httpx.Response(401, text="invalid key sk-secret")) with pytest.raises(LLMError) as info: @@ -122,6 +144,8 @@ def handler(request: httpx.Request) -> httpx.Response: "code": "rate_limit_exceeded", } } + + GROQ_MINUTE = { "error": { "message": "Rate limit reached on tokens per minute (TPM): Limit 8000. Please try again in 41.5s.", diff --git a/evaluation/report.py b/evaluation/report.py index ba11a19..0f86d54 100644 --- a/evaluation/report.py +++ b/evaluation/report.py @@ -94,6 +94,8 @@ def of(behaviour: str) -> list[dict[str, Any]]: "code_commit": first.get("code_commit"), "dataset_commit": first.get("dataset_commit"), "answers": first.get("answers"), + "llm_host": first.get("llm_host"), + "reasoning_effort": first.get("reasoning_effort"), "max_rows": first.get("max_rows"), "planned": len(planned_ids), "scored": len(scored), @@ -149,7 +151,10 @@ def render(m: dict[str, Any]) -> str: status = "complete" if m["complete"] else f"PARTIAL: {m['not_run']} of {m['planned']} not run" lines = [ f"Run {m['run_id']} ({m['suite']}, {status})", - f"Model: {m['model']} | answers: {m['answers']}", + f"Model: {m['model']}" + + (f" via {m['llm_host']}" if m.get("llm_host") else "") + + (f" | reasoning: {m['reasoning_effort']}" if m.get("reasoning_effort") else "") + + f" | answers: {m['answers']}", *( [f"Dataset: BIRD {m['split']} | scope: {m['scope']} | evidence: {m['evidence']}"] if m["dataset"] == "bird" diff --git a/evaluation/run.py b/evaluation/run.py index 4240d23..4ffcf6a 100644 --- a/evaluation/run.py +++ b/evaluation/run.py @@ -42,6 +42,7 @@ from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any +from urllib.parse import urlsplit from sqlalchemy.engine import make_url @@ -310,10 +311,14 @@ def token_estimate(pending: int, per_question: tuple[float, str] | None, daily_t def measured_tokens_per_question( - dataset: str, model: str, answers: str, results_dir: Path = RESULTS_DIR + dataset: str, + model: str, + answers: str, + reasoning_effort: str | None = None, + results_dir: Path = RESULTS_DIR, ) -> tuple[float, str] | None: """Mean tokens per scored question in earlier runs of this model on this dataset, with the same - answer mode (LLM answers cost one more call).""" + answer mode (LLM answers cost one more call) and reasoning effort (more reasoning, more tokens).""" totals: list[int] = [] runs: set[str] = set() for path in sorted(results_dir.glob("*.jsonl")): @@ -322,6 +327,7 @@ def measured_tokens_per_question( record.get("dataset", "owid") == dataset and record.get("model") == model and record.get("answers") == answers + and record.get("reasoning_effort") == reasoning_effort and record.get("correct") is not None and record.get("prompt_tokens") is not None ): @@ -430,7 +436,11 @@ def main(argv: list[str] | None = None) -> int: default_rows = bird.BIRD_MAX_ROWS if args.dataset == "bird" else settings.max_rows max_rows = args.max_rows or default_rows - meta: dict[str, Any] = {"dataset": args.dataset, "max_rows": max_rows} + meta: dict[str, Any] = { + "dataset": args.dataset, + "max_rows": max_rows, + "reasoning_effort": settings.llm_reasoning_effort, + } resume = f"python -m evaluation.run --suite {args.suite}" if max_rows != default_rows: resume += f" --max-rows {max_rows}" @@ -509,7 +519,11 @@ def main(argv: list[str] | None = None) -> int: print("Set LLM_API_KEY to run the question suite.", file=sys.stderr) return 2 llm: LLMClient = OpenAICompatibleClient( - settings.llm_base_url, settings.llm_api_key, settings.llm_model + settings.llm_base_url, + settings.llm_api_key, + settings.llm_model, + max_output_tokens=settings.llm_max_output_tokens, + reasoning_effort=settings.llm_reasoning_effort, ) controller = AgentController( llm, @@ -545,6 +559,7 @@ def main(argv: list[str] | None = None) -> int: run_id=run_id, suite=args.suite, model=model, + llm_host=None if args.suite == "sql-safety" else urlsplit(settings.llm_base_url).hostname, answers=args.answers, max_retries=settings.max_retries, code_commit=_git_commit(), @@ -553,7 +568,7 @@ def main(argv: list[str] | None = None) -> int: per_question = ( None if args.suite == "sql-safety" - else measured_tokens_per_question(args.dataset, model, args.answers) + else measured_tokens_per_question(args.dataset, model, args.answers, settings.llm_reasoning_effort) ) with ExitStack() as stack: official = stack.enter_context(bird.official_scorer(url)) if args.dataset == "bird" else None diff --git a/evaluation/tests/test_run.py b/evaluation/tests/test_run.py index ef54406..f7d4e46 100644 --- a/evaluation/tests/test_run.py +++ b/evaluation/tests/test_run.py @@ -291,14 +291,21 @@ def test_token_estimate_uses_measured_use_only(tmp_path: Path) -> None: "prompt_tokens": 9000, "completion_tokens": 0}, {"id": "B4", "dataset": "bird", "model": "m", "answers": "template", "correct": False, "prompt_tokens": 2600, "completion_tokens": 400}, + {"id": "B5", "dataset": "bird", "model": "m", "answers": "template", "correct": True, + "reasoning_effort": "high", "prompt_tokens": 2000, "completion_tokens": 3000}, ] # fmt: skip (tmp_path / "old.jsonl").write_text("\n".join(json.dumps(r) for r in old) + "\n") - assert measured_tokens_per_question("bird", "m", "template", tmp_path) == ( + assert measured_tokens_per_question("bird", "m", "template", results_dir=tmp_path) == ( 2500.0, "2 questions of 1 earlier run(s)", ) - assert measured_tokens_per_question("bird", "m", "llm", tmp_path) is None - assert measured_tokens_per_question("owid", "m", "template", tmp_path) is None + # More reasoning costs more tokens: only runs with the same effort count. + assert measured_tokens_per_question("bird", "m", "template", "high", tmp_path) == ( + 5000.0, + "1 questions of 1 earlier run(s)", + ) + assert measured_tokens_per_question("bird", "m", "llm", results_dir=tmp_path) is None + assert measured_tokens_per_question("owid", "m", "template", results_dir=tmp_path) is None def test_bird_report_leads_with_the_official_metric_and_shows_the_review_beside_it(tmp_path: Path) -> None: From 8ec81ab86583f5cf90795d41cac05ea84479aa1c Mon Sep 17 00:00:00 2001 From: parth Date: Sun, 11 Oct 2026 23:15:17 +0530 Subject: [PATCH 2/3] Tell the SQL generator never to concatenate columns In the partial Mini-Dev run, 4 of 87 questions were missed only because the model joined two stored columns into one (a definition named them as one term). The rule is general: concatenating is reformatting stored values, which the prompt already discourages. Prompt version sql-generator/7. --- backend/app/agent/sql_generator.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/app/agent/sql_generator.py b/backend/app/agent/sql_generator.py index 5c421e2..f575073 100644 --- a/backend/app/agent/sql_generator.py +++ b/backend/app/agent/sql_generator.py @@ -17,7 +17,7 @@ from app.agent.schema_retriever import SchemaContext from app.llm.client import LLMCall, LLMClient -PROMPT_VERSION = "sql-generator/6" +PROMPT_VERSION = "sql-generator/7" MAX_HISTORY_TURNS = 3 SYSTEM_PROMPT = """\ @@ -45,6 +45,9 @@ count. - Return stored values as they are. Do not round, cast or reformat a returned value unless the question asks for it; cast only where a calculation needs it (e.g. to avoid integer division). +- Return each stored column as its own result column. Never concatenate columns into one, even + when the question or a definition names them together ("address means street, city" + returns street and city as two columns). - When listing entities through a join that can repeat them, return each entity once. - Do not add conditions the question does not state (such as only the latest record, or only rows where another column is not NULL). From 6f85d1453a09a6b003beda508912dbb280d64489 Mon Sep 17 00:00:00 2001 From: parth Date: Mon, 12 Oct 2026 11:41:36 +0530 Subject: [PATCH 3/3] Wire candidate voting into settings and the runner; pace LLM calls - SQL_CANDIDATES (API) and --candidates (evaluation runs) pass the number of candidate queries to AgentController. Runs record it, resume refuses a mismatch, the token estimate only counts runs with the same number, and the report shows it. - LLM_REQUESTS_PER_MINUTE spaces LLM calls (retries included) so a small per-minute limit such as the Cerebras trial's is not hit; with it set the runner's --delay defaults to 0. Pacing waits are not counted in call duration. - Evaluation runs wait out per-minute 429s inside the LLM call (up to 120 s), keeping the question's earlier calls and candidates instead of re-asking. - A 429 asking to wait more than ten minutes counts as a used-up quota even without a body (Cerebras can answer with only Retry-After). --- .env.example | 15 ++++++++--- EVALUATION_PLAN.md | 8 ++++++ README.md | 2 ++ backend/app/api/routes.py | 1 + backend/app/config.py | 6 +++++ backend/app/llm/client.py | 34 +++++++++++++++++++++++-- backend/app/main.py | 1 + backend/tests/test_api.py | 9 +++++++ backend/tests/test_llm_client.py | 43 ++++++++++++++++++++++++++++++++ evaluation/report.py | 2 ++ evaluation/run.py | 39 +++++++++++++++++++++++------ evaluation/tests/test_run.py | 7 ++++++ 12 files changed, 154 insertions(+), 13 deletions(-) diff --git a/.env.example b/.env.example index a53826e..d7d4135 100644 --- a/.env.example +++ b/.env.example @@ -25,6 +25,9 @@ MAX_ROWS=1000 # llm: answers written by the LLM and checked against the result (one extra call per question) # template: answers built from the rows, no extra LLM call (e.g. for evaluation runs) ANSWER_MODE=llm +# Candidate queries per question (1-5). Above 1, the result most candidates agree on wins: about +# 2-3x the LLM calls and tokens of a single candidate. Evaluation runs can set it with --candidates. +SQL_CANDIDATES=1 # --- Rate limits on POST /api/query (the endpoint that spends LLM quota; 0 = off) --- # Per client address per minute, and across all clients per 24 hours. Each question makes @@ -45,14 +48,18 @@ LLM_MODEL=gemini-3.8-flash # Groq free tier (key: https://console.groq.com/keys): # LLM_BASE_URL=https://api.groq.com/openai/v1 LLM_MODEL=openai/gpt-oss-120b # (the evaluated model; free tier: 8K tokens/minute, 200K tokens/day) -# Cerebras (key: https://cloud.cerebras.ai; trial: $5 credit for 30 days, 5 requests/minute, -# 1M tokens/day): +# Cerebras (key: https://cloud.cerebras.ai; trial for gpt-oss-120b: $5 credit for 30 days, +# 5 requests/minute, 30K uncached tokens/minute, 1M tokens/hour and per day): # LLM_BASE_URL=https://api.cerebras.ai/v1 LLM_MODEL=gpt-oss-120b LLM_MAX_OUTPUT_TOKENS=4096 +# LLM_REQUESTS_PER_MINUTE=4 (each call reserves prompt + output cap, ~7.5k tokens, against +# the 30K per minute: 4 calls fit, 5 may not) # Optional: cap on each reply, reasoning included (Cerebras reserves the cap against its rate -# limits before running a request), and reasoning effort for reasoning models (low|medium|high). -# Both unset: the provider's defaults. +# limits before running a request), reasoning effort for reasoning models (low|medium|high), and +# spacing of LLM calls to stay under a per-minute request limit. Unset: the provider's defaults, +# calls not spaced. # LLM_MAX_OUTPUT_TOKENS= # LLM_REASONING_EFFORT= +# LLM_REQUESTS_PER_MINUTE= # --- Host ports published by docker compose (change when 5432 / 8000 are already in use; # then use the same port in DATABASE_URL and ADMIN_DATABASE_URL) --- diff --git a/EVALUATION_PLAN.md b/EVALUATION_PLAN.md index 2b7a86f..2215f8b 100644 --- a/EVALUATION_PLAN.md +++ b/EVALUATION_PLAN.md @@ -129,6 +129,14 @@ compares it with --daily-tokens when given (Groq's daily token limit is not in i headers). Provider failures are handled by kind: a per-minute limit is waited out when the provider asks for at most two minutes and the question asked again; a used-up daily or account quota stops the run at once and says when to resume; other failures stop it after three in a row. +A 429 asking for a wait of more than ten minutes counts as a used-up quota even without a message +saying so (Cerebras can answer with no body, only Retry-After). Within a run, a short per-minute +wait is first waited out inside the LLM call, so the question's earlier calls are kept; with +LLM_REQUESTS_PER_MINUTE set the calls are spaced out so the limit is not hit at all, and --delay +defaults to 0. --candidates N (default SQL_CANDIDATES, 1) generates up to N queries per question and +keeps the result most agree on; it stops early once the majority is decided, so it costs 2-3x the +calls and tokens of one candidate at N = 3. Runs record the number, and the token estimate only +counts earlier runs with the same number. βΈ» diff --git a/README.md b/README.md index c27c188..e2357d9 100644 --- a/README.md +++ b/README.md @@ -104,9 +104,11 @@ All settings are environment variables; [.env.example](.env.example) documents e | Variable | Purpose | |---|---| | `LLM_BASE_URL`, `LLM_API_KEY`, `LLM_MODEL` | Any OpenAI-compatible provider | +| `LLM_MAX_OUTPUT_TOKENS`, `LLM_REASONING_EFFORT`, `LLM_REQUESTS_PER_MINUTE` | Optional: reply cap, reasoning effort, spacing of LLM calls for small per-minute limits | | `DATABASE_URL` / `DATABASES_CONFIG` | One database, or several via [databases.example.toml](databases.example.toml) | | `QUERY_TIMEOUT_SECONDS`, `MAX_ROWS`, `MAX_RETRIES` | Query guardrails | | `ANSWER_MODE` | `llm` (checked against the rows) or `template` (no extra LLM call) | +| `SQL_CANDIDATES` | Candidate queries per question; above 1 the majority result wins (2-3x the LLM calls) | | `RATE_LIMIT_PER_MINUTE`, `RATE_LIMIT_PER_DAY`, `TRUSTED_PROXY_HOPS` | Limits on `/api/query` | | `ALLOW_UI_CONNECTIONS` | Local use only | diff --git a/backend/app/api/routes.py b/backend/app/api/routes.py index eda66a7..5fdd43f 100644 --- a/backend/app/api/routes.py +++ b/backend/app/api/routes.py @@ -47,6 +47,7 @@ def get_agent(request: Request, settings: Annotated[Settings, Depends(get_settin max_rows=settings.max_rows, max_retries=settings.max_retries, answer_llm=llm if settings.answer_mode == "llm" else None, + candidates=settings.sql_candidates, ) diff --git a/backend/app/config.py b/backend/app/config.py index abf170c..b35553a 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -28,6 +28,9 @@ class Settings(BaseSettings): # "llm": natural-language answers written by the LLM (one extra call per question) and checked # against the rows; "template": answers built from the rows without the LLM. answer_mode: Literal["llm", "template"] = "llm" + # Candidate queries per question: above 1, the result most candidates agree on wins (about + # 2-3x the LLM calls and tokens of one candidate). See app/agent/controller.py. + sql_candidates: int = Field(default=1, ge=1, le=5) # Any OpenAI-compatible chat-completions endpoint (Groq, Gemini, OpenRouter, Ollama, ...). llm_base_url: str = "https://generativelanguage.googleapis.com/v1beta/openai/" @@ -38,6 +41,9 @@ class Settings(BaseSettings): llm_max_output_tokens: int | None = Field(default=None, ge=256, le=65_536) # For reasoning models (gpt-oss: low, medium, high). Unset: the provider's default. llm_reasoning_effort: Literal["low", "medium", "high"] | None = None + # Spaces LLM calls out to stay under a small per-minute request limit (a trial tier). + # Unset: calls are not spaced (the provider's 429s are still retried or reported). + llm_requests_per_minute: int | None = Field(default=None, ge=1, le=10_000) # Limits on POST /api/query, the endpoint that spends LLM quota (see app/api/rate_limit.py). # 0 turns a limit off. Per process: several workers or instances each enforce their own. diff --git a/backend/app/llm/client.py b/backend/app/llm/client.py index 497ffcc..88ab862 100644 --- a/backend/app/llm/client.py +++ b/backend/app/llm/client.py @@ -8,6 +8,8 @@ with a short wait, so a free-tier hiccup does not fail the whole question. A failure that waiting will not fix (a daily or account quota used up) is not retried, and is reported as such: LLMError.code says what kind of failure it was, and retry_after_seconds when the provider said how long to wait. +With requests_per_minute set, calls are spaced out so a small per-minute limit (a trial tier) +is not hit in the first place. """ from __future__ import annotations @@ -15,6 +17,7 @@ import json import logging import re +import threading import time from collections.abc import Callable from dataclasses import dataclass @@ -41,6 +44,9 @@ # Provider error codes are reported only when they look like an identifier ("rate_limit_exceeded"). _PROVIDER_CODE = re.compile(r"[A-Za-z][A-Za-z0-9_.-]{0,39}") MAX_RETRY_AFTER_SECONDS = 2 * 24 * 3600 +# A 429 that asks for a longer wait than this is a used-up quota for practical purposes, even when +# the body does not say so (Cerebras can answer 429 with no body and only a Retry-After header). +QUOTA_WAIT_SECONDS = 600 class LLMError(RuntimeError): @@ -132,6 +138,8 @@ def __init__( sleep: Callable[[float], None] = time.sleep, max_output_tokens: int | None = None, reasoning_effort: str | None = None, + requests_per_minute: int | None = None, + clock: Callable[[], float] = time.monotonic, ) -> None: """max_output_tokens caps each reply, reasoning included (some providers reserve the cap against their rate limits before a request runs); reasoning_effort ("low", "medium", @@ -142,6 +150,11 @@ def __init__( self.max_attempts = max_attempts self.max_wait_seconds = max_wait_seconds self._sleep = sleep + self._clock = clock + # Calls start at least this far apart (retries included); shared by every thread. + self._interval = 60.0 / requests_per_minute if requests_per_minute else 0.0 + self._next_start = 0.0 + self._pace_lock = threading.Lock() self._http = httpx.Client( base_url=base_url.rstrip("/") + "/", headers={"Authorization": f"Bearer {api_key.get_secret_value()}"}, @@ -163,9 +176,11 @@ def complete_json( if self.reasoning_effort is not None: payload["reasoning_effort"] = self.reasoning_effort started = time.perf_counter() + paced = 0.0 # time spent waiting for a slot: not part of the call's duration attempt = 0 while True: attempt += 1 + paced += self._pace() try: response = self._http.post("chat/completions", json=payload) except httpx.HTTPError as exc: @@ -178,7 +193,7 @@ def complete_json( if wait > self.max_wait_seconds: break # e.g. a per-minute quota: waiting that long inside a request is worse than failing self._sleep(wait) - duration_ms = int((time.perf_counter() - started) * 1000) + duration_ms = max(0, int((time.perf_counter() - started - paced) * 1000)) log_event( logger, "llm call", @@ -201,6 +216,18 @@ def complete_json( ) return parse_output(content, output), call + def _pace(self) -> float: + """Wait for this call's slot when calls are spaced out (requests_per_minute); the seconds waited.""" + if not self._interval: + return 0.0 + with self._pace_lock: + now = self._clock() + start = max(now, self._next_start) + self._next_start = start + self._interval + if start > now: + self._sleep(start - now) + return start - now + @staticmethod def _retry_wait(response: httpx.Response, attempt: int) -> float: """Seconds to wait: the provider's Retry-After if given, else 1s, 2s, 4s, ...""" @@ -211,7 +238,10 @@ def _retry_wait(response: httpx.Response, attempt: int) -> float: def _quota_exhausted(response: httpx.Response) -> bool: - return response.status_code == 429 and bool(_EXHAUSTED_QUOTA.search(response.text)) + if response.status_code != 429: + return False + wait = _retry_after(response) + return bool(_EXHAUSTED_QUOTA.search(response.text)) or (wait is not None and wait > QUOTA_WAIT_SECONDS) def _retry_after(response: httpx.Response) -> int | None: diff --git a/backend/app/main.py b/backend/app/main.py index 3da9019..b5c81b5 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -34,6 +34,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: settings.llm_model, max_output_tokens=settings.llm_max_output_tokens, reasoning_effort=settings.llm_reasoning_effort, + requests_per_minute=settings.llm_requests_per_minute, ) if settings.llm_api_key and settings.llm_api_key.get_secret_value() else None diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 10d3bd6..91591ff 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -2,12 +2,14 @@ import json from collections.abc import Iterator +from types import SimpleNamespace import pytest from fastapi.testclient import TestClient from pydantic import SecretStr from app.api.rate_limit import RateLimiter +from app.api.routes import get_agent from app.config import Settings, get_settings from app.database.connections import ConnectionConfig, ConnectionRegistry from app.llm.client import ScriptedLLMClient @@ -33,6 +35,13 @@ def allow_ui_connections(enabled: bool) -> None: app.dependency_overrides[get_settings] = lambda: settings +def test_the_agent_uses_the_configured_number_of_candidates() -> None: + llm = ScriptedLLMClient(lambda system, user: "{}") + request = SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace(llm=llm))) + assert get_agent(request, Settings()).candidates == 1 # type: ignore[arg-type] + assert get_agent(request, Settings(sql_candidates=3)).candidates == 3 # type: ignore[arg-type] + + def test_health_without_databases_is_degraded(client: TestClient) -> None: response = client.get("/api/health") assert response.status_code == 503 diff --git a/backend/tests/test_llm_client.py b/backend/tests/test_llm_client.py index 546581e..0587aba 100644 --- a/backend/tests/test_llm_client.py +++ b/backend/tests/test_llm_client.py @@ -112,6 +112,47 @@ def test_long_retry_after_fails_fast_instead_of_blocking_the_request() -> None: assert sleeps == [] +def test_calls_are_spaced_out_to_stay_under_a_requests_per_minute_limit() -> None: + now = [100.0] + sleeps: list[float] = [] + + def sleep(seconds: float) -> None: + sleeps.append(seconds) + now[0] += seconds + + client = OpenAICompatibleClient( + "https://llm.example.test/v1", + SecretStr("sk-secret"), + "test-model", + transport=httpx.MockTransport(lambda request: httpx.Response(200, json=_reply(_OK))), + sleep=sleep, + requests_per_minute=4, + clock=lambda: now[0], + ) + client.complete_json("s", "u", GeneratedSQL) # the first call goes at once + now[0] += 5.0 + client.complete_json("s", "u", GeneratedSQL) # 5 s later: waits for its slot, 15 s after the first + now[0] += 30.0 + client.complete_json("s", "u", GeneratedSQL) # its slot has passed: no wait + assert sleeps == [10.0] + + +def test_retries_wait_for_their_slot_too() -> None: + sleeps: list[float] = [] + handler = _sequence(httpx.Response(503), httpx.Response(200, json=_reply(_OK))) + client = OpenAICompatibleClient( + "https://llm.example.test/v1", + SecretStr("sk-secret"), + "test-model", + transport=httpx.MockTransport(handler), + sleep=sleeps.append, + requests_per_minute=6, + clock=lambda: 0.0, # the backoff sleep does not move this clock: the slot wait is all of it + ) + client.complete_json("s", "u", GeneratedSQL) + assert sleeps == [1.0, 10.0] # the backoff, then the rest of the 10 s slot + + def test_retries_are_bounded() -> None: calls: list[int] = [] @@ -183,6 +224,8 @@ def test_per_minute_limit_is_a_rate_limit_with_the_providers_wait() -> None: ("response", "code", "retry_after"), [ (httpx.Response(429, headers={"Retry-After": "40"}), "rate_limit", 40), + # No body, only a wait of hours (as Cerebras answers a used-up quota): a used-up quota. + (httpx.Response(429, headers={"Retry-After": "11726"}), "quota_exhausted", 11726), (httpx.Response(429, json={"error": {"code": "insufficient_quota"}}), "quota_exhausted", None), ( httpx.Response( diff --git a/evaluation/report.py b/evaluation/report.py index 0f86d54..d487206 100644 --- a/evaluation/report.py +++ b/evaluation/report.py @@ -96,6 +96,7 @@ def of(behaviour: str) -> list[dict[str, Any]]: "answers": first.get("answers"), "llm_host": first.get("llm_host"), "reasoning_effort": first.get("reasoning_effort"), + "candidates": first.get("candidates", 1), "max_rows": first.get("max_rows"), "planned": len(planned_ids), "scored": len(scored), @@ -154,6 +155,7 @@ def render(m: dict[str, Any]) -> str: f"Model: {m['model']}" + (f" via {m['llm_host']}" if m.get("llm_host") else "") + (f" | reasoning: {m['reasoning_effort']}" if m.get("reasoning_effort") else "") + + (f" | candidates: {m['candidates']}" if m.get("candidates", 1) > 1 else "") + f" | answers: {m['answers']}", *( [f"Dataset: BIRD {m['split']} | scope: {m['scope']} | evidence: {m['evidence']}"] diff --git a/evaluation/run.py b/evaluation/run.py index 4ffcf6a..8f4f481 100644 --- a/evaluation/run.py +++ b/evaluation/run.py @@ -11,6 +11,8 @@ python -m evaluation.run --dataset bird --scope all --evidence off # BIRD's other dev questions, for tuning: a fixed sample stratified by database and difficulty python -m evaluation.run --dataset bird --split dev --sample 300 --daily-tokens 200000 + # several candidate queries per question, the result most agree on wins (2-3x the tokens) + python -m evaluation.run --dataset bird --split dev --sample 30 --candidates 3 # metrics python -m evaluation.report evaluation/results/.jsonl @@ -19,8 +21,9 @@ refused when its row counts differ from those the ground truth was built on. Provider failures (rate limits, outages) are recorded as "not run", never as wrong answers; after several in a row the run stops and can be resumed later. A used-up daily quota stops the run at once; a short -per-minute limit is waited out (as long as the provider asks, up to two minutes) and the question -is asked again. +per-minute limit is waited out (as long as the provider asks, up to two minutes), within the LLM +call where possible, else by asking the question again. With LLM_REQUESTS_PER_MINUTE set, the LLM +calls themselves are spaced out and --delay defaults to 0. Before running, the expected token use is printed: tokens per question as measured in earlier runs of the same model and dataset (never assumed), against --daily-tokens when given. @@ -316,9 +319,11 @@ def measured_tokens_per_question( answers: str, reasoning_effort: str | None = None, results_dir: Path = RESULTS_DIR, + candidates: int = 1, ) -> tuple[float, str] | None: """Mean tokens per scored question in earlier runs of this model on this dataset, with the same - answer mode (LLM answers cost one more call) and reasoning effort (more reasoning, more tokens).""" + answer mode (LLM answers cost one more call), reasoning effort (more reasoning, more tokens) and + number of candidate queries.""" totals: list[int] = [] runs: set[str] = set() for path in sorted(results_dir.glob("*.jsonl")): @@ -328,6 +333,7 @@ def measured_tokens_per_question( and record.get("model") == model and record.get("answers") == answers and record.get("reasoning_effort") == reasoning_effort + and record.get("candidates", 1) == candidates and record.get("correct") is not None and record.get("prompt_tokens") is not None ): @@ -386,8 +392,15 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--category", help="only questions of this category") parser.add_argument("--limit", type=int, help="run at most this many pending questions") parser.add_argument( - "--delay", type=float, default=None, help="seconds between questions (default 15; 0 offline)" + "--delay", + type=float, + default=None, + help="seconds between questions (default 15; 0 offline or with LLM_REQUESTS_PER_MINUTE set)", ) + parser.add_argument( + "--candidates", type=int, choices=range(1, 6), metavar="{1..5}", + help="candidate queries per question, the result most agree on wins (default: SQL_CANDIDATES)", + ) # fmt: skip parser.add_argument("--answers", choices=["template", "llm"], default="template", help="template saves one LLM call per question (default)") # fmt: skip parser.add_argument("--max-provider-failures", type=int, default=3) @@ -436,14 +449,18 @@ def main(argv: list[str] | None = None) -> int: default_rows = bird.BIRD_MAX_ROWS if args.dataset == "bird" else settings.max_rows max_rows = args.max_rows or default_rows + candidates = 1 if args.suite == "sql-safety" else args.candidates or settings.sql_candidates meta: dict[str, Any] = { "dataset": args.dataset, "max_rows": max_rows, "reasoning_effort": settings.llm_reasoning_effort, + "candidates": candidates, } resume = f"python -m evaluation.run --suite {args.suite}" if max_rows != default_rows: resume += f" --max-rows {max_rows}" + if candidates != 1: + resume += f" --candidates {candidates}" if args.dataset == "bird": url = bird.bird_url(settings.database_url) expected_doc = bird.load_expected(bird.expected_path(args.split)) @@ -500,7 +517,7 @@ def main(argv: list[str] | None = None) -> int: if args.run_id: first = next(iter(load_records(path).values()), {}) # Runs from before a setting was recorded used its default then. - defaults = {"dataset": "owid", "max_rows": settings.max_rows, "split": "test"} + defaults = {"dataset": "owid", "max_rows": settings.max_rows, "split": "test", "candidates": 1} settings_then = {k: first.get(k, defaults.get(k)) for k in meta} if settings_then != meta: print(f"Run {run_id} was started with {settings_then}; resume it with the same options.", @@ -524,12 +541,17 @@ def main(argv: list[str] | None = None) -> int: settings.llm_model, max_output_tokens=settings.llm_max_output_tokens, reasoning_effort=settings.llm_reasoning_effort, + requests_per_minute=settings.llm_requests_per_minute, + # Unlike a web request, a run can wait out a per-minute limit inside the call, which + # keeps the question's earlier calls (and candidates) instead of asking it again. + max_wait_seconds=MAX_RATE_LIMIT_WAIT_SECONDS, ) controller = AgentController( llm, max_rows=max_rows, max_retries=settings.max_retries, answer_llm=llm if args.answers == "llm" else None, + candidates=candidates, ) if args.dataset == "bird": with_evidence = args.evidence == "on" @@ -543,7 +565,8 @@ def main(argv: list[str] | None = None) -> int: ask = lambda item: controller.run( # noqa: E731 item["question"], connection, [Turn(**turn) for turn in item.get("history", [])] ) - model, delay = settings.llm_model, 15.0 if args.delay is None else args.delay + default_delay = 0.0 if settings.llm_requests_per_minute else 15.0 + model, delay = settings.llm_model, default_delay if args.delay is None else args.delay if args.ids: wanted = set(args.ids.split(",")) @@ -568,7 +591,9 @@ def main(argv: list[str] | None = None) -> int: per_question = ( None if args.suite == "sql-safety" - else measured_tokens_per_question(args.dataset, model, args.answers, settings.llm_reasoning_effort) + else measured_tokens_per_question( + args.dataset, model, args.answers, settings.llm_reasoning_effort, candidates=candidates + ) ) with ExitStack() as stack: official = stack.enter_context(bird.official_scorer(url)) if args.dataset == "bird" else None diff --git a/evaluation/tests/test_run.py b/evaluation/tests/test_run.py index f7d4e46..167dc9b 100644 --- a/evaluation/tests/test_run.py +++ b/evaluation/tests/test_run.py @@ -293,6 +293,8 @@ def test_token_estimate_uses_measured_use_only(tmp_path: Path) -> None: "prompt_tokens": 2600, "completion_tokens": 400}, {"id": "B5", "dataset": "bird", "model": "m", "answers": "template", "correct": True, "reasoning_effort": "high", "prompt_tokens": 2000, "completion_tokens": 3000}, + {"id": "B6", "dataset": "bird", "model": "m", "answers": "template", "correct": True, + "candidates": 3, "prompt_tokens": 6000, "completion_tokens": 1000}, ] # fmt: skip (tmp_path / "old.jsonl").write_text("\n".join(json.dumps(r) for r in old) + "\n") assert measured_tokens_per_question("bird", "m", "template", results_dir=tmp_path) == ( @@ -304,6 +306,11 @@ def test_token_estimate_uses_measured_use_only(tmp_path: Path) -> None: 5000.0, "1 questions of 1 earlier run(s)", ) + # Several candidates cost several times the calls: only runs with as many count. + assert measured_tokens_per_question("bird", "m", "template", results_dir=tmp_path, candidates=3) == ( + 7000.0, + "1 questions of 1 earlier run(s)", + ) assert measured_tokens_per_question("bird", "m", "llm", results_dir=tmp_path) is None assert measured_tokens_per_question("owid", "m", "template", results_dir=tmp_path) is None