BIRD: output cap, reasoning effort, candidate voting settings and LLM call pacing

.env.example

@@ -25,6 +25,9 @@ MAX_ROWS=1000
 # llm: answers written by the LLM and checked against the result (one extra call per question)
 # template: answers built from the rows, no extra LLM call (e.g. for evaluation runs)
 ANSWER_MODE=llm
+# Candidate queries per question (1-5). Above 1, the result most candidates agree on wins: about
+# 2-3x the LLM calls and tokens of a single candidate. Evaluation runs can set it with --candidates.
+SQL_CANDIDATES=1
 
 # --- Rate limits on POST /api/query (the endpoint that spends LLM quota; 0 = off) ---
 # Per client address per minute, and across all clients per 24 hours. Each question makes
@@ -45,6 +48,18 @@ LLM_MODEL=gemini-3.8-flash
 # Groq free tier (key: https://console.groq.com/keys):
 #   LLM_BASE_URL=https://api.groq.com/openai/v1   LLM_MODEL=openai/gpt-oss-120b
 #   (the evaluated model; free tier: 8K tokens/minute, 200K tokens/day)
+# Cerebras (key: https://cloud.cerebras.ai; trial for gpt-oss-120b: $5 credit for 30 days,
+# 5 requests/minute, 30K uncached tokens/minute, 1M tokens/hour and per day):
+#   LLM_BASE_URL=https://api.cerebras.ai/v1   LLM_MODEL=gpt-oss-120b   LLM_MAX_OUTPUT_TOKENS=4096
+#   LLM_REQUESTS_PER_MINUTE=4   (each call reserves prompt + output cap, ~7.5k tokens, against
+#   the 30K per minute: 4 calls fit, 5 may not)
+# Optional: cap on each reply, reasoning included (Cerebras reserves the cap against its rate
+# limits before running a request), reasoning effort for reasoning models (low|medium|high), and
+# spacing of LLM calls to stay under a per-minute request limit. Unset: the provider's defaults,
+# calls not spaced.
+# LLM_MAX_OUTPUT_TOKENS=
+# LLM_REASONING_EFFORT=
+# LLM_REQUESTS_PER_MINUTE=
 
 # --- Host ports published by docker compose (change when 5432 / 8000 are already in use;
 # then use the same port in DATABASE_URL and ADMIN_DATABASE_URL) ---

EVALUATION_PLAN.md

@@ -129,6 +129,14 @@ compares it with --daily-tokens when given (Groq's daily token limit is not in i
 headers). Provider failures are handled by kind: a per-minute limit is waited out when the
 provider asks for at most two minutes and the question asked again; a used-up daily or account
 quota stops the run at once and says when to resume; other failures stop it after three in a row.
+A 429 asking for a wait of more than ten minutes counts as a used-up quota even without a message
+saying so (Cerebras can answer with no body, only Retry-After). Within a run, a short per-minute
+wait is first waited out inside the LLM call, so the question's earlier calls are kept; with
+LLM_REQUESTS_PER_MINUTE set the calls are spaced out so the limit is not hit at all, and --delay
+defaults to 0. --candidates N (default SQL_CANDIDATES, 1) generates up to N queries per question and
+keeps the result most agree on; it stops early once the majority is decided, so it costs 2-3x the
+calls and tokens of one candidate at N = 3. Runs record the number, and the token estimate only
+counts earlier runs with the same number.
 
 ⸻
 

README.md

@@ -104,9 +104,11 @@ All settings are environment variables; [.env.example](.env.example) documents e
 | Variable | Purpose |
 |---|---|
 | `LLM_BASE_URL`, `LLM_API_KEY`, `LLM_MODEL` | Any OpenAI-compatible provider |
+| `LLM_MAX_OUTPUT_TOKENS`, `LLM_REASONING_EFFORT`, `LLM_REQUESTS_PER_MINUTE` | Optional: reply cap, reasoning effort, spacing of LLM calls for small per-minute limits |
 | `DATABASE_URL` / `DATABASES_CONFIG` | One database, or several via [databases.example.toml](databases.example.toml) |
 | `QUERY_TIMEOUT_SECONDS`, `MAX_ROWS`, `MAX_RETRIES` | Query guardrails |
 | `ANSWER_MODE` | `llm` (checked against the rows) or `template` (no extra LLM call) |
+| `SQL_CANDIDATES` | Candidate queries per question; above 1 the majority result wins (2-3x the LLM calls) |
 | `RATE_LIMIT_PER_MINUTE`, `RATE_LIMIT_PER_DAY`, `TRUSTED_PROXY_HOPS` | Limits on `/api/query` |
 | `ALLOW_UI_CONNECTIONS` | Local use only |
 

backend/app/agent/sql_generator.py

@@ -17,7 +17,7 @@
 from app.agent.schema_retriever import SchemaContext
 from app.llm.client import LLMCall, LLMClient
 
-PROMPT_VERSION = "sql-generator/6"
+PROMPT_VERSION = "sql-generator/7"
 MAX_HISTORY_TURNS = 3
 
 SYSTEM_PROMPT = """\
@@ -45,6 +45,9 @@
   count.
 - Return stored values as they are. Do not round, cast or reformat a returned value unless the
   question asks for it; cast only where a calculation needs it (e.g. to avoid integer division).
+- Return each stored column as its own result column. Never concatenate columns into one, even
+  when the question or a definition names them together ("address means street, city"
+  returns street and city as two columns).
 - When listing entities through a join that can repeat them, return each entity once.
 - Do not add conditions the question does not state (such as only the latest record, or only
   rows where another column is not NULL).

backend/app/api/routes.py

@@ -47,6 +47,7 @@ def get_agent(request: Request, settings: Annotated[Settings, Depends(get_settin
         max_rows=settings.max_rows,
         max_retries=settings.max_retries,
         answer_llm=llm if settings.answer_mode == "llm" else None,
+        candidates=settings.sql_candidates,
     )
 
 

backend/app/config.py

@@ -28,11 +28,22 @@ class Settings(BaseSettings):
     # "llm": natural-language answers written by the LLM (one extra call per question) and checked
     # against the rows; "template": answers built from the rows without the LLM.
     answer_mode: Literal["llm", "template"] = "llm"
+    # Candidate queries per question: above 1, the result most candidates agree on wins (about
+    # 2-3x the LLM calls and tokens of one candidate). See app/agent/controller.py.
+    sql_candidates: int = Field(default=1, ge=1, le=5)
 
     # Any OpenAI-compatible chat-completions endpoint (Groq, Gemini, OpenRouter, Ollama, ...).
     llm_base_url: str = "https://generativelanguage.googleapis.com/v1beta/openai/"
     llm_api_key: SecretStr | None = None
     llm_model: str = "gemini-3.8-flash"
+    # Cap on each reply, reasoning included. Unset: the provider's default. Set it where the
+    # provider reserves the cap against its rate limits before running a request (Cerebras).
+    llm_max_output_tokens: int | None = Field(default=None, ge=256, le=65_536)
+    # For reasoning models (gpt-oss: low, medium, high). Unset: the provider's default.
+    llm_reasoning_effort: Literal["low", "medium", "high"] | None = None
+    # Spaces LLM calls out to stay under a small per-minute request limit (a trial tier).
+    # Unset: calls are not spaced (the provider's 429s are still retried or reported).
+    llm_requests_per_minute: int | None = Field(default=None, ge=1, le=10_000)
 
     # Limits on POST /api/query, the endpoint that spends LLM quota (see app/api/rate_limit.py).
     # 0 turns a limit off. Per process: several workers or instances each enforce their own.

backend/app/llm/client.py

@@ -8,13 +8,16 @@
 with a short wait, so a free-tier hiccup does not fail the whole question. A failure that waiting
 will not fix (a daily or account quota used up) is not retried, and is reported as such: LLMError.code says
 what kind of failure it was, and retry_after_seconds when the provider said how long to wait.
+With requests_per_minute set, calls are spaced out so a small per-minute limit (a trial tier)
+is not hit in the first place.
 """
 
 from __future__ import annotations
 
 import json
 import logging
 import re
+import threading
 import time
 from collections.abc import Callable
 from dataclasses import dataclass
@@ -41,6 +44,9 @@
 # Provider error codes are reported only when they look like an identifier ("rate_limit_exceeded").
 _PROVIDER_CODE = re.compile(r"[A-Za-z][A-Za-z0-9_.-]{0,39}")
 MAX_RETRY_AFTER_SECONDS = 2 * 24 * 3600
+# A 429 that asks for a longer wait than this is a used-up quota for practical purposes, even when
+# the body does not say so (Cerebras can answer 429 with no body and only a Retry-After header).
+QUOTA_WAIT_SECONDS = 600
 
 
 class LLMError(RuntimeError):
@@ -130,11 +136,25 @@ def __init__(
         max_wait_seconds: float = 8.0,
         transport: httpx.BaseTransport | None = None,
         sleep: Callable[[float], None] = time.sleep,
+        max_output_tokens: int | None = None,
+        reasoning_effort: str | None = None,
+        requests_per_minute: int | None = None,
+        clock: Callable[[], float] = time.monotonic,
     ) -> None:
+        """max_output_tokens caps each reply, reasoning included (some providers reserve the cap
+        against their rate limits before a request runs); reasoning_effort ("low", "medium",
+        "high") is passed to reasoning models. Either is sent only when set."""
         self.model = model
+        self.max_output_tokens = max_output_tokens
+        self.reasoning_effort = reasoning_effort
         self.max_attempts = max_attempts
         self.max_wait_seconds = max_wait_seconds
         self._sleep = sleep
+        self._clock = clock
+        # Calls start at least this far apart (retries included); shared by every thread.
+        self._interval = 60.0 / requests_per_minute if requests_per_minute else 0.0
+        self._next_start = 0.0
+        self._pace_lock = threading.Lock()
         self._http = httpx.Client(
             base_url=base_url.rstrip("/") + "/",
             headers={"Authorization": f"Bearer {api_key.get_secret_value()}"},
@@ -151,10 +171,16 @@ def complete_json(
             "response_format": {"type": "json_object"},
             "messages": [{"role": "system", "content": system}, {"role": "user", "content": user}],
         }
+        if self.max_output_tokens is not None:
+            payload["max_tokens"] = self.max_output_tokens  # the name every compatible API accepts
+        if self.reasoning_effort is not None:
+            payload["reasoning_effort"] = self.reasoning_effort
         started = time.perf_counter()
+        paced = 0.0  # time spent waiting for a slot: not part of the call's duration
         attempt = 0
         while True:
             attempt += 1
+            paced += self._pace()
             try:
                 response = self._http.post("chat/completions", json=payload)
             except httpx.HTTPError as exc:
@@ -167,7 +193,7 @@ def complete_json(
             if wait > self.max_wait_seconds:
                 break  # e.g. a per-minute quota: waiting that long inside a request is worse than failing
             self._sleep(wait)
-        duration_ms = int((time.perf_counter() - started) * 1000)
+        duration_ms = max(0, int((time.perf_counter() - started - paced) * 1000))
         log_event(
             logger,
             "llm call",
@@ -190,6 +216,18 @@ def complete_json(
         )
         return parse_output(content, output), call
 
+    def _pace(self) -> float:
+        """Wait for this call's slot when calls are spaced out (requests_per_minute); the seconds waited."""
+        if not self._interval:
+            return 0.0
+        with self._pace_lock:
+            now = self._clock()
+            start = max(now, self._next_start)
+            self._next_start = start + self._interval
+        if start > now:
+            self._sleep(start - now)
+        return start - now
+
     @staticmethod
     def _retry_wait(response: httpx.Response, attempt: int) -> float:
         """Seconds to wait: the provider's Retry-After if given, else 1s, 2s, 4s, ..."""
@@ -200,7 +238,10 @@ def _retry_wait(response: httpx.Response, attempt: int) -> float:
 
 
 def _quota_exhausted(response: httpx.Response) -> bool:
-    return response.status_code == 429 and bool(_EXHAUSTED_QUOTA.search(response.text))
+    if response.status_code != 429:
+        return False
+    wait = _retry_after(response)
+    return bool(_EXHAUSTED_QUOTA.search(response.text)) or (wait is not None and wait > QUOTA_WAIT_SECONDS)
 
 
 def _retry_after(response: httpx.Response) -> int | None:

backend/app/main.py

@@ -28,7 +28,14 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     )
     app.state.registry = registry
     app.state.llm = (
-        OpenAICompatibleClient(settings.llm_base_url, settings.llm_api_key, settings.llm_model)
+        OpenAICompatibleClient(
+            settings.llm_base_url,
+            settings.llm_api_key,
+            settings.llm_model,
+            max_output_tokens=settings.llm_max_output_tokens,
+            reasoning_effort=settings.llm_reasoning_effort,
+            requests_per_minute=settings.llm_requests_per_minute,
+        )
         if settings.llm_api_key and settings.llm_api_key.get_secret_value()
         else None
     )

backend/tests/test_api.py

@@ -2,12 +2,14 @@
 
 import json
 from collections.abc import Iterator
+from types import SimpleNamespace
 
 import pytest
 from fastapi.testclient import TestClient
 from pydantic import SecretStr
 
 from app.api.rate_limit import RateLimiter
+from app.api.routes import get_agent
 from app.config import Settings, get_settings
 from app.database.connections import ConnectionConfig, ConnectionRegistry
 from app.llm.client import ScriptedLLMClient
@@ -33,6 +35,13 @@ def allow_ui_connections(enabled: bool) -> None:
     app.dependency_overrides[get_settings] = lambda: settings
 
 
+def test_the_agent_uses_the_configured_number_of_candidates() -> None:
+    llm = ScriptedLLMClient(lambda system, user: "{}")
+    request = SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace(llm=llm)))
+    assert get_agent(request, Settings()).candidates == 1  # type: ignore[arg-type]
+    assert get_agent(request, Settings(sql_candidates=3)).candidates == 3  # type: ignore[arg-type]
+
+
 def test_health_without_databases_is_degraded(client: TestClient) -> None:
     response = client.get("/api/health")
     assert response.status_code == 503

backend/tests/test_llm_client.py

@@ -54,6 +54,28 @@ def handler(request: httpx.Request) -> httpx.Response:
     assert [m["role"] for m in seen["body"]["messages"]] == ["system", "user"]
 
 
+def test_output_cap_and_reasoning_effort_are_sent_only_when_set() -> None:
+    bodies: list[dict[str, Any]] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        bodies.append(json.loads(request.content))
+        return httpx.Response(200, json=_reply('{"sql": "SELECT 1", "explanation": "x"}'))
+
+    _client(handler).complete_json("s", "u", GeneratedSQL)
+    configured = OpenAICompatibleClient(
+        "https://llm.example.test/v1",
+        SecretStr("sk-secret"),
+        "test-model",
+        transport=httpx.MockTransport(handler),
+        max_output_tokens=4096,
+        reasoning_effort="high",
+    )
+    configured.complete_json("s", "u", GeneratedSQL)
+    assert "max_tokens" not in bodies[0] and "reasoning_effort" not in bodies[0]
+    assert (bodies[1]["max_tokens"], bodies[1]["reasoning_effort"]) == (4096, "high")
+    assert "max_completion_tokens" not in bodies[1]  # an alias of max_tokens: never send both
+
+
 def test_http_errors_do_not_leak_the_response_body_or_key() -> None:
     client = _client(lambda request: httpx.Response(401, text="invalid key sk-secret"))
     with pytest.raises(LLMError) as info:
@@ -90,6 +112,47 @@ def test_long_retry_after_fails_fast_instead_of_blocking_the_request() -> None:
     assert sleeps == []
 
 
+def test_calls_are_spaced_out_to_stay_under_a_requests_per_minute_limit() -> None:
+    now = [100.0]
+    sleeps: list[float] = []
+
+    def sleep(seconds: float) -> None:
+        sleeps.append(seconds)
+        now[0] += seconds
+
+    client = OpenAICompatibleClient(
+        "https://llm.example.test/v1",
+        SecretStr("sk-secret"),
+        "test-model",
+        transport=httpx.MockTransport(lambda request: httpx.Response(200, json=_reply(_OK))),
+        sleep=sleep,
+        requests_per_minute=4,
+        clock=lambda: now[0],
+    )
+    client.complete_json("s", "u", GeneratedSQL)  # the first call goes at once
+    now[0] += 5.0
+    client.complete_json("s", "u", GeneratedSQL)  # 5 s later: waits for its slot, 15 s after the first
+    now[0] += 30.0
+    client.complete_json("s", "u", GeneratedSQL)  # its slot has passed: no wait
+    assert sleeps == [10.0]
+
+
+def test_retries_wait_for_their_slot_too() -> None:
+    sleeps: list[float] = []
+    handler = _sequence(httpx.Response(503), httpx.Response(200, json=_reply(_OK)))
+    client = OpenAICompatibleClient(
+        "https://llm.example.test/v1",
+        SecretStr("sk-secret"),
+        "test-model",
+        transport=httpx.MockTransport(handler),
+        sleep=sleeps.append,
+        requests_per_minute=6,
+        clock=lambda: 0.0,  # the backoff sleep does not move this clock: the slot wait is all of it
+    )
+    client.complete_json("s", "u", GeneratedSQL)
+    assert sleeps == [1.0, 10.0]  # the backoff, then the rest of the 10 s slot
+
+
 def test_retries_are_bounded() -> None:
     calls: list[int] = []
 
@@ -122,6 +185,8 @@ def handler(request: httpx.Request) -> httpx.Response:
         "code": "rate_limit_exceeded",
     }
 }
+
+
 GROQ_MINUTE = {
     "error": {
         "message": "Rate limit reached on tokens per minute (TPM): Limit 8000. Please try again in 41.5s.",
@@ -159,6 +224,8 @@ def test_per_minute_limit_is_a_rate_limit_with_the_providers_wait() -> None:
     ("response", "code", "retry_after"),
     [
         (httpx.Response(429, headers={"Retry-After": "40"}), "rate_limit", 40),
+        # No body, only a wait of hours (as Cerebras answers a used-up quota): a used-up quota.
+        (httpx.Response(429, headers={"Retry-After": "11726"}), "quota_exhausted", 11726),
         (httpx.Response(429, json={"error": {"code": "insufficient_quota"}}), "quota_exhausted", None),
         (
             httpx.Response(

evaluation/report.py

@@ -94,6 +94,9 @@ def of(behaviour: str) -> list[dict[str, Any]]:
         "code_commit": first.get("code_commit"),
         "dataset_commit": first.get("dataset_commit"),
         "answers": first.get("answers"),
+        "llm_host": first.get("llm_host"),
+        "reasoning_effort": first.get("reasoning_effort"),
+        "candidates": first.get("candidates", 1),
         "max_rows": first.get("max_rows"),
         "planned": len(planned_ids),
         "scored": len(scored),
@@ -149,7 +152,11 @@ def render(m: dict[str, Any]) -> str:
     status = "complete" if m["complete"] else f"PARTIAL: {m['not_run']} of {m['planned']} not run"
     lines = [
         f"Run {m['run_id']} ({m['suite']}, {status})",
-        f"Model: {m['model']} | answers: {m['answers']}",
+        f"Model: {m['model']}"
+        + (f" via {m['llm_host']}" if m.get("llm_host") else "")
+        + (f" | reasoning: {m['reasoning_effort']}" if m.get("reasoning_effort") else "")
+        + (f" | candidates: {m['candidates']}" if m.get("candidates", 1) > 1 else "")
+        + f" | answers: {m['answers']}",
         *(
             [f"Dataset: BIRD {m['split']} | scope: {m['scope']} | evidence: {m['evidence']}"]
             if m["dataset"] == "bird"