Skip to content

Kill hung runs early when a pinned pool model is delisted #621

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
seonghobae wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Kill hung runs early when a pinned pool model is delisted #621

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 15 additions & 7 deletions scripts/ci/run_opencode_review_model_pool.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,9 @@ write_prompt() {
else
intro="Review PR #\${PR_NUMBER} in \${OPENCODE_SOURCE_WORKDIR} with \${model_candidate}."
fi
contract_file="$OPENCODE_REVIEW_WORKDIR/opencode-review-contract-${model_candidate//\//-}.md"
# Colon-safe: OpenRouter ":free" candidates would otherwise produce file
# names that Windows and actions/upload-artifact reject.
contract_file="$OPENCODE_REVIEW_WORKDIR/opencode-review-contract-${model_candidate//[\/:]/-}.md"
evidence_excerpt_file="$OPENCODE_REVIEW_WORKDIR/bounded-review-evidence-excerpt.md"
evidence_file_in_workdir="$OPENCODE_REVIEW_WORKDIR/bounded-review-evidence.md"
cp "$GITHUB_WORKSPACE/scripts/ci/opencode_review_prompt_template.md" "$contract_file"
Expand Down Expand Up @@ -243,7 +245,7 @@ is_fatal_provider_failure() {
return 0
fi
[ -s "$opencode_json_file" ] || return 1
grep -Eiq 'budget limit|insufficient_quota' "$opencode_json_file"
grep -Eiq 'budget limit|insufficient_quota|model_not_found|model not found|ModelNotFoundError|not a valid model|no endpoints' "$opencode_json_file"
}

has_fatal_provider_error_event() {
Expand All @@ -253,8 +255,12 @@ has_fatal_provider_error_event() {
# Only structured "type":"error" events count while the process is still
# running: model prose or tool output quoting these signatures is
# JSON-escaped inside event strings, so a healthy streaming run is never
# killed for merely discussing context windows or quota errors.
awk 'tolower($0) ~ /"type"[[:space:]]*:[[:space:]]*"error"/ && tolower($0) ~ /contextoverflowerror|tokens_limit_reached|request body too large|context window|budget limit|insufficient_quota/ { found = 1; exit } END { exit !found }' "$opencode_json_file"
# killed for merely discussing context windows, quota errors, or missing
# models. Model-unavailable signatures (OpenRouter "No endpoints found" /
# "not a valid model ID", OpenAI-style model_not_found) matter because a
# delisted pinned free model would otherwise hang and burn the whole
# candidate run budget.
awk 'tolower($0) ~ /"type"[[:space:]]*:[[:space:]]*"error"/ && tolower($0) ~ /contextoverflowerror|tokens_limit_reached|request body too large|context window|budget limit|insufficient_quota|model_not_found|model not found|modelnotfounderror|not a valid model|no endpoints/ { found = 1; exit } END { exit !found }' "$opencode_json_file"
}

emit_sanitized_opencode_failure_detail() {
Expand All @@ -276,6 +282,8 @@ emit_sanitized_opencode_failure_detail() {
failure_class="context-window"
elif grep -Eiq 'budget limit|insufficient_quota|quota exceeded' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then
failure_class="quota-or-budget"
elif grep -Eiq 'model_not_found|model not found|ModelNotFoundError|not a valid model|no endpoints' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then
failure_class="model-unavailable"
elif grep -Eiq 'rate.?limit|too many requests|(^|[^0-9])429([^0-9]|$)' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then
failure_class="rate-limit"
elif grep -Eiq 'permission denied|authentication|authorization|(^|[^0-9])(401|403)([^0-9]|$)' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then
Expand Down Expand Up @@ -427,7 +435,7 @@ run_one_model_attempt() {
printf 'OpenCode %s attempt %s/%s timed out after %ss; falling through within the remaining retry budget instead of blocking the org queue.\n' "$model_candidate" "$attempt" "$attempts" "$run_timeout_seconds"
fi
if is_fatal_provider_failure "$opencode_json_file"; then
printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, or quota); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts"
printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, quota, or model unavailable); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts"
return 2
fi
return 1
Expand All @@ -438,7 +446,7 @@ run_one_model_attempt() {
printf 'OpenCode %s attempt %s/%s JSON output did not include a session id.\n' "$model_candidate" "$attempt" "$attempts"
emit_rejected_opencode_artifact_metadata "sessionless-json" "$opencode_json_file"
if is_fatal_provider_failure "$opencode_json_file"; then
printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, or quota); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts"
printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, quota, or model unavailable); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts"
return 2
fi
return 1
Expand Down Expand Up @@ -533,7 +541,7 @@ main() {
continue
fi
assert_reasoning_effort_for_candidate "$model_candidate"
safe_model="${model_candidate//\//-}"
safe_model="${model_candidate//[\/:]/-}"
prompt_file="${RUNNER_TEMP}/opencode-review-${safe_model}-prompt.md"
candidate_output_file="${RUNNER_TEMP}/opencode-review-${safe_model}.md"
opencode_json_file="${candidate_output_file}.jsonl"
Expand Down
26 changes: 26 additions & 0 deletions tests/test_opencode_model_pool_runner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -576,6 +576,32 @@ def test_model_text_quoting_error_signatures_does_not_kill_run(tmp_path: Path) -
assert "logged a fatal provider error while still running" not in result.stdout


def test_delisted_openrouter_model_error_kills_hung_run_early(tmp_path: Path) -> None:
"""A delisted pinned OpenRouter model dies seconds after a model-unavailable error."""
start = time.monotonic()
result = run_failed_model(
tmp_path,
json_line=(
'{"type":"error","error":{"name":"ProviderModelNotFoundError","data":'
'{"message":"No endpoints found for nvidia/nemotron-3-ultra-550b-a55b:free."}}}'
),
model_candidates="openrouter/nvidia/nemotron-3-ultra-550b-a55b:free",
extra_env={
"OPENROUTER_API_KEY": "fake-openrouter-key",
"FAKE_OPENCODE_HANG_SECONDS": "120",
"OPENCODE_RUN_TIMEOUT_SECONDS": "120",
"OPENCODE_TOTAL_RETRY_BUDGET_SECONDS": "240",
},
)
elapsed = time.monotonic() - start

assert result.returncode == 1
assert "logged a fatal provider error while still running" in result.stdout
assert "skipping remaining attempts for this model" in result.stdout
assert "class=model-unavailable" in result.stdout
assert elapsed < 25


def test_dynamic_review_cadence_uses_small_change_timeout(tmp_path: Path) -> None:
"""Small PRs fail through hung/unavailable providers quickly with a visible budget reason."""
result = run_failed_model(
Expand Down
Loading