From c653803345ae8616ea694d8875f1eef340c5c616 Mon Sep 17 00:00:00 2001 From: Renxiong Wang Date: Tue, 18 Aug 2026 09:55:53 -0700 Subject: [PATCH 1/2] feat(run_config): Gemini via Google's Antigravity CLI, all three tracks Adds gemini-3p6-flash_antigravity.sh for qa, tw and rf, following the existing opus-4p6_claude-code.sh pattern. Runs Gemini under Google's own agent scaffold rather than Claude Code or mini-swe-agent, which is the comparison people benchmarking Gemini here are usually after. Each track keeps its own harness settings (keepalive for qa/tw, modal_vm_runtime for rf, the allowlisted agent host for tw). Two caveats documented in the README rather than left to be rediscovered: Harbor's antigravity-cli agent needs harbor-framework/harbor#2780 before it authenticates headlessly or emits a trajectory at all, and agy rejects model ids outside its client-side allowlist before any request is sent, so --model failures are not quota or auth problems. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 18 ++++++++++ run_config/qa/gemini-3p6-flash_antigravity.sh | 30 ++++++++++++++++ run_config/rf/gemini-3p6-flash_antigravity.sh | 30 ++++++++++++++++ run_config/tw/gemini-3p6-flash_antigravity.sh | 35 +++++++++++++++++++ 4 files changed, 113 insertions(+) create mode 100755 run_config/qa/gemini-3p6-flash_antigravity.sh create mode 100755 run_config/rf/gemini-3p6-flash_antigravity.sh create mode 100755 run_config/tw/gemini-3p6-flash_antigravity.sh diff --git a/README.md b/README.md index 3e64122..9c4f0e5 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,24 @@ bash run_config/tw/opus-4p6_claude-code.sh To create your own config, copy an existing one and modify the agent, model, and parameters. +### Gemini via the Antigravity CLI + +`run_config/*/gemini-3p6-flash_antigravity.sh` runs Gemini under Google's own +Antigravity CLI (`agy`) instead of Claude Code or mini-swe-agent. Set +`GEMINI_API_KEY` in `.env`; the allowed agent host is +`generativelanguage.googleapis.com`. + +Two things to know before trusting the numbers: + +- Harbor's `antigravity-cli` agent needs the fixes in + [harbor#2780](https://github.com/harbor-framework/harbor/pull/2780). Without + them settings are written where `agy` does not read them, `GEMINI_API_KEY` is + ignored in favour of an interactive login that hangs headless, and no + trajectory is collected. +- `agy` rejects model ids outside its client-side allowlist before any request + leaves the process, so `--model` failures are not quota or auth problems. + Which ids are accepted depends on your key's entitlement. + ## Citation If you use SWE Atlas in your research, please cite our paper: diff --git a/run_config/qa/gemini-3p6-flash_antigravity.sh b/run_config/qa/gemini-3p6-flash_antigravity.sh new file mode 100755 index 0000000..48ddc98 --- /dev/null +++ b/run_config/qa/gemini-3p6-flash_antigravity.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Gemini under Google's own Antigravity CLI (agy) rather than Claude Code or +# mini-swe-agent. Requires GEMINI_API_KEY in .env. +# +# The model id is what agy's client-side allowlist accepts in Gemini Developer +# API mode; newer ids depend on your key's entitlement and are rejected before +# any request is sent, so change it only if you have verified yours. +set -euo pipefail + +# Load credentials +set -a +source "$(dirname "$0")/../../.env" +set +a + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +: "${GEMINI_API_KEY:?Set GEMINI_API_KEY in .env}" + +harbor run \ + -p ./data/qa \ + -a antigravity-cli \ + -m "google/gemini-3.6-flash" \ + -e modal \ + --ek 'keepalive=["-c","sleep infinity"]' \ + -k 1 \ + -n 12 \ + --ae "GEMINI_API_KEY=${GEMINI_API_KEY}" \ + --allow-agent-host generativelanguage.googleapis.com \ + --agent-include-logs '**' \ + -o results/qa/ \ + --job-name "gemini-3p6-flash_antigravity" diff --git a/run_config/rf/gemini-3p6-flash_antigravity.sh b/run_config/rf/gemini-3p6-flash_antigravity.sh new file mode 100755 index 0000000..c1826e2 --- /dev/null +++ b/run_config/rf/gemini-3p6-flash_antigravity.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Gemini under Google's own Antigravity CLI (agy) rather than Claude Code or +# mini-swe-agent. Requires GEMINI_API_KEY in .env. +# +# The model id is what agy's client-side allowlist accepts in Gemini Developer +# API mode; newer ids depend on your key's entitlement and are rejected before +# any request is sent, so change it only if you have verified yours. +set -euo pipefail + +# Load credentials +set -a +source "$(dirname "$0")/../../.env" +set +a + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +: "${GEMINI_API_KEY:?Set GEMINI_API_KEY in .env}" + +harbor run \ + -p ./data/rf \ + -a antigravity-cli \ + -m "google/gemini-3.6-flash" \ + -e modal \ + --ek modal_vm_runtime=true \ + -k 1 \ + -n 12 \ + --ae "GEMINI_API_KEY=${GEMINI_API_KEY}" \ + --allow-agent-host generativelanguage.googleapis.com \ + --agent-include-logs '**' \ + -o results/rf/ \ + --job-name "gemini-3p6-flash_antigravity" diff --git a/run_config/tw/gemini-3p6-flash_antigravity.sh b/run_config/tw/gemini-3p6-flash_antigravity.sh new file mode 100755 index 0000000..7112fd8 --- /dev/null +++ b/run_config/tw/gemini-3p6-flash_antigravity.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Gemini under Google's own Antigravity CLI (agy) rather than Claude Code or +# mini-swe-agent. Requires GEMINI_API_KEY in .env. +# +# The model id is what agy's client-side allowlist accepts in Gemini Developer +# API mode; newer ids depend on your key's entitlement and are rejected before +# any request is sent, so change it only if you have verified yours. +# +# TW runs with network access restricted to an allowlist, so the agent's own +# API host has to be named explicitly -- see HARBOR_AGENT_ALLOWED_HOST in the +# README. For agy that host is generativelanguage.googleapis.com. +set -euo pipefail + +# Load credentials +set -a +source "$(dirname "$0")/../../.env" +set +a + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +HARBOR_BIN="${HARBOR_BIN:-harbor}" +: "${GEMINI_API_KEY:?Set GEMINI_API_KEY in .env}" + +"$HARBOR_BIN" run \ + -p ./data/tw \ + -a antigravity-cli \ + -m "google/gemini-3.6-flash" \ + -e modal \ + --ek 'keepalive=["-c","sleep infinity"]' \ + -k 1 \ + -n 12 \ + --ae "GEMINI_API_KEY=${GEMINI_API_KEY}" \ + --allow-agent-host "${HARBOR_AGENT_ALLOWED_HOST:-generativelanguage.googleapis.com}" \ + --agent-include-logs '**' \ + -o results/tw/ \ + --job-name "gemini-3p6-flash_antigravity" From 5a4c415bca7c5cf7ae2417409fa6ea9b133f9632 Mon Sep 17 00:00:00 2001 From: Renxiong Wang Date: Tue, 18 Aug 2026 11:20:02 -0700 Subject: [PATCH 2/2] docs: point at harbor#2776 and #2687 instead of the closed #2780 #2780 was superseded before review; the auth/settings fixes live in #2776 and trajectory export in #2687. Also notes the --add-dir finding from #2776, which is the one most likely to affect Test Writing scores. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9c4f0e5..b641b56 100644 --- a/README.md +++ b/README.md @@ -76,11 +76,14 @@ Antigravity CLI (`agy`) instead of Claude Code or mini-swe-agent. Set Two things to know before trusting the numbers: -- Harbor's `antigravity-cli` agent needs the fixes in - [harbor#2780](https://github.com/harbor-framework/harbor/pull/2780). Without - them settings are written where `agy` does not read them, `GEMINI_API_KEY` is - ignored in favour of an interactive login that hangs headless, and no - trajectory is collected. +- Harbor's `antigravity-cli` agent needs + [harbor#2776](https://github.com/harbor-framework/harbor/pull/2776) and + [harbor#2687](https://github.com/harbor-framework/harbor/pull/2687), both open + but unmerged as of 2026-08-18. Without them, settings are written where `agy` + does not read them, `GEMINI_API_KEY` is ignored in favour of an interactive + login that hangs headless, no trajectory is collected, and — absent + `--add-dir` — file-producing tasks can write outside the workdir and score + zero. Test Writing is the track most exposed to that last one. - `agy` rejects model ids outside its client-side allowlist before any request leaves the process, so `--model` failures are not quota or auth problems. Which ids are accepted depends on your key's entitlement.