From 96d909918526c75d415b171f84592f20b29d8774 Mon Sep 17 00:00:00 2001 From: Intron7 Date: Tue, 4 Aug 2026 11:18:28 +0200 Subject: [PATCH 01/15] add codex and change cost calc Signed-off-by: Intron7 --- .vscode/settings.json | 6 +- CHANGELOG.md | 33 + README.md | 65 + docs/api.md | 2 + pyproject.toml | 13 +- src/acumen/__init__.py | 33 +- src/acumen/_skills/data/SKILL.md | 23 +- .../_skills/data/references/authoring.md | 6 +- .../_skills/data/references/benchmark.md | 28 +- .../_skills/data/references/python-api.md | 50 +- src/acumen/_skills/data/references/setup.md | 10 +- src/acumen/_skills/install.py | 16 +- src/acumen/agents.py | 486 +++ src/acumen/bench.py | 9 +- src/acumen/cli.py | 166 +- src/acumen/config.py | 26 + src/acumen/draft.py | 37 +- src/acumen/env.py | 183 +- src/acumen/improve.py | 52 +- src/acumen/logs.py | 134 +- src/acumen/pricefeed.py | 255 ++ src/acumen/prices.py | 240 ++ src/acumen/prompts.py | 7 +- src/acumen/runner.py | 97 +- src/acumen/sandbox.py | 31 +- src/acumen/scaffold.py | 2 +- src/acumen/ship.py | 63 +- src/acumen/taskgen.py | 59 +- src/acumen/transcript.py | 168 +- tests/conftest.py | 4 +- tests/test_cli.py | 22 +- tests/test_core.py | 626 +++- umask.save | 1 + uv.lock | 3054 +++++++++++++++++ 34 files changed, 5672 insertions(+), 335 deletions(-) create mode 100644 src/acumen/agents.py create mode 100644 src/acumen/pricefeed.py create mode 100644 src/acumen/prices.py create mode 100644 umask.save create mode 100644 uv.lock diff --git a/.vscode/settings.json b/.vscode/settings.json index e034b91..a6bcf49 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -14,5 +14,9 @@ }, "python.analysis.typeCheckingMode": "basic", "python.testing.pytestEnabled": true, - "python.testing.pytestArgs": ["-vv", "--color=yes"], + "python.testing.pytestArgs": [ + "-vv", + "--color=yes" + ], + "cursorpyright.analysis.typeCheckingMode": "basic", } diff --git a/CHANGELOG.md b/CHANGELOG.md index 6700dde..78063dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,3 +9,36 @@ and this project adheres to [Semantic Versioning][]. [semantic versioning]: https://semver.org/ ## [0.0.1dev] + +### Added + +- Run Claude and Codex models side by side in benchmark matrices and use either + provider for drafting, improving, task generation, and shipping. +- Compute `cost_usd` from each run's token breakdown rather than the provider's own + figure, so both providers are priced by one arithmetic path and cached input is billed + at its own rate. The rates used are frozen into `result.json`. +- Add `acumen prices` to show the rate table and `acumen prices --refresh` to diff it + against the providers' published pricing, plus a `prices:` config key to override it. +- Render an HTML transcript for Codex runs too, from the `codex exec` event stream. + +### Changed + +- Make both backends optional, so a Claude-only and a Codex-only install are each complete: + the Claude Agent SDK moves to the `claude` extra (`pip install acumen[claude]`, or + `acumen[all]`) and Codex needs only its CLI on `PATH`. Selecting a model whose backend is + not installed fails preflight with the command that installs it. +- Enforce `max_turns` and `max_usd` for Codex, which has no cap of its own, from its event + stream. Turns are counted in completed model actions rather than `codex exec` invocations — + one invocation is a single Codex turn, so the old count was always 1 — and the run is + stopped at the cap. `max_usd` can only mark the outcome: Codex reports usage when a turn + ends, so an over-budget run is recorded as a `budget` failure after the spend. + +### Fixed + +- Stop feeding Codex transcripts to `claude-code-log`, which reads the SDK-native format + only: it skipped every line, exited 0, and wrote an empty page that was then recorded as a + successfully rendered transcript. +- Drop `Claude` from the drafting and improving prompts, which described the artifact as a + "Claude Skill" even when a Codex agent was writing it for a non-Claude skills directory. +- Remove `check_auth`/`auth_available`, which only ever looked for Claude credentials and + would report a Codex-only setup as unauthenticated. `resolve_auth_mode` replaced them. diff --git a/README.md b/README.md index cecf89e..3054a71 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,62 @@ train/test isolation, and for `draft`/`improve` it is recorded in the version's shown in the report. (Don't paste held-out test answers into `improve` feedback — that would defeat the split.) +Claude and Codex can run side by side. Put both model families in `models` to compare them +in one matrix; model IDs beginning with `claude` use Claude Code, while `gpt-*`, `o1`, +`o3`, `o4`, and `codex-*` use Codex: + +```yaml +models: + - claude-opus-5 + - claude-sonnet-5 + - claude-haiku-4-5-20251001 + - gpt-5.6-sol + - gpt-5.6-terra + - gpt-5.6-luna +``` + +This spans each provider's quality/cost range; it is not a claim that the tiers are +one-to-one equivalents. + +Neither backend is required. Claude is an optional dependency and Codex is an external CLI, +so install only the one you run — `pip install acumen[claude]`, or plain `acumen` plus the +`codex` CLI on `PATH`. Selecting a model whose backend is missing fails immediately, with the +install command, before acumen prepares a target or spends anything. + +Claude API runs use `ANTHROPIC_API_KEY`; Codex API runs use `CODEX_API_KEY` (or +`OPENAI_API_KEY`). The meta-agent commands also accept a Codex model through their +`*_model` config keys or `--model`, and `--auth auto` prefers that provider's logged-in +subscription. + +`max_turns` and `max_usd` apply to both providers, but they are not equally strict for Codex, +which has no cap of its own — acumen enforces both against its event stream: + +- **`max_turns` bounds the run.** One `codex exec` is a single Codex turn however much work + happens inside it, so turns are counted in completed model actions (a message, a command, a + file change, a tool or search call) and the agent is stopped at the cap. +- **`max_usd` cannot.** Codex reports usage once, when the turn ends, so a breach is only + visible after the money is spent. The run is recorded as a budget failure — the same outcome + Claude gives it — but bound Codex spend with `max_turns`. acumen prints this before the pass. + +**Cost is computed from tokens, not taken from the provider.** Only one of the two +providers reports a billed dollar figure, so a mixed matrix would be comparing a real +number against a zero. Every run instead records its token breakdown — fresh input, cache +reads, cache writes, output, which are priced up to 10x apart — and `cost_usd` is derived +from a rate table, identical arithmetic for both providers. The rates used are frozen into +each `result.json`, so a pass benchmarked in January is not silently re-priced in July. + +```bash +acumen prices # the rates in use, and where each came from +acumen prices --refresh # re-check them against the providers' pricing pages +``` + +`--refresh` fetches both providers' published tables and prints a diff for you to accept — +it never rewrites anything, because picking the wrong tier or context band would silently +misprice every future run. Adopt changes by pasting the emitted block into `config.yaml` +under `prices:`, which is also how you price a model acumen doesn't ship a rate for, or +override rates for a gateway. A model with no rate records its tokens and leaves `cost_usd` +unset — never zero, which would read as free. + `draft`, `improve`, `tasks`, and `ship` each drive a long autonomous agent. Every run writes a live `logs/acumen--.jsonl` (one event per step, flushed as it goes — so you can watch progress by reading the file) and a rendered `.html` transcript. Add `--stream` to @@ -82,6 +138,15 @@ in particular, the [API documentation][]. You need to have Python 3.12 or newer installed on your system. If you don't have Python installed, we recommend installing [uv][]. +Install the backend you actually run — both are optional, and either alone is a complete +install: + +| you run | install | also needs | +|---|---|---| +| Claude only | `pip install acumen[claude]` | an Anthropic key or a `claude` login | +| Codex only | `pip install acumen` | the `codex` CLI on `PATH`, plus a Codex login or key | +| both | `pip install acumen[all]` | both of the above | +